accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject [10/10] git commit: ACCUMULO-1000 compressed iterator config in conditional writer RPC calls
Date Wed, 17 Jul 2013 12:58:24 GMT
ACCUMULO-1000 compressed iterator config in conditional writer RPC calls

Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/be54c959
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/be54c959
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/be54c959

Branch: refs/heads/ACCUMULO-1000
Commit: be54c959d2e04556401d62acceaae6a2ddbc62bc
Parents: 3a2fca3
Author: Keith Turner <kturner@apache.org>
Authored: Tue Jul 16 19:17:44 2013 -0400
Committer: Keith Turner <kturner@apache.org>
Committed: Tue Jul 16 19:17:44 2013 -0400

----------------------------------------------------------------------
 .../core/client/impl/CompressedIterators.java   | 131 +++++
 .../core/client/impl/ConditionalWriterImpl.java |  37 +-
 .../accumulo/core/data/ArrayByteSequence.java   |  13 +
 .../org/apache/accumulo/core/data/Mutation.java | 130 +----
 .../accumulo/core/data/thrift/TCondition.java   | 379 ++-----------
 .../accumulo/core/file/rfile/RelativeKey.java   |  33 +-
 .../thrift/TabletClientService.java             | 556 ++++++++++++-------
 .../core/util/UnsynchronizedBuffer.java         | 195 +++++++
 core/src/main/thrift/data.thrift                |   3 +-
 core/src/main/thrift/tabletserver.thrift        |   2 +-
 .../core/file/rfile/RelativeKeyTest.java        |  31 +-
 .../server/tabletserver/TabletServer.java       |  21 +-
 .../test/performance/thrift/NullTserver.java    |   2 +-
 .../accumulo/test/ConditionalWriterTest.java    |  57 +-
 14 files changed, 865 insertions(+), 725 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/be54c959/core/src/main/java/org/apache/accumulo/core/client/impl/CompressedIterators.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/CompressedIterators.java
b/core/src/main/java/org/apache/accumulo/core/client/impl/CompressedIterators.java
new file mode 100644
index 0000000..549322e
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/CompressedIterators.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.impl;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.util.UnsynchronizedBuffer;
+
+public class CompressedIterators {
+  private Map<String,Integer> symbolMap;
+  private List<String> symbolTable;
+  private Map<ByteSequence,IterConfig> cache;
+  
+  public static class IterConfig {
+    public List<IterInfo> ssiList = new ArrayList<IterInfo>();
+    public Map<String,Map<String,String>> ssio = new HashMap<String,Map<String,String>>();
+  }
+
+  public CompressedIterators() {
+    symbolMap = new HashMap<String,Integer>();
+    symbolTable = new ArrayList<String>();
+  }
+  
+  public CompressedIterators(List<String> symbols) {
+    this.symbolTable = symbols;
+    this.cache = new HashMap<ByteSequence,IterConfig>();
+  }
+
+  private int getSymbolID(String symbol) {
+    Integer id = symbolMap.get(symbol);
+    if (id == null) {
+      id = symbolTable.size();
+      symbolTable.add(symbol);
+      symbolMap.put(symbol, id);
+    }
+    
+    return id;
+  }
+  
+  public ByteBuffer compress(IteratorSetting[] iterators) {
+    
+    UnsynchronizedBuffer.Writer out = new UnsynchronizedBuffer.Writer(iterators.length *
8);
+    
+    out.writeVInt(iterators.length);
+
+    for (IteratorSetting is : iterators) {
+      out.writeVInt(getSymbolID(is.getName()));
+      out.writeVInt(getSymbolID(is.getIteratorClass()));
+      out.writeVInt(is.getPriority());
+      
+      Map<String,String> opts = is.getOptions();
+      out.writeVInt(opts.size());
+      
+      for (Entry<String,String> entry : opts.entrySet()) {
+        out.writeVInt(getSymbolID(entry.getKey()));
+        out.writeVInt(getSymbolID(entry.getValue()));
+      }
+    }
+    
+    return out.toByteBuffer();
+    
+  }
+  
+  public IterConfig decompress(ByteBuffer iterators) {
+    
+    ByteSequence iterKey = new ArrayByteSequence(iterators);
+    IterConfig config = cache.get(iterKey);
+    if (config != null) {
+      return config;
+    }
+
+    config = new IterConfig();
+
+    UnsynchronizedBuffer.Reader in = new UnsynchronizedBuffer.Reader(iterators);
+
+    int num = in.readVInt();
+    
+    for (int i = 0; i < num; i++) {
+      String name = symbolTable.get(in.readVInt());
+      String iterClass = symbolTable.get(in.readVInt());
+      int prio = in.readVInt();
+      
+      config.ssiList.add(new IterInfo(prio, iterClass, name));
+      
+      int numOpts = in.readVInt();
+      
+      HashMap<String,String> opts = new HashMap<String,String>();
+      
+      for (int j = 0; j < numOpts; j++) {
+        String key = symbolTable.get(in.readVInt());
+        String val = symbolTable.get(in.readVInt());
+        
+        opts.put(key, val);
+      }
+      
+      config.ssio.put(name, opts);
+      
+    }
+
+    cache.put(iterKey, config);
+    return config;
+  }
+
+  public List<String> getSymbolTable() {
+    return symbolTable;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/be54c959/core/src/main/java/org/apache/accumulo/core/client/impl/ConditionalWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/ConditionalWriterImpl.java
b/core/src/main/java/org/apache/accumulo/core/client/impl/ConditionalWriterImpl.java
index b8238ac..3b6f8a5 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/ConditionalWriterImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ConditionalWriterImpl.java
@@ -17,6 +17,7 @@
 
 package org.apache.accumulo.core.client.impl;
 
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -37,14 +38,12 @@ import org.apache.accumulo.core.client.AccumuloException;
 import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.ConditionalWriter;
 import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.client.IteratorSetting;
 import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.client.impl.TabletLocator.TabletServerMutations;
 import org.apache.accumulo.core.data.ByteSequence;
 import org.apache.accumulo.core.data.Condition;
 import org.apache.accumulo.core.data.ConditionalMutation;
 import org.apache.accumulo.core.data.KeyExtent;
-import org.apache.accumulo.core.data.thrift.IterInfo;
 import org.apache.accumulo.core.data.thrift.TCMResult;
 import org.apache.accumulo.core.data.thrift.TCMStatus;
 import org.apache.accumulo.core.data.thrift.TCondition;
@@ -346,9 +345,11 @@ class ConditionalWriterImpl implements ConditionalWriter {
 
       Map<TKeyExtent,List<TConditionalMutation>> tmutations = new HashMap<TKeyExtent,List<TConditionalMutation>>();
 
-      convertMutations(mutations, cmidToCm, cmid, tmutations);
+      CompressedIterators compressedIters = new CompressedIterators();
+      convertMutations(mutations, cmidToCm, cmid, tmutations, compressedIters);
 
-      List<TCMResult> tresults = client.conditionalUpdate(tinfo, credentials, ByteBufferUtil.toByteBuffers(auths.getAuthorizations()),
tmutations);
+      List<TCMResult> tresults = client.conditionalUpdate(tinfo, credentials, ByteBufferUtil.toByteBuffers(auths.getAuthorizations()),
tmutations,
+          compressedIters.getSymbolTable());
 
       HashSet<KeyExtent> extentsToInvalidate = new HashSet<KeyExtent>();
 
@@ -406,9 +407,7 @@ class ConditionalWriterImpl implements ConditionalWriter {
   }
 
   private void convertMutations(TabletServerMutations<QCMutation> mutations, Map<Long,CMK>
cmidToCm, MutableLong cmid,
-      Map<TKeyExtent,List<TConditionalMutation>> tmutations) {
-
-    // TODO compress repeated iterator configurations
+      Map<TKeyExtent,List<TConditionalMutation>> tmutations, CompressedIterators
compressedIters) {
 
     for (Entry<KeyExtent,List<QCMutation>> entry : mutations.getMutations().entrySet())
{
       TKeyExtent tke = entry.getKey().toThrift();
@@ -419,7 +418,7 @@ class ConditionalWriterImpl implements ConditionalWriter {
       for (QCMutation cm : condMutations) {
         TMutation tm = cm.toThrift();
 
-        List<TCondition> conditions = convertConditions(cm);
+        List<TCondition> conditions = convertConditions(cm, compressedIters);
 
         cmidToCm.put(cmid.longValue(), new CMK(entry.getKey(), cm));
         TConditionalMutation tcm = new TConditionalMutation(conditions, tm, cmid.longValue());
@@ -431,7 +430,7 @@ class ConditionalWriterImpl implements ConditionalWriter {
     }
   }
 
-  private List<TCondition> convertConditions(ConditionalMutation cm) {
+  private List<TCondition> convertConditions(ConditionalMutation cm, CompressedIterators
compressedIters) {
     List<TCondition> conditions = new ArrayList<TCondition>(cm.getConditions().size());
     
     for (Condition cond : cm.getConditions()) {
@@ -443,26 +442,10 @@ class ConditionalWriterImpl implements ConditionalWriter {
         hasTs = true;
       }
       
-      IteratorSetting[] iters = cond.getIterators();
-      
-      List<IterInfo> ssiList = new ArrayList<IterInfo>(iters.length);
-      Map<String,Map<String,String>> sso = new HashMap<String,Map<String,String>>();
-      
-      if (iters.length == 0) {
-        ssiList = Collections.emptyList();
-        sso = Collections.emptyMap();
-      } else {
-        ssiList = new ArrayList<IterInfo>(iters.length);
-        sso = new HashMap<String,Map<String,String>>();
-        
-        for (IteratorSetting is : iters) {
-          ssiList.add(new IterInfo(is.getPriority(), is.getIteratorClass(), is.getName()));
-          sso.put(is.getName(), is.getOptions());
-        }
-      }
+      ByteBuffer iters = compressedIters.compress(cond.getIterators());
       
       TCondition tc = new TCondition(ByteBufferUtil.toByteBuffers(cond.getFamily()), ByteBufferUtil.toByteBuffers(cond.getQualifier()),
-          ByteBufferUtil.toByteBuffers(cond.getVisibility()), ts, hasTs, ByteBufferUtil.toByteBuffers(cond.getValue()),
ssiList, sso);
+          ByteBufferUtil.toByteBuffers(cond.getVisibility()), ts, hasTs, ByteBufferUtil.toByteBuffers(cond.getValue()),
iters);
       
       conditions.add(tc);
     }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/be54c959/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java b/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
index d44a7a6..eaa61b9 100644
--- a/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
@@ -17,6 +17,7 @@
 package org.apache.accumulo.core.data;
 
 import java.io.Serializable;
+import java.nio.ByteBuffer;
 
 public class ArrayByteSequence extends ByteSequence implements Serializable {
   
@@ -48,6 +49,18 @@ public class ArrayByteSequence extends ByteSequence implements Serializable
{
     this(s.getBytes());
   }
   
+  public ArrayByteSequence(ByteBuffer buffer) {
+    if (buffer.hasArray()) {
+      this.data = buffer.array();
+      this.offset = buffer.arrayOffset();
+      this.length = buffer.limit();
+    } else {
+      this.data = new byte[buffer.remaining()];
+      this.offset = 0;
+      buffer.get(data);
+    }
+  }
+
   @Override
   public byte byteAt(int i) {
     

http://git-wip-us.apache.org/repos/asf/accumulo/blob/be54c959/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/data/Mutation.java b/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
index 00cefbf..4ac3f0c 100644
--- a/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
@@ -27,6 +27,7 @@ import java.util.List;
 import org.apache.accumulo.core.data.thrift.TMutation;
 import org.apache.accumulo.core.security.ColumnVisibility;
 import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.UnsynchronizedBuffer;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableUtils;
@@ -65,121 +66,8 @@ public class Mutation implements Writable {
   private byte[] data;
   private int entries;
   private List<byte[]> values;
-
-  // created this little class instead of using ByteArrayOutput stream and DataOutputStream
-  // because both are synchronized... lots of small syncs slow things down
-  private static class ByteBuffer {
-    
-    int offset;
-    byte data[] = new byte[64];
-    
-    private void reserve(int l) {
-      if (offset + l > data.length) {
-        int newSize = data.length * 2;
-        while (newSize <= offset + l)
-          newSize = newSize * 2;
-        
-        byte[] newData = new byte[newSize];
-        System.arraycopy(data, 0, newData, 0, offset);
-        data = newData;
-      }
-      
-    }
-    
-    public void add(byte[] bytes, int off, int length) {
-      reserve(length);
-      System.arraycopy(bytes, off, data, offset, length);
-      offset += length;
-    }
-    
-    void add(boolean b) {
-      reserve(1);
-      if (b)
-        data[offset++] = 1;
-      else
-        data[offset++] = 0;
-    }
-    
-    public byte[] toArray() {
-      byte ret[] = new byte[offset];
-      System.arraycopy(data, 0, ret, 0, offset);
-      return ret;
-    }
-    
-    public void writeVLong(long i) {
-      reserve(9);
-      if (i >= -112 && i <= 127) {
-        data[offset++] = (byte)i;
-        return;
-      }
-        
-      int len = -112;
-      if (i < 0) {
-        i ^= -1L; // take one's complement'
-        len = -120;
-      }
-        
-      long tmp = i;
-      while (tmp != 0) {
-        tmp = tmp >> 8;
-        len--;
-      }
-        
-      data[offset++] = (byte)len;
-        
-      len = (len < -120) ? -(len + 120) : -(len + 112);
-        
-      for (int idx = len; idx != 0; idx--) {
-        int shiftbits = (idx - 1) * 8;
-        long mask = 0xFFL << shiftbits;
-        data[offset++] = (byte)((i & mask) >> shiftbits);
-      }
-    }
-  }
-  
-  private static class SimpleReader {
-    int offset;
-    byte data[];
-    
-    SimpleReader(byte b[]) {
-      this.data = b;
-    }
-
-    int readInt() {
-      return (data[offset++] << 24) + ((data[offset++] & 255) << 16) + ((data[offset++]
& 255) << 8) + ((data[offset++] & 255) << 0);
-    }
-    
-    long readLong() {
-      return (((long) data[offset++] << 56) + ((long) (data[offset++] & 255) <<
48) + ((long) (data[offset++] & 255) << 40)
-          + ((long) (data[offset++] & 255) << 32) + ((long) (data[offset++] &
255) << 24) + ((data[offset++] & 255) << 16) + ((data[offset++] & 255)
<< 8) + ((data[offset++] & 255) << 0));
-    }
-    
-    void readBytes(byte b[]) {
-      System.arraycopy(data, offset, b, 0, b.length);
-      offset += b.length;
-    }
-    
-    boolean readBoolean() {
-      return (data[offset++] == 1);
-    }
-    
-    long readVLong() {
-      byte firstByte = data[offset++];
-      int len =  WritableUtils.decodeVIntSize(firstByte);
-      if (len == 1) {
-        return firstByte;
-      }
-      long i = 0;
-      for (int idx = 0; idx < len-1; idx++) {
-        byte b = data[offset++];
-        i = i << 8;
-        i = i | (b & 0xFF);
-      }
-      return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i);
-    }
-  }
   
-  private ByteBuffer buffer;
+  private UnsynchronizedBuffer.Writer buffer;
   
   private List<ColumnUpdate> updates;
   
@@ -205,7 +93,7 @@ public class Mutation implements Writable {
   public Mutation(byte[] row, int start, int length) {
     this.row = new byte[length];
     System.arraycopy(row, start, this.row, 0, length);
-    buffer = new ByteBuffer();
+    buffer = new UnsynchronizedBuffer.Writer();
   }
   
   public Mutation(Text row) {
@@ -445,7 +333,7 @@ public class Mutation implements Writable {
     put(columnFamily, columnQualifier, columnVisibility.getExpression(), true, timestamp,
true, EMPTY_BYTES);
   }
 
-  private byte[] oldReadBytes(SimpleReader in) {
+  private byte[] oldReadBytes(UnsynchronizedBuffer.Reader in) {
     int len = in.readInt();
     if (len == 0)
       return EMPTY_BYTES;
@@ -455,7 +343,7 @@ public class Mutation implements Writable {
     return bytes;
   }
   
-  private byte[] readBytes(SimpleReader in) {
+  private byte[] readBytes(UnsynchronizedBuffer.Reader in) {
     int len = (int)in.readVLong();
     if (len == 0)
       return EMPTY_BYTES;
@@ -468,7 +356,7 @@ public class Mutation implements Writable {
   public List<ColumnUpdate> getUpdates() {
     serialize();
     
-    SimpleReader in = new SimpleReader(data);
+    UnsynchronizedBuffer.Reader in = new UnsynchronizedBuffer.Reader(data);
     
     if (updates == null) {
       if (entries == 1) {
@@ -490,7 +378,7 @@ public class Mutation implements Writable {
     return new ColumnUpdate(cf, cq, cv, hasts, ts, deleted, val);
   }
 
-  private ColumnUpdate deserializeColumnUpdate(SimpleReader in) {
+  private ColumnUpdate deserializeColumnUpdate(UnsynchronizedBuffer.Reader in) {
     byte[] cf = readBytes(in);
     byte[] cq = readBytes(in);
     byte[] cv = readBytes(in);
@@ -623,8 +511,8 @@ public class Mutation implements Writable {
     }
     
     // convert data to new format
-    SimpleReader din = new SimpleReader(localData);
-    buffer = new ByteBuffer();
+    UnsynchronizedBuffer.Reader din = new UnsynchronizedBuffer.Reader(localData);
+    buffer = new UnsynchronizedBuffer.Writer();
     for (int i = 0; i < localEntries; i++) {
       byte[] cf = oldReadBytes(din);
       byte[] cq = oldReadBytes(din);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/be54c959/core/src/main/java/org/apache/accumulo/core/data/thrift/TCondition.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/data/thrift/TCondition.java b/core/src/main/java/org/apache/accumulo/core/data/thrift/TCondition.java
index 95de004..40a12be 100644
--- a/core/src/main/java/org/apache/accumulo/core/data/thrift/TCondition.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/thrift/TCondition.java
@@ -55,8 +55,7 @@ import org.slf4j.LoggerFactory;
   private static final org.apache.thrift.protocol.TField TS_FIELD_DESC = new org.apache.thrift.protocol.TField("ts",
org.apache.thrift.protocol.TType.I64, (short)4);
   private static final org.apache.thrift.protocol.TField HAS_TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("hasTimestamp",
org.apache.thrift.protocol.TType.BOOL, (short)5);
   private static final org.apache.thrift.protocol.TField VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("val",
org.apache.thrift.protocol.TType.STRING, (short)6);
-  private static final org.apache.thrift.protocol.TField SSI_LIST_FIELD_DESC = new org.apache.thrift.protocol.TField("ssiList",
org.apache.thrift.protocol.TType.LIST, (short)7);
-  private static final org.apache.thrift.protocol.TField SSIO_FIELD_DESC = new org.apache.thrift.protocol.TField("ssio",
org.apache.thrift.protocol.TType.MAP, (short)8);
+  private static final org.apache.thrift.protocol.TField ITERATORS_FIELD_DESC = new org.apache.thrift.protocol.TField("iterators",
org.apache.thrift.protocol.TType.STRING, (short)7);
 
   private static final Map<Class<? extends IScheme>, SchemeFactory> schemes =
new HashMap<Class<? extends IScheme>, SchemeFactory>();
   static {
@@ -70,8 +69,7 @@ import org.slf4j.LoggerFactory;
   public long ts; // required
   public boolean hasTimestamp; // required
   public ByteBuffer val; // required
-  public List<IterInfo> ssiList; // required
-  public Map<String,Map<String,String>> ssio; // required
+  public ByteBuffer iterators; // required
 
   /** The set of fields this struct contains, along with convenience methods for finding
and manipulating them. */
   @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum
{
@@ -81,8 +79,7 @@ import org.slf4j.LoggerFactory;
     TS((short)4, "ts"),
     HAS_TIMESTAMP((short)5, "hasTimestamp"),
     VAL((short)6, "val"),
-    SSI_LIST((short)7, "ssiList"),
-    SSIO((short)8, "ssio");
+    ITERATORS((short)7, "iterators");
 
     private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
 
@@ -109,10 +106,8 @@ import org.slf4j.LoggerFactory;
           return HAS_TIMESTAMP;
         case 6: // VAL
           return VAL;
-        case 7: // SSI_LIST
-          return SSI_LIST;
-        case 8: // SSIO
-          return SSIO;
+        case 7: // ITERATORS
+          return ITERATORS;
         default:
           return null;
       }
@@ -171,15 +166,8 @@ import org.slf4j.LoggerFactory;
         new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
     tmpMap.put(_Fields.VAL, new org.apache.thrift.meta_data.FieldMetaData("val", org.apache.thrift.TFieldRequirementType.DEFAULT,

         new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING
       , true)));
-    tmpMap.put(_Fields.SSI_LIST, new org.apache.thrift.meta_data.FieldMetaData("ssiList",
org.apache.thrift.TFieldRequirementType.DEFAULT, 
-        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,

-            new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT,
IterInfo.class))));
-    tmpMap.put(_Fields.SSIO, new org.apache.thrift.meta_data.FieldMetaData("ssio", org.apache.thrift.TFieldRequirementType.DEFAULT,

-        new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,

-            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),

-            new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,

-                new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),

-                new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))));
+    tmpMap.put(_Fields.ITERATORS, new org.apache.thrift.meta_data.FieldMetaData("iterators",
org.apache.thrift.TFieldRequirementType.DEFAULT, 
+        new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING
       , true)));
     metaDataMap = Collections.unmodifiableMap(tmpMap);
     org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TCondition.class, metaDataMap);
   }
@@ -194,8 +182,7 @@ import org.slf4j.LoggerFactory;
     long ts,
     boolean hasTimestamp,
     ByteBuffer val,
-    List<IterInfo> ssiList,
-    Map<String,Map<String,String>> ssio)
+    ByteBuffer iterators)
   {
     this();
     this.cf = cf;
@@ -206,8 +193,7 @@ import org.slf4j.LoggerFactory;
     this.hasTimestamp = hasTimestamp;
     setHasTimestampIsSet(true);
     this.val = val;
-    this.ssiList = ssiList;
-    this.ssio = ssio;
+    this.iterators = iterators;
   }
 
   /**
@@ -233,38 +219,9 @@ import org.slf4j.LoggerFactory;
       this.val = org.apache.thrift.TBaseHelper.copyBinary(other.val);
 ;
     }
-    if (other.isSetSsiList()) {
-      List<IterInfo> __this__ssiList = new ArrayList<IterInfo>();
-      for (IterInfo other_element : other.ssiList) {
-        __this__ssiList.add(new IterInfo(other_element));
-      }
-      this.ssiList = __this__ssiList;
-    }
-    if (other.isSetSsio()) {
-      Map<String,Map<String,String>> __this__ssio = new HashMap<String,Map<String,String>>();
-      for (Map.Entry<String, Map<String,String>> other_element : other.ssio.entrySet())
{
-
-        String other_element_key = other_element.getKey();
-        Map<String,String> other_element_value = other_element.getValue();
-
-        String __this__ssio_copy_key = other_element_key;
-
-        Map<String,String> __this__ssio_copy_value = new HashMap<String,String>();
-        for (Map.Entry<String, String> other_element_value_element : other_element_value.entrySet())
{
-
-          String other_element_value_element_key = other_element_value_element.getKey();
-          String other_element_value_element_value = other_element_value_element.getValue();
-
-          String __this__ssio_copy_value_copy_key = other_element_value_element_key;
-
-          String __this__ssio_copy_value_copy_value = other_element_value_element_value;
-
-          __this__ssio_copy_value.put(__this__ssio_copy_value_copy_key, __this__ssio_copy_value_copy_value);
-        }
-
-        __this__ssio.put(__this__ssio_copy_key, __this__ssio_copy_value);
-      }
-      this.ssio = __this__ssio;
+    if (other.isSetIterators()) {
+      this.iterators = org.apache.thrift.TBaseHelper.copyBinary(other.iterators);
+;
     }
   }
 
@@ -282,8 +239,7 @@ import org.slf4j.LoggerFactory;
     setHasTimestampIsSet(false);
     this.hasTimestamp = false;
     this.val = null;
-    this.ssiList = null;
-    this.ssio = null;
+    this.iterators = null;
   }
 
   public byte[] getCf() {
@@ -468,77 +424,37 @@ import org.slf4j.LoggerFactory;
     }
   }
 
-  public int getSsiListSize() {
-    return (this.ssiList == null) ? 0 : this.ssiList.size();
-  }
-
-  public java.util.Iterator<IterInfo> getSsiListIterator() {
-    return (this.ssiList == null) ? null : this.ssiList.iterator();
-  }
-
-  public void addToSsiList(IterInfo elem) {
-    if (this.ssiList == null) {
-      this.ssiList = new ArrayList<IterInfo>();
-    }
-    this.ssiList.add(elem);
+  public byte[] getIterators() {
+    setIterators(org.apache.thrift.TBaseHelper.rightSize(iterators));
+    return iterators == null ? null : iterators.array();
   }
 
-  public List<IterInfo> getSsiList() {
-    return this.ssiList;
+  public ByteBuffer bufferForIterators() {
+    return iterators;
   }
 
-  public TCondition setSsiList(List<IterInfo> ssiList) {
-    this.ssiList = ssiList;
+  public TCondition setIterators(byte[] iterators) {
+    setIterators(iterators == null ? (ByteBuffer)null : ByteBuffer.wrap(iterators));
     return this;
   }
 
-  public void unsetSsiList() {
-    this.ssiList = null;
-  }
-
-  /** Returns true if field ssiList is set (has been assigned a value) and false otherwise
*/
-  public boolean isSetSsiList() {
-    return this.ssiList != null;
-  }
-
-  public void setSsiListIsSet(boolean value) {
-    if (!value) {
-      this.ssiList = null;
-    }
-  }
-
-  public int getSsioSize() {
-    return (this.ssio == null) ? 0 : this.ssio.size();
-  }
-
-  public void putToSsio(String key, Map<String,String> val) {
-    if (this.ssio == null) {
-      this.ssio = new HashMap<String,Map<String,String>>();
-    }
-    this.ssio.put(key, val);
-  }
-
-  public Map<String,Map<String,String>> getSsio() {
-    return this.ssio;
-  }
-
-  public TCondition setSsio(Map<String,Map<String,String>> ssio) {
-    this.ssio = ssio;
+  public TCondition setIterators(ByteBuffer iterators) {
+    this.iterators = iterators;
     return this;
   }
 
-  public void unsetSsio() {
-    this.ssio = null;
+  public void unsetIterators() {
+    this.iterators = null;
   }
 
-  /** Returns true if field ssio is set (has been assigned a value) and false otherwise */
-  public boolean isSetSsio() {
-    return this.ssio != null;
+  /** Returns true if field iterators is set (has been assigned a value) and false otherwise
*/
+  public boolean isSetIterators() {
+    return this.iterators != null;
   }
 
-  public void setSsioIsSet(boolean value) {
+  public void setIteratorsIsSet(boolean value) {
     if (!value) {
-      this.ssio = null;
+      this.iterators = null;
     }
   }
 
@@ -592,19 +508,11 @@ import org.slf4j.LoggerFactory;
       }
       break;
 
-    case SSI_LIST:
+    case ITERATORS:
       if (value == null) {
-        unsetSsiList();
+        unsetIterators();
       } else {
-        setSsiList((List<IterInfo>)value);
-      }
-      break;
-
-    case SSIO:
-      if (value == null) {
-        unsetSsio();
-      } else {
-        setSsio((Map<String,Map<String,String>>)value);
+        setIterators((ByteBuffer)value);
       }
       break;
 
@@ -631,11 +539,8 @@ import org.slf4j.LoggerFactory;
     case VAL:
       return getVal();
 
-    case SSI_LIST:
-      return getSsiList();
-
-    case SSIO:
-      return getSsio();
+    case ITERATORS:
+      return getIterators();
 
     }
     throw new IllegalStateException();
@@ -660,10 +565,8 @@ import org.slf4j.LoggerFactory;
       return isSetHasTimestamp();
     case VAL:
       return isSetVal();
-    case SSI_LIST:
-      return isSetSsiList();
-    case SSIO:
-      return isSetSsio();
+    case ITERATORS:
+      return isSetIterators();
     }
     throw new IllegalStateException();
   }
@@ -735,21 +638,12 @@ import org.slf4j.LoggerFactory;
         return false;
     }
 
-    boolean this_present_ssiList = true && this.isSetSsiList();
-    boolean that_present_ssiList = true && that.isSetSsiList();
-    if (this_present_ssiList || that_present_ssiList) {
-      if (!(this_present_ssiList && that_present_ssiList))
-        return false;
-      if (!this.ssiList.equals(that.ssiList))
-        return false;
-    }
-
-    boolean this_present_ssio = true && this.isSetSsio();
-    boolean that_present_ssio = true && that.isSetSsio();
-    if (this_present_ssio || that_present_ssio) {
-      if (!(this_present_ssio && that_present_ssio))
+    boolean this_present_iterators = true && this.isSetIterators();
+    boolean that_present_iterators = true && that.isSetIterators();
+    if (this_present_iterators || that_present_iterators) {
+      if (!(this_present_iterators && that_present_iterators))
         return false;
-      if (!this.ssio.equals(that.ssio))
+      if (!this.iterators.equals(that.iterators))
         return false;
     }
 
@@ -829,22 +723,12 @@ import org.slf4j.LoggerFactory;
         return lastComparison;
       }
     }
-    lastComparison = Boolean.valueOf(isSetSsiList()).compareTo(typedOther.isSetSsiList());
+    lastComparison = Boolean.valueOf(isSetIterators()).compareTo(typedOther.isSetIterators());
     if (lastComparison != 0) {
       return lastComparison;
     }
-    if (isSetSsiList()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ssiList, typedOther.ssiList);
-      if (lastComparison != 0) {
-        return lastComparison;
-      }
-    }
-    lastComparison = Boolean.valueOf(isSetSsio()).compareTo(typedOther.isSetSsio());
-    if (lastComparison != 0) {
-      return lastComparison;
-    }
-    if (isSetSsio()) {
-      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ssio, typedOther.ssio);
+    if (isSetIterators()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.iterators, typedOther.iterators);
       if (lastComparison != 0) {
         return lastComparison;
       }
@@ -909,19 +793,11 @@ import org.slf4j.LoggerFactory;
     }
     first = false;
     if (!first) sb.append(", ");
-    sb.append("ssiList:");
-    if (this.ssiList == null) {
+    sb.append("iterators:");
+    if (this.iterators == null) {
       sb.append("null");
     } else {
-      sb.append(this.ssiList);
-    }
-    first = false;
-    if (!first) sb.append(", ");
-    sb.append("ssio:");
-    if (this.ssio == null) {
-      sb.append("null");
-    } else {
-      sb.append(this.ssio);
+      org.apache.thrift.TBaseHelper.toString(this.iterators, sb);
     }
     first = false;
     sb.append(")");
@@ -1017,53 +893,10 @@ import org.slf4j.LoggerFactory;
               org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
             }
             break;
-          case 7: // SSI_LIST
-            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
-              {
-                org.apache.thrift.protocol.TList _list78 = iprot.readListBegin();
-                struct.ssiList = new ArrayList<IterInfo>(_list78.size);
-                for (int _i79 = 0; _i79 < _list78.size; ++_i79)
-                {
-                  IterInfo _elem80; // required
-                  _elem80 = new IterInfo();
-                  _elem80.read(iprot);
-                  struct.ssiList.add(_elem80);
-                }
-                iprot.readListEnd();
-              }
-              struct.setSsiListIsSet(true);
-            } else { 
-              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
-            }
-            break;
-          case 8: // SSIO
-            if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
-              {
-                org.apache.thrift.protocol.TMap _map81 = iprot.readMapBegin();
-                struct.ssio = new HashMap<String,Map<String,String>>(2*_map81.size);
-                for (int _i82 = 0; _i82 < _map81.size; ++_i82)
-                {
-                  String _key83; // required
-                  Map<String,String> _val84; // required
-                  _key83 = iprot.readString();
-                  {
-                    org.apache.thrift.protocol.TMap _map85 = iprot.readMapBegin();
-                    _val84 = new HashMap<String,String>(2*_map85.size);
-                    for (int _i86 = 0; _i86 < _map85.size; ++_i86)
-                    {
-                      String _key87; // required
-                      String _val88; // required
-                      _key87 = iprot.readString();
-                      _val88 = iprot.readString();
-                      _val84.put(_key87, _val88);
-                    }
-                    iprot.readMapEnd();
-                  }
-                  struct.ssio.put(_key83, _val84);
-                }
-                iprot.readMapEnd();
-              }
-              struct.setSsioIsSet(true);
+          case 7: // ITERATORS
+            if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+              struct.iterators = iprot.readBinary();
+              struct.setIteratorsIsSet(true);
             } else { 
               org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
             }
@@ -1109,37 +942,9 @@ import org.slf4j.LoggerFactory;
         oprot.writeBinary(struct.val);
         oprot.writeFieldEnd();
       }
-      if (struct.ssiList != null) {
-        oprot.writeFieldBegin(SSI_LIST_FIELD_DESC);
-        {
-          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT,
struct.ssiList.size()));
-          for (IterInfo _iter89 : struct.ssiList)
-          {
-            _iter89.write(oprot);
-          }
-          oprot.writeListEnd();
-        }
-        oprot.writeFieldEnd();
-      }
-      if (struct.ssio != null) {
-        oprot.writeFieldBegin(SSIO_FIELD_DESC);
-        {
-          oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING,
org.apache.thrift.protocol.TType.MAP, struct.ssio.size()));
-          for (Map.Entry<String, Map<String,String>> _iter90 : struct.ssio.entrySet())
-          {
-            oprot.writeString(_iter90.getKey());
-            {
-              oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING,
org.apache.thrift.protocol.TType.STRING, _iter90.getValue().size()));
-              for (Map.Entry<String, String> _iter91 : _iter90.getValue().entrySet())
-              {
-                oprot.writeString(_iter91.getKey());
-                oprot.writeString(_iter91.getValue());
-              }
-              oprot.writeMapEnd();
-            }
-          }
-          oprot.writeMapEnd();
-        }
+      if (struct.iterators != null) {
+        oprot.writeFieldBegin(ITERATORS_FIELD_DESC);
+        oprot.writeBinary(struct.iterators);
         oprot.writeFieldEnd();
       }
       oprot.writeFieldStop();
@@ -1178,13 +983,10 @@ import org.slf4j.LoggerFactory;
       if (struct.isSetVal()) {
         optionals.set(5);
       }
-      if (struct.isSetSsiList()) {
+      if (struct.isSetIterators()) {
         optionals.set(6);
       }
-      if (struct.isSetSsio()) {
-        optionals.set(7);
-      }
-      oprot.writeBitSet(optionals, 8);
+      oprot.writeBitSet(optionals, 7);
       if (struct.isSetCf()) {
         oprot.writeBinary(struct.cf);
       }
@@ -1203,38 +1005,15 @@ import org.slf4j.LoggerFactory;
       if (struct.isSetVal()) {
         oprot.writeBinary(struct.val);
       }
-      if (struct.isSetSsiList()) {
-        {
-          oprot.writeI32(struct.ssiList.size());
-          for (IterInfo _iter92 : struct.ssiList)
-          {
-            _iter92.write(oprot);
-          }
-        }
-      }
-      if (struct.isSetSsio()) {
-        {
-          oprot.writeI32(struct.ssio.size());
-          for (Map.Entry<String, Map<String,String>> _iter93 : struct.ssio.entrySet())
-          {
-            oprot.writeString(_iter93.getKey());
-            {
-              oprot.writeI32(_iter93.getValue().size());
-              for (Map.Entry<String, String> _iter94 : _iter93.getValue().entrySet())
-              {
-                oprot.writeString(_iter94.getKey());
-                oprot.writeString(_iter94.getValue());
-              }
-            }
-          }
-        }
+      if (struct.isSetIterators()) {
+        oprot.writeBinary(struct.iterators);
       }
     }
 
     @Override
     public void read(org.apache.thrift.protocol.TProtocol prot, TCondition struct) throws
org.apache.thrift.TException {
       TTupleProtocol iprot = (TTupleProtocol) prot;
-      BitSet incoming = iprot.readBitSet(8);
+      BitSet incoming = iprot.readBitSet(7);
       if (incoming.get(0)) {
         struct.cf = iprot.readBinary();
         struct.setCfIsSet(true);
@@ -1260,44 +1039,8 @@ import org.slf4j.LoggerFactory;
         struct.setValIsSet(true);
       }
       if (incoming.get(6)) {
-        {
-          org.apache.thrift.protocol.TList _list95 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT,
iprot.readI32());
-          struct.ssiList = new ArrayList<IterInfo>(_list95.size);
-          for (int _i96 = 0; _i96 < _list95.size; ++_i96)
-          {
-            IterInfo _elem97; // required
-            _elem97 = new IterInfo();
-            _elem97.read(iprot);
-            struct.ssiList.add(_elem97);
-          }
-        }
-        struct.setSsiListIsSet(true);
-      }
-      if (incoming.get(7)) {
-        {
-          org.apache.thrift.protocol.TMap _map98 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING,
org.apache.thrift.protocol.TType.MAP, iprot.readI32());
-          struct.ssio = new HashMap<String,Map<String,String>>(2*_map98.size);
-          for (int _i99 = 0; _i99 < _map98.size; ++_i99)
-          {
-            String _key100; // required
-            Map<String,String> _val101; // required
-            _key100 = iprot.readString();
-            {
-              org.apache.thrift.protocol.TMap _map102 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING,
org.apache.thrift.protocol.TType.STRING, iprot.readI32());
-              _val101 = new HashMap<String,String>(2*_map102.size);
-              for (int _i103 = 0; _i103 < _map102.size; ++_i103)
-              {
-                String _key104; // required
-                String _val105; // required
-                _key104 = iprot.readString();
-                _val105 = iprot.readString();
-                _val101.put(_key104, _val105);
-              }
-            }
-            struct.ssio.put(_key100, _val101);
-          }
-        }
-        struct.setSsioIsSet(true);
+        struct.iterators = iprot.readBinary();
+        struct.setIteratorsIsSet(true);
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/be54c959/core/src/main/java/org/apache/accumulo/core/file/rfile/RelativeKey.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/RelativeKey.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/RelativeKey.java
index 4ba0eb6..97001ee 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/RelativeKey.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/RelativeKey.java
@@ -23,6 +23,7 @@ import java.io.IOException;
 import org.apache.accumulo.core.data.ArrayByteSequence;
 import org.apache.accumulo.core.data.ByteSequence;
 import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.util.UnsynchronizedBuffer;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableUtils;
 
@@ -472,37 +473,9 @@ public class RelativeKey implements Writable {
     read(in, mbseq, len);
   }
   
-  /**
-   * Determines what next array size should be by rounding up to next power of two.
-   * 
-   */
-  static int nextArraySize(int i) {
-    if (i < 0)
-      throw new IllegalArgumentException();
-    
-    if (i > (1 << 30))
-      return Integer.MAX_VALUE; // this is the next power of 2 minus one... a special case
-
-    if (i == 0) {
-      return 1;
-    }
-    
-    // round up to next power of two
-    int ret = i;
-    ret--;
-    ret |= ret >> 1;
-    ret |= ret >> 2;
-    ret |= ret >> 4;
-    ret |= ret >> 8;
-    ret |= ret >> 16;
-    ret++;
-    
-    return ret;
-  }
-
   private static void read(DataInput in, MByteSequence mbseqDestination, int len) throws
IOException {
     if (mbseqDestination.getBackingArray().length < len) {
-      mbseqDestination.setArray(new byte[nextArraySize(len)]);
+      mbseqDestination.setArray(new byte[UnsynchronizedBuffer.nextArraySize(len)]);
     }
     
     in.readFully(mbseqDestination.getBackingArray(), 0, len);
@@ -529,7 +502,7 @@ public class RelativeKey implements Writable {
     int remainingLen = WritableUtils.readVInt(in);
     int len = prefixLen + remainingLen;
     if (dest.getBackingArray().length < len) {
-      dest.setArray(new byte[nextArraySize(len)]);
+      dest.setArray(new byte[UnsynchronizedBuffer.nextArraySize(len)]);
     }
     if (prefixSource.isBackedByArray()) {
       System.arraycopy(prefixSource.getBackingArray(), prefixSource.offset(), dest.getBackingArray(),
0, prefixLen);


Mime
View raw message