accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject svn commit: r1342798 - in /accumulo/branches/ACCUMULO-578: ./ core/ core/src/main/java/org/apache/accumulo/core/client/admin/ core/src/main/java/org/apache/accumulo/core/client/mock/ core/src/main/java/org/apache/accumulo/core/iterators/ core/src/main/...
Date Fri, 25 May 2012 20:24:34 GMT
Author: ecn
Date: Fri May 25 20:24:33 2012
New Revision: 1342798

URL: http://svn.apache.org/viewvc?rev=1342798&view=rev
Log:
ACCUMULO-578 finished merging in trunk changes

Added:
    accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/IteratorConfig.java
      - copied unchanged from r1342236, accumulo/trunk/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/IteratorConfig.java
    accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TIteratorSetting.java
      - copied unchanged from r1342236, accumulo/trunk/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TIteratorSetting.java
Modified:
    accumulo/branches/ACCUMULO-578/   (props changed)
    accumulo/branches/ACCUMULO-578/core/   (props changed)
    accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
    accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
    accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java
    accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
    accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
    accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletMutations.java
    accumulo/branches/ACCUMULO-578/core/src/main/thrift/tabletserver.thrift
    accumulo/branches/ACCUMULO-578/core/src/test/java/org/apache/accumulo/core/client/admin/TableOperationsHelperTest.java
    accumulo/branches/ACCUMULO-578/server/   (props changed)
    accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/master/Master.java
    accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java
    accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
    accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
    accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
    accumulo/branches/ACCUMULO-578/src/   (props changed)
    accumulo/branches/ACCUMULO-578/test/system/auto/simple/dynamic.py
    accumulo/branches/ACCUMULO-578/test/system/auto/simple/rename.py
    accumulo/branches/ACCUMULO-578/test/system/auto/simple/table.py

Propchange: accumulo/branches/ACCUMULO-578/
------------------------------------------------------------------------------
  Merged /accumulo/trunk:r1341000,1341572-1342236,1342774-1342794

Propchange: accumulo/branches/ACCUMULO-578/core/
------------------------------------------------------------------------------
  Merged /accumulo/trunk/core:r1341000,1341572-1342236,1342774-1342794

Modified: accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java?rev=1342798&r1=1342797&r2=1342798&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java (original)
+++ accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java Fri May 25 20:24:33 2012
@@ -19,6 +19,7 @@ package org.apache.accumulo.core.client.
 import java.io.IOException;
 import java.util.Collection;
 import java.util.EnumSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
@@ -195,6 +196,25 @@ public interface TableOperations {
       AccumuloException;
   
   /**
+   * Starts a full major compaction of the tablets in the range (start, end]. The compaction is preformed even for tablets that have only one file.
+   * 
+   * @param tableName
+   *          the table to compact
+   * @param start
+   *          first tablet to be compacted contains the row after this row, null means the first tablet in table
+   * @param end
+   *          last tablet to be merged contains this row, null means the last tablet in table
+   * @param iterators
+   *          A set of iterators that will be applied to each tablet compacted
+   * @param flush
+   *          when true, table memory is flushed before compaction starts
+   * @param wait
+   *          when true, the call will not return until compactions are finished
+   */
+  public void compact(String tableName, Text start, Text end, List<IteratorSetting> iterators, boolean flush, boolean wait) throws AccumuloSecurityException,
+      TableNotFoundException, AccumuloException;
+  
+  /**
    * Delete a table
    * 
    * @param tableName

Modified: accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java?rev=1342798&r1=1342797&r2=1342798&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java (original)
+++ accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java Fri May 25 20:24:33 2012
@@ -32,11 +32,17 @@ import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeMap;
 import java.util.TreeSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.client.AccumuloException;
 import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
 import org.apache.accumulo.core.client.Scanner;
 import org.apache.accumulo.core.client.TableDeletedException;
 import org.apache.accumulo.core.client.TableExistsException;
@@ -301,6 +307,62 @@ public class TableOperationsImpl extends
     }
   }
   
+  private static class SplitEnv {
+    private String tableName;
+    private String tableId;
+    private ExecutorService executor;
+    private CountDownLatch latch;
+    private AtomicReference<Exception> exception;
+    
+    SplitEnv(String tableName, String tableId, ExecutorService executor, CountDownLatch latch, AtomicReference<Exception> exception) {
+      this.tableName = tableName;
+      this.tableId = tableId;
+      this.executor = executor;
+      this.latch = latch;
+      this.exception = exception;
+    }
+  }
+  
+  private class SplitTask implements Runnable {
+    
+    private List<Text> splits;
+    private SplitEnv env;
+    
+    SplitTask(SplitEnv env, List<Text> splits) {
+      this.env = env;
+      this.splits = splits;
+    }
+    
+    @Override
+    public void run() {
+      try {
+        if (env.exception.get() != null)
+          return;
+        
+        if (splits.size() <= 2) {
+          addSplits(env.tableName, new TreeSet<Text>(splits), env.tableId);
+          for (int i = 0; i < splits.size(); i++)
+            env.latch.countDown();
+          return;
+        }
+        
+        int mid = splits.size() / 2;
+        
+        // split the middle split point to ensure that child task split different tablets and can therefore
+        // run in parallel
+        addSplits(env.tableName, new TreeSet<Text>(splits.subList(mid, mid + 1)), env.tableId);
+        env.latch.countDown();
+        
+        env.executor.submit(new SplitTask(env, splits.subList(0, mid)));
+        env.executor.submit(new SplitTask(env, splits.subList(mid + 1, splits.size())));
+        
+      } catch (Exception e) {
+        env.exception.compareAndSet(null, e);
+      }
+    }
+    
+  }
+
   /**
    * @param tableName
    *          the name of the table
@@ -315,7 +377,40 @@ public class TableOperationsImpl extends
    */
   public void addSplits(String tableName, SortedSet<Text> partitionKeys) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
     String tableId = Tables.getTableId(instance, tableName);
-    addSplits(tableName, partitionKeys, tableId);
+    
+    List<Text> splits = new ArrayList<Text>(partitionKeys);
+    // should be sorted because we copied from a sorted set, but that makes assumptions about
+    // how the copy was done so resort to be sure.
+    Collections.sort(splits);
+
+    CountDownLatch latch = new CountDownLatch(splits.size());
+    AtomicReference<Exception> exception = new AtomicReference<Exception>(null);
+    
+    ExecutorService executor = Executors.newFixedThreadPool(16);
+    try {
+      executor.submit(new SplitTask(new SplitEnv(tableName, tableId, executor, latch, exception), splits));
+
+      while (!latch.await(100, TimeUnit.MILLISECONDS)) {
+        if (exception.get() != null) {
+          executor.shutdownNow();
+          Exception excep = exception.get();
+          if (excep instanceof TableNotFoundException)
+            throw (TableNotFoundException) excep;
+          else if (excep instanceof AccumuloException)
+            throw (AccumuloException) excep;
+          else if (excep instanceof AccumuloSecurityException)
+            throw (AccumuloSecurityException) excep;
+          else if (excep instanceof RuntimeException)
+            throw (RuntimeException) excep;
+          else
+            throw new RuntimeException(excep);
+        }
+      }
+    } catch (InterruptedException e) {
+      throw new RuntimeException(e);
+    } finally {
+      executor.shutdown();
+    }
   }
   
   private void addSplits(String tableName, SortedSet<Text> partitionKeys, String tableId) throws AccumuloException, AccumuloSecurityException,
@@ -590,6 +685,11 @@ public class TableOperationsImpl extends
   
   public void compact(String tableName, Text start, Text end, boolean flush, boolean wait) throws AccumuloSecurityException, TableNotFoundException,
       AccumuloException {
+    compact(tableName, start, end, new ArrayList<IteratorSetting>(), flush, wait);
+  }
+  
+  public void compact(String tableName, Text start, Text end, List<IteratorSetting> iterators, boolean flush, boolean wait) throws AccumuloSecurityException,
+      TableNotFoundException, AccumuloException {
     ArgumentChecker.notNull(tableName);
     ByteBuffer EMPTY = ByteBuffer.allocate(0);
     
@@ -599,7 +699,8 @@ public class TableOperationsImpl extends
       _flush(tableId, start, end, true);
     
     List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.getBytes()), start == null ? EMPTY : TextUtil.getByteBuffer(start), end == null ? EMPTY
-        : TextUtil.getByteBuffer(end));
+        : TextUtil.getByteBuffer(end), ByteBuffer.wrap(IteratorUtil.encodeIteratorSettings(iterators)));
+
     Map<String,String> opts = new HashMap<String,String>();
     try {
       doTableOperation(TableOperation.COMPACT, args, opts, wait);

Modified: accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java?rev=1342798&r1=1342797&r2=1342798&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java (original)
+++ accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java Fri May 25 20:24:33 2012
@@ -20,6 +20,7 @@ import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
@@ -29,6 +30,7 @@ import java.util.TreeSet;
 import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.client.AccumuloException;
 import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.IteratorSetting;
 import org.apache.accumulo.core.client.TableExistsException;
 import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.client.admin.FindMax;
@@ -200,6 +202,12 @@ public class MockTableOperations extends
   }
   
   @Override
+  public void compact(String tableName, Text start, Text end, List<IteratorSetting> iterators, boolean flush, boolean wait) throws AccumuloSecurityException,
+      TableNotFoundException, AccumuloException {
+    throw new NotImplementedException();
+  }
+  
+  @Override
   public void clone(String srcTableName, String newTableName, boolean flush, Map<String,String> propertiesToSet, Set<String> propertiesToExclude)
       throws AccumuloException, AccumuloSecurityException, TableNotFoundException, TableExistsException {
     throw new NotImplementedException();

Modified: accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java?rev=1342798&r1=1342797&r2=1342798&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java (original)
+++ accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java Fri May 25 20:24:33 2012
@@ -27,6 +27,7 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.TreeMap;
 
+import org.apache.accumulo.core.client.IteratorSetting;
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.Key;
@@ -35,10 +36,16 @@ import org.apache.accumulo.core.data.Ran
 import org.apache.accumulo.core.data.thrift.IterInfo;
 import org.apache.accumulo.core.iterators.system.SynchronizedIterator;
 import org.apache.accumulo.core.iterators.user.VersioningIterator;
+import org.apache.accumulo.core.tabletserver.thrift.IteratorConfig;
+import org.apache.accumulo.core.tabletserver.thrift.TIteratorSetting;
 import org.apache.accumulo.start.classloader.AccumuloClassLoader;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.log4j.Logger;
+import org.apache.thrift.TDeserializer;
+import org.apache.thrift.TException;
+import org.apache.thrift.TSerializer;
+import org.apache.thrift.protocol.TBinaryProtocol;
 public class IteratorUtil {
   
   private static final Logger log = Logger.getLogger(IteratorUtil.class);
@@ -155,6 +162,21 @@ public class IteratorUtil {
   }
   
   public static <K extends WritableComparable<?>,V extends Writable> SortedKeyValueIterator<K,V> loadIterators(IteratorScope scope,
+      SortedKeyValueIterator<K,V> source, KeyExtent extent, AccumuloConfiguration conf, List<IteratorSetting> iterators, IteratorEnvironment env)
+      throws IOException {
+    
+    List<IterInfo> ssiList = new ArrayList<IterInfo>();
+    Map<String,Map<String,String>> ssio = new HashMap<String,Map<String,String>>();
+    
+    for (IteratorSetting is : iterators) {
+      ssiList.add(new IterInfo(is.getPriority(), is.getIteratorClass(), is.getName()));
+      ssio.put(is.getName(), is.getOptions());
+    }
+    
+    return loadIterators(scope, source, extent, conf, ssiList, ssio, env, true);
+  }
+  
+  public static <K extends WritableComparable<?>,V extends Writable> SortedKeyValueIterator<K,V> loadIterators(IteratorScope scope,
       SortedKeyValueIterator<K,V> source, KeyExtent extent, AccumuloConfiguration conf, List<IterInfo> ssiList, Map<String,Map<String,String>> ssio,
       IteratorEnvironment env) throws IOException {
     return loadIterators(scope, source, extent, conf, ssiList, ssio, env, true);
@@ -243,4 +265,57 @@ public class IteratorUtil {
     
     return seekRange;
   }
+  
+  public static TIteratorSetting toTIteratorSetting(IteratorSetting is) {
+    return new TIteratorSetting(is.getPriority(), is.getName(), is.getIteratorClass(), is.getOptions());
+  }
+  
+  public static IteratorSetting toIteratorSetting(TIteratorSetting tis) {
+    return new IteratorSetting(tis.getPriority(), tis.getName(), tis.getIteratorClass(), tis.getProperties());
+  }
+
+  public static IteratorConfig toIteratorConfig(List<IteratorSetting> iterators) {
+    ArrayList<TIteratorSetting> tisList = new ArrayList<TIteratorSetting>();
+    
+    for (IteratorSetting iteratorSetting : iterators) {
+      tisList.add(toTIteratorSetting(iteratorSetting));
+    }
+    
+    return new IteratorConfig(tisList);
+  }
+  
+  public static List<IteratorSetting> toIteratorSettings(IteratorConfig ic) {
+    List<IteratorSetting> ret = new ArrayList<IteratorSetting>();
+    for (TIteratorSetting tIteratorSetting : ic.getIterators()) {
+      ret.add(toIteratorSetting(tIteratorSetting));
+    }
+    
+    return ret;
+  }
+
+  public static byte[] encodeIteratorSettings(IteratorConfig iterators) {
+    TSerializer tser = new TSerializer(new TBinaryProtocol.Factory());
+    
+    try {
+      return tser.serialize(iterators);
+    } catch (TException e) {
+      throw new RuntimeException(e);
+    }
+  }
+  
+  public static byte[] encodeIteratorSettings(List<IteratorSetting> iterators) {
+    return encodeIteratorSettings(toIteratorConfig(iterators));
+  }
+
+
+  public static List<IteratorSetting> decodeIteratorSettings(byte[] enc) {
+    TDeserializer tdser = new TDeserializer(new TBinaryProtocol.Factory());
+    IteratorConfig ic = new IteratorConfig();
+    try {
+      tdser.deserialize(ic, enc);
+    } catch (TException e) {
+      throw new RuntimeException(e);
+    }
+    return toIteratorSettings(ic);
+  }
 }

Modified: accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java?rev=1342798&r1=1342797&r2=1342798&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java (original)
+++ accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java Fri May 25 20:24:33 2012
@@ -4018,14 +4018,14 @@ import org.slf4j.LoggerFactory;
           case 4: // COLUMNS
             if (field.type == org.apache.thrift.protocol.TType.LIST) {
               {
-                org.apache.thrift.protocol.TList _list26 = iprot.readListBegin();
-                this.columns = new ArrayList<org.apache.accumulo.core.data.thrift.TColumn>(_list26.size);
-                for (int _i27 = 0; _i27 < _list26.size; ++_i27)
+                org.apache.thrift.protocol.TList _list35 = iprot.readListBegin();
+                this.columns = new ArrayList<org.apache.accumulo.core.data.thrift.TColumn>(_list35.size);
+                for (int _i36 = 0; _i36 < _list35.size; ++_i36)
                 {
-                  org.apache.accumulo.core.data.thrift.TColumn _elem28;
-                  _elem28 = new org.apache.accumulo.core.data.thrift.TColumn();
-                  _elem28.read(iprot);
-                  this.columns.add(_elem28);
+                  org.apache.accumulo.core.data.thrift.TColumn _elem37;
+                  _elem37 = new org.apache.accumulo.core.data.thrift.TColumn();
+                  _elem37.read(iprot);
+                  this.columns.add(_elem37);
                 }
                 iprot.readListEnd();
               }
@@ -4044,14 +4044,14 @@ import org.slf4j.LoggerFactory;
           case 6: // SSI_LIST
             if (field.type == org.apache.thrift.protocol.TType.LIST) {
               {
-                org.apache.thrift.protocol.TList _list29 = iprot.readListBegin();
-                this.ssiList = new ArrayList<org.apache.accumulo.core.data.thrift.IterInfo>(_list29.size);
-                for (int _i30 = 0; _i30 < _list29.size; ++_i30)
+                org.apache.thrift.protocol.TList _list38 = iprot.readListBegin();
+                this.ssiList = new ArrayList<org.apache.accumulo.core.data.thrift.IterInfo>(_list38.size);
+                for (int _i39 = 0; _i39 < _list38.size; ++_i39)
                 {
-                  org.apache.accumulo.core.data.thrift.IterInfo _elem31;
-                  _elem31 = new org.apache.accumulo.core.data.thrift.IterInfo();
-                  _elem31.read(iprot);
-                  this.ssiList.add(_elem31);
+                  org.apache.accumulo.core.data.thrift.IterInfo _elem40;
+                  _elem40 = new org.apache.accumulo.core.data.thrift.IterInfo();
+                  _elem40.read(iprot);
+                  this.ssiList.add(_elem40);
                 }
                 iprot.readListEnd();
               }
@@ -4062,27 +4062,27 @@ import org.slf4j.LoggerFactory;
           case 7: // SSIO
             if (field.type == org.apache.thrift.protocol.TType.MAP) {
               {
-                org.apache.thrift.protocol.TMap _map32 = iprot.readMapBegin();
-                this.ssio = new HashMap<String,Map<String,String>>(2*_map32.size);
-                for (int _i33 = 0; _i33 < _map32.size; ++_i33)
+                org.apache.thrift.protocol.TMap _map41 = iprot.readMapBegin();
+                this.ssio = new HashMap<String,Map<String,String>>(2*_map41.size);
+                for (int _i42 = 0; _i42 < _map41.size; ++_i42)
                 {
-                  String _key34;
-                  Map<String,String> _val35;
-                  _key34 = iprot.readString();
+                  String _key43;
+                  Map<String,String> _val44;
+                  _key43 = iprot.readString();
                   {
-                    org.apache.thrift.protocol.TMap _map36 = iprot.readMapBegin();
-                    _val35 = new HashMap<String,String>(2*_map36.size);
-                    for (int _i37 = 0; _i37 < _map36.size; ++_i37)
+                    org.apache.thrift.protocol.TMap _map45 = iprot.readMapBegin();
+                    _val44 = new HashMap<String,String>(2*_map45.size);
+                    for (int _i46 = 0; _i46 < _map45.size; ++_i46)
                     {
-                      String _key38;
-                      String _val39;
-                      _key38 = iprot.readString();
-                      _val39 = iprot.readString();
-                      _val35.put(_key38, _val39);
+                      String _key47;
+                      String _val48;
+                      _key47 = iprot.readString();
+                      _val48 = iprot.readString();
+                      _val44.put(_key47, _val48);
                     }
                     iprot.readMapEnd();
                   }
-                  this.ssio.put(_key34, _val35);
+                  this.ssio.put(_key43, _val44);
                 }
                 iprot.readMapEnd();
               }
@@ -4093,13 +4093,13 @@ import org.slf4j.LoggerFactory;
           case 8: // AUTHORIZATIONS
             if (field.type == org.apache.thrift.protocol.TType.LIST) {
               {
-                org.apache.thrift.protocol.TList _list40 = iprot.readListBegin();
-                this.authorizations = new ArrayList<ByteBuffer>(_list40.size);
-                for (int _i41 = 0; _i41 < _list40.size; ++_i41)
+                org.apache.thrift.protocol.TList _list49 = iprot.readListBegin();
+                this.authorizations = new ArrayList<ByteBuffer>(_list49.size);
+                for (int _i50 = 0; _i50 < _list49.size; ++_i50)
                 {
-                  ByteBuffer _elem42;
-                  _elem42 = iprot.readBinary();
-                  this.authorizations.add(_elem42);
+                  ByteBuffer _elem51;
+                  _elem51 = iprot.readBinary();
+                  this.authorizations.add(_elem51);
                 }
                 iprot.readListEnd();
               }
@@ -4157,9 +4157,9 @@ import org.slf4j.LoggerFactory;
         oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
         {
           oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, this.columns.size()));
-          for (org.apache.accumulo.core.data.thrift.TColumn _iter43 : this.columns)
+          for (org.apache.accumulo.core.data.thrift.TColumn _iter52 : this.columns)
           {
-            _iter43.write(oprot);
+            _iter52.write(oprot);
           }
           oprot.writeListEnd();
         }
@@ -4172,9 +4172,9 @@ import org.slf4j.LoggerFactory;
         oprot.writeFieldBegin(SSI_LIST_FIELD_DESC);
         {
           oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, this.ssiList.size()));
-          for (org.apache.accumulo.core.data.thrift.IterInfo _iter44 : this.ssiList)
+          for (org.apache.accumulo.core.data.thrift.IterInfo _iter53 : this.ssiList)
           {
-            _iter44.write(oprot);
+            _iter53.write(oprot);
           }
           oprot.writeListEnd();
         }
@@ -4184,15 +4184,15 @@ import org.slf4j.LoggerFactory;
         oprot.writeFieldBegin(SSIO_FIELD_DESC);
         {
           oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.MAP, this.ssio.size()));
-          for (Map.Entry<String, Map<String,String>> _iter45 : this.ssio.entrySet())
+          for (Map.Entry<String, Map<String,String>> _iter54 : this.ssio.entrySet())
           {
-            oprot.writeString(_iter45.getKey());
+            oprot.writeString(_iter54.getKey());
             {
-              oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, _iter45.getValue().size()));
-              for (Map.Entry<String, String> _iter46 : _iter45.getValue().entrySet())
+              oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, _iter54.getValue().size()));
+              for (Map.Entry<String, String> _iter55 : _iter54.getValue().entrySet())
               {
-                oprot.writeString(_iter46.getKey());
-                oprot.writeString(_iter46.getValue());
+                oprot.writeString(_iter55.getKey());
+                oprot.writeString(_iter55.getValue());
               }
               oprot.writeMapEnd();
             }
@@ -4205,9 +4205,9 @@ import org.slf4j.LoggerFactory;
         oprot.writeFieldBegin(AUTHORIZATIONS_FIELD_DESC);
         {
           oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, this.authorizations.size()));
-          for (ByteBuffer _iter47 : this.authorizations)
+          for (ByteBuffer _iter56 : this.authorizations)
           {
-            oprot.writeBinary(_iter47);
+            oprot.writeBinary(_iter56);
           }
           oprot.writeListEnd();
         }
@@ -7082,27 +7082,27 @@ import org.slf4j.LoggerFactory;
           case 2: // BATCH
             if (field.type == org.apache.thrift.protocol.TType.MAP) {
               {
-                org.apache.thrift.protocol.TMap _map48 = iprot.readMapBegin();
-                this.batch = new HashMap<org.apache.accumulo.core.data.thrift.TKeyExtent,List<org.apache.accumulo.core.data.thrift.TRange>>(2*_map48.size);
-                for (int _i49 = 0; _i49 < _map48.size; ++_i49)
+                org.apache.thrift.protocol.TMap _map57 = iprot.readMapBegin();
+                this.batch = new HashMap<org.apache.accumulo.core.data.thrift.TKeyExtent,List<org.apache.accumulo.core.data.thrift.TRange>>(2*_map57.size);
+                for (int _i58 = 0; _i58 < _map57.size; ++_i58)
                 {
-                  org.apache.accumulo.core.data.thrift.TKeyExtent _key50;
-                  List<org.apache.accumulo.core.data.thrift.TRange> _val51;
-                  _key50 = new org.apache.accumulo.core.data.thrift.TKeyExtent();
-                  _key50.read(iprot);
+                  org.apache.accumulo.core.data.thrift.TKeyExtent _key59;
+                  List<org.apache.accumulo.core.data.thrift.TRange> _val60;
+                  _key59 = new org.apache.accumulo.core.data.thrift.TKeyExtent();
+                  _key59.read(iprot);
                   {
-                    org.apache.thrift.protocol.TList _list52 = iprot.readListBegin();
-                    _val51 = new ArrayList<org.apache.accumulo.core.data.thrift.TRange>(_list52.size);
-                    for (int _i53 = 0; _i53 < _list52.size; ++_i53)
+                    org.apache.thrift.protocol.TList _list61 = iprot.readListBegin();
+                    _val60 = new ArrayList<org.apache.accumulo.core.data.thrift.TRange>(_list61.size);
+                    for (int _i62 = 0; _i62 < _list61.size; ++_i62)
                     {
-                      org.apache.accumulo.core.data.thrift.TRange _elem54;
-                      _elem54 = new org.apache.accumulo.core.data.thrift.TRange();
-                      _elem54.read(iprot);
-                      _val51.add(_elem54);
+                      org.apache.accumulo.core.data.thrift.TRange _elem63;
+                      _elem63 = new org.apache.accumulo.core.data.thrift.TRange();
+                      _elem63.read(iprot);
+                      _val60.add(_elem63);
                     }
                     iprot.readListEnd();
                   }
-                  this.batch.put(_key50, _val51);
+                  this.batch.put(_key59, _val60);
                 }
                 iprot.readMapEnd();
               }
@@ -7113,14 +7113,14 @@ import org.slf4j.LoggerFactory;
           case 3: // COLUMNS
             if (field.type == org.apache.thrift.protocol.TType.LIST) {
               {
-                org.apache.thrift.protocol.TList _list55 = iprot.readListBegin();
-                this.columns = new ArrayList<org.apache.accumulo.core.data.thrift.TColumn>(_list55.size);
-                for (int _i56 = 0; _i56 < _list55.size; ++_i56)
+                org.apache.thrift.protocol.TList _list64 = iprot.readListBegin();
+                this.columns = new ArrayList<org.apache.accumulo.core.data.thrift.TColumn>(_list64.size);
+                for (int _i65 = 0; _i65 < _list64.size; ++_i65)
                 {
-                  org.apache.accumulo.core.data.thrift.TColumn _elem57;
-                  _elem57 = new org.apache.accumulo.core.data.thrift.TColumn();
-                  _elem57.read(iprot);
-                  this.columns.add(_elem57);
+                  org.apache.accumulo.core.data.thrift.TColumn _elem66;
+                  _elem66 = new org.apache.accumulo.core.data.thrift.TColumn();
+                  _elem66.read(iprot);
+                  this.columns.add(_elem66);
                 }
                 iprot.readListEnd();
               }
@@ -7131,14 +7131,14 @@ import org.slf4j.LoggerFactory;
           case 4: // SSI_LIST
             if (field.type == org.apache.thrift.protocol.TType.LIST) {
               {
-                org.apache.thrift.protocol.TList _list58 = iprot.readListBegin();
-                this.ssiList = new ArrayList<org.apache.accumulo.core.data.thrift.IterInfo>(_list58.size);
-                for (int _i59 = 0; _i59 < _list58.size; ++_i59)
+                org.apache.thrift.protocol.TList _list67 = iprot.readListBegin();
+                this.ssiList = new ArrayList<org.apache.accumulo.core.data.thrift.IterInfo>(_list67.size);
+                for (int _i68 = 0; _i68 < _list67.size; ++_i68)
                 {
-                  org.apache.accumulo.core.data.thrift.IterInfo _elem60;
-                  _elem60 = new org.apache.accumulo.core.data.thrift.IterInfo();
-                  _elem60.read(iprot);
-                  this.ssiList.add(_elem60);
+                  org.apache.accumulo.core.data.thrift.IterInfo _elem69;
+                  _elem69 = new org.apache.accumulo.core.data.thrift.IterInfo();
+                  _elem69.read(iprot);
+                  this.ssiList.add(_elem69);
                 }
                 iprot.readListEnd();
               }
@@ -7149,27 +7149,27 @@ import org.slf4j.LoggerFactory;
           case 5: // SSIO
             if (field.type == org.apache.thrift.protocol.TType.MAP) {
               {
-                org.apache.thrift.protocol.TMap _map61 = iprot.readMapBegin();
-                this.ssio = new HashMap<String,Map<String,String>>(2*_map61.size);
-                for (int _i62 = 0; _i62 < _map61.size; ++_i62)
+                org.apache.thrift.protocol.TMap _map70 = iprot.readMapBegin();
+                this.ssio = new HashMap<String,Map<String,String>>(2*_map70.size);
+                for (int _i71 = 0; _i71 < _map70.size; ++_i71)
                 {
-                  String _key63;
-                  Map<String,String> _val64;
-                  _key63 = iprot.readString();
+                  String _key72;
+                  Map<String,String> _val73;
+                  _key72 = iprot.readString();
                   {
-                    org.apache.thrift.protocol.TMap _map65 = iprot.readMapBegin();
-                    _val64 = new HashMap<String,String>(2*_map65.size);
-                    for (int _i66 = 0; _i66 < _map65.size; ++_i66)
+                    org.apache.thrift.protocol.TMap _map74 = iprot.readMapBegin();
+                    _val73 = new HashMap<String,String>(2*_map74.size);
+                    for (int _i75 = 0; _i75 < _map74.size; ++_i75)
                     {
-                      String _key67;
-                      String _val68;
-                      _key67 = iprot.readString();
-                      _val68 = iprot.readString();
-                      _val64.put(_key67, _val68);
+                      String _key76;
+                      String _val77;
+                      _key76 = iprot.readString();
+                      _val77 = iprot.readString();
+                      _val73.put(_key76, _val77);
                     }
                     iprot.readMapEnd();
                   }
-                  this.ssio.put(_key63, _val64);
+                  this.ssio.put(_key72, _val73);
                 }
                 iprot.readMapEnd();
               }
@@ -7180,13 +7180,13 @@ import org.slf4j.LoggerFactory;
           case 6: // AUTHORIZATIONS
             if (field.type == org.apache.thrift.protocol.TType.LIST) {
               {
-                org.apache.thrift.protocol.TList _list69 = iprot.readListBegin();
-                this.authorizations = new ArrayList<ByteBuffer>(_list69.size);
-                for (int _i70 = 0; _i70 < _list69.size; ++_i70)
+                org.apache.thrift.protocol.TList _list78 = iprot.readListBegin();
+                this.authorizations = new ArrayList<ByteBuffer>(_list78.size);
+                for (int _i79 = 0; _i79 < _list78.size; ++_i79)
                 {
-                  ByteBuffer _elem71;
-                  _elem71 = iprot.readBinary();
-                  this.authorizations.add(_elem71);
+                  ByteBuffer _elem80;
+                  _elem80 = iprot.readBinary();
+                  this.authorizations.add(_elem80);
                 }
                 iprot.readListEnd();
               }
@@ -7226,14 +7226,14 @@ import org.slf4j.LoggerFactory;
         oprot.writeFieldBegin(BATCH_FIELD_DESC);
         {
           oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRUCT, org.apache.thrift.protocol.TType.LIST, this.batch.size()));
-          for (Map.Entry<org.apache.accumulo.core.data.thrift.TKeyExtent, List<org.apache.accumulo.core.data.thrift.TRange>> _iter72 : this.batch.entrySet())
+          for (Map.Entry<org.apache.accumulo.core.data.thrift.TKeyExtent, List<org.apache.accumulo.core.data.thrift.TRange>> _iter81 : this.batch.entrySet())
           {
-            _iter72.getKey().write(oprot);
+            _iter81.getKey().write(oprot);
             {
-              oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, _iter72.getValue().size()));
-              for (org.apache.accumulo.core.data.thrift.TRange _iter73 : _iter72.getValue())
+              oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, _iter81.getValue().size()));
+              for (org.apache.accumulo.core.data.thrift.TRange _iter82 : _iter81.getValue())
               {
-                _iter73.write(oprot);
+                _iter82.write(oprot);
               }
               oprot.writeListEnd();
             }
@@ -7246,9 +7246,9 @@ import org.slf4j.LoggerFactory;
         oprot.writeFieldBegin(COLUMNS_FIELD_DESC);
         {
           oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, this.columns.size()));
-          for (org.apache.accumulo.core.data.thrift.TColumn _iter74 : this.columns)
+          for (org.apache.accumulo.core.data.thrift.TColumn _iter83 : this.columns)
           {
-            _iter74.write(oprot);
+            _iter83.write(oprot);
           }
           oprot.writeListEnd();
         }
@@ -7258,9 +7258,9 @@ import org.slf4j.LoggerFactory;
         oprot.writeFieldBegin(SSI_LIST_FIELD_DESC);
         {
           oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, this.ssiList.size()));
-          for (org.apache.accumulo.core.data.thrift.IterInfo _iter75 : this.ssiList)
+          for (org.apache.accumulo.core.data.thrift.IterInfo _iter84 : this.ssiList)
           {
-            _iter75.write(oprot);
+            _iter84.write(oprot);
           }
           oprot.writeListEnd();
         }
@@ -7270,15 +7270,15 @@ import org.slf4j.LoggerFactory;
         oprot.writeFieldBegin(SSIO_FIELD_DESC);
         {
           oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.MAP, this.ssio.size()));
-          for (Map.Entry<String, Map<String,String>> _iter76 : this.ssio.entrySet())
+          for (Map.Entry<String, Map<String,String>> _iter85 : this.ssio.entrySet())
           {
-            oprot.writeString(_iter76.getKey());
+            oprot.writeString(_iter85.getKey());
             {
-              oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, _iter76.getValue().size()));
-              for (Map.Entry<String, String> _iter77 : _iter76.getValue().entrySet())
+              oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, _iter85.getValue().size()));
+              for (Map.Entry<String, String> _iter86 : _iter85.getValue().entrySet())
               {
-                oprot.writeString(_iter77.getKey());
-                oprot.writeString(_iter77.getValue());
+                oprot.writeString(_iter86.getKey());
+                oprot.writeString(_iter86.getValue());
               }
               oprot.writeMapEnd();
             }
@@ -7291,9 +7291,9 @@ import org.slf4j.LoggerFactory;
         oprot.writeFieldBegin(AUTHORIZATIONS_FIELD_DESC);
         {
           oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, this.authorizations.size()));
-          for (ByteBuffer _iter78 : this.authorizations)
+          for (ByteBuffer _iter87 : this.authorizations)
           {
-            oprot.writeBinary(_iter78);
+            oprot.writeBinary(_iter87);
           }
           oprot.writeListEnd();
         }
@@ -10504,14 +10504,14 @@ import org.slf4j.LoggerFactory;
           case 4: // MUTATIONS
             if (field.type == org.apache.thrift.protocol.TType.LIST) {
               {
-                org.apache.thrift.protocol.TList _list79 = iprot.readListBegin();
-                this.mutations = new ArrayList<org.apache.accumulo.core.data.thrift.TMutation>(_list79.size);
-                for (int _i80 = 0; _i80 < _list79.size; ++_i80)
+                org.apache.thrift.protocol.TList _list88 = iprot.readListBegin();
+                this.mutations = new ArrayList<org.apache.accumulo.core.data.thrift.TMutation>(_list88.size);
+                for (int _i89 = 0; _i89 < _list88.size; ++_i89)
                 {
-                  org.apache.accumulo.core.data.thrift.TMutation _elem81;
-                  _elem81 = new org.apache.accumulo.core.data.thrift.TMutation();
-                  _elem81.read(iprot);
-                  this.mutations.add(_elem81);
+                  org.apache.accumulo.core.data.thrift.TMutation _elem90;
+                  _elem90 = new org.apache.accumulo.core.data.thrift.TMutation();
+                  _elem90.read(iprot);
+                  this.mutations.add(_elem90);
                 }
                 iprot.readListEnd();
               }
@@ -10551,9 +10551,9 @@ import org.slf4j.LoggerFactory;
         oprot.writeFieldBegin(MUTATIONS_FIELD_DESC);
         {
           oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, this.mutations.size()));
-          for (org.apache.accumulo.core.data.thrift.TMutation _iter82 : this.mutations)
+          for (org.apache.accumulo.core.data.thrift.TMutation _iter91 : this.mutations)
           {
-            _iter82.write(oprot);
+            _iter91.write(oprot);
           }
           oprot.writeListEnd();
         }
@@ -12984,29 +12984,29 @@ import org.slf4j.LoggerFactory;
           case 2: // FILES
             if (field.type == org.apache.thrift.protocol.TType.MAP) {
               {
-                org.apache.thrift.protocol.TMap _map83 = iprot.readMapBegin();
-                this.files = new HashMap<org.apache.accumulo.core.data.thrift.TKeyExtent,Map<String,org.apache.accumulo.core.data.thrift.MapFileInfo>>(2*_map83.size);
-                for (int _i84 = 0; _i84 < _map83.size; ++_i84)
+                org.apache.thrift.protocol.TMap _map92 = iprot.readMapBegin();
+                this.files = new HashMap<org.apache.accumulo.core.data.thrift.TKeyExtent,Map<String,org.apache.accumulo.core.data.thrift.MapFileInfo>>(2*_map92.size);
+                for (int _i93 = 0; _i93 < _map92.size; ++_i93)
                 {
-                  org.apache.accumulo.core.data.thrift.TKeyExtent _key85;
-                  Map<String,org.apache.accumulo.core.data.thrift.MapFileInfo> _val86;
-                  _key85 = new org.apache.accumulo.core.data.thrift.TKeyExtent();
-                  _key85.read(iprot);
+                  org.apache.accumulo.core.data.thrift.TKeyExtent _key94;
+                  Map<String,org.apache.accumulo.core.data.thrift.MapFileInfo> _val95;
+                  _key94 = new org.apache.accumulo.core.data.thrift.TKeyExtent();
+                  _key94.read(iprot);
                   {
-                    org.apache.thrift.protocol.TMap _map87 = iprot.readMapBegin();
-                    _val86 = new HashMap<String,org.apache.accumulo.core.data.thrift.MapFileInfo>(2*_map87.size);
-                    for (int _i88 = 0; _i88 < _map87.size; ++_i88)
+                    org.apache.thrift.protocol.TMap _map96 = iprot.readMapBegin();
+                    _val95 = new HashMap<String,org.apache.accumulo.core.data.thrift.MapFileInfo>(2*_map96.size);
+                    for (int _i97 = 0; _i97 < _map96.size; ++_i97)
                     {
-                      String _key89;
-                      org.apache.accumulo.core.data.thrift.MapFileInfo _val90;
-                      _key89 = iprot.readString();
-                      _val90 = new org.apache.accumulo.core.data.thrift.MapFileInfo();
-                      _val90.read(iprot);
-                      _val86.put(_key89, _val90);
+                      String _key98;
+                      org.apache.accumulo.core.data.thrift.MapFileInfo _val99;
+                      _key98 = iprot.readString();
+                      _val99 = new org.apache.accumulo.core.data.thrift.MapFileInfo();
+                      _val99.read(iprot);
+                      _val95.put(_key98, _val99);
                     }
                     iprot.readMapEnd();
                   }
-                  this.files.put(_key85, _val86);
+                  this.files.put(_key94, _val95);
                 }
                 iprot.readMapEnd();
               }
@@ -13046,15 +13046,15 @@ import org.slf4j.LoggerFactory;
         oprot.writeFieldBegin(FILES_FIELD_DESC);
         {
           oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRUCT, org.apache.thrift.protocol.TType.MAP, this.files.size()));
-          for (Map.Entry<org.apache.accumulo.core.data.thrift.TKeyExtent, Map<String,org.apache.accumulo.core.data.thrift.MapFileInfo>> _iter91 : this.files.entrySet())
+          for (Map.Entry<org.apache.accumulo.core.data.thrift.TKeyExtent, Map<String,org.apache.accumulo.core.data.thrift.MapFileInfo>> _iter100 : this.files.entrySet())
           {
-            _iter91.getKey().write(oprot);
+            _iter100.getKey().write(oprot);
             {
-              oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, _iter91.getValue().size()));
-              for (Map.Entry<String, org.apache.accumulo.core.data.thrift.MapFileInfo> _iter92 : _iter91.getValue().entrySet())
+              oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, _iter100.getValue().size()));
+              for (Map.Entry<String, org.apache.accumulo.core.data.thrift.MapFileInfo> _iter101 : _iter100.getValue().entrySet())
               {
-                oprot.writeString(_iter92.getKey());
-                _iter92.getValue().write(oprot);
+                oprot.writeString(_iter101.getKey());
+                _iter101.getValue().write(oprot);
               }
               oprot.writeMapEnd();
             }
@@ -13462,14 +13462,14 @@ import org.slf4j.LoggerFactory;
           case 0: // SUCCESS
             if (field.type == org.apache.thrift.protocol.TType.LIST) {
               {
-                org.apache.thrift.protocol.TList _list93 = iprot.readListBegin();
-                this.success = new ArrayList<org.apache.accumulo.core.data.thrift.TKeyExtent>(_list93.size);
-                for (int _i94 = 0; _i94 < _list93.size; ++_i94)
+                org.apache.thrift.protocol.TList _list102 = iprot.readListBegin();
+                this.success = new ArrayList<org.apache.accumulo.core.data.thrift.TKeyExtent>(_list102.size);
+                for (int _i103 = 0; _i103 < _list102.size; ++_i103)
                 {
-                  org.apache.accumulo.core.data.thrift.TKeyExtent _elem95;
-                  _elem95 = new org.apache.accumulo.core.data.thrift.TKeyExtent();
-                  _elem95.read(iprot);
-                  this.success.add(_elem95);
+                  org.apache.accumulo.core.data.thrift.TKeyExtent _elem104;
+                  _elem104 = new org.apache.accumulo.core.data.thrift.TKeyExtent();
+                  _elem104.read(iprot);
+                  this.success.add(_elem104);
                 }
                 iprot.readListEnd();
               }
@@ -13503,9 +13503,9 @@ import org.slf4j.LoggerFactory;
         oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
         {
           oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, this.success.size()));
-          for (org.apache.accumulo.core.data.thrift.TKeyExtent _iter96 : this.success)
+          for (org.apache.accumulo.core.data.thrift.TKeyExtent _iter105 : this.success)
           {
-            _iter96.write(oprot);
+            _iter105.write(oprot);
           }
           oprot.writeListEnd();
         }
@@ -19989,14 +19989,14 @@ import org.slf4j.LoggerFactory;
           case 0: // SUCCESS
             if (field.type == org.apache.thrift.protocol.TType.LIST) {
               {
-                org.apache.thrift.protocol.TList _list97 = iprot.readListBegin();
-                this.success = new ArrayList<TabletStats>(_list97.size);
-                for (int _i98 = 0; _i98 < _list97.size; ++_i98)
+                org.apache.thrift.protocol.TList _list106 = iprot.readListBegin();
+                this.success = new ArrayList<TabletStats>(_list106.size);
+                for (int _i107 = 0; _i107 < _list106.size; ++_i107)
                 {
-                  TabletStats _elem99;
-                  _elem99 = new TabletStats();
-                  _elem99.read(iprot);
-                  this.success.add(_elem99);
+                  TabletStats _elem108;
+                  _elem108 = new TabletStats();
+                  _elem108.read(iprot);
+                  this.success.add(_elem108);
                 }
                 iprot.readListEnd();
               }
@@ -20030,9 +20030,9 @@ import org.slf4j.LoggerFactory;
         oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
         {
           oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, this.success.size()));
-          for (TabletStats _iter100 : this.success)
+          for (TabletStats _iter109 : this.success)
           {
-            _iter100.write(oprot);
+            _iter109.write(oprot);
           }
           oprot.writeListEnd();
         }
@@ -22825,14 +22825,14 @@ import org.slf4j.LoggerFactory;
           case 0: // SUCCESS
             if (field.type == org.apache.thrift.protocol.TType.LIST) {
               {
-                org.apache.thrift.protocol.TList _list101 = iprot.readListBegin();
-                this.success = new ArrayList<ActiveScan>(_list101.size);
-                for (int _i102 = 0; _i102 < _list101.size; ++_i102)
+                org.apache.thrift.protocol.TList _list110 = iprot.readListBegin();
+                this.success = new ArrayList<ActiveScan>(_list110.size);
+                for (int _i111 = 0; _i111 < _list110.size; ++_i111)
                 {
-                  ActiveScan _elem103;
-                  _elem103 = new ActiveScan();
-                  _elem103.read(iprot);
-                  this.success.add(_elem103);
+                  ActiveScan _elem112;
+                  _elem112 = new ActiveScan();
+                  _elem112.read(iprot);
+                  this.success.add(_elem112);
                 }
                 iprot.readListEnd();
               }
@@ -22866,9 +22866,9 @@ import org.slf4j.LoggerFactory;
         oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
         {
           oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, this.success.size()));
-          for (ActiveScan _iter104 : this.success)
+          for (ActiveScan _iter113 : this.success)
           {
-            _iter104.write(oprot);
+            _iter113.write(oprot);
           }
           oprot.writeListEnd();
         }
@@ -24284,13 +24284,13 @@ import org.slf4j.LoggerFactory;
           case 3: // FILENAMES
             if (field.type == org.apache.thrift.protocol.TType.LIST) {
               {
-                org.apache.thrift.protocol.TList _list105 = iprot.readListBegin();
-                this.filenames = new ArrayList<String>(_list105.size);
-                for (int _i106 = 0; _i106 < _list105.size; ++_i106)
+                org.apache.thrift.protocol.TList _list114 = iprot.readListBegin();
+                this.filenames = new ArrayList<String>(_list114.size);
+                for (int _i115 = 0; _i115 < _list114.size; ++_i115)
                 {
-                  String _elem107;
-                  _elem107 = iprot.readString();
-                  this.filenames.add(_elem107);
+                  String _elem116;
+                  _elem116 = iprot.readString();
+                  this.filenames.add(_elem116);
                 }
                 iprot.readListEnd();
               }
@@ -24327,9 +24327,9 @@ import org.slf4j.LoggerFactory;
         oprot.writeFieldBegin(FILENAMES_FIELD_DESC);
         {
           oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, this.filenames.size()));
-          for (String _iter108 : this.filenames)
+          for (String _iter117 : this.filenames)
           {
-            oprot.writeString(_iter108);
+            oprot.writeString(_iter117);
           }
           oprot.writeListEnd();
         }

Modified: accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletMutations.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletMutations.java?rev=1342798&r1=1342797&r2=1342798&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletMutations.java (original)
+++ accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletMutations.java Fri May 25 20:24:33 2012
@@ -428,14 +428,14 @@ import org.slf4j.LoggerFactory;
         case 3: // MUTATIONS
           if (field.type == org.apache.thrift.protocol.TType.LIST) {
             {
-              org.apache.thrift.protocol.TList _list22 = iprot.readListBegin();
-              this.mutations = new ArrayList<org.apache.accumulo.core.data.thrift.TMutation>(_list22.size);
-              for (int _i23 = 0; _i23 < _list22.size; ++_i23)
+              org.apache.thrift.protocol.TList _list31 = iprot.readListBegin();
+              this.mutations = new ArrayList<org.apache.accumulo.core.data.thrift.TMutation>(_list31.size);
+              for (int _i32 = 0; _i32 < _list31.size; ++_i32)
               {
-                org.apache.accumulo.core.data.thrift.TMutation _elem24;
-                _elem24 = new org.apache.accumulo.core.data.thrift.TMutation();
-                _elem24.read(iprot);
-                this.mutations.add(_elem24);
+                org.apache.accumulo.core.data.thrift.TMutation _elem33;
+                _elem33 = new org.apache.accumulo.core.data.thrift.TMutation();
+                _elem33.read(iprot);
+                this.mutations.add(_elem33);
               }
               iprot.readListEnd();
             }
@@ -468,9 +468,9 @@ import org.slf4j.LoggerFactory;
       oprot.writeFieldBegin(MUTATIONS_FIELD_DESC);
       {
         oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, this.mutations.size()));
-        for (org.apache.accumulo.core.data.thrift.TMutation _iter25 : this.mutations)
+        for (org.apache.accumulo.core.data.thrift.TMutation _iter34 : this.mutations)
         {
-          _iter25.write(oprot);
+          _iter34.write(oprot);
         }
         oprot.writeListEnd();
       }

Modified: accumulo/branches/ACCUMULO-578/core/src/main/thrift/tabletserver.thrift
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/core/src/main/thrift/tabletserver.thrift?rev=1342798&r1=1342797&r2=1342798&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/core/src/main/thrift/tabletserver.thrift (original)
+++ accumulo/branches/ACCUMULO-578/core/src/main/thrift/tabletserver.thrift Fri May 25 20:24:33 2012
@@ -85,6 +85,17 @@ struct ActiveScan {
     12:map<string, map<string, string>> ssio  /* Server Side Iterator Options */
 }
 
+struct TIteratorSetting {
+    1:i32 priority;
+    2:string name;
+    3:string iteratorClass;
+    4:map<string,string> properties;
+}
+
+struct IteratorConfig {
+   1:list<TIteratorSetting> iterators;
+}
+
 service TabletClientService extends client.ClientService {
   // scan a range of keys
   data.InitialScan startScan(11:cloudtrace.TInfo tinfo,

Modified: accumulo/branches/ACCUMULO-578/core/src/test/java/org/apache/accumulo/core/client/admin/TableOperationsHelperTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/core/src/test/java/org/apache/accumulo/core/client/admin/TableOperationsHelperTest.java?rev=1342798&r1=1342797&r2=1342798&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/core/src/test/java/org/apache/accumulo/core/client/admin/TableOperationsHelperTest.java (original)
+++ accumulo/branches/ACCUMULO-578/core/src/test/java/org/apache/accumulo/core/client/admin/TableOperationsHelperTest.java Fri May 25 20:24:33 2012
@@ -21,6 +21,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
@@ -97,6 +98,10 @@ public class TableOperationsHelperTest {
         AccumuloException {}
     
     @Override
+    public void compact(String tableName, Text start, Text end, List<IteratorSetting> iterators, boolean flush, boolean wait) throws AccumuloSecurityException,
+        TableNotFoundException, AccumuloException {}
+    
+    @Override
     public void delete(String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {}
     
     @Override
@@ -177,7 +182,6 @@ public class TableOperationsHelperTest {
       }
       Assert.assertEquals(expected, settings.get(tablename));
     }
-    
   }
   
   @Test

Propchange: accumulo/branches/ACCUMULO-578/server/
------------------------------------------------------------------------------
  Merged /accumulo/trunk/server:r1341000,1341572-1342236,1342774-1342794

Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1342798&r1=1342797&r2=1342798&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/master/Master.java (original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/master/Master.java Fri May 25 20:24:33 2012
@@ -47,6 +47,7 @@ import org.apache.accumulo.core.client.B
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.Instance;
 import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
 import org.apache.accumulo.core.client.MutationsRejectedException;
 import org.apache.accumulo.core.client.RowIterator;
 import org.apache.accumulo.core.client.Scanner;
@@ -66,6 +67,7 @@ import org.apache.accumulo.core.data.Ran
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.data.thrift.TKeyExtent;
 import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil;
 import org.apache.accumulo.core.master.state.tables.TableState;
 import org.apache.accumulo.core.master.thrift.MasterClientService;
 import org.apache.accumulo.core.master.thrift.MasterClientService.Processor;
@@ -1022,11 +1024,12 @@ public class Master implements LiveTServ
           String tableId = ByteBufferUtil.toString(arguments.get(0));
           byte[] startRow = ByteBufferUtil.toBytes(arguments.get(1));
           byte[] endRow = ByteBufferUtil.toBytes(arguments.get(2));
+          List<IteratorSetting> iterators = IteratorUtil.decodeIteratorSettings(ByteBufferUtil.toBytes(arguments.get(3)));
           
           verify(c, tableId, TableOperation.COMPACT,
               check(c, tableId, TablePermission.WRITE) || check(c, tableId, TablePermission.ALTER_TABLE) || check(c, SystemPermission.ALTER_TABLE));
           
-          fate.seedTransaction(opid, new TraceRepo<Master>(new CompactRange(tableId, startRow, endRow)), autoCleanup);
+          fate.seedTransaction(opid, new TraceRepo<Master>(new CompactRange(tableId, startRow, endRow, iterators)), autoCleanup);
           break;
         }
         default:

Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java?rev=1342798&r1=1342797&r2=1342798&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java (original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/master/tableOps/CompactRange.java Fri May 25 20:24:33 2012
@@ -18,12 +18,14 @@ package org.apache.accumulo.server.maste
 
 import java.util.Collections;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map.Entry;
 
 import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.Instance;
 import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
 import org.apache.accumulo.core.client.RowIterator;
 import org.apache.accumulo.core.client.Scanner;
 import org.apache.accumulo.core.client.impl.Tables;
@@ -34,7 +36,10 @@ import org.apache.accumulo.core.data.Key
 import org.apache.accumulo.core.data.KeyExtent;
 import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorUtil;
 import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.tabletserver.thrift.IteratorConfig;
+import org.apache.accumulo.core.tabletserver.thrift.TIteratorSetting;
 import org.apache.accumulo.core.util.ColumnFQ;
 import org.apache.accumulo.server.client.HdfsZooInstance;
 import org.apache.accumulo.server.fate.Repo;
@@ -46,6 +51,7 @@ import org.apache.accumulo.server.util.M
 import org.apache.accumulo.server.zookeeper.IZooReaderWriter;
 import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 import org.apache.accumulo.server.zookeeper.ZooReaderWriter.Mutator;
+import org.apache.commons.codec.binary.Hex;
 import org.apache.hadoop.io.Text;
 import org.apache.log4j.Logger;
 import org.apache.thrift.TException;
@@ -161,6 +167,7 @@ class CompactionDriver extends MasterRep
   
   @Override
   public Repo<Master> call(long tid, Master environment) throws Exception {
+    CompactRange.removeIterators(tid, tableId);
     Utils.getReadLock(tableId, tid).unlock();
     return null;
   }
@@ -178,11 +185,14 @@ public class CompactRange extends Master
   private String tableId;
   private byte[] startRow;
   private byte[] endRow;
+  private IteratorConfig iterators;
   
-  public CompactRange(String tableId, byte[] startRow, byte[] endRow) throws ThriftTableOperationException {
+  public CompactRange(String tableId, byte[] startRow, byte[] endRow, List<IteratorSetting> iterators) throws ThriftTableOperationException {
     this.tableId = tableId;
     this.startRow = startRow.length == 0 ? null : startRow;
     this.endRow = endRow.length == 0 ? null : endRow;
+    // store as IteratorConfig because its serializable
+    this.iterators = IteratorUtil.toIteratorConfig(iterators);
     
     if (this.startRow != null && this.endRow != null && new Text(startRow).compareTo(new Text(endRow)) >= 0)
       throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.BAD_RANGE,
@@ -195,7 +205,7 @@ public class CompactRange extends Master
   }
   
   @Override
-  public Repo<Master> call(long tid, Master environment) throws Exception {
+  public Repo<Master> call(final long tid, Master environment) throws Exception {
     String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_COMPACT_ID;
     
     IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
@@ -204,22 +214,78 @@ public class CompactRange extends Master
       cid = zoo.mutate(zTablePath, null, null, new Mutator() {
         @Override
         public byte[] mutate(byte[] currentValue) throws Exception {
-          long flushID = Long.parseLong(new String(currentValue));
+          String cvs = new String(currentValue);
+          String[] tokens = cvs.split(",");
+          long flushID = Long.parseLong(new String(tokens[0]));
           flushID++;
-          return ("" + flushID).getBytes();
+          
+          String txidString = String.format("%016x", tid);
+          StringBuilder encodedIterators = new StringBuilder();
+          for (int i = 1; i < tokens.length; i++) {
+            if (tokens[i].startsWith(txidString))
+              continue; // skip self
+            encodedIterators.append(",");
+            encodedIterators.append(tokens[i]);
+          }
+
+          if (iterators != null && iterators.getIterators().size() > 0) {
+            Hex hex = new Hex();
+            encodedIterators.append(",");
+            encodedIterators.append(txidString);
+            encodedIterators.append("=");
+            for (TIteratorSetting tis : iterators.getIterators()) {
+              if (tis.iteratorClass != null)
+                tis.name = txidString + tis.name; // give a unique name to avoid collisions with other running compactions
+            }
+            encodedIterators.append(new String(hex.encode(IteratorUtil.encodeIteratorSettings(iterators))));
+          }
+          
+          return ("" + flushID + encodedIterators).getBytes();
         }
       });
       
-      return new CompactionDriver(Long.parseLong(new String(cid)), tableId, startRow, endRow);
+      return new CompactionDriver(Long.parseLong(new String(cid).split(",")[0]), tableId, startRow, endRow);
     } catch (NoNodeException nne) {
       throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.NOTFOUND, null);
     }
     
   }
   
+  static void removeIterators(final long txid, String tableId) throws Exception {
+    String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_COMPACT_ID;
+    
+    IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance();
+    
+    zoo.mutate(zTablePath, null, null, new Mutator() {
+      @Override
+      public byte[] mutate(byte[] currentValue) throws Exception {
+        String cvs = new String(currentValue);
+        String[] tokens = cvs.split(",");
+        long flushID = Long.parseLong(new String(tokens[0]));
+
+        String txidString = String.format("%016x", txid);
+        
+        StringBuilder encodedIterators = new StringBuilder();
+        for (int i = 1; i < tokens.length; i++) {
+          if (tokens[i].startsWith(txidString))
+            continue;
+          encodedIterators.append(",");
+          encodedIterators.append(tokens[i]);
+        }
+        
+        return ("" + flushID + encodedIterators).getBytes();
+      }
+    });
+
+  }
+
   @Override
   public void undo(long tid, Master environment) throws Exception {
-    Utils.unreserveTable(tableId, tid, false);
+    try {
+      removeIterators(tid, tableId);
+    } finally {
+      Utils.unreserveTable(tableId, tid, false);
+    }
   }
   
 }

Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java?rev=1342798&r1=1342797&r2=1342798&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java (original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java Fri May 25 20:24:33 2012
@@ -27,6 +27,7 @@ import java.util.concurrent.Callable;
 
 import org.apache.accumulo.cloudtrace.instrument.Span;
 import org.apache.accumulo.cloudtrace.instrument.Trace;
+import org.apache.accumulo.core.client.IteratorSetting;
 import org.apache.accumulo.core.data.ByteSequence;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.KeyExtent;
@@ -38,7 +39,6 @@ import org.apache.accumulo.core.iterator
 import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
-import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 import org.apache.accumulo.core.iterators.system.CountingIterator;
 import org.apache.accumulo.core.iterators.system.DeletingIterator;
 import org.apache.accumulo.core.iterators.system.MultiIterator;
@@ -80,9 +80,10 @@ public class Compactor implements Callab
   private Configuration conf;
   private FileSystem fs;
   private KeyExtent extent;
+  private List<IteratorSetting> iterators;
   
   Compactor(Configuration conf, FileSystem fs, Map<String,DataFileValue> files, InMemoryMap imm, String outputFile, boolean propogateDeletes,
-      TableConfiguration acuTableConf, KeyExtent extent, CompactionEnv env) {
+      TableConfiguration acuTableConf, KeyExtent extent, CompactionEnv env, List<IteratorSetting> iterators) {
     this.extent = extent;
     this.conf = conf;
     this.fs = fs;
@@ -92,6 +93,12 @@ public class Compactor implements Callab
     this.propogateDeletes = propogateDeletes;
     this.acuTableConf = acuTableConf;
     this.env = env;
+    this.iterators = iterators;
+  }
+  
+  Compactor(Configuration conf, FileSystem fs, Map<String,DataFileValue> files, InMemoryMap imm, String outputFile, boolean propogateDeletes,
+      TableConfiguration acuTableConf, KeyExtent extent, CompactionEnv env) {
+    this(conf, fs, files, imm, outputFile, propogateDeletes, acuTableConf, extent, env, new ArrayList<IteratorSetting>());
   }
   
   public FileSystem getFileSystem() {
@@ -258,7 +265,7 @@ public class Compactor implements Callab
         throw new IllegalArgumentException();
       
       SortedKeyValueIterator<Key,Value> itr = iterEnv.getTopLevelIterator(IteratorUtil.loadIterators(env.getIteratorScope(), cfsi, extent, acuTableConf,
-          iterEnv));
+          iterators, iterEnv));
       
       itr.seek(extent.toDataRange(), columnFamilies, inclusive);
       

Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java?rev=1342798&r1=1342797&r2=1342798&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java (original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java Fri May 25 20:24:33 2012
@@ -49,6 +49,7 @@ import org.apache.accumulo.cloudtrace.in
 import org.apache.accumulo.cloudtrace.instrument.Trace;
 import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
 import org.apache.accumulo.core.client.impl.ScannerImpl;
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.ConfigurationObserver;
@@ -122,6 +123,8 @@ import org.apache.accumulo.server.util.M
 import org.apache.accumulo.server.util.TabletOperations;
 import org.apache.accumulo.server.zookeeper.IZooReaderWriter;
 import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -2395,11 +2398,22 @@ public class Tablet {
     }
   }
   
-  long getCompactionID() throws NoNodeException {
+  Pair<Long,List<IteratorSetting>> getCompactionID() throws NoNodeException {
     try {
       String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" + extent.getTableId()
           + Constants.ZTABLE_COMPACT_ID;
-      return Long.parseLong(new String(ZooReaderWriter.getRetryingInstance().getData(zTablePath, null)));
+      
+      String[] tokens = new String(ZooReaderWriter.getRetryingInstance().getData(zTablePath, null)).split(",");
+      long compactID = Long.parseLong(tokens[0]);
+      
+      List<IteratorSetting> allIters = new ArrayList<IteratorSetting>();
+      for (int i = 1; i < tokens.length; i++) {
+        Hex hex = new Hex();
+        List<IteratorSetting> iters = IteratorUtil.decodeIteratorSettings(hex.decode(tokens[i].split("=")[1].getBytes()));
+        allIters.addAll(iters);
+      }
+      
+      return new Pair<Long,List<IteratorSetting>>(compactID, allIters);
     } catch (InterruptedException e) {
       throw new RuntimeException(e);
     } catch (NumberFormatException nfe) {
@@ -2410,6 +2424,8 @@ public class Tablet {
       } else {
         throw new RuntimeException(ke);
       }
+    } catch (DecoderException e) {
+      throw new RuntimeException(e);
     }
   }
   
@@ -3171,7 +3187,7 @@ public class Tablet {
       
       log.debug(String.format("MajC initiate lock %.2f secs, wait %.2f secs", (t3 - t2) / 1000.0, (t2 - t1) / 1000.0));
       
-      Long compactionId = null;
+      Pair<Long,List<IteratorSetting>> compactionId = null;
       if (!propogateDeletes) {
         // compacting everything, so update the compaction id in !METADATA
         try {
@@ -3181,6 +3197,11 @@ public class Tablet {
         }
       }
       
+      List<IteratorSetting> compactionIterators = new ArrayList<IteratorSetting>();
+      if (compactionId != null) {
+        compactionIterators = compactionId.getSecond();
+      }
+
       // need to handle case where only one file is being major compacted
       while (filesToCompact.size() > 0) {
         
@@ -3226,7 +3247,7 @@ public class Tablet {
                                                                                                                                           // unless
                                                                                                                                           // last
                                                                                                                                           // batch
-              acuTableConf, extent, cenv);
+              acuTableConf, extent, cenv, compactionIterators);
           
           CompactionStats mcs = compactor.call();
           
@@ -3235,8 +3256,7 @@ public class Tablet {
           span.data("written", "" + mcs.getEntriesWritten());
           majCStats.add(mcs);
           
-          datafileManager.bringMajorCompactionOnline(smallestFiles, compactTmpName, fileName,
-              filesToCompact.size() == 0 && compactionId != null ? compactionId.getFirst() : null,
+          datafileManager.bringMajorCompactionOnline(smallestFiles, compactTmpName, fileName, filesToCompact.size() == 0 ? compactionId.getFirst() : null,
               new DataFileValue(mcs.getFileSize(), mcs.getEntriesWritten()));
           
           // when major compaction produces a file w/ zero entries, it will be deleted... do not want

Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1342798&r1=1342797&r2=1342798&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java (original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java Fri May 25 20:24:33 2012
@@ -2030,7 +2030,7 @@ public class TabletServer extends Abstra
         // compaction id once
         if (compactionId == null)
           try {
-            compactionId = tablet.getCompactionID();
+            compactionId = tablet.getCompactionID().getFirst();
           } catch (NoNodeException e) {
             log.info("Asked to compact table with no compaction id " + ke + " " + e.getMessage());
             return;

Propchange: accumulo/branches/ACCUMULO-578/src/
------------------------------------------------------------------------------
  Merged /accumulo/trunk/src:r1341000,1341572-1342236,1342774-1342794

Modified: accumulo/branches/ACCUMULO-578/test/system/auto/simple/dynamic.py
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/test/system/auto/simple/dynamic.py?rev=1342798&r1=1342797&r2=1342798&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/test/system/auto/simple/dynamic.py (original)
+++ accumulo/branches/ACCUMULO-578/test/system/auto/simple/dynamic.py Fri May 25 20:24:33 2012
@@ -85,7 +85,7 @@ public class StringSummation%s implement
 
         # initialize the database
         aggregator = 'accumulo.test.StringSummation%s' % rand
-        cmd = 'createtable test -a cf=' + aggregator
+        cmd = 'createtable test\nsetiter -agg -minc -majc -scan -p 10 -t test\ncf ' + aggregator + '\n\n'
         out, err, code = self.rootShell(self.masterHost(),"%s\n" % cmd)
         self.assert_(code == 0)
 

Modified: accumulo/branches/ACCUMULO-578/test/system/auto/simple/rename.py
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/test/system/auto/simple/rename.py?rev=1342798&r1=1342797&r2=1342798&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/test/system/auto/simple/rename.py (original)
+++ accumulo/branches/ACCUMULO-578/test/system/auto/simple/rename.py Fri May 25 20:24:33 2012
@@ -44,7 +44,7 @@ class RenameTest(readwrite.SunnyDayTest)
                                      self.options.rows,
                                      self.options.rows), 100)
         self.shell(self.masterHost(),
-                   'deletetable test_ingest\n'
+                   'deletetable test_ingest\nyes\n'
                    'renametable renamed test_ingest\n')
         self.waitForStop(self.verify(self.masterHost(), self.options.rows), 100)
 

Modified: accumulo/branches/ACCUMULO-578/test/system/auto/simple/table.py
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/test/system/auto/simple/table.py?rev=1342798&r1=1342797&r2=1342798&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/test/system/auto/simple/table.py (original)
+++ accumulo/branches/ACCUMULO-578/test/system/auto/simple/table.py Fri May 25 20:24:33 2012
@@ -74,7 +74,7 @@ class TableTest(readwrite.SunnyDayTest):
         self.assert_(out.find('%s/tables/%s' % (ACCUMULO_DIR,firstTID)) >= 0)
 
         #delete the table
-        out, err, code = self.sshell("deletetable test_ingest\n")
+        out, err, code = self.sshell("deletetable test_ingest\nyes\n")
         self.assert_(code == 0)
         self.shutdown_accumulo()
         self.start_accumulo()



Mime
View raw message