accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject [42/64] [abbrv] Merge branch '1.4.6-SNAPSHOT' into 1.5.2-SNAPSHOT
Date Wed, 09 Apr 2014 17:58:13 GMT
http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
index 8835b1c,0000000..53ea0e8
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java
@@@ -1,676 -1,0 +1,672 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.iterators.user;
 +
 +import java.io.IOException;
 +import java.util.ArrayList;
 +import java.util.Collection;
 +import java.util.Collections;
 +import java.util.Comparator;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.NoSuchElementException;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.data.ByteSequence;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.PartialKey;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.IteratorEnvironment;
 +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 +import org.apache.accumulo.core.iterators.OptionDescriber;
 +import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 +import org.apache.accumulo.core.iterators.WrappingIterator;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.security.ColumnVisibility;
 +import org.apache.accumulo.core.security.VisibilityEvaluator;
 +import org.apache.accumulo.core.security.VisibilityParseException;
 +import org.apache.accumulo.core.util.BadArgumentException;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.commons.collections.BufferOverflowException;
 +import org.apache.commons.collections.map.LRUMap;
 +import org.apache.hadoop.io.Text;
 +import org.apache.log4j.Logger;
 +
 +/**
 + * The TransformingIterator allows portions of a key (except for the row) to be transformed. This iterator handles the details that come with modifying keys
 + * (i.e., that the sort order could change). In order to do so, however, the iterator must put all keys sharing the same prefix in memory. Prefix is defined as
 + * the parts of the key that are not modified by this iterator. That is, if the iterator modifies column qualifier and timestamp, then the prefix is row and
 + * column family. In that case, the iterator must load all column qualifiers for each row/column family pair into memory. Given this constraint, care must be
 + * taken by users of this iterator to ensure it is not run in such a way that will overrun memory in a tablet server.
 + * <p>
 + * If the implementing iterator is transforming column families, then it must also override {@code untransformColumnFamilies(Collection)} to handle the case
 + * when column families are fetched at scan time. The fetched column families will/must be in the transformed space, and the untransformed column families need
 + * to be passed to this iterator's source. If it is not possible to write a reverse transformation (e.g., the column family transformation depends on the row
 + * value or something like that), then the iterator must not fetch specific column families (or only fetch column families that are known to not transform at
 + * all).
 + * <p>
 + * If the implementing iterator is transforming column visibilities, then users must be careful NOT to fetch column qualifiers from the scanner. The reason for
 + * this is due to ACCUMULO-??? (insert issue number).
 + * <p>
 + * If the implementing iterator is transforming column visibilities, then the user should be sure to supply authorizations via the {@link #AUTH_OPT} iterator
 + * option (note that this is only necessary for scan scope iterators). The supplied authorizations should be in the transformed space, but the authorizations
 + * supplied to the scanner should be in the untransformed space. That is, if the iterator transforms A to 1, B to 2, C to 3, etc, then the auths supplied when
 + * the scanner is constructed should be A,B,C,... and the auths supplied to the iterator should be 1,2,3,... The reason for this is that the scanner performs
 + * security filtering before this iterator is called, so the authorizations need to be in the original untransformed space. Since the iterator can transform
 + * visibilities, it is possible that it could produce visibilities that the user cannot see, so the transformed keys must be tested to ensure the user is
 + * allowed to view them. Note that this test is not necessary when the iterator is not used in the scan scope since no security filtering is performed during
 + * major and minor compactions. It should also be noted that this iterator implements the security filtering rather than relying on a follow-on iterator to do
 + * it so that we ensure the test is performed.
 + */
 +abstract public class TransformingIterator extends WrappingIterator implements OptionDescriber {
 +  public static final String AUTH_OPT = "authorizations";
 +  public static final String MAX_BUFFER_SIZE_OPT = "maxBufferSize";
 +  private static final long DEFAULT_MAX_BUFFER_SIZE = 10000000;
-   
++
 +  protected Logger log = Logger.getLogger(getClass());
-   
++
 +  protected ArrayList<Pair<Key,Value>> keys = new ArrayList<Pair<Key,Value>>();
 +  protected int keyPos = -1;
 +  protected boolean scanning;
 +  protected Range seekRange;
 +  protected Collection<ByteSequence> seekColumnFamilies;
 +  protected boolean seekColumnFamiliesInclusive;
-   
++
 +  private VisibilityEvaluator ve = null;
 +  private LRUMap visibleCache = null;
 +  private LRUMap parsedVisibilitiesCache = null;
 +  private long maxBufferSize;
-   
++
 +  private static Comparator<Pair<Key,Value>> keyComparator = new Comparator<Pair<Key,Value>>() {
 +    @Override
 +    public int compare(Pair<Key,Value> o1, Pair<Key,Value> o2) {
 +      return o1.getFirst().compareTo(o2.getFirst());
 +    }
 +  };
-   
++
 +  public TransformingIterator() {}
-   
++
 +  @Override
 +  public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
 +    super.init(source, options, env);
 +    scanning = IteratorScope.scan.equals(env.getIteratorScope());
 +    if (scanning) {
 +      String auths = options.get(AUTH_OPT);
 +      if (auths != null && !auths.isEmpty()) {
 +        ve = new VisibilityEvaluator(new Authorizations(auths.getBytes(Constants.UTF8)));
 +        visibleCache = new LRUMap(100);
 +      }
 +    }
-     
++
 +    if (options.containsKey(MAX_BUFFER_SIZE_OPT)) {
 +      maxBufferSize = AccumuloConfiguration.getMemoryInBytes(options.get(MAX_BUFFER_SIZE_OPT));
 +    } else {
 +      maxBufferSize = DEFAULT_MAX_BUFFER_SIZE;
 +    }
-     
++
 +    parsedVisibilitiesCache = new LRUMap(100);
 +  }
-   
++
 +  @Override
 +  public IteratorOptions describeOptions() {
 +    String desc = "This iterator allows ranges of key to be transformed (with the exception of row transformations).";
 +    String authDesc = "Comma-separated list of user's scan authorizations.  "
 +        + "If excluded or empty, then no visibility check is performed on transformed keys.";
 +    String bufferDesc = "Maximum buffer size (in accumulo memory spec) to use for buffering keys before throwing a BufferOverflowException.  "
 +        + "Users should keep this limit in mind when deciding what to transform.  That is, if transforming the column family for example, then all "
 +        + "keys sharing the same row and column family must fit within this limit (along with their associated values)";
 +    HashMap<String,String> namedOptions = new HashMap<String,String>();
 +    namedOptions.put(AUTH_OPT, authDesc);
 +    namedOptions.put(MAX_BUFFER_SIZE_OPT, bufferDesc);
 +    return new IteratorOptions(getClass().getSimpleName(), desc, namedOptions, null);
 +  }
-   
++
 +  @Override
 +  public boolean validateOptions(Map<String,String> options) {
-     
++
 +    for (Entry<String,String> option : options.entrySet()) {
 +      try {
 +        if (option.getKey().equals(AUTH_OPT)) {
 +          new Authorizations(option.getValue().getBytes(Constants.UTF8));
 +        } else if (option.getKey().equals(MAX_BUFFER_SIZE_OPT)) {
 +          AccumuloConfiguration.getMemoryInBytes(option.getValue());
 +        }
 +      } catch (Exception e) {
 +        throw new IllegalArgumentException("Failed to parse opt " + option.getKey() + " " + option.getValue(), e);
 +      }
 +    }
-     
++
 +    return true;
 +  }
 +
 +  @Override
 +  public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
 +    TransformingIterator copy;
-     
++
 +    try {
 +      copy = getClass().newInstance();
 +    } catch (Exception e) {
 +      throw new RuntimeException(e);
 +    }
-     
++
 +    copy.setSource(getSource().deepCopy(env));
-     
++
 +    copy.scanning = scanning;
 +    copy.keyPos = keyPos;
 +    copy.keys.addAll(keys);
 +    copy.seekRange = (seekRange == null) ? null : new Range(seekRange);
 +    copy.seekColumnFamilies = (seekColumnFamilies == null) ? null : new HashSet<ByteSequence>(seekColumnFamilies);
 +    copy.seekColumnFamiliesInclusive = seekColumnFamiliesInclusive;
-     
++
 +    copy.ve = ve;
 +    if (visibleCache != null) {
 +      copy.visibleCache = new LRUMap(visibleCache.maxSize());
 +      copy.visibleCache.putAll(visibleCache);
 +    }
-     
++
 +    if (parsedVisibilitiesCache != null) {
 +      copy.parsedVisibilitiesCache = new LRUMap(parsedVisibilitiesCache.maxSize());
 +      copy.parsedVisibilitiesCache.putAll(parsedVisibilitiesCache);
 +    }
-     
++
 +    copy.maxBufferSize = maxBufferSize;
-     
++
 +    return copy;
 +  }
-   
++
 +  @Override
 +  public boolean hasTop() {
 +    return keyPos >= 0 && keyPos < keys.size();
 +  }
-   
++
 +  @Override
 +  public Key getTopKey() {
 +    return hasTop() ? keys.get(keyPos).getFirst() : null;
 +  }
-   
++
 +  @Override
 +  public Value getTopValue() {
 +    return hasTop() ? keys.get(keyPos).getSecond() : null;
 +  }
-   
++
 +  @Override
 +  public void next() throws IOException {
 +    // Move on to the next entry since we returned the entry at keyPos before
 +    if (keyPos >= 0)
 +      keyPos++;
-     
++
 +    // If we emptied out the transformed key map then transform the next key
 +    // set from the source. It’s possible that transformation could produce keys
 +    // that are outside of our range or are not visible to the end user, so after the
 +    // call below we might not have added any keys to the map. Keep going until
 +    // we either get some keys in the map or exhaust the source iterator.
 +    while (!hasTop() && super.hasTop())
 +      transformKeys();
 +  }
-   
++
 +  @Override
 +  public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
 +    seekRange = new Range(range);
 +    seekColumnFamilies = columnFamilies;
 +    seekColumnFamiliesInclusive = inclusive;
-     
++
 +    // Seek the source iterator, but use a recalculated range that ensures
 +    // we see all keys with the same "prefix." We need to do this since
 +    // transforming could change the sort order and transformed keys that
 +    // are before the range start could be inside the range after transformation.
 +    super.seek(computeReseekRange(range), untransformColumnFamilies(columnFamilies), inclusive);
-     
++
 +    // Range clipping could cause us to trim out all the keys we transformed.
 +    // Keep looping until we either have some keys in the output range, or have
 +    // exhausted the source iterator.
 +    keyPos = -1; // “Clear” list so hasTop returns false to get us into the loop (transformKeys actually clears)
 +    while (!hasTop() && super.hasTop()) {
 +      // Build up a sorted list of all keys for the same prefix. When
 +      // people ask for keys, return from this list first until it is empty
 +      // before incrementing the source iterator.
 +      transformKeys();
 +    }
 +  }
-   
++
 +  private static class RangeIterator implements SortedKeyValueIterator<Key,Value> {
-     
++
 +    private SortedKeyValueIterator<Key,Value> source;
 +    private Key prefixKey;
 +    private PartialKey keyPrefix;
 +    private boolean hasTop = false;
-     
++
 +    RangeIterator(SortedKeyValueIterator<Key,Value> source, Key prefixKey, PartialKey keyPrefix) {
 +      this.source = source;
 +      this.prefixKey = prefixKey;
 +      this.keyPrefix = keyPrefix;
 +    }
-     
++
 +    @Override
 +    public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
 +      throw new UnsupportedOperationException();
 +    }
-     
++
 +    @Override
 +    public boolean hasTop() {
 +      // only have a top if the prefix matches
 +      return hasTop = source.hasTop() && source.getTopKey().equals(prefixKey, keyPrefix);
 +    }
-     
++
 +    @Override
 +    public void next() throws IOException {
 +      // do not let user advance too far and try to avoid reexecuting hasTop()
 +      if (!hasTop && !hasTop())
 +        throw new NoSuchElementException();
 +      hasTop = false;
 +      source.next();
 +    }
-     
++
 +    @Override
 +    public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
 +      throw new UnsupportedOperationException();
 +    }
-     
++
 +    @Override
 +    public Key getTopKey() {
 +      return source.getTopKey();
 +    }
-     
++
 +    @Override
 +    public Value getTopValue() {
 +      return source.getTopValue();
 +    }
-     
++
 +    @Override
 +    public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
 +      throw new UnsupportedOperationException();
 +    }
-     
++
 +  }
-   
++
 +  /**
 +   * Reads all keys matching the first key's prefix from the source iterator, transforms them, and sorts the resulting keys. Transformed keys that fall outside
 +   * of our seek range or can't be seen by the user are excluded.
 +   */
 +  protected void transformKeys() throws IOException {
 +    keyPos = -1;
 +    keys.clear();
 +    final Key prefixKey = super.hasTop() ? new Key(super.getTopKey()) : null;
-     
++
 +    transformRange(new RangeIterator(getSource(), prefixKey, getKeyPrefix()), new KVBuffer() {
-       
++
 +      long appened = 0;
-       
++
 +      @Override
 +      public void append(Key key, Value val) {
 +        // ensure the key provided by the user has the correct prefix
 +        if (!key.equals(prefixKey, getKeyPrefix()))
 +          throw new IllegalArgumentException("Key prefixes are not equal " + key + " " + prefixKey);
-         
++
 +        // Transformation could have produced a key that falls outside
 +        // of the seek range, or one that the user cannot see. Check
 +        // these before adding it to the output list.
 +        if (includeTransformedKey(key)) {
-           
++
 +          // try to defend against a scan or compaction using all memory in a tablet server
 +          if (appened > maxBufferSize)
 +            throw new BufferOverflowException("Exceeded buffer size of " + maxBufferSize + ", prefixKey: " + prefixKey);
-           
++
 +          if (getSource().hasTop() && key == getSource().getTopKey())
 +            key = new Key(key);
 +          keys.add(new Pair<Key,Value>(key, new Value(val)));
 +          appened += (key.getSize() + val.getSize() + 128);
 +        }
 +      }
 +    });
-     
++
 +    // consume any key in range that user did not consume
 +    while (super.hasTop() && super.getTopKey().equals(prefixKey, getKeyPrefix())) {
 +      super.next();
 +    }
-     
++
 +    if (!keys.isEmpty()) {
 +      Collections.sort(keys, keyComparator);
 +      keyPos = 0;
 +    }
 +  }
-   
++
 +  /**
 +   * Determines whether or not to include {@code transformedKey} in the output. It is possible that transformation could have produced a key that falls outside
 +   * of the seek range, a key with a visibility the user can't see, a key with a visibility that doesn't parse, or a key with a column family that wasn't
 +   * fetched. We only do some checks (outside the range, user can see) if we're scanning. The range check is not done for major/minor compaction since seek
 +   * ranges won't be in our transformed key space and we will never change the row so we can't produce keys that would fall outside the tablet anyway.
-    * 
++   *
 +   * @param transformedKey
 +   *          the key to check
 +   * @return {@code true} if the key should be included and {@code false} if not
 +   */
 +  protected boolean includeTransformedKey(Key transformedKey) {
 +    boolean include = canSee(transformedKey);
 +    if (scanning && seekRange != null) {
 +      include = include && seekRange.contains(transformedKey);
 +    }
 +    return include;
 +  }
-   
++
 +  /**
 +   * Indicates whether or not the user is able to see {@code key}. If the user has not supplied authorizations, or the iterator is not in the scan scope, then
 +   * this method simply returns {@code true}. Otherwise, {@code key}'s column visibility is tested against the user-supplied authorizations, and the test result
 +   * is returned. For performance, the test results are cached so that the same visibility is not tested multiple times.
-    * 
++   *
 +   * @param key
 +   *          the key to test
 +   * @return {@code true} if the key is visible or iterator is not scanning, and {@code false} if not
 +   */
 +  protected boolean canSee(Key key) {
 +    // Ensure that the visibility (which could have been transformed) parses. Must always do this check, even if visibility is not evaluated.
 +    ByteSequence visibility = key.getColumnVisibilityData();
 +    ColumnVisibility colVis = null;
 +    Boolean parsed = (Boolean) parsedVisibilitiesCache.get(visibility);
 +    if (parsed == null) {
 +      try {
 +        colVis = new ColumnVisibility(visibility.toArray());
 +        parsedVisibilitiesCache.put(visibility, Boolean.TRUE);
 +      } catch (BadArgumentException e) {
 +        log.error("Parse error after transformation : " + visibility);
 +        parsedVisibilitiesCache.put(visibility, Boolean.FALSE);
 +        if (scanning) {
 +          return false;
 +        } else {
 +          throw e;
 +        }
 +      }
 +    } else if (!parsed) {
 +      if (scanning)
 +        return false;
 +      else
 +        throw new IllegalStateException();
 +    }
-     
++
 +    Boolean visible = canSeeColumnFamily(key);
-     
++
 +    if (!scanning || !visible || ve == null || visibleCache == null || visibility.length() == 0)
 +      return visible;
-     
++
 +    visible = (Boolean) visibleCache.get(visibility);
 +    if (visible == null) {
 +      try {
 +        if (colVis == null)
 +          colVis = new ColumnVisibility(visibility.toArray());
 +        visible = ve.evaluate(colVis);
 +        visibleCache.put(visibility, visible);
 +      } catch (VisibilityParseException e) {
 +        log.error("Parse Error", e);
 +        visible = Boolean.FALSE;
 +      } catch (BadArgumentException e) {
 +        log.error("Parse Error", e);
 +        visible = Boolean.FALSE;
 +      }
 +    }
-     
++
 +    return visible;
 +  }
-   
++
 +  /**
 +   * Indicates whether or not {@code key} can be seen, according to the fetched column families for this iterator.
-    * 
++   *
 +   * @param key
 +   *          the key whose column family is to be tested
 +   * @return {@code true} if {@code key}'s column family is one of those fetched in the set passed to our {@link #seek(Range, Collection, boolean)} method
 +   */
 +  protected boolean canSeeColumnFamily(Key key) {
 +    boolean visible = true;
 +    if (seekColumnFamilies != null) {
 +      ByteSequence columnFamily = key.getColumnFamilyData();
 +      if (seekColumnFamiliesInclusive)
 +        visible = seekColumnFamilies.contains(columnFamily);
 +      else
 +        visible = !seekColumnFamilies.contains(columnFamily);
 +    }
 +    return visible;
 +  }
-   
++
 +  /**
 +   * Possibly expand {@code range} to include everything for the key prefix we are working with. That is, if our prefix is ROW_COLFAM, then we need to expand
 +   * the range so we're sure to include all entries having the same row and column family as the start/end of the range.
-    * 
++   *
 +   * @param range
 +   *          the range to expand
 +   * @return the modified range
 +   */
 +  protected Range computeReseekRange(Range range) {
 +    Key startKey = range.getStartKey();
 +    boolean startKeyInclusive = range.isStartKeyInclusive();
 +    // If anything after the prefix is set, then clip the key so we include
 +    // everything for the prefix.
 +    if (isSetAfterPart(startKey, getKeyPrefix())) {
 +      startKey = copyPartialKey(startKey, getKeyPrefix());
 +      startKeyInclusive = true;
 +    }
 +    Key endKey = range.getEndKey();
 +    boolean endKeyInclusive = range.isEndKeyInclusive();
 +    if (isSetAfterPart(endKey, getKeyPrefix())) {
 +      endKey = endKey.followingKey(getKeyPrefix());
 +      endKeyInclusive = true;
 +    }
 +    return new Range(startKey, startKeyInclusive, endKey, endKeyInclusive);
 +  }
-   
++
 +  /**
 +   * Indicates whether or not any part of {@code key} excluding {@code part} is set. For example, if part is ROW_COLFAM_COLQUAL, then this method determines
 +   * whether or not the column visibility, timestamp, or delete flag is set on {@code key}.
-    * 
++   *
 +   * @param key
 +   *          the key to check
 +   * @param part
 +   *          the part of the key that doesn't need to be checked (everything after does)
 +   * @return {@code true} if anything after {@code part} is set on {@code key}, and {@code false} if not
 +   */
 +  protected boolean isSetAfterPart(Key key, PartialKey part) {
 +    boolean isSet = false;
 +    if (key != null) {
 +      // Breaks excluded on purpose.
 +      switch (part) {
 +        case ROW:
 +          isSet = isSet || key.getColumnFamilyData().length() > 0;
 +        case ROW_COLFAM:
 +          isSet = isSet || key.getColumnQualifierData().length() > 0;
 +        case ROW_COLFAM_COLQUAL:
 +          isSet = isSet || key.getColumnVisibilityData().length() > 0;
 +        case ROW_COLFAM_COLQUAL_COLVIS:
 +          isSet = isSet || key.getTimestamp() < Long.MAX_VALUE;
 +        case ROW_COLFAM_COLQUAL_COLVIS_TIME:
 +          isSet = isSet || key.isDeleted();
 +        case ROW_COLFAM_COLQUAL_COLVIS_TIME_DEL:
 +          break;
 +      }
 +    }
 +    return isSet;
 +  }
-   
++
 +  /**
 +   * Creates a copy of {@code key}, copying only the parts of the key specified in {@code part}. For example, if {@code part} is ROW_COLFAM_COLQUAL, then this
 +   * method would copy the row, column family, and column qualifier from {@code key} into a new key.
-    * 
++   *
 +   * @param key
 +   *          the key to copy
 +   * @param part
 +   *          the parts of {@code key} to copy
 +   * @return the new key containing {@code part} of {@code key}
 +   */
 +  protected Key copyPartialKey(Key key, PartialKey part) {
 +    Key keyCopy;
 +    switch (part) {
 +      case ROW:
 +        keyCopy = new Key(key.getRow());
 +        break;
 +      case ROW_COLFAM:
 +        keyCopy = new Key(key.getRow(), key.getColumnFamily());
 +        break;
 +      case ROW_COLFAM_COLQUAL:
 +        keyCopy = new Key(key.getRow(), key.getColumnFamily(), key.getColumnQualifier());
 +        break;
 +      case ROW_COLFAM_COLQUAL_COLVIS:
 +        keyCopy = new Key(key.getRow(), key.getColumnFamily(), key.getColumnQualifier(), key.getColumnVisibility());
 +        break;
 +      case ROW_COLFAM_COLQUAL_COLVIS_TIME:
 +        keyCopy = new Key(key.getRow(), key.getColumnFamily(), key.getColumnQualifier(), key.getColumnVisibility(), key.getTimestamp());
 +        break;
 +      default:
 +        throw new IllegalArgumentException("Unsupported key part: " + part);
 +    }
 +    return keyCopy;
 +  }
-   
++
 +  /**
 +   * Make a new key with all parts (including delete flag) coming from {@code originalKey} but use {@code newColFam} as the column family.
 +   */
 +  protected Key replaceColumnFamily(Key originalKey, Text newColFam) {
 +    byte[] row = originalKey.getRowData().toArray();
 +    byte[] cf = newColFam.getBytes();
 +    byte[] cq = originalKey.getColumnQualifierData().toArray();
 +    byte[] cv = originalKey.getColumnVisibilityData().toArray();
 +    long timestamp = originalKey.getTimestamp();
 +    Key newKey = new Key(row, 0, row.length, cf, 0, newColFam.getLength(), cq, 0, cq.length, cv, 0, cv.length, timestamp);
 +    newKey.setDeleted(originalKey.isDeleted());
 +    return newKey;
 +  }
-   
++
 +  /**
 +   * Make a new key with all parts (including delete flag) coming from {@code originalKey} but use {@code newColQual} as the column qualifier.
 +   */
 +  protected Key replaceColumnQualifier(Key originalKey, Text newColQual) {
 +    byte[] row = originalKey.getRowData().toArray();
 +    byte[] cf = originalKey.getColumnFamilyData().toArray();
 +    byte[] cq = newColQual.getBytes();
 +    byte[] cv = originalKey.getColumnVisibilityData().toArray();
 +    long timestamp = originalKey.getTimestamp();
 +    Key newKey = new Key(row, 0, row.length, cf, 0, cf.length, cq, 0, newColQual.getLength(), cv, 0, cv.length, timestamp);
 +    newKey.setDeleted(originalKey.isDeleted());
 +    return newKey;
 +  }
-   
++
 +  /**
 +   * Make a new key with all parts (including delete flag) coming from {@code originalKey} but use {@code newColVis} as the column visibility.
 +   */
 +  protected Key replaceColumnVisibility(Key originalKey, Text newColVis) {
 +    byte[] row = originalKey.getRowData().toArray();
 +    byte[] cf = originalKey.getColumnFamilyData().toArray();
 +    byte[] cq = originalKey.getColumnQualifierData().toArray();
 +    byte[] cv = newColVis.getBytes();
 +    long timestamp = originalKey.getTimestamp();
 +    Key newKey = new Key(row, 0, row.length, cf, 0, cf.length, cq, 0, cq.length, cv, 0, newColVis.getLength(), timestamp);
 +    newKey.setDeleted(originalKey.isDeleted());
 +    return newKey;
 +  }
-   
++
 +  /**
 +   * Make a new key with a column family, column qualifier, and column visibility. Copy the rest of the parts of the key (including delete flag) from
 +   * {@code originalKey}.
 +   */
 +  protected Key replaceKeyParts(Key originalKey, Text newColFam, Text newColQual, Text newColVis) {
 +    byte[] row = originalKey.getRowData().toArray();
 +    byte[] cf = newColFam.getBytes();
 +    byte[] cq = newColQual.getBytes();
 +    byte[] cv = newColVis.getBytes();
 +    long timestamp = originalKey.getTimestamp();
 +    Key newKey = new Key(row, 0, row.length, cf, 0, newColFam.getLength(), cq, 0, newColQual.getLength(), cv, 0, newColVis.getLength(), timestamp);
 +    newKey.setDeleted(originalKey.isDeleted());
 +    return newKey;
 +  }
-   
++
 +  /**
 +   * Make a new key with a column qualifier, and column visibility. Copy the rest of the parts of the key (including delete flag) from {@code originalKey}.
 +   */
 +  protected Key replaceKeyParts(Key originalKey, Text newColQual, Text newColVis) {
 +    byte[] row = originalKey.getRowData().toArray();
 +    byte[] cf = originalKey.getColumnFamilyData().toArray();
 +    byte[] cq = newColQual.getBytes();
 +    byte[] cv = newColVis.getBytes();
 +    long timestamp = originalKey.getTimestamp();
 +    Key newKey = new Key(row, 0, row.length, cf, 0, cf.length, cq, 0, newColQual.getLength(), cv, 0, newColVis.getLength(), timestamp);
 +    newKey.setDeleted(originalKey.isDeleted());
 +    return newKey;
 +  }
-   
++
 +  /**
 +   * Reverses the transformation applied to column families that are fetched at seek time. If this iterator is transforming column families, then this method
 +   * should be overridden to reverse the transformation on the supplied collection of column families. This is necessary since the fetch/seek will be performed
 +   * in the transformed space, but when passing the column family set on to the source, the column families need to be in the untransformed space.
-    * 
++   *
 +   * @param columnFamilies
 +   *          the column families that have been fetched at seek time
 +   * @return the untransformed column families that would transform info {@code columnFamilies}
 +   */
 +  protected Collection<ByteSequence> untransformColumnFamilies(Collection<ByteSequence> columnFamilies) {
 +    return columnFamilies;
 +  }
-   
++
 +  /**
 +   * Indicates the prefix of keys that will be transformed by this iterator. In other words, this is the part of the key that will <i>not</i> be transformed by
 +   * this iterator. For example, if this method returns ROW_COLFAM, then {@link #transformKeys()} may be changing the column qualifier, column visibility, or
 +   * timestamp, but it won't be changing the row or column family.
-    * 
++   *
 +   * @return the part of the key this iterator is not transforming
 +   */
 +  abstract protected PartialKey getKeyPrefix();
-   
++
 +  public static interface KVBuffer {
 +    void append(Key key, Value val);
 +  }
-   
++
 +  /**
 +   * Transforms {@code input}. This method must not change the row part of the key, and must only change the parts of the key after the return value of
 +   * {@link #getKeyPrefix()}. Implementors must also remember to copy the delete flag from {@code originalKey} onto the new key. Or, implementors should use one
 +   * of the helper methods to produce the new key. See any of the replaceKeyParts methods.
-    * 
++   *
 +   * @param input
 +   *          An iterator over a group of keys with the same prefix. This iterator provides an efficient view, bounded by the prefix, of the underlying iterator
 +   *          and can not be seeked.
 +   * @param output
 +   *          An output buffer that holds transformed key values. All key values added to the buffer must have the same prefix as the input keys.
-    * @throws IOException
 +   * @see #replaceColumnFamily(Key, Text)
 +   * @see #replaceColumnQualifier(Key, Text)
 +   * @see #replaceColumnVisibility(Key, Text)
 +   * @see #replaceKeyParts(Key, Text, Text)
 +   * @see #replaceKeyParts(Key, Text, Text, Text)
 +   */
 +  abstract protected void transformRange(SortedKeyValueIterator<Key,Value> input, KVBuffer output) throws IOException;
-   
++
 +  /**
-    * Configure authoriations used for post transformation filtering.
-    * 
-    * @param config
-    * @param auths
++   * Configure authorizations used for post transformation filtering.
++   *
 +   */
 +  public static void setAuthorizations(IteratorSetting config, Authorizations auths) {
 +    config.addOption(AUTH_OPT, auths.serialize());
 +  }
-   
++
 +  /**
 +   * Configure the maximum amount of memory that can be used for transformation. If this memory is exceeded an exception will be thrown.
-    * 
-    * @param config
++   *
 +   * @param maxBufferSize
 +   *          size in bytes
 +   */
 +  public static void setMaxBufferSize(IteratorSetting config, long maxBufferSize) {
 +    config.addOption(MAX_BUFFER_SIZE_OPT, maxBufferSize + "");
 +  }
-   
++
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java
index 7804aa4,0000000..2fc3a27
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java
@@@ -1,172 -1,0 +1,169 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.iterators.user;
 +
 +import java.io.IOException;
 +import java.util.Collection;
 +import java.util.Collections;
 +import java.util.Map;
 +
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.data.ByteSequence;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.PartialKey;
 +import org.apache.accumulo.core.data.Range;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.IteratorEnvironment;
 +import org.apache.accumulo.core.iterators.IteratorUtil;
 +import org.apache.accumulo.core.iterators.OptionDescriber;
 +import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 +import org.apache.accumulo.core.iterators.WrappingIterator;
 +
 +public class VersioningIterator extends WrappingIterator implements OptionDescriber {
 +  private final int maxCount = 10;
 +  
 +  private Key currentKey = new Key();
 +  private int numVersions;
 +  protected int maxVersions;
 +  
 +  private Range range;
 +  private Collection<ByteSequence> columnFamilies;
 +  private boolean inclusive;
 +  
 +  @Override
 +  public VersioningIterator deepCopy(IteratorEnvironment env) {
 +    VersioningIterator copy = new VersioningIterator();
 +    copy.setSource(getSource().deepCopy(env));
 +    copy.maxVersions = maxVersions;
 +    return copy;
 +  }
 +  
 +  @Override
 +  public void next() throws IOException {
 +    if (numVersions >= maxVersions) {
 +      skipRowColumn();
 +      resetVersionCount();
 +      return;
 +    }
 +    
 +    super.next();
 +    if (getSource().hasTop()) {
 +      if (getSource().getTopKey().equals(currentKey, PartialKey.ROW_COLFAM_COLQUAL_COLVIS)) {
 +        numVersions++;
 +      } else {
 +        resetVersionCount();
 +      }
 +    }
 +  }
 +  
 +  @Override
 +  public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
 +    // do not want to seek to the middle of a row
 +    Range seekRange = IteratorUtil.maximizeStartKeyTimeStamp(range);
 +    this.range = seekRange;
 +    this.columnFamilies = columnFamilies;
 +    this.inclusive = inclusive;
 +    
 +    super.seek(seekRange, columnFamilies, inclusive);
 +    resetVersionCount();
 +    
 +    if (range.getStartKey() != null)
 +      while (hasTop() && range.beforeStartKey(getTopKey()))
 +        next();
 +  }
 +  
 +  private void resetVersionCount() {
 +    if (super.hasTop())
 +      currentKey.set(getSource().getTopKey());
 +    numVersions = 1;
 +  }
 +  
 +  private void skipRowColumn() throws IOException {
 +    Key keyToSkip = currentKey;
 +    super.next();
 +    
 +    int count = 0;
 +    while (getSource().hasTop() && getSource().getTopKey().equals(keyToSkip, PartialKey.ROW_COLFAM_COLQUAL_COLVIS)) {
 +      if (count < maxCount) {
 +        // it is quicker to call next if we are close, but we never know if we are close
 +        // so give next a try a few times
 +        getSource().next();
 +        count++;
 +      } else {
 +        reseek(keyToSkip.followingKey(PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
 +        count = 0;
 +      }
 +    }
 +  }
 +  
 +  protected void reseek(Key key) throws IOException {
 +    if (key == null)
 +      return;
 +    if (range.afterEndKey(key)) {
 +      range = new Range(range.getEndKey(), true, range.getEndKey(), range.isEndKeyInclusive());
 +      getSource().seek(range, columnFamilies, inclusive);
 +    } else {
 +      range = new Range(key, true, range.getEndKey(), range.isEndKeyInclusive());
 +      getSource().seek(range, columnFamilies, inclusive);
 +    }
 +  }
 +  
 +  @Override
 +  public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
 +    super.init(source, options, env);
 +    this.numVersions = 0;
 +    
 +    String maxVerString = options.get("maxVersions");
 +    if (maxVerString != null)
 +      this.maxVersions = Integer.parseInt(maxVerString);
 +    else
 +      this.maxVersions = 1;
 +    
 +    if (maxVersions < 1)
 +      throw new IllegalArgumentException("maxVersions for versioning iterator must be >= 1");
 +  }
 +  
 +  @Override
 +  public IteratorOptions describeOptions() {
 +    return new IteratorOptions("vers", "The VersioningIterator keeps a fixed number of versions for each key", Collections.singletonMap("maxVersions",
 +        "number of versions to keep for a particular key (with differing timestamps)"), null);
 +  }
 +  
 +  private static final String MAXVERSIONS_OPT = "maxVersions";
 +  
 +  @Override
 +  public boolean validateOptions(Map<String,String> options) {
 +    int i;
 +    try {
 +      i = Integer.parseInt(options.get(MAXVERSIONS_OPT));
 +    } catch (Exception e) {
 +      throw new IllegalArgumentException("bad integer " + MAXVERSIONS_OPT + ":" + options.get(MAXVERSIONS_OPT));
 +    }
 +    if (i < 1)
 +      throw new IllegalArgumentException(MAXVERSIONS_OPT + " for versioning iterator must be >= 1");
 +    return true;
 +  }
 +  
 +  /**
 +   * Encode the maximum number of versions to return onto the ScanIterator
-    * 
-    * @param cfg
-    * @param maxVersions
 +   */
 +  public static void setMaxVersions(IteratorSetting cfg, int maxVersions) {
 +    if (maxVersions < 1)
 +      throw new IllegalArgumentException(MAXVERSIONS_OPT + " for versioning iterator must be >= 1");
 +    cfg.addOption(MAXVERSIONS_OPT, Integer.toString(maxVersions));
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/security/SecurityUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/SecurityUtil.java
index 672e784,0000000..65cb7ed
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/security/SecurityUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/SecurityUtil.java
@@@ -1,89 -1,0 +1,88 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.security;
 +
 +import java.io.IOException;
 +import java.net.InetAddress;
 +
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.hadoop.security.UserGroupInformation;
 +import org.apache.log4j.Logger;
 +
 +/**
 + * 
 + */
 +public class SecurityUtil {
 +  private static final Logger log = Logger.getLogger(SecurityUtil.class);
 +  private static final String ACCUMULO_HOME = "ACCUMULO_HOME", ACCUMULO_CONF_DIR = "ACCUMULO_CONF_DIR";
 +  public static boolean usingKerberos = false;
 +
 +  /**
 +   * This method is for logging a server in kerberos. If this is used in client code, it will fail unless run as the accumulo keytab's owner. Instead, use
 +   * {@link #login(String, String)}
 +   */
 +  public static void serverLogin() {
 +    @SuppressWarnings("deprecation")
 +    AccumuloConfiguration acuConf = AccumuloConfiguration.getSiteConfiguration();
 +    String keyTab = acuConf.get(Property.GENERAL_KERBEROS_KEYTAB);
 +    if (keyTab == null || keyTab.length() == 0)
 +      return;
 +    
 +    usingKerberos = true;
 +    if (keyTab.contains("$" + ACCUMULO_HOME) && System.getenv(ACCUMULO_HOME) != null)
 +      keyTab = keyTab.replace("$" + ACCUMULO_HOME, System.getenv(ACCUMULO_HOME));
 +    
 +    if (keyTab.contains("$" + ACCUMULO_CONF_DIR) && System.getenv(ACCUMULO_CONF_DIR) != null)
 +      keyTab = keyTab.replace("$" + ACCUMULO_CONF_DIR, System.getenv(ACCUMULO_CONF_DIR));
 +    
 +    String principalConfig = acuConf.get(Property.GENERAL_KERBEROS_PRINCIPAL);
 +    if (principalConfig == null || principalConfig.length() == 0)
 +      return;
 +    
 +    if (login(principalConfig, keyTab)) {
 +      try {
 +        // This spawns a thread to periodically renew the logged in (accumulo) user
 +        UserGroupInformation.getLoginUser();
 +      } catch (IOException io) {
 +        log.error("Error starting up renewal thread. This shouldn't be happenining.", io);
 +      }
 +    }
 +  }
 +  
 +  /**
 +   * This will log in the given user in kerberos.
 +   * 
 +   * @param principalConfig
 +   *          This is the principals name in the format NAME/HOST@REALM. {@link org.apache.hadoop.security.SecurityUtil#HOSTNAME_PATTERN} will automatically be
 +   *          replaced by the systems host name.
-    * @param keyTabPath
 +   * @return true if login succeeded, otherwise false
 +   */
 +  public static boolean login(String principalConfig, String keyTabPath) {
 +    try {
 +      String principalName = org.apache.hadoop.security.SecurityUtil.getServerPrincipal(principalConfig, InetAddress.getLocalHost().getCanonicalHostName());
 +      if (keyTabPath != null && principalName != null && keyTabPath.length() != 0 && principalName.length() != 0) {
 +        UserGroupInformation.loginUserFromKeytab(principalName, keyTabPath);
 +        log.info("Succesfully logged in as user " + principalConfig);
 +        return true;
 +      }
 +    } catch (IOException io) {
 +      log.error("Error logging in user " + principalConfig + " using keytab at " + keyTabPath, io);
 +    }
 +    return false;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModule.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModule.java
index fca7d22,0000000..40d3ab7
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModule.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModule.java
@@@ -1,110 -1,0 +1,108 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.security.crypto;
 +
 +import java.io.IOException;
 +import java.io.InputStream;
 +import java.io.OutputStream;
 +import java.util.Map;
 +
 +/**
 + * Classes that obey this interface may be used to provide encrypting and decrypting streams to the rest of Accumulo. Classes that obey this interface may be
 + * configured as the crypto module by setting the property crypto.module.class in the accumulo-site.xml file.
 + * 
 + * Note that this first iteration of this API is considered deprecated because we anticipate it changing in non-backwards compatible ways as we explore the
 + * requirements for encryption in Accumulo. So, your mileage is gonna vary a lot as we go forward.
 + * 
 + */
 +@Deprecated
 +public interface CryptoModule {
 +  
 +  public enum CryptoInitProperty {
 +    ALGORITHM_NAME("algorithm.name"), CIPHER_SUITE("cipher.suite"), INITIALIZATION_VECTOR("initialization.vector"), PLAINTEXT_SESSION_KEY(
 +        "plaintext.session.key");
 +    
 +    private CryptoInitProperty(String name) {
 +      key = name;
 +    }
 +    
 +    private String key;
 +    
 +    public String getKey() {
 +      return key;
 +    }
 +  }
 +  
 +  /**
 +   * Wraps an OutputStream in an encrypting OutputStream. The given map contains the settings for the cryptographic algorithm to use. <b>Callers of this method
 +   * should expect that the given OutputStream will be written to before cryptographic writes occur.</b> These writes contain the cryptographic information used
 +   * to encrypt the following bytes (these data include the initialization vector, encrypted session key, and so on). If writing arbitrarily to the underlying
 +   * stream is not desirable, users should call the other flavor of getEncryptingOutputStream which accepts these data as parameters.
 +   * 
 +   * @param out
 +   *          the OutputStream to wrap
 +   * @param cryptoOpts
 +   *          the cryptographic parameters to use; specific string names to look for will depend on the various implementations
 +   * @return an OutputStream that wraps the given parameter
-    * @throws IOException
 +   */
 +  public OutputStream getEncryptingOutputStream(OutputStream out, Map<String,String> cryptoOpts) throws IOException;
 +  
 +  /**
 +   * Wraps an InputStream and returns a decrypting input stream. The given map contains the settings for the intended cryptographic operations, but implementors
 +   * should take care to ensure that the crypto from the given input stream matches their expectations about what they will use to decrypt it, as the parameters
 +   * may have changed. Also, care should be taken around transitioning between non-encrypting and encrypting streams; implementors should handle the case where
 +   * the given input stream is <b>not</b> encrypted at all.
 +   * 
 +   * It is expected that this version of getDecryptingInputStream is called in conjunction with the getEncryptingOutputStream from above. It should expect its
 +   * input streams to contain the data written by getEncryptingOutputStream.
 +   * 
 +   * @param in
 +   *          the InputStream to wrap
 +   * @param cryptoOpts
 +   *          the cryptographic parameters to use; specific string names to look for will depend on the various implementations
 +   * @return an InputStream that wraps the given parameter
-    * @throws IOException
 +   */
 +  public InputStream getDecryptingInputStream(InputStream in, Map<String,String> cryptoOpts) throws IOException;
 +  
 +  /**
 +   * Wraps an OutputStream in an encrypting OutputStream. The given map contains the settings for the cryptographic algorithm to use. The cryptoInitParams map
 +   * contains all the cryptographic details to construct a key (or keys), initialization vectors, etc. and use them to properly initialize the stream for
 +   * writing. These initialization parameters must be persisted elsewhere, along with the cryptographic configuration (algorithm, mode, etc.), so that they may
 +   * be read in at the time of reading the encrypted content.
 +   * 
 +   * @param out
 +   *          the OutputStream to wrap
 +   * @param conf
 +   *          the cryptographic algorithm configuration
 +   * @param cryptoInitParams
 +   *          the initialization parameters for the algorithm, usually including initialization vector and session key
 +   * @return a wrapped output stream
 +   */
 +  public OutputStream getEncryptingOutputStream(OutputStream out, Map<String,String> conf, Map<CryptoModule.CryptoInitProperty,Object> cryptoInitParams);
 +  
 +  /**
 +   * Wraps an InputStream and returns a decrypting input stream. The given map contains the settings for the intended cryptographic operations, but implementors
 +   * should take care to ensure that the crypto from the given input stream matches their expectations about what they will use to decrypt it, as the parameters
 +   * may have changed. Also, care should be taken around transitioning between non-encrypting and encrypting streams; implementors should handle the case where
 +   * the given input stream is <b>not</b> encrypted at all.
 +   * 
 +   * The cryptoInitParams contains all necessary information to properly initialize the given cipher, usually including things like initialization vector and
 +   * secret key.
 +   */
 +  public InputStream getDecryptingInputStream(InputStream in, Map<String,String> cryptoOpts, Map<CryptoModule.CryptoInitProperty,Object> cryptoInitParams)
 +      throws IOException;
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java
index 2f03e02,0000000..956c961
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java
@@@ -1,254 -1,0 +1,253 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.security.crypto;
 +
 +import java.io.IOException;
 +import java.io.InputStream;
 +import java.io.OutputStream;
 +import java.util.Map;
 +
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
 +import org.apache.log4j.Logger;
 +
 +/**
 + * This factory module exists to assist other classes in loading crypto modules.
 + * 
 + * @deprecated This feature is experimental and may go away in future versions.
 + */
 +@Deprecated
 +public class CryptoModuleFactory {
 +  
 +  private static Logger log = Logger.getLogger(CryptoModuleFactory.class);
 +  
 +  /**
 +   * This method returns a crypto module based on settings in the given configuration parameter.
 +   * 
-    * @param conf
 +   * @return a class implementing the CryptoModule interface. It will *never* return null; rather, it will return a class which obeys the interface but makes no
 +   *         changes to the underlying data.
 +   */
 +  public static CryptoModule getCryptoModule(AccumuloConfiguration conf) {
 +    String cryptoModuleClassname = conf.get(Property.CRYPTO_MODULE_CLASS);
 +    return getCryptoModule(cryptoModuleClassname);
 +  }
 +  
 +  @SuppressWarnings({"rawtypes"})
 +  public static CryptoModule getCryptoModule(String cryptoModuleClassname) {
 +    log.debug(String.format("About to instantiate crypto module %s", cryptoModuleClassname));
 +    
 +    if (cryptoModuleClassname.equals("NullCryptoModule")) {
 +      return new NullCryptoModule();
 +    }
 +    
 +    CryptoModule cryptoModule = null;
 +    Class cryptoModuleClazz = null;
 +    try {
 +      cryptoModuleClazz = AccumuloVFSClassLoader.loadClass(cryptoModuleClassname);
 +    } catch (ClassNotFoundException e1) {
 +      log.warn(String.format("Could not find configured crypto module \"%s\".  NO ENCRYPTION WILL BE USED.", cryptoModuleClassname));
 +      return new NullCryptoModule();
 +    }
 +    
 +    // Check if the given class implements the CryptoModule interface
 +    Class[] interfaces = cryptoModuleClazz.getInterfaces();
 +    boolean implementsCryptoModule = false;
 +    
 +    for (Class clazz : interfaces) {
 +      if (clazz.equals(CryptoModule.class)) {
 +        implementsCryptoModule = true;
 +        break;
 +      }
 +    }
 +    
 +    if (!implementsCryptoModule) {
 +      log.warn("Configured Accumulo crypto module \"%s\" does not implement the CryptoModule interface. NO ENCRYPTION WILL BE USED.");
 +      return new NullCryptoModule();
 +    } else {
 +      try {
 +        cryptoModule = (CryptoModule) cryptoModuleClazz.newInstance();
 +        
 +        log.debug("Successfully instantiated crypto module");
 +        
 +      } catch (InstantiationException e) {
 +        log.warn(String.format("Got instantiation exception %s when instantiating crypto module \"%s\".  NO ENCRYPTION WILL BE USED.", e.getCause().getClass()
 +            .getCanonicalName(), cryptoModuleClassname));
 +        log.warn(e.getCause());
 +        return new NullCryptoModule();
 +      } catch (IllegalAccessException e) {
 +        log.warn(String.format("Got illegal access exception when trying to instantiate crypto module \"%s\".  NO ENCRYPTION WILL BE USED.",
 +            cryptoModuleClassname));
 +        log.warn(e);
 +        return new NullCryptoModule();
 +      }
 +    }
 +    return cryptoModule;
 +  }
 +  
 +  public static SecretKeyEncryptionStrategy getSecretKeyEncryptionStrategy(AccumuloConfiguration conf) {
 +    String className = conf.get(Property.CRYPTO_SECRET_KEY_ENCRYPTION_STRATEGY_CLASS);
 +    return getSecretKeyEncryptionStrategy(className);
 +  }
 +  
 +  @SuppressWarnings("rawtypes")
 +  public static SecretKeyEncryptionStrategy getSecretKeyEncryptionStrategy(String className) {
 +    if (className == null || className.equals("NullSecretKeyEncryptionStrategy")) {
 +      return new NullSecretKeyEncryptionStrategy();
 +    }
 +    
 +    SecretKeyEncryptionStrategy strategy = null;
 +    Class keyEncryptionStrategyClazz = null;
 +    try {
 +      keyEncryptionStrategyClazz = AccumuloVFSClassLoader.loadClass(className);
 +    } catch (ClassNotFoundException e1) {
 +      log.warn(String.format("Could not find configured secret key encryption strategy \"%s\".  NO ENCRYPTION WILL BE USED.", className));
 +      return new NullSecretKeyEncryptionStrategy();
 +    }
 +    
 +    // Check if the given class implements the CryptoModule interface
 +    Class[] interfaces = keyEncryptionStrategyClazz.getInterfaces();
 +    boolean implementsSecretKeyStrategy = false;
 +    
 +    for (Class clazz : interfaces) {
 +      if (clazz.equals(SecretKeyEncryptionStrategy.class)) {
 +        implementsSecretKeyStrategy = true;
 +        break;
 +      }
 +    }
 +    
 +    if (!implementsSecretKeyStrategy) {
 +      log.warn("Configured Accumulo secret key encryption strategy \"%s\" does not implement the SecretKeyEncryptionStrategy interface. NO ENCRYPTION WILL BE USED.");
 +      return new NullSecretKeyEncryptionStrategy();
 +    } else {
 +      try {
 +        strategy = (SecretKeyEncryptionStrategy) keyEncryptionStrategyClazz.newInstance();
 +        
 +        log.debug("Successfully instantiated secret key encryption strategy");
 +        
 +      } catch (InstantiationException e) {
 +        log.warn(String.format("Got instantiation exception %s when instantiating secret key encryption strategy \"%s\".  NO ENCRYPTION WILL BE USED.", e
 +            .getCause().getClass().getCanonicalName(), className));
 +        log.warn(e.getCause());
 +        return new NullSecretKeyEncryptionStrategy();
 +      } catch (IllegalAccessException e) {
 +        log.warn(String.format("Got illegal access exception when trying to instantiate secret key encryption strategy \"%s\".  NO ENCRYPTION WILL BE USED.",
 +            className));
 +        log.warn(e);
 +        return new NullSecretKeyEncryptionStrategy();
 +      }
 +    }
 +    
 +    return strategy;
 +  }
 +  
 +  private static class NullSecretKeyEncryptionStrategy implements SecretKeyEncryptionStrategy {
 +    
 +    @Override
 +    public SecretKeyEncryptionStrategyContext encryptSecretKey(SecretKeyEncryptionStrategyContext context) {
 +      context.setEncryptedSecretKey(context.getPlaintextSecretKey());
 +      context.setOpaqueKeyEncryptionKeyID("");
 +      
 +      return context;
 +    }
 +    
 +    @Override
 +    public SecretKeyEncryptionStrategyContext decryptSecretKey(SecretKeyEncryptionStrategyContext context) {
 +      context.setPlaintextSecretKey(context.getEncryptedSecretKey());
 +      
 +      return context;
 +    }
 +    
 +    @Override
 +    public SecretKeyEncryptionStrategyContext getNewContext() {
 +      return new SecretKeyEncryptionStrategyContext() {
 +        
 +        @Override
 +        public byte[] getPlaintextSecretKey() {
 +          return plaintextSecretKey;
 +        }
 +        
 +        @Override
 +        public void setPlaintextSecretKey(byte[] plaintextSecretKey) {
 +          this.plaintextSecretKey = plaintextSecretKey;
 +        }
 +        
 +        @Override
 +        public byte[] getEncryptedSecretKey() {
 +          return encryptedSecretKey;
 +        }
 +        
 +        @Override
 +        public void setEncryptedSecretKey(byte[] encryptedSecretKey) {
 +          this.encryptedSecretKey = encryptedSecretKey;
 +        }
 +        
 +        @Override
 +        public String getOpaqueKeyEncryptionKeyID() {
 +          return opaqueKeyEncryptionKeyID;
 +        }
 +        
 +        @Override
 +        public void setOpaqueKeyEncryptionKeyID(String opaqueKeyEncryptionKeyID) {
 +          this.opaqueKeyEncryptionKeyID = opaqueKeyEncryptionKeyID;
 +        }
 +        
 +        @Override
 +        public Map<String,String> getContext() {
 +          return context;
 +        }
 +        
 +        @Override
 +        public void setContext(Map<String,String> context) {
 +          this.context = context;
 +        }
 +        
 +        private byte[] plaintextSecretKey;
 +        private byte[] encryptedSecretKey;
 +        private String opaqueKeyEncryptionKeyID;
 +        private Map<String,String> context;
 +      };
 +    }
 +    
 +  }
 +  
 +  private static class NullCryptoModule implements CryptoModule {
 +    
 +    @Override
 +    public OutputStream getEncryptingOutputStream(OutputStream out, Map<String,String> cryptoOpts) throws IOException {
 +      return out;
 +    }
 +    
 +    @Override
 +    public InputStream getDecryptingInputStream(InputStream in, Map<String,String> cryptoOpts) throws IOException {
 +      return in;
 +    }
 +    
 +    @Override
 +    public OutputStream getEncryptingOutputStream(OutputStream out, Map<String,String> conf, Map<CryptoInitProperty,Object> cryptoInitParams) {
 +      return out;
 +    }
 +    
 +    @Override
 +    public InputStream getDecryptingInputStream(InputStream in, Map<String,String> cryptoOpts, Map<CryptoInitProperty,Object> cryptoInitParams)
 +        throws IOException {
 +      return in;
 +    }
 +    
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/test/java/org/apache/accumulo/core/client/impl/ScannerOptionsTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/impl/ScannerOptionsTest.java
index 07bd518,0000000..463822a
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/impl/ScannerOptionsTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/impl/ScannerOptionsTest.java
@@@ -1,59 -1,0 +1,57 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.impl;
 +
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.fail;
 +
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.iterators.DebugIterator;
 +import org.apache.accumulo.core.iterators.user.WholeRowIterator;
 +import org.junit.Test;
 +
 +/**
 + * Test that scanner options are set/unset correctly
 + */
 +public class ScannerOptionsTest {
 +  
 +  /**
 +   * Test that you properly add and remove iterators from a scanner
-    * 
-    * @throws Throwable
 +   */
 +  @Test
 +  public void testAddRemoveIterator() throws Throwable {
 +    ScannerOptions options = new ScannerOptions();
 +    options.addScanIterator(new IteratorSetting(1, "NAME", WholeRowIterator.class));
 +    assertEquals(1, options.serverSideIteratorList.size());
 +    options.removeScanIterator("NAME");
 +    assertEquals(0, options.serverSideIteratorList.size());
 +  }
 +  
 +  @Test
 +  public void testIteratorConflict() {
 +    ScannerOptions options = new ScannerOptions();
 +    options.addScanIterator(new IteratorSetting(1, "NAME", DebugIterator.class));
 +    try {
 +      options.addScanIterator(new IteratorSetting(2, "NAME", DebugIterator.class));
 +      fail();
 +    } catch (IllegalArgumentException e) {}
 +    try {
 +      options.addScanIterator(new IteratorSetting(1, "NAME2", DebugIterator.class));
 +      fail();
 +    } catch (IllegalArgumentException e) {}
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/92613388/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
index 4f527e1,0000000..3ec9bb1
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
@@@ -1,284 -1,0 +1,280 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.core.client.mapred;
 +
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertNull;
 +import static org.junit.Assert.assertTrue;
 +
 +import java.io.ByteArrayOutputStream;
 +import java.io.DataOutputStream;
 +import java.io.IOException;
 +import java.util.List;
 +
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.IteratorSetting;
 +import org.apache.accumulo.core.client.mock.MockInstance;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.iterators.user.RegExFilter;
 +import org.apache.accumulo.core.iterators.user.WholeRowIterator;
 +import org.apache.accumulo.core.util.CachedConfiguration;
 +import org.apache.commons.codec.binary.Base64;
 +import org.apache.hadoop.conf.Configured;
 +import org.apache.hadoop.io.Text;
 +import org.apache.hadoop.mapred.JobClient;
 +import org.apache.hadoop.mapred.JobConf;
 +import org.apache.hadoop.mapred.Mapper;
 +import org.apache.hadoop.mapred.OutputCollector;
 +import org.apache.hadoop.mapred.Reporter;
 +import org.apache.hadoop.mapred.lib.NullOutputFormat;
 +import org.apache.hadoop.util.Tool;
 +import org.apache.hadoop.util.ToolRunner;
 +import org.junit.Test;
 +
 +public class AccumuloInputFormatTest {
 +  
 +  private static final String PREFIX = AccumuloInputFormatTest.class.getSimpleName();
 +  private static final String INSTANCE_NAME = PREFIX + "_mapred_instance";
 +  private static final String TEST_TABLE_1 = PREFIX + "_mapred_table_1";
 +  
 +  /**
 +   * Check that the iterator configuration is getting stored in the Job conf correctly.
-    * 
-    * @throws IOException
 +   */
 +  @Test
 +  public void testSetIterator() throws IOException {
 +    JobConf job = new JobConf();
 +    
 +    IteratorSetting is = new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator");
 +    AccumuloInputFormat.addIterator(job, is);
 +    ByteArrayOutputStream baos = new ByteArrayOutputStream();
 +    is.write(new DataOutputStream(baos));
 +    String iterators = job.get("AccumuloInputFormat.ScanOpts.Iterators");
 +    assertEquals(new String(Base64.encodeBase64(baos.toByteArray())), iterators);
 +  }
 +  
 +  @Test
 +  public void testAddIterator() throws IOException {
 +    JobConf job = new JobConf();
 +    
 +    AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", WholeRowIterator.class));
 +    AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
 +    IteratorSetting iter = new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator");
 +    iter.addOption("v1", "1");
 +    iter.addOption("junk", "\0omg:!\\xyzzy");
 +    AccumuloInputFormat.addIterator(job, iter);
 +    
 +    List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
 +    
 +    // Check the list size
 +    assertTrue(list.size() == 3);
 +    
 +    // Walk the list and make sure our settings are correct
 +    IteratorSetting setting = list.get(0);
 +    assertEquals(1, setting.getPriority());
 +    assertEquals("org.apache.accumulo.core.iterators.user.WholeRowIterator", setting.getIteratorClass());
 +    assertEquals("WholeRow", setting.getName());
 +    assertEquals(0, setting.getOptions().size());
 +    
 +    setting = list.get(1);
 +    assertEquals(2, setting.getPriority());
 +    assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
 +    assertEquals("Versions", setting.getName());
 +    assertEquals(0, setting.getOptions().size());
 +    
 +    setting = list.get(2);
 +    assertEquals(3, setting.getPriority());
 +    assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
 +    assertEquals("Count", setting.getName());
 +    assertEquals(2, setting.getOptions().size());
 +    assertEquals("1", setting.getOptions().get("v1"));
 +    assertEquals("\0omg:!\\xyzzy", setting.getOptions().get("junk"));
 +  }
 +  
 +  /**
 +   * Test adding iterator options where the keys and values contain both the FIELD_SEPARATOR character (':') and ITERATOR_SEPARATOR (',') characters. There
 +   * should be no exceptions thrown when trying to parse these types of option entries.
 +   * 
 +   * This test makes sure that the expected raw values, as appears in the Job, are equal to what's expected.
 +   */
 +  @Test
 +  public void testIteratorOptionEncoding() throws Throwable {
 +    String key = "colon:delimited:key";
 +    String value = "comma,delimited,value";
 +    IteratorSetting someSetting = new IteratorSetting(1, "iterator", "Iterator.class");
 +    someSetting.addOption(key, value);
 +    JobConf job = new JobConf();
 +    AccumuloInputFormat.addIterator(job, someSetting);
 +    
 +    List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
 +    assertEquals(1, list.size());
 +    assertEquals(1, list.get(0).getOptions().size());
 +    assertEquals(list.get(0).getOptions().get(key), value);
 +    
 +    someSetting.addOption(key + "2", value);
 +    someSetting.setPriority(2);
 +    someSetting.setName("it2");
 +    AccumuloInputFormat.addIterator(job, someSetting);
 +    list = AccumuloInputFormat.getIterators(job);
 +    assertEquals(2, list.size());
 +    assertEquals(1, list.get(0).getOptions().size());
 +    assertEquals(list.get(0).getOptions().get(key), value);
 +    assertEquals(2, list.get(1).getOptions().size());
 +    assertEquals(list.get(1).getOptions().get(key), value);
 +    assertEquals(list.get(1).getOptions().get(key + "2"), value);
 +  }
 +  
 +  /**
 +   * Test getting iterator settings for multiple iterators set
-    * 
-    * @throws IOException
 +   */
 +  @Test
 +  public void testGetIteratorSettings() throws IOException {
 +    JobConf job = new JobConf();
 +    
 +    AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator"));
 +    AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
 +    AccumuloInputFormat.addIterator(job, new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"));
 +    
 +    List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
 +    
 +    // Check the list size
 +    assertTrue(list.size() == 3);
 +    
 +    // Walk the list and make sure our settings are correct
 +    IteratorSetting setting = list.get(0);
 +    assertEquals(1, setting.getPriority());
 +    assertEquals("org.apache.accumulo.core.iterators.WholeRowIterator", setting.getIteratorClass());
 +    assertEquals("WholeRow", setting.getName());
 +    
 +    setting = list.get(1);
 +    assertEquals(2, setting.getPriority());
 +    assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
 +    assertEquals("Versions", setting.getName());
 +    
 +    setting = list.get(2);
 +    assertEquals(3, setting.getPriority());
 +    assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
 +    assertEquals("Count", setting.getName());
 +    
 +  }
 +  
 +  @Test
 +  public void testSetRegex() throws IOException {
 +    JobConf job = new JobConf();
 +    
 +    String regex = ">\"*%<>\'\\";
 +    
 +    IteratorSetting is = new IteratorSetting(50, regex, RegExFilter.class);
 +    RegExFilter.setRegexs(is, regex, null, null, null, false);
 +    AccumuloInputFormat.addIterator(job, is);
 +    
 +    assertTrue(regex.equals(AccumuloInputFormat.getIterators(job).get(0).getName()));
 +  }
 +  
 +  private static AssertionError e1 = null;
 +  private static AssertionError e2 = null;
 +  
 +  private static class MRTester extends Configured implements Tool {
 +    private static class TestMapper implements Mapper<Key,Value,Key,Value> {
 +      Key key = null;
 +      int count = 0;
 +      
 +      @Override
 +      public void map(Key k, Value v, OutputCollector<Key,Value> output, Reporter reporter) throws IOException {
 +        try {
 +          if (key != null)
 +            assertEquals(key.getRow().toString(), new String(v.get()));
 +          assertEquals(k.getRow(), new Text(String.format("%09x", count + 1)));
 +          assertEquals(new String(v.get()), String.format("%09x", count));
 +        } catch (AssertionError e) {
 +          e1 = e;
 +        }
 +        key = new Key(k);
 +        count++;
 +      }
 +      
 +      @Override
 +      public void configure(JobConf job) {}
 +      
 +      @Override
 +      public void close() throws IOException {
 +        try {
 +          assertEquals(100, count);
 +        } catch (AssertionError e) {
 +          e2 = e;
 +        }
 +      }
 +      
 +    }
 +    
 +    @Override
 +    public int run(String[] args) throws Exception {
 +      
 +      if (args.length != 3) {
 +        throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user> <pass> <table>");
 +      }
 +      
 +      String user = args[0];
 +      String pass = args[1];
 +      String table = args[2];
 +      
 +      JobConf job = new JobConf(getConf());
 +      job.setJarByClass(this.getClass());
 +      
 +      job.setInputFormat(AccumuloInputFormat.class);
 +      
 +      AccumuloInputFormat.setConnectorInfo(job, user, new PasswordToken(pass));
 +      AccumuloInputFormat.setInputTableName(job, table);
 +      AccumuloInputFormat.setMockInstance(job, INSTANCE_NAME);
 +      
 +      job.setMapperClass(TestMapper.class);
 +      job.setMapOutputKeyClass(Key.class);
 +      job.setMapOutputValueClass(Value.class);
 +      job.setOutputFormat(NullOutputFormat.class);
 +      
 +      job.setNumReduceTasks(0);
 +      
 +      return JobClient.runJob(job).isSuccessful() ? 0 : 1;
 +    }
 +    
 +    public static void main(String[] args) throws Exception {
 +      assertEquals(0, ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args));
 +    }
 +  }
 +  
 +  @Test
 +  public void testMap() throws Exception {
 +    MockInstance mockInstance = new MockInstance(INSTANCE_NAME);
 +    Connector c = mockInstance.getConnector("root", new PasswordToken(""));
 +    c.tableOperations().create(TEST_TABLE_1);
 +    BatchWriter bw = c.createBatchWriter(TEST_TABLE_1, new BatchWriterConfig());
 +    for (int i = 0; i < 100; i++) {
 +      Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
 +      m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
 +      bw.addMutation(m);
 +    }
 +    bw.close();
 +    
 +    MRTester.main(new String[] {"root", "", TEST_TABLE_1});
 +    assertNull(e1);
 +    assertNull(e2);
 +  }
 +}


Mime
View raw message