accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject [28/61] [abbrv] [partial] accumulo git commit: ACCUMULO-722 put trunk in my sandbox
Date Thu, 03 Mar 2016 21:59:53 GMT
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7bdbfccb/1.5/core/src/main/java/org/apache/accumulo/core/data/thrift/UpdateErrors.java
----------------------------------------------------------------------
diff --git a/1.5/core/src/main/java/org/apache/accumulo/core/data/thrift/UpdateErrors.java b/1.5/core/src/main/java/org/apache/accumulo/core/data/thrift/UpdateErrors.java
new file mode 100644
index 0000000..a61e3ca
--- /dev/null
+++ b/1.5/core/src/main/java/org/apache/accumulo/core/data/thrift/UpdateErrors.java
@@ -0,0 +1,755 @@
+/**
+ * Autogenerated by Thrift Compiler (0.8.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ *  @generated
+ */
+package org.apache.accumulo.core.data.thrift;
+
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+
+import org.apache.thrift.scheme.TupleScheme;
+import org.apache.thrift.protocol.TTupleProtocol;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.EnumMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.EnumSet;
+import java.util.Collections;
+import java.util.BitSet;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@SuppressWarnings("all") public class UpdateErrors implements org.apache.thrift.TBase<UpdateErrors, UpdateErrors._Fields>, java.io.Serializable, Cloneable {
+  private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("UpdateErrors");
+
+  private static final org.apache.thrift.protocol.TField FAILED_EXTENTS_FIELD_DESC = new org.apache.thrift.protocol.TField("failedExtents", org.apache.thrift.protocol.TType.MAP, (short)1);
+  private static final org.apache.thrift.protocol.TField VIOLATION_SUMMARIES_FIELD_DESC = new org.apache.thrift.protocol.TField("violationSummaries", org.apache.thrift.protocol.TType.LIST, (short)2);
+  private static final org.apache.thrift.protocol.TField AUTHORIZATION_FAILURES_FIELD_DESC = new org.apache.thrift.protocol.TField("authorizationFailures", org.apache.thrift.protocol.TType.LIST, (short)3);
+
+  private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+  static {
+    schemes.put(StandardScheme.class, new UpdateErrorsStandardSchemeFactory());
+    schemes.put(TupleScheme.class, new UpdateErrorsTupleSchemeFactory());
+  }
+
+  public Map<TKeyExtent,Long> failedExtents; // required
+  public List<TConstraintViolationSummary> violationSummaries; // required
+  public List<TKeyExtent> authorizationFailures; // required
+
+  /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+  public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+    FAILED_EXTENTS((short)1, "failedExtents"),
+    VIOLATION_SUMMARIES((short)2, "violationSummaries"),
+    AUTHORIZATION_FAILURES((short)3, "authorizationFailures");
+
+    private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+    static {
+      for (_Fields field : EnumSet.allOf(_Fields.class)) {
+        byName.put(field.getFieldName(), field);
+      }
+    }
+
+    /**
+     * Find the _Fields constant that matches fieldId, or null if its not found.
+     */
+    public static _Fields findByThriftId(int fieldId) {
+      switch(fieldId) {
+        case 1: // FAILED_EXTENTS
+          return FAILED_EXTENTS;
+        case 2: // VIOLATION_SUMMARIES
+          return VIOLATION_SUMMARIES;
+        case 3: // AUTHORIZATION_FAILURES
+          return AUTHORIZATION_FAILURES;
+        default:
+          return null;
+      }
+    }
+
+    /**
+     * Find the _Fields constant that matches fieldId, throwing an exception
+     * if it is not found.
+     */
+    public static _Fields findByThriftIdOrThrow(int fieldId) {
+      _Fields fields = findByThriftId(fieldId);
+      if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+      return fields;
+    }
+
+    /**
+     * Find the _Fields constant that matches name, or null if its not found.
+     */
+    public static _Fields findByName(String name) {
+      return byName.get(name);
+    }
+
+    private final short _thriftId;
+    private final String _fieldName;
+
+    _Fields(short thriftId, String fieldName) {
+      _thriftId = thriftId;
+      _fieldName = fieldName;
+    }
+
+    public short getThriftFieldId() {
+      return _thriftId;
+    }
+
+    public String getFieldName() {
+      return _fieldName;
+    }
+  }
+
+  // isset id assignments
+  public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+  static {
+    Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+    tmpMap.put(_Fields.FAILED_EXTENTS, new org.apache.thrift.meta_data.FieldMetaData("failedExtents", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+        new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, 
+            new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TKeyExtent.class), 
+            new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))));
+    tmpMap.put(_Fields.VIOLATION_SUMMARIES, new org.apache.thrift.meta_data.FieldMetaData("violationSummaries", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
+            new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TConstraintViolationSummary.class))));
+    tmpMap.put(_Fields.AUTHORIZATION_FAILURES, new org.apache.thrift.meta_data.FieldMetaData("authorizationFailures", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+        new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
+            new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TKeyExtent.class))));
+    metaDataMap = Collections.unmodifiableMap(tmpMap);
+    org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(UpdateErrors.class, metaDataMap);
+  }
+
+  public UpdateErrors() {
+  }
+
+  public UpdateErrors(
+    Map<TKeyExtent,Long> failedExtents,
+    List<TConstraintViolationSummary> violationSummaries,
+    List<TKeyExtent> authorizationFailures)
+  {
+    this();
+    this.failedExtents = failedExtents;
+    this.violationSummaries = violationSummaries;
+    this.authorizationFailures = authorizationFailures;
+  }
+
+  /**
+   * Performs a deep copy on <i>other</i>.
+   */
+  public UpdateErrors(UpdateErrors other) {
+    if (other.isSetFailedExtents()) {
+      Map<TKeyExtent,Long> __this__failedExtents = new HashMap<TKeyExtent,Long>();
+      for (Map.Entry<TKeyExtent, Long> other_element : other.failedExtents.entrySet()) {
+
+        TKeyExtent other_element_key = other_element.getKey();
+        Long other_element_value = other_element.getValue();
+
+        TKeyExtent __this__failedExtents_copy_key = new TKeyExtent(other_element_key);
+
+        Long __this__failedExtents_copy_value = other_element_value;
+
+        __this__failedExtents.put(__this__failedExtents_copy_key, __this__failedExtents_copy_value);
+      }
+      this.failedExtents = __this__failedExtents;
+    }
+    if (other.isSetViolationSummaries()) {
+      List<TConstraintViolationSummary> __this__violationSummaries = new ArrayList<TConstraintViolationSummary>();
+      for (TConstraintViolationSummary other_element : other.violationSummaries) {
+        __this__violationSummaries.add(new TConstraintViolationSummary(other_element));
+      }
+      this.violationSummaries = __this__violationSummaries;
+    }
+    if (other.isSetAuthorizationFailures()) {
+      List<TKeyExtent> __this__authorizationFailures = new ArrayList<TKeyExtent>();
+      for (TKeyExtent other_element : other.authorizationFailures) {
+        __this__authorizationFailures.add(new TKeyExtent(other_element));
+      }
+      this.authorizationFailures = __this__authorizationFailures;
+    }
+  }
+
+  public UpdateErrors deepCopy() {
+    return new UpdateErrors(this);
+  }
+
+  @Override
+  public void clear() {
+    this.failedExtents = null;
+    this.violationSummaries = null;
+    this.authorizationFailures = null;
+  }
+
+  public int getFailedExtentsSize() {
+    return (this.failedExtents == null) ? 0 : this.failedExtents.size();
+  }
+
+  public void putToFailedExtents(TKeyExtent key, long val) {
+    if (this.failedExtents == null) {
+      this.failedExtents = new HashMap<TKeyExtent,Long>();
+    }
+    this.failedExtents.put(key, val);
+  }
+
+  public Map<TKeyExtent,Long> getFailedExtents() {
+    return this.failedExtents;
+  }
+
+  public UpdateErrors setFailedExtents(Map<TKeyExtent,Long> failedExtents) {
+    this.failedExtents = failedExtents;
+    return this;
+  }
+
+  public void unsetFailedExtents() {
+    this.failedExtents = null;
+  }
+
+  /** Returns true if field failedExtents is set (has been assigned a value) and false otherwise */
+  public boolean isSetFailedExtents() {
+    return this.failedExtents != null;
+  }
+
+  public void setFailedExtentsIsSet(boolean value) {
+    if (!value) {
+      this.failedExtents = null;
+    }
+  }
+
+  public int getViolationSummariesSize() {
+    return (this.violationSummaries == null) ? 0 : this.violationSummaries.size();
+  }
+
+  public java.util.Iterator<TConstraintViolationSummary> getViolationSummariesIterator() {
+    return (this.violationSummaries == null) ? null : this.violationSummaries.iterator();
+  }
+
+  public void addToViolationSummaries(TConstraintViolationSummary elem) {
+    if (this.violationSummaries == null) {
+      this.violationSummaries = new ArrayList<TConstraintViolationSummary>();
+    }
+    this.violationSummaries.add(elem);
+  }
+
+  public List<TConstraintViolationSummary> getViolationSummaries() {
+    return this.violationSummaries;
+  }
+
+  public UpdateErrors setViolationSummaries(List<TConstraintViolationSummary> violationSummaries) {
+    this.violationSummaries = violationSummaries;
+    return this;
+  }
+
+  public void unsetViolationSummaries() {
+    this.violationSummaries = null;
+  }
+
+  /** Returns true if field violationSummaries is set (has been assigned a value) and false otherwise */
+  public boolean isSetViolationSummaries() {
+    return this.violationSummaries != null;
+  }
+
+  public void setViolationSummariesIsSet(boolean value) {
+    if (!value) {
+      this.violationSummaries = null;
+    }
+  }
+
+  public int getAuthorizationFailuresSize() {
+    return (this.authorizationFailures == null) ? 0 : this.authorizationFailures.size();
+  }
+
+  public java.util.Iterator<TKeyExtent> getAuthorizationFailuresIterator() {
+    return (this.authorizationFailures == null) ? null : this.authorizationFailures.iterator();
+  }
+
+  public void addToAuthorizationFailures(TKeyExtent elem) {
+    if (this.authorizationFailures == null) {
+      this.authorizationFailures = new ArrayList<TKeyExtent>();
+    }
+    this.authorizationFailures.add(elem);
+  }
+
+  public List<TKeyExtent> getAuthorizationFailures() {
+    return this.authorizationFailures;
+  }
+
+  public UpdateErrors setAuthorizationFailures(List<TKeyExtent> authorizationFailures) {
+    this.authorizationFailures = authorizationFailures;
+    return this;
+  }
+
+  public void unsetAuthorizationFailures() {
+    this.authorizationFailures = null;
+  }
+
+  /** Returns true if field authorizationFailures is set (has been assigned a value) and false otherwise */
+  public boolean isSetAuthorizationFailures() {
+    return this.authorizationFailures != null;
+  }
+
+  public void setAuthorizationFailuresIsSet(boolean value) {
+    if (!value) {
+      this.authorizationFailures = null;
+    }
+  }
+
+  public void setFieldValue(_Fields field, Object value) {
+    switch (field) {
+    case FAILED_EXTENTS:
+      if (value == null) {
+        unsetFailedExtents();
+      } else {
+        setFailedExtents((Map<TKeyExtent,Long>)value);
+      }
+      break;
+
+    case VIOLATION_SUMMARIES:
+      if (value == null) {
+        unsetViolationSummaries();
+      } else {
+        setViolationSummaries((List<TConstraintViolationSummary>)value);
+      }
+      break;
+
+    case AUTHORIZATION_FAILURES:
+      if (value == null) {
+        unsetAuthorizationFailures();
+      } else {
+        setAuthorizationFailures((List<TKeyExtent>)value);
+      }
+      break;
+
+    }
+  }
+
+  public Object getFieldValue(_Fields field) {
+    switch (field) {
+    case FAILED_EXTENTS:
+      return getFailedExtents();
+
+    case VIOLATION_SUMMARIES:
+      return getViolationSummaries();
+
+    case AUTHORIZATION_FAILURES:
+      return getAuthorizationFailures();
+
+    }
+    throw new IllegalStateException();
+  }
+
+  /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+  public boolean isSet(_Fields field) {
+    if (field == null) {
+      throw new IllegalArgumentException();
+    }
+
+    switch (field) {
+    case FAILED_EXTENTS:
+      return isSetFailedExtents();
+    case VIOLATION_SUMMARIES:
+      return isSetViolationSummaries();
+    case AUTHORIZATION_FAILURES:
+      return isSetAuthorizationFailures();
+    }
+    throw new IllegalStateException();
+  }
+
+  @Override
+  public boolean equals(Object that) {
+    if (that == null)
+      return false;
+    if (that instanceof UpdateErrors)
+      return this.equals((UpdateErrors)that);
+    return false;
+  }
+
+  public boolean equals(UpdateErrors that) {
+    if (that == null)
+      return false;
+
+    boolean this_present_failedExtents = true && this.isSetFailedExtents();
+    boolean that_present_failedExtents = true && that.isSetFailedExtents();
+    if (this_present_failedExtents || that_present_failedExtents) {
+      if (!(this_present_failedExtents && that_present_failedExtents))
+        return false;
+      if (!this.failedExtents.equals(that.failedExtents))
+        return false;
+    }
+
+    boolean this_present_violationSummaries = true && this.isSetViolationSummaries();
+    boolean that_present_violationSummaries = true && that.isSetViolationSummaries();
+    if (this_present_violationSummaries || that_present_violationSummaries) {
+      if (!(this_present_violationSummaries && that_present_violationSummaries))
+        return false;
+      if (!this.violationSummaries.equals(that.violationSummaries))
+        return false;
+    }
+
+    boolean this_present_authorizationFailures = true && this.isSetAuthorizationFailures();
+    boolean that_present_authorizationFailures = true && that.isSetAuthorizationFailures();
+    if (this_present_authorizationFailures || that_present_authorizationFailures) {
+      if (!(this_present_authorizationFailures && that_present_authorizationFailures))
+        return false;
+      if (!this.authorizationFailures.equals(that.authorizationFailures))
+        return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public int hashCode() {
+    return 0;
+  }
+
+  public int compareTo(UpdateErrors other) {
+    if (!getClass().equals(other.getClass())) {
+      return getClass().getName().compareTo(other.getClass().getName());
+    }
+
+    int lastComparison = 0;
+    UpdateErrors typedOther = (UpdateErrors)other;
+
+    lastComparison = Boolean.valueOf(isSetFailedExtents()).compareTo(typedOther.isSetFailedExtents());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetFailedExtents()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.failedExtents, typedOther.failedExtents);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
+    lastComparison = Boolean.valueOf(isSetViolationSummaries()).compareTo(typedOther.isSetViolationSummaries());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetViolationSummaries()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.violationSummaries, typedOther.violationSummaries);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
+    lastComparison = Boolean.valueOf(isSetAuthorizationFailures()).compareTo(typedOther.isSetAuthorizationFailures());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    if (isSetAuthorizationFailures()) {
+      lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.authorizationFailures, typedOther.authorizationFailures);
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
+    return 0;
+  }
+
+  public _Fields fieldForId(int fieldId) {
+    return _Fields.findByThriftId(fieldId);
+  }
+
+  public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+    schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+  }
+
+  public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+    schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder("UpdateErrors(");
+    boolean first = true;
+
+    sb.append("failedExtents:");
+    if (this.failedExtents == null) {
+      sb.append("null");
+    } else {
+      sb.append(this.failedExtents);
+    }
+    first = false;
+    if (!first) sb.append(", ");
+    sb.append("violationSummaries:");
+    if (this.violationSummaries == null) {
+      sb.append("null");
+    } else {
+      sb.append(this.violationSummaries);
+    }
+    first = false;
+    if (!first) sb.append(", ");
+    sb.append("authorizationFailures:");
+    if (this.authorizationFailures == null) {
+      sb.append("null");
+    } else {
+      sb.append(this.authorizationFailures);
+    }
+    first = false;
+    sb.append(")");
+    return sb.toString();
+  }
+
+  public void validate() throws org.apache.thrift.TException {
+    // check for required fields
+  }
+
+  private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+    try {
+      write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+    } catch (org.apache.thrift.TException te) {
+      throw new java.io.IOException(te);
+    }
+  }
+
+  private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+    try {
+      read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+    } catch (org.apache.thrift.TException te) {
+      throw new java.io.IOException(te);
+    }
+  }
+
+  private static class UpdateErrorsStandardSchemeFactory implements SchemeFactory {
+    public UpdateErrorsStandardScheme getScheme() {
+      return new UpdateErrorsStandardScheme();
+    }
+  }
+
+  private static class UpdateErrorsStandardScheme extends StandardScheme<UpdateErrors> {
+
+    public void read(org.apache.thrift.protocol.TProtocol iprot, UpdateErrors struct) throws org.apache.thrift.TException {
+      org.apache.thrift.protocol.TField schemeField;
+      iprot.readStructBegin();
+      while (true)
+      {
+        schemeField = iprot.readFieldBegin();
+        if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { 
+          break;
+        }
+        switch (schemeField.id) {
+          case 1: // FAILED_EXTENTS
+            if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
+              {
+                org.apache.thrift.protocol.TMap _map50 = iprot.readMapBegin();
+                struct.failedExtents = new HashMap<TKeyExtent,Long>(2*_map50.size);
+                for (int _i51 = 0; _i51 < _map50.size; ++_i51)
+                {
+                  TKeyExtent _key52; // required
+                  long _val53; // required
+                  _key52 = new TKeyExtent();
+                  _key52.read(iprot);
+                  _val53 = iprot.readI64();
+                  struct.failedExtents.put(_key52, _val53);
+                }
+                iprot.readMapEnd();
+              }
+              struct.setFailedExtentsIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
+          case 2: // VIOLATION_SUMMARIES
+            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
+              {
+                org.apache.thrift.protocol.TList _list54 = iprot.readListBegin();
+                struct.violationSummaries = new ArrayList<TConstraintViolationSummary>(_list54.size);
+                for (int _i55 = 0; _i55 < _list54.size; ++_i55)
+                {
+                  TConstraintViolationSummary _elem56; // required
+                  _elem56 = new TConstraintViolationSummary();
+                  _elem56.read(iprot);
+                  struct.violationSummaries.add(_elem56);
+                }
+                iprot.readListEnd();
+              }
+              struct.setViolationSummariesIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
+          case 3: // AUTHORIZATION_FAILURES
+            if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
+              {
+                org.apache.thrift.protocol.TList _list57 = iprot.readListBegin();
+                struct.authorizationFailures = new ArrayList<TKeyExtent>(_list57.size);
+                for (int _i58 = 0; _i58 < _list57.size; ++_i58)
+                {
+                  TKeyExtent _elem59; // required
+                  _elem59 = new TKeyExtent();
+                  _elem59.read(iprot);
+                  struct.authorizationFailures.add(_elem59);
+                }
+                iprot.readListEnd();
+              }
+              struct.setAuthorizationFailuresIsSet(true);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+            }
+            break;
+          default:
+            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+        }
+        iprot.readFieldEnd();
+      }
+      iprot.readStructEnd();
+
+      // check for required fields of primitive type, which can't be checked in the validate method
+      struct.validate();
+    }
+
+    public void write(org.apache.thrift.protocol.TProtocol oprot, UpdateErrors struct) throws org.apache.thrift.TException {
+      struct.validate();
+
+      oprot.writeStructBegin(STRUCT_DESC);
+      if (struct.failedExtents != null) {
+        oprot.writeFieldBegin(FAILED_EXTENTS_FIELD_DESC);
+        {
+          oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRUCT, org.apache.thrift.protocol.TType.I64, struct.failedExtents.size()));
+          for (Map.Entry<TKeyExtent, Long> _iter60 : struct.failedExtents.entrySet())
+          {
+            _iter60.getKey().write(oprot);
+            oprot.writeI64(_iter60.getValue());
+          }
+          oprot.writeMapEnd();
+        }
+        oprot.writeFieldEnd();
+      }
+      if (struct.violationSummaries != null) {
+        oprot.writeFieldBegin(VIOLATION_SUMMARIES_FIELD_DESC);
+        {
+          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.violationSummaries.size()));
+          for (TConstraintViolationSummary _iter61 : struct.violationSummaries)
+          {
+            _iter61.write(oprot);
+          }
+          oprot.writeListEnd();
+        }
+        oprot.writeFieldEnd();
+      }
+      if (struct.authorizationFailures != null) {
+        oprot.writeFieldBegin(AUTHORIZATION_FAILURES_FIELD_DESC);
+        {
+          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.authorizationFailures.size()));
+          for (TKeyExtent _iter62 : struct.authorizationFailures)
+          {
+            _iter62.write(oprot);
+          }
+          oprot.writeListEnd();
+        }
+        oprot.writeFieldEnd();
+      }
+      oprot.writeFieldStop();
+      oprot.writeStructEnd();
+    }
+
+  }
+
+  private static class UpdateErrorsTupleSchemeFactory implements SchemeFactory {
+    public UpdateErrorsTupleScheme getScheme() {
+      return new UpdateErrorsTupleScheme();
+    }
+  }
+
+  private static class UpdateErrorsTupleScheme extends TupleScheme<UpdateErrors> {
+
+    @Override
+    public void write(org.apache.thrift.protocol.TProtocol prot, UpdateErrors struct) throws org.apache.thrift.TException {
+      TTupleProtocol oprot = (TTupleProtocol) prot;
+      BitSet optionals = new BitSet();
+      if (struct.isSetFailedExtents()) {
+        optionals.set(0);
+      }
+      if (struct.isSetViolationSummaries()) {
+        optionals.set(1);
+      }
+      if (struct.isSetAuthorizationFailures()) {
+        optionals.set(2);
+      }
+      oprot.writeBitSet(optionals, 3);
+      if (struct.isSetFailedExtents()) {
+        {
+          oprot.writeI32(struct.failedExtents.size());
+          for (Map.Entry<TKeyExtent, Long> _iter63 : struct.failedExtents.entrySet())
+          {
+            _iter63.getKey().write(oprot);
+            oprot.writeI64(_iter63.getValue());
+          }
+        }
+      }
+      if (struct.isSetViolationSummaries()) {
+        {
+          oprot.writeI32(struct.violationSummaries.size());
+          for (TConstraintViolationSummary _iter64 : struct.violationSummaries)
+          {
+            _iter64.write(oprot);
+          }
+        }
+      }
+      if (struct.isSetAuthorizationFailures()) {
+        {
+          oprot.writeI32(struct.authorizationFailures.size());
+          for (TKeyExtent _iter65 : struct.authorizationFailures)
+          {
+            _iter65.write(oprot);
+          }
+        }
+      }
+    }
+
+    @Override
+    public void read(org.apache.thrift.protocol.TProtocol prot, UpdateErrors struct) throws org.apache.thrift.TException {
+      TTupleProtocol iprot = (TTupleProtocol) prot;
+      BitSet incoming = iprot.readBitSet(3);
+      if (incoming.get(0)) {
+        {
+          org.apache.thrift.protocol.TMap _map66 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRUCT, org.apache.thrift.protocol.TType.I64, iprot.readI32());
+          struct.failedExtents = new HashMap<TKeyExtent,Long>(2*_map66.size);
+          for (int _i67 = 0; _i67 < _map66.size; ++_i67)
+          {
+            TKeyExtent _key68; // required
+            long _val69; // required
+            _key68 = new TKeyExtent();
+            _key68.read(iprot);
+            _val69 = iprot.readI64();
+            struct.failedExtents.put(_key68, _val69);
+          }
+        }
+        struct.setFailedExtentsIsSet(true);
+      }
+      if (incoming.get(1)) {
+        {
+          org.apache.thrift.protocol.TList _list70 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+          struct.violationSummaries = new ArrayList<TConstraintViolationSummary>(_list70.size);
+          for (int _i71 = 0; _i71 < _list70.size; ++_i71)
+          {
+            TConstraintViolationSummary _elem72; // required
+            _elem72 = new TConstraintViolationSummary();
+            _elem72.read(iprot);
+            struct.violationSummaries.add(_elem72);
+          }
+        }
+        struct.setViolationSummariesIsSet(true);
+      }
+      if (incoming.get(2)) {
+        {
+          org.apache.thrift.protocol.TList _list73 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+          struct.authorizationFailures = new ArrayList<TKeyExtent>(_list73.size);
+          for (int _i74 = 0; _i74 < _list73.size; ++_i74)
+          {
+            TKeyExtent _elem75; // required
+            _elem75 = new TKeyExtent();
+            _elem75.read(iprot);
+            struct.authorizationFailures.add(_elem75);
+          }
+        }
+        struct.setAuthorizationFailuresIsSet(true);
+      }
+    }
+  }
+
+}
+

http://git-wip-us.apache.org/repos/asf/accumulo/blob/7bdbfccb/1.5/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
----------------------------------------------------------------------
diff --git a/1.5/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java b/1.5/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
new file mode 100644
index 0000000..380461b
--- /dev/null
+++ b/1.5/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
@@ -0,0 +1,502 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.core.file;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.accumulo.core.bloomfilter.DynamicBloomFilter;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.keyfunctor.KeyFunctor;
+import org.apache.accumulo.core.file.rfile.RFile;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.LoggingRunnable;
+import org.apache.accumulo.core.util.NamingThreadFactory;
+import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.util.bloom.Key;
+import org.apache.hadoop.util.hash.Hash;
+import org.apache.log4j.Logger;
+
+/**
+ * A class that sits on top of different accumulo file formats and provides bloom filter functionality.
+ * 
+ */
+public class BloomFilterLayer {
+  private static final Logger LOG = Logger.getLogger(BloomFilterLayer.class);
+  public static final String BLOOM_FILE_NAME = "acu_bloom";
+  public static final int HASH_COUNT = 5;
+  
+  private static ExecutorService loadThreadPool = null;
+  
+  private static synchronized ExecutorService getLoadThreadPool(int maxLoadThreads) {
+    if (loadThreadPool != null) {
+      return loadThreadPool;
+    }
+    
+    if (maxLoadThreads > 0) {
+      BlockingQueue<Runnable> q = new LinkedBlockingQueue<Runnable>();
+      loadThreadPool = new ThreadPoolExecutor(0, maxLoadThreads, 60, TimeUnit.SECONDS, q, new NamingThreadFactory("bloom-loader"));
+    }
+    
+    return loadThreadPool;
+  }
+  
+  public static class Writer implements FileSKVWriter {
+    private DynamicBloomFilter bloomFilter;
+    private int numKeys;
+    private int vectorSize;
+    
+    private FileSKVWriter writer;
+    private KeyFunctor transformer = null;
+    private boolean closed = false;
+    
+    Writer(FileSKVWriter writer, AccumuloConfiguration acuconf) {
+      this.writer = writer;
+      initBloomFilter(acuconf);
+    }
+    
+    private synchronized void initBloomFilter(AccumuloConfiguration acuconf) {
+      
+      numKeys = acuconf.getCount(Property.TABLE_BLOOM_SIZE);
+      // vector size should be <code>-kn / (ln(1 - c^(1/k)))</code> bits for
+      // single key, where <code> is the number of hash functions,
+      // <code>n</code> is the number of keys and <code>c</code> is the desired
+      // max. error rate.
+      // Our desired error rate is by default 0.005, i.e. 0.5%
+      double errorRate = acuconf.getFraction(Property.TABLE_BLOOM_ERRORRATE);
+      vectorSize = (int) Math.ceil(-HASH_COUNT * numKeys / Math.log(1.0 - Math.pow(errorRate, 1.0 / HASH_COUNT)));
+      bloomFilter = new DynamicBloomFilter(vectorSize, HASH_COUNT, Hash.parseHashType(acuconf.get(Property.TABLE_BLOOM_HASHTYPE)), numKeys);
+      
+      /**
+       * load KeyFunctor
+       */
+      try {
+        Class<? extends KeyFunctor> clazz = AccumuloClassLoader.loadClass(acuconf.get(Property.TABLE_BLOOM_KEY_FUNCTOR), KeyFunctor.class);
+        transformer = clazz.newInstance();
+        
+      } catch (Exception e) {
+        LOG.error("Failed to find KeyFunctor: " + acuconf.get(Property.TABLE_BLOOM_KEY_FUNCTOR), e);
+        throw new IllegalArgumentException("Failed to find KeyFunctor: " + acuconf.get(Property.TABLE_BLOOM_KEY_FUNCTOR));
+        
+      }
+      
+    }
+    
+    @Override
+    public synchronized void append(org.apache.accumulo.core.data.Key key, Value val) throws IOException {
+      writer.append(key, val);
+      Key bloomKey = transformer.transform(key);
+      if (bloomKey.getBytes().length > 0)
+        bloomFilter.add(bloomKey);
+    }
+    
+    @Override
+    public synchronized void close() throws IOException {
+      
+      if (closed)
+        return;
+      
+      DataOutputStream out = writer.createMetaStore(BLOOM_FILE_NAME);
+      out.writeUTF(transformer.getClass().getCanonicalName());
+      bloomFilter.write(out);
+      out.flush();
+      out.close();
+      writer.close();
+      closed = true;
+    }
+    
+    @Override
+    public DataOutputStream createMetaStore(String name) throws IOException {
+      return writer.createMetaStore(name);
+    }
+    
+    @Override
+    public void startDefaultLocalityGroup() throws IOException {
+      writer.startDefaultLocalityGroup();
+      
+    }
+    
+    @Override
+    public void startNewLocalityGroup(String name, Set<ByteSequence> columnFamilies) throws IOException {
+      writer.startNewLocalityGroup(name, columnFamilies);
+    }
+    
+    @Override
+    public boolean supportsLocalityGroups() {
+      return writer.supportsLocalityGroups();
+    }
+  }
+  
+  static class BloomFilterLoader {
+    
+    private volatile DynamicBloomFilter bloomFilter;
+    private int loadRequest = 0;
+    private int loadThreshold = 1;
+    private int maxLoadThreads;
+    private Runnable loadTask;
+    private volatile KeyFunctor transformer = null;
+    private volatile boolean closed = false;
+    
+    BloomFilterLoader(final FileSKVIterator reader, AccumuloConfiguration acuconf) {
+      
+      maxLoadThreads = acuconf.getCount(Property.TSERV_BLOOM_LOAD_MAXCONCURRENT);
+      
+      loadThreshold = acuconf.getCount(Property.TABLE_BLOOM_LOAD_THRESHOLD);
+      
+      loadTask = new Runnable() {
+        public void run() {
+          
+          // no need to load the bloom filter if the map file is closed
+          if (closed)
+            return;
+          String ClassName = null;
+          DataInputStream in = null;
+          
+          try {
+            in = reader.getMetaStore(BLOOM_FILE_NAME);
+            DynamicBloomFilter tmpBloomFilter = new DynamicBloomFilter();
+            
+            // check for closed again after open but before reading the bloom filter in
+            if (closed)
+              return;
+            
+            /**
+             * Load classname for keyFunctor
+             */
+            ClassName = in.readUTF();
+            
+            Class<? extends KeyFunctor> clazz = AccumuloClassLoader.loadClass(ClassName, KeyFunctor.class);
+            transformer = clazz.newInstance();
+            
+            /**
+             * read in bloom filter
+             */
+            
+            tmpBloomFilter.readFields(in);
+            // only set the bloom filter after it is fully constructed
+            bloomFilter = tmpBloomFilter;
+          } catch (NoSuchMetaStoreException nsme) {
+            // file does not have a bloom filter, ignore it
+          } catch (IOException ioe) {
+            if (!closed)
+              LOG.warn("Can't open BloomFilter", ioe);
+            else
+              LOG.debug("Can't open BloomFilter, file closed : " + ioe.getMessage());
+            
+            bloomFilter = null;
+          } catch (ClassNotFoundException e) {
+            LOG.error("Failed to find KeyFunctor in config: " + ClassName, e);
+            bloomFilter = null;
+          } catch (InstantiationException e) {
+            LOG.error("Could not instantiate KeyFunctor: " + ClassName, e);
+            bloomFilter = null;
+          } catch (IllegalAccessException e) {
+            LOG.error("Illegal acess exception", e);
+            bloomFilter = null;
+          } catch (NullPointerException npe) {
+            if (!closed)
+              throw npe;
+            else
+              LOG.debug("Can't open BloomFilter, NPE after closed ", npe);
+            
+          } finally {
+            if (in != null) {
+              try {
+                in.close();
+              } catch (IOException e) {
+                LOG.warn("Failed to close ", e);
+              }
+            }
+          }
+        }
+      };
+      
+      initiateLoad(maxLoadThreads);
+      
+    }
+    
+    private synchronized void initiateLoad(int maxLoadThreads) {
+      // ensure only one thread initiates loading of bloom filter by
+      // only taking action when loadTask != null
+      if (loadTask != null && loadRequest >= loadThreshold) {
+        try {
+          ExecutorService ltp = getLoadThreadPool(maxLoadThreads);
+          if (ltp == null) {
+            // load the bloom filter in the foreground
+            loadTask.run();
+          } else {
+            // load the bloom filter in the background
+            ltp.execute(new LoggingRunnable(LOG, loadTask));
+          }
+        } finally {
+          // set load task to null so no one else can initiate the load
+          loadTask = null;
+        }
+      }
+      
+      loadRequest++;
+    }
+    
+    /**
+     * Checks if this MyMapFile has the indicated key. The membership test is performed using a Bloom filter, so the result has always non-zero probability of
+     * false positives.
+     * 
+     * @param key
+     *          key to check
+     * @return false iff key doesn't exist, true if key probably exists.
+     * @throws IOException
+     */
+    boolean probablyHasKey(Range range) throws IOException {
+      if (bloomFilter == null) {
+        initiateLoad(maxLoadThreads);
+        if (bloomFilter == null)
+          return true;
+      }
+      
+      Key bloomKey = transformer.transform(range);
+      
+      if (bloomKey == null || bloomKey.getBytes().length == 0)
+        return true;
+      
+      return bloomFilter.membershipTest(bloomKey);
+    }
+    
+    public void close() {
+      this.closed = true;
+    }
+  }
+  
+  public static class Reader implements FileSKVIterator {
+    
+    private BloomFilterLoader bfl;
+    private FileSKVIterator reader;
+    
+    public Reader(FileSKVIterator reader, AccumuloConfiguration acuconf) {
+      this.reader = reader;
+      bfl = new BloomFilterLoader(reader, acuconf);
+    }
+    
+    private Reader(FileSKVIterator src, BloomFilterLoader bfl) {
+      this.reader = src;
+      this.bfl = bfl;
+    }
+    
+    private boolean checkSuper = true;
+    
+    @Override
+    public boolean hasTop() {
+      return checkSuper ? reader.hasTop() : false;
+    }
+    
+    @Override
+    public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+      
+      if (!bfl.probablyHasKey(range)) {
+        checkSuper = false;
+      } else {
+        reader.seek(range, columnFamilies, inclusive);
+        checkSuper = true;
+      }
+    }
+    
+    public synchronized void close() throws IOException {
+      bfl.close();
+      reader.close();
+    }
+    
+    @Override
+    public org.apache.accumulo.core.data.Key getFirstKey() throws IOException {
+      return reader.getFirstKey();
+    }
+    
+    @Override
+    public org.apache.accumulo.core.data.Key getLastKey() throws IOException {
+      return reader.getLastKey();
+    }
+    
+    @Override
+    public SortedKeyValueIterator<org.apache.accumulo.core.data.Key,Value> deepCopy(IteratorEnvironment env) {
+      return new BloomFilterLayer.Reader((FileSKVIterator) reader.deepCopy(env), bfl);
+    }
+    
+    @Override
+    public org.apache.accumulo.core.data.Key getTopKey() {
+      return reader.getTopKey();
+    }
+    
+    @Override
+    public Value getTopValue() {
+      return reader.getTopValue();
+    }
+    
+    @Override
+    public void init(SortedKeyValueIterator<org.apache.accumulo.core.data.Key,Value> source, Map<String,String> options, IteratorEnvironment env)
+        throws IOException {
+      throw new UnsupportedOperationException();
+      
+    }
+    
+    @Override
+    public void next() throws IOException {
+      reader.next();
+    }
+    
+    @Override
+    public DataInputStream getMetaStore(String name) throws IOException {
+      return reader.getMetaStore(name);
+    }
+    
+    @Override
+    public void closeDeepCopies() throws IOException {
+      reader.closeDeepCopies();
+    }
+    
+    @Override
+    public void setInterruptFlag(AtomicBoolean flag) {
+      reader.setInterruptFlag(flag);
+    }
+    
+  }
+  
+  public static void main(String[] args) throws IOException {
+    PrintStream out = System.out;
+    
+    Random r = new Random();
+    
+    HashSet<Integer> valsSet = new HashSet<Integer>();
+    
+    for (int i = 0; i < 100000; i++) {
+      valsSet.add(r.nextInt(Integer.MAX_VALUE));
+    }
+    
+    ArrayList<Integer> vals = new ArrayList<Integer>(valsSet);
+    Collections.sort(vals);
+    
+    ConfigurationCopy acuconf = new ConfigurationCopy(AccumuloConfiguration.getDefaultConfiguration());
+    acuconf.set(Property.TABLE_BLOOM_ENABLED, "true");
+    acuconf.set(Property.TABLE_BLOOM_KEY_FUNCTOR, "accumulo.core.file.keyfunctor.ColumnFamilyFunctor");
+    acuconf.set(Property.TABLE_FILE_TYPE, RFile.EXTENSION);
+    acuconf.set(Property.TABLE_BLOOM_LOAD_THRESHOLD, "1");
+    acuconf.set(Property.TSERV_BLOOM_LOAD_MAXCONCURRENT, "1");
+    
+    Configuration conf = CachedConfiguration.getInstance();
+    FileSystem fs = FileSystem.get(conf);
+    
+    String suffix = FileOperations.getNewFileExtension(acuconf);
+    String fname = "/tmp/test." + suffix;
+    FileSKVWriter bmfw = FileOperations.getInstance().openWriter(fname, fs, conf, acuconf);
+    
+    long t1 = System.currentTimeMillis();
+    
+    bmfw.startDefaultLocalityGroup();
+    
+    for (Integer i : vals) {
+      String fi = String.format("%010d", i);
+      bmfw.append(new org.apache.accumulo.core.data.Key(new Text("r" + fi), new Text("cf1")), new Value(("v" + fi).getBytes()));
+      bmfw.append(new org.apache.accumulo.core.data.Key(new Text("r" + fi), new Text("cf2")), new Value(("v" + fi).getBytes()));
+    }
+    
+    long t2 = System.currentTimeMillis();
+    
+    out.printf("write rate %6.2f\n", vals.size() / ((t2 - t1) / 1000.0));
+    
+    bmfw.close();
+    
+    t1 = System.currentTimeMillis();
+    FileSKVIterator bmfr = FileOperations.getInstance().openReader(fname, false, fs, conf, acuconf);
+    t2 = System.currentTimeMillis();
+    out.println("Opened " + fname + " in " + (t2 - t1));
+    
+    t1 = System.currentTimeMillis();
+    
+    int hits = 0;
+    for (int i = 0; i < 5000; i++) {
+      int row = r.nextInt(Integer.MAX_VALUE);
+      String fi = String.format("%010d", row);
+      // bmfr.seek(new Range(new Text("r"+fi)));
+      org.apache.accumulo.core.data.Key k1 = new org.apache.accumulo.core.data.Key(new Text("r" + fi), new Text("cf1"));
+      bmfr.seek(new Range(k1, true, k1.followingKey(PartialKey.ROW_COLFAM), false), new ArrayList<ByteSequence>(), false);
+      if (valsSet.contains(row)) {
+        hits++;
+        if (!bmfr.hasTop()) {
+          out.println("ERROR " + row);
+        }
+      }
+    }
+    
+    t2 = System.currentTimeMillis();
+    
+    out.printf("random lookup rate : %6.2f\n", 5000 / ((t2 - t1) / 1000.0));
+    out.println("hits = " + hits);
+    
+    int count = 0;
+    
+    t1 = System.currentTimeMillis();
+    
+    for (Integer row : valsSet) {
+      String fi = String.format("%010d", row);
+      // bmfr.seek(new Range(new Text("r"+fi)));
+      
+      org.apache.accumulo.core.data.Key k1 = new org.apache.accumulo.core.data.Key(new Text("r" + fi), new Text("cf1"));
+      bmfr.seek(new Range(k1, true, k1.followingKey(PartialKey.ROW_COLFAM), false), new ArrayList<ByteSequence>(), false);
+      
+      if (!bmfr.hasTop()) {
+        out.println("ERROR 2 " + row);
+      }
+      
+      count++;
+      
+      if (count >= 500) {
+        break;
+      }
+    }
+    
+    t2 = System.currentTimeMillis();
+    
+    out.printf("existant lookup rate %6.2f\n", 500 / ((t2 - t1) / 1000.0));
+    out.println("expected hits 500.  Receive hits: " + count);
+    bmfr.close();
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/7bdbfccb/1.5/core/src/main/java/org/apache/accumulo/core/file/FileOperations.java
----------------------------------------------------------------------
diff --git a/1.5/core/src/main/java/org/apache/accumulo/core/file/FileOperations.java b/1.5/core/src/main/java/org/apache/accumulo/core/file/FileOperations.java
new file mode 100644
index 0000000..9f60725
--- /dev/null
+++ b/1.5/core/src/main/java/org/apache/accumulo/core/file/FileOperations.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.file.blockfile.cache.BlockCache;
+import org.apache.accumulo.core.file.map.MapFileOperations;
+import org.apache.accumulo.core.file.rfile.RFile;
+import org.apache.accumulo.core.file.rfile.RFileOperations;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+class DispatchingFileFactory extends FileOperations {
+  
+  private FileOperations findFileFactory(String file) {
+    
+    Path p = new Path(file);
+    String name = p.getName();
+    
+    if (name.startsWith(Constants.MAPFILE_EXTENSION + "_")) {
+      return new MapFileOperations();
+    }
+    
+    String[] sp = name.split("\\.");
+    
+    if (sp.length != 2) {
+      throw new IllegalArgumentException("File name " + name + " has no extension");
+    }
+    
+    String extension = sp[1];
+    
+    if (extension.equals(Constants.MAPFILE_EXTENSION) || extension.equals(Constants.MAPFILE_EXTENSION + "_tmp")) {
+      return new MapFileOperations();
+    } else if (extension.equals(RFile.EXTENSION) || extension.equals(RFile.EXTENSION + "_tmp")) {
+      return new RFileOperations();
+    } else {
+      throw new IllegalArgumentException("File type " + extension + " not supported");
+    }
+  }
+  
+  @Override
+  public FileSKVIterator openIndex(String file, FileSystem fs, Configuration conf, AccumuloConfiguration acuconf) throws IOException {
+    return findFileFactory(file).openIndex(file, fs, conf, acuconf, null, null);
+  }
+  
+  @Override
+  public FileSKVIterator openReader(String file, boolean seekToBeginning, FileSystem fs, Configuration conf, AccumuloConfiguration acuconf) throws IOException {
+    FileSKVIterator iter = findFileFactory(file).openReader(file, seekToBeginning, fs, conf, acuconf, null, null);
+    if (acuconf.getBoolean(Property.TABLE_BLOOM_ENABLED)) {
+      return new BloomFilterLayer.Reader(iter, acuconf);
+    }
+    return iter;
+  }
+  
+  @Override
+  public FileSKVWriter openWriter(String file, FileSystem fs, Configuration conf, AccumuloConfiguration acuconf) throws IOException {
+    FileSKVWriter writer = findFileFactory(file).openWriter(file, fs, conf, acuconf);
+    if (acuconf.getBoolean(Property.TABLE_BLOOM_ENABLED)) {
+      return new BloomFilterLayer.Writer(writer, acuconf);
+    }
+    return writer;
+  }
+  
+  @Override
+  public long getFileSize(String file, FileSystem fs, Configuration conf, AccumuloConfiguration acuconf) throws IOException {
+    return findFileFactory(file).getFileSize(file, fs, conf, acuconf);
+  }
+  
+  @Override
+  public FileSKVIterator openReader(String file, Range range, Set<ByteSequence> columnFamilies, boolean inclusive, FileSystem fs, Configuration conf,
+      AccumuloConfiguration tableConf) throws IOException {
+    return findFileFactory(file).openReader(file, range, columnFamilies, inclusive, fs, conf, tableConf, null, null);
+  }
+  
+  @Override
+  public FileSKVIterator openReader(String file, Range range, Set<ByteSequence> columnFamilies, boolean inclusive, FileSystem fs, Configuration conf,
+      AccumuloConfiguration tableConf, BlockCache dataCache, BlockCache indexCache) throws IOException {
+    
+    if (!tableConf.getBoolean(Property.TABLE_INDEXCACHE_ENABLED))
+      indexCache = null;
+    if (!tableConf.getBoolean(Property.TABLE_BLOCKCACHE_ENABLED))
+      dataCache = null;
+    
+    return findFileFactory(file).openReader(file, range, columnFamilies, inclusive, fs, conf, tableConf, dataCache, indexCache);
+  }
+  
+  @Override
+  public FileSKVIterator openReader(String file, boolean seekToBeginning, FileSystem fs, Configuration conf, AccumuloConfiguration acuconf,
+      BlockCache dataCache, BlockCache indexCache) throws IOException {
+    
+    if (!acuconf.getBoolean(Property.TABLE_INDEXCACHE_ENABLED))
+      indexCache = null;
+    if (!acuconf.getBoolean(Property.TABLE_BLOCKCACHE_ENABLED))
+      dataCache = null;
+    
+    FileSKVIterator iter = findFileFactory(file).openReader(file, seekToBeginning, fs, conf, acuconf, dataCache, indexCache);
+    if (acuconf.getBoolean(Property.TABLE_BLOOM_ENABLED)) {
+      return new BloomFilterLayer.Reader(iter, acuconf);
+    }
+    return iter;
+  }
+  
+  @Override
+  public FileSKVIterator openIndex(String file, FileSystem fs, Configuration conf, AccumuloConfiguration acuconf, BlockCache dCache, BlockCache iCache)
+      throws IOException {
+    
+    if (!acuconf.getBoolean(Property.TABLE_INDEXCACHE_ENABLED))
+      iCache = null;
+    if (!acuconf.getBoolean(Property.TABLE_BLOCKCACHE_ENABLED))
+      dCache = null;
+    
+    return findFileFactory(file).openIndex(file, fs, conf, acuconf, dCache, iCache);
+  }
+  
+}
+
+public abstract class FileOperations {
+  
+  private static final HashSet<String> validExtensions = new HashSet<String>(Arrays.asList(Constants.MAPFILE_EXTENSION, RFile.EXTENSION));
+  
+  public static Set<String> getValidExtensions() {
+    return validExtensions;
+  }
+  
+  public static String getNewFileExtension(AccumuloConfiguration acuconf) {
+    return acuconf.get(Property.TABLE_FILE_TYPE);
+  }
+  
+  public static FileOperations getInstance() {
+    return new DispatchingFileFactory();
+  }
+  
+  /**
+   * Open a reader that will not be seeked giving an initial seek location. This is useful for file operations that only need to scan data within a range and do
+   * not need to seek. Therefore file metadata such as indexes does not need to be kept in memory while the file is scanned. Also seek optimizations like bloom
+   * filters do not need to be loaded.
+   * 
+   */
+  
+  public abstract FileSKVIterator openReader(String file, Range range, Set<ByteSequence> columnFamilies, boolean inclusive, FileSystem fs, Configuration conf,
+      AccumuloConfiguration tableConf) throws IOException;
+  
+  public abstract FileSKVIterator openReader(String file, Range range, Set<ByteSequence> columnFamilies, boolean inclusive, FileSystem fs, Configuration conf,
+      AccumuloConfiguration tableConf, BlockCache dataCache, BlockCache indexCache) throws IOException;
+  
+  /**
+   * Open a reader that fully support seeking and also enable any optimizations related to seeking, like bloom filters.
+   * 
+   */
+  
+  public abstract FileSKVIterator openReader(String file, boolean seekToBeginning, FileSystem fs, Configuration conf, AccumuloConfiguration acuconf)
+      throws IOException;
+  
+  public abstract FileSKVIterator openReader(String file, boolean seekToBeginning, FileSystem fs, Configuration conf, AccumuloConfiguration acuconf,
+      BlockCache dataCache, BlockCache indexCache) throws IOException;
+  
+  public abstract FileSKVWriter openWriter(String file, FileSystem fs, Configuration conf, AccumuloConfiguration acuconf) throws IOException;
+  
+  public abstract FileSKVIterator openIndex(String file, FileSystem fs, Configuration conf, AccumuloConfiguration acuconf) throws IOException;
+  
+  public abstract FileSKVIterator openIndex(String file, FileSystem fs, Configuration conf, AccumuloConfiguration acuconf, BlockCache dCache, BlockCache iCache)
+      throws IOException;
+  
+  public abstract long getFileSize(String file, FileSystem fs, Configuration conf, AccumuloConfiguration acuconf) throws IOException;
+  
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/7bdbfccb/1.5/core/src/main/java/org/apache/accumulo/core/file/FileSKVIterator.java
----------------------------------------------------------------------
diff --git a/1.5/core/src/main/java/org/apache/accumulo/core/file/FileSKVIterator.java b/1.5/core/src/main/java/org/apache/accumulo/core/file/FileSKVIterator.java
new file mode 100644
index 0000000..b5674bf
--- /dev/null
+++ b/1.5/core/src/main/java/org/apache/accumulo/core/file/FileSKVIterator.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.iterators.system.InterruptibleIterator;
+
+public interface FileSKVIterator extends InterruptibleIterator {
+  public Key getFirstKey() throws IOException;
+  
+  public Key getLastKey() throws IOException;
+  
+  public DataInputStream getMetaStore(String name) throws IOException, NoSuchMetaStoreException;
+  
+  public void closeDeepCopies() throws IOException;
+  
+  public void close() throws IOException;
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/7bdbfccb/1.5/core/src/main/java/org/apache/accumulo/core/file/FileSKVWriter.java
----------------------------------------------------------------------
diff --git a/1.5/core/src/main/java/org/apache/accumulo/core/file/FileSKVWriter.java b/1.5/core/src/main/java/org/apache/accumulo/core/file/FileSKVWriter.java
new file mode 100644
index 0000000..8718515
--- /dev/null
+++ b/1.5/core/src/main/java/org/apache/accumulo/core/file/FileSKVWriter.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+
+public interface FileSKVWriter {
+  boolean supportsLocalityGroups();
+  
+  void startNewLocalityGroup(String name, Set<ByteSequence> columnFamilies) throws IOException;
+  
+  void startDefaultLocalityGroup() throws IOException;
+  
+  void append(Key key, Value value) throws IOException;
+  
+  DataOutputStream createMetaStore(String name) throws IOException;
+  
+  void close() throws IOException;
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/7bdbfccb/1.5/core/src/main/java/org/apache/accumulo/core/file/FileUtil.java
----------------------------------------------------------------------
diff --git a/1.5/core/src/main/java/org/apache/accumulo/core/file/FileUtil.java b/1.5/core/src/main/java/org/apache/accumulo/core/file/FileUtil.java
new file mode 100644
index 0000000..12ca368
--- /dev/null
+++ b/1.5/core/src/main/java/org/apache/accumulo/core/file/FileUtil.java
@@ -0,0 +1,558 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.rfile.RFile;
+import org.apache.accumulo.core.file.rfile.RFileOperations;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.system.MultiIterator;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.LocalityGroupUtil;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.log4j.Logger;
+
+public class FileUtil {
+  
+  public static class FileInfo {
+    Key firstKey = new Key();
+    Key lastKey = new Key();
+    
+    public FileInfo(Key firstKey, Key lastKey) {
+      this.firstKey = firstKey;
+      this.lastKey = lastKey;
+    }
+    
+    public Text getFirstRow() {
+      return firstKey.getRow();
+    }
+    
+    public Text getLastRow() {
+      return lastKey.getRow();
+    }
+  }
+  
+  private static final Logger log = Logger.getLogger(FileUtil.class);
+  
+  private static String createTmpDir(AccumuloConfiguration acuConf, FileSystem fs) throws IOException {
+    String accumuloDir = acuConf.get(Property.INSTANCE_DFS_DIR);
+    
+    String tmpDir = null;
+    while (tmpDir == null) {
+      tmpDir = accumuloDir + "/tmp/idxReduce_" + String.format("%09d", (int) (Math.random() * Integer.MAX_VALUE));
+      
+      try {
+        fs.getFileStatus(new Path(tmpDir));
+        tmpDir = null;
+        continue;
+      } catch (FileNotFoundException fne) {
+        // found an unused temp directory
+      }
+      
+      fs.mkdirs(new Path(tmpDir));
+      
+      // try to reserve the tmp dir
+      if (!fs.createNewFile(new Path(tmpDir + "/__reserve")))
+        tmpDir = null;
+    }
+    
+    return tmpDir;
+  }
+  
+  public static Collection<String> reduceFiles(AccumuloConfiguration acuConf, Configuration conf, FileSystem fs, Text prevEndRow, Text endRow,
+      Collection<String> mapFiles, int maxFiles, String tmpDir, int pass) throws IOException {
+    ArrayList<String> paths = new ArrayList<String>(mapFiles);
+    
+    if (paths.size() <= maxFiles)
+      return paths;
+    
+    String newDir = String.format("%s/pass_%04d", tmpDir, pass);
+    
+    int start = 0;
+    
+    ArrayList<String> outFiles = new ArrayList<String>();
+    
+    int count = 0;
+    
+    while (start < paths.size()) {
+      int end = Math.min(maxFiles + start, paths.size());
+      List<String> inFiles = paths.subList(start, end);
+      
+      start = end;
+      
+      String newMapFile = String.format("%s/%04d." + RFile.EXTENSION, newDir, count++);
+      
+      outFiles.add(newMapFile);
+      
+      FileSKVWriter writer = new RFileOperations().openWriter(newMapFile, fs, conf, acuConf);
+      writer.startDefaultLocalityGroup();
+      List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(inFiles.size());
+      
+      FileSKVIterator reader = null;
+      try {
+        for (String s : inFiles) {
+          reader = FileOperations.getInstance().openIndex(s, fs, conf, acuConf);
+          iters.add(reader);
+        }
+        
+        MultiIterator mmfi = new MultiIterator(iters, true);
+        
+        while (mmfi.hasTop()) {
+          Key key = mmfi.getTopKey();
+          
+          boolean gtPrevEndRow = prevEndRow == null || key.compareRow(prevEndRow) > 0;
+          boolean lteEndRow = endRow == null || key.compareRow(endRow) <= 0;
+          
+          if (gtPrevEndRow && lteEndRow)
+            writer.append(key, new Value(new byte[0]));
+          
+          if (!lteEndRow)
+            break;
+          
+          mmfi.next();
+        }
+      } finally {
+        try {
+          if (reader != null)
+            reader.close();
+        } catch (IOException e) {
+          log.error(e, e);
+        }
+        
+        for (SortedKeyValueIterator<Key,Value> r : iters)
+          try {
+            if (r != null)
+              ((FileSKVIterator) r).close();
+          } catch (IOException e) {
+            // continue closing
+            log.error(e, e);
+          }
+        
+        try {
+          if (writer != null)
+            writer.close();
+        } catch (IOException e) {
+          log.error(e, e);
+          throw e;
+        }
+      }
+    }
+    
+    return reduceFiles(acuConf, conf, fs, prevEndRow, endRow, outFiles, maxFiles, tmpDir, pass + 1);
+  }
+
+  public static SortedMap<Double,Key> findMidPoint(FileSystem fs, AccumuloConfiguration acuConf, Text prevEndRow, Text endRow, Collection<String> mapFiles,
+      double minSplit) throws IOException {
+    return findMidPoint(fs, acuConf, prevEndRow, endRow, mapFiles, minSplit, true);
+  }
+  
+  public static double estimatePercentageLTE(FileSystem fs, AccumuloConfiguration acuconf, Text prevEndRow, Text endRow, Collection<String> mapFiles,
+      Text splitRow) throws IOException {
+    
+    Configuration conf = CachedConfiguration.getInstance();
+    
+    String tmpDir = null;
+    
+    int maxToOpen = acuconf.getCount(Property.TSERV_TABLET_SPLIT_FINDMIDPOINT_MAXOPEN);
+    ArrayList<FileSKVIterator> readers = new ArrayList<FileSKVIterator>(mapFiles.size());
+    
+    try {
+      if (mapFiles.size() > maxToOpen) {
+        tmpDir = createTmpDir(acuconf, fs);
+        
+        log.debug("Too many indexes (" + mapFiles.size() + ") to open at once for " + endRow + " " + prevEndRow + ", reducing in tmpDir = " + tmpDir);
+        
+        long t1 = System.currentTimeMillis();
+        mapFiles = reduceFiles(acuconf, conf, fs, prevEndRow, endRow, mapFiles, maxToOpen, tmpDir, 0);
+        long t2 = System.currentTimeMillis();
+        
+        log.debug("Finished reducing indexes for " + endRow + " " + prevEndRow + " in " + String.format("%6.2f secs", (t2 - t1) / 1000.0));
+      }
+      
+      if (prevEndRow == null)
+        prevEndRow = new Text();
+      
+      long numKeys = 0;
+      
+      numKeys = countIndexEntries(acuconf, prevEndRow, endRow, mapFiles, true, conf, fs, readers);
+      
+      if (numKeys == 0) {
+        // not enough info in the index to answer the question, so instead of going to
+        // the data just punt and return .5
+        return .5;
+      }
+      
+      List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(readers);
+      MultiIterator mmfi = new MultiIterator(iters, true);
+      
+      // skip the prevendrow
+      while (mmfi.hasTop() && mmfi.getTopKey().compareRow(prevEndRow) <= 0) {
+        mmfi.next();
+      }
+      
+      int numLte = 0;
+      
+      while (mmfi.hasTop() && mmfi.getTopKey().compareRow(splitRow) <= 0) {
+        numLte++;
+        mmfi.next();
+      }
+      
+      if (numLte > numKeys) {
+        // something went wrong
+        throw new RuntimeException("numLte > numKeys " + numLte + " " + numKeys + " " + prevEndRow + " " + endRow + " " + splitRow + " " + mapFiles);
+      }
+      
+      // do not want to return 0% or 100%, so add 1 and 2 below
+      return (numLte + 1) / (double) (numKeys + 2);
+      
+    } finally {
+      cleanupIndexOp(acuconf, tmpDir, fs, readers);
+    }
+  }
+  
+  /**
+   * 
+   * @param mapFiles
+   *          - list MapFiles to find the mid point key
+   * 
+   *          ISSUES : This method used the index files to find the mid point. If the map files have different index intervals this method will not return an
+   *          accurate mid point. Also, it would be tricky to use this method in conjunction with an in memory map because the indexing interval is unknown.
+   */
+  public static SortedMap<Double,Key> findMidPoint(FileSystem fs, AccumuloConfiguration acuConf, Text prevEndRow, Text endRow, Collection<String> mapFiles,
+      double minSplit, boolean useIndex) throws IOException {
+    Configuration conf = CachedConfiguration.getInstance();
+    
+    Collection<String> origMapFiles = mapFiles;
+    
+    String tmpDir = null;
+    
+    int maxToOpen = acuConf.getCount(Property.TSERV_TABLET_SPLIT_FINDMIDPOINT_MAXOPEN);
+    ArrayList<FileSKVIterator> readers = new ArrayList<FileSKVIterator>(mapFiles.size());
+    
+    try {
+      if (mapFiles.size() > maxToOpen) {
+        if (!useIndex)
+          throw new IOException("Cannot find mid point using data files, too many " + mapFiles.size());
+        tmpDir = createTmpDir(acuConf, fs);
+        
+        log.debug("Too many indexes (" + mapFiles.size() + ") to open at once for " + endRow + " " + prevEndRow + ", reducing in tmpDir = " + tmpDir);
+        
+        long t1 = System.currentTimeMillis();
+        mapFiles = reduceFiles(acuConf, conf, fs, prevEndRow, endRow, mapFiles, maxToOpen, tmpDir, 0);
+        long t2 = System.currentTimeMillis();
+        
+        log.debug("Finished reducing indexes for " + endRow + " " + prevEndRow + " in " + String.format("%6.2f secs", (t2 - t1) / 1000.0));
+      }
+      
+      if (prevEndRow == null)
+        prevEndRow = new Text();
+      
+      long t1 = System.currentTimeMillis();
+      
+      long numKeys = 0;
+      
+      numKeys = countIndexEntries(acuConf, prevEndRow, endRow, mapFiles, tmpDir == null ? useIndex : false, conf, fs, readers);
+      
+      if (numKeys == 0) {
+        if (useIndex) {
+          log.warn("Failed to find mid point using indexes, falling back to data files which is slower. No entries between " + prevEndRow + " and " + endRow
+              + " for " + mapFiles);
+          // need to pass original map files, not possibly reduced indexes
+          return findMidPoint(fs, acuConf, prevEndRow, endRow, origMapFiles, minSplit, false);
+        }
+        throw new IOException("Failed to find mid point, no entries between " + prevEndRow + " and " + endRow + " for " + mapFiles);
+      }
+      
+      List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(readers);
+      MultiIterator mmfi = new MultiIterator(iters, true);
+      
+      // skip the prevendrow
+      while (mmfi.hasTop() && mmfi.getTopKey().compareRow(prevEndRow) <= 0)
+        mmfi.next();
+      
+      // read half of the keys in the index
+      TreeMap<Double,Key> ret = new TreeMap<Double,Key>();
+      Key lastKey = null;
+      long keysRead = 0;
+      
+      Key keyBeforeMidPoint = null;
+      long keyBeforeMidPointPosition = 0;
+      
+      while (keysRead < numKeys / 2) {
+        if (lastKey != null && !lastKey.equals(mmfi.getTopKey(), PartialKey.ROW) && (keysRead - 1) / (double) numKeys >= minSplit) {
+          keyBeforeMidPoint = new Key(lastKey);
+          keyBeforeMidPointPosition = keysRead - 1;
+        }
+        
+        if (lastKey == null)
+          lastKey = new Key();
+        
+        lastKey.set(mmfi.getTopKey());
+        
+        keysRead++;
+        
+        // consume minimum
+        mmfi.next();
+      }
+      
+      if (keyBeforeMidPoint != null)
+        ret.put(keyBeforeMidPointPosition / (double) numKeys, keyBeforeMidPoint);
+      
+      long t2 = System.currentTimeMillis();
+      
+      log.debug(String.format("Found midPoint from indexes in %6.2f secs.\n", ((t2 - t1) / 1000.0)));
+      
+      ret.put(.5, mmfi.getTopKey());
+      
+      // sanity check
+      for (Key key : ret.values()) {
+        boolean inRange = (key.compareRow(prevEndRow) > 0 && (endRow == null || key.compareRow(endRow) < 1));
+        if (!inRange) {
+          throw new IOException("Found mid point is not in range " + key + " " + prevEndRow + " " + endRow + " " + mapFiles);
+        }
+      }
+      
+      return ret;
+    } finally {
+      cleanupIndexOp(acuConf, tmpDir, fs, readers);
+    }
+  }
+  
+  private static void cleanupIndexOp(AccumuloConfiguration acuConf, String tmpDir, FileSystem fs, ArrayList<FileSKVIterator> readers) throws IOException {
+    // close all of the index sequence files
+    for (FileSKVIterator r : readers) {
+      try {
+        if (r != null)
+          r.close();
+      } catch (IOException e) {
+        // okay, try to close the rest anyway
+        log.error(e, e);
+      }
+    }
+    
+    if (tmpDir != null) {
+      String tmpPrefix = acuConf.get(Property.INSTANCE_DFS_DIR) + "/tmp";
+      if (tmpDir.startsWith(tmpPrefix))
+        fs.delete(new Path(tmpDir), true);
+      else
+        log.error("Did not delete tmp dir because it wasn't a tmp dir " + tmpDir);
+    }
+  }
+  
+  private static long countIndexEntries(AccumuloConfiguration acuConf, Text prevEndRow, Text endRow, Collection<String> mapFiles, boolean useIndex,
+      Configuration conf, FileSystem fs, ArrayList<FileSKVIterator> readers) throws IOException {
+    
+    long numKeys = 0;
+    
+    // count the total number of index entries
+    for (String path : mapFiles) {
+      FileSKVIterator reader = null;
+      try {
+        if (useIndex)
+          reader = FileOperations.getInstance().openIndex(path, fs, conf, acuConf);
+        else
+          reader = FileOperations.getInstance().openReader(path, new Range(prevEndRow, false, null, true), LocalityGroupUtil.EMPTY_CF_SET, false, fs, conf,
+              acuConf);
+        
+        while (reader.hasTop()) {
+          Key key = reader.getTopKey();
+          if (endRow != null && key.compareRow(endRow) > 0)
+            break;
+          else if (prevEndRow == null || key.compareRow(prevEndRow) > 0)
+            numKeys++;
+          
+          reader.next();
+        }
+      } finally {
+        try {
+          if (reader != null)
+            reader.close();
+        } catch (IOException e) {
+          log.error(e, e);
+        }
+      }
+      
+      if (useIndex)
+        readers.add(FileOperations.getInstance().openIndex(path, fs, conf, acuConf));
+      else
+        readers.add(FileOperations.getInstance().openReader(path, new Range(prevEndRow, false, null, true), LocalityGroupUtil.EMPTY_CF_SET, false, fs, conf,
+            acuConf));
+      
+    }
+    return numKeys;
+  }
+  
+  public static Map<String,FileInfo> tryToGetFirstAndLastRows(FileSystem fs, AccumuloConfiguration acuConf, Set<String> mapfiles) {
+    
+    HashMap<String,FileInfo> mapFilesInfo = new HashMap<String,FileInfo>();
+    
+    Configuration conf = CachedConfiguration.getInstance();
+    
+    long t1 = System.currentTimeMillis();
+    
+    for (String mapfile : mapfiles) {
+      
+      FileSKVIterator reader = null;
+      try {
+        reader = FileOperations.getInstance().openReader(mapfile, false, fs, conf, acuConf);
+        
+        Key firstKey = reader.getFirstKey();
+        if (firstKey != null) {
+          mapFilesInfo.put(mapfile, new FileInfo(firstKey, reader.getLastKey()));
+        }
+        
+      } catch (IOException ioe) {
+        log.warn("Failed to read map file to determine first and last key : " + mapfile, ioe);
+      } finally {
+        if (reader != null) {
+          try {
+            reader.close();
+          } catch (IOException ioe) {
+            log.warn("failed to close " + mapfile, ioe);
+          }
+        }
+      }
+      
+    }
+    
+    long t2 = System.currentTimeMillis();
+    
+    log.debug(String.format("Found first and last keys for %d map files in %6.2f secs", mapfiles.size(), (t2 - t1) / 1000.0));
+    
+    return mapFilesInfo;
+  }
+  
+  public static WritableComparable<Key> findLastKey(FileSystem fs, AccumuloConfiguration acuConf, Collection<String> mapFiles) throws IOException {
+    Key lastKey = null;
+    
+    Configuration conf = CachedConfiguration.getInstance();
+    
+    for (String path : mapFiles) {
+      FileSKVIterator reader = FileOperations.getInstance().openReader(path, true, fs, conf, acuConf);
+      
+      try {
+        if (!reader.hasTop())
+          // file is empty, so there is no last key
+          continue;
+        
+        Key key = reader.getLastKey();
+        
+        if (lastKey == null || key.compareTo(lastKey) > 0)
+          lastKey = key;
+      } finally {
+        try {
+          if (reader != null)
+            reader.close();
+        } catch (IOException e) {
+          log.error(e, e);
+        }
+      }
+    }
+    
+    return lastKey;
+    
+  }
+  
+  private static class MLong {
+    public MLong(long i) {
+      l = i;
+    }
+    
+    long l;
+  }
+  
+  public static Map<KeyExtent,Long> estimateSizes(AccumuloConfiguration acuConf, Path mapFile, long fileSize, List<KeyExtent> extents, Configuration conf,
+      FileSystem fs) throws IOException {
+    
+    long totalIndexEntries = 0;
+    Map<KeyExtent,MLong> counts = new TreeMap<KeyExtent,MLong>();
+    for (KeyExtent keyExtent : extents)
+      counts.put(keyExtent, new MLong(0));
+    
+    Text row = new Text();
+    
+    FileSKVIterator index = FileOperations.getInstance().openIndex(mapFile.toString(), fs, conf, acuConf);
+    
+    try {
+      while (index.hasTop()) {
+        Key key = index.getTopKey();
+        totalIndexEntries++;
+        key.getRow(row);
+        
+        for (Entry<KeyExtent,MLong> entry : counts.entrySet())
+          if (entry.getKey().contains(row))
+            entry.getValue().l++;
+        
+        index.next();
+      }
+    } finally {
+      try {
+        if (index != null)
+          index.close();
+      } catch (IOException e) {
+        // continue with next file
+        log.error(e, e);
+      }
+    }
+    
+    Map<KeyExtent,Long> results = new TreeMap<KeyExtent,Long>();
+    for (KeyExtent keyExtent : extents) {
+      double numEntries = counts.get(keyExtent).l;
+      if (numEntries == 0)
+        numEntries = 1;
+      long estSize = (long) ((numEntries / totalIndexEntries) * fileSize);
+      results.put(keyExtent, estSize);
+    }
+    return results;
+  }
+  
+  public static FileSystem getFileSystem(Configuration conf, AccumuloConfiguration acuconf) throws IOException {
+    String uri = acuconf.get(Property.INSTANCE_DFS_URI);
+    if ("".equals(uri))
+      return FileSystem.get(conf);
+    else
+      try {
+        return FileSystem.get(new URI(uri), conf);
+      } catch (URISyntaxException e) {
+        throw new IOException(e);
+      }
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/7bdbfccb/1.5/core/src/main/java/org/apache/accumulo/core/file/NoSuchMetaStoreException.java
----------------------------------------------------------------------
diff --git a/1.5/core/src/main/java/org/apache/accumulo/core/file/NoSuchMetaStoreException.java b/1.5/core/src/main/java/org/apache/accumulo/core/file/NoSuchMetaStoreException.java
new file mode 100644
index 0000000..7de78b7
--- /dev/null
+++ b/1.5/core/src/main/java/org/apache/accumulo/core/file/NoSuchMetaStoreException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file;
+
+import java.io.IOException;
+
+public class NoSuchMetaStoreException extends IOException {
+  
+  public NoSuchMetaStoreException(String msg, Throwable e) {
+    super(msg, e);
+  }
+  
+  public NoSuchMetaStoreException(String msg) {
+    super(msg);
+  }
+  
+  /**
+	 * 
+	 */
+  private static final long serialVersionUID = 1L;
+  
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/7bdbfccb/1.5/core/src/main/java/org/apache/accumulo/core/file/blockfile/ABlockReader.java
----------------------------------------------------------------------
diff --git a/1.5/core/src/main/java/org/apache/accumulo/core/file/blockfile/ABlockReader.java b/1.5/core/src/main/java/org/apache/accumulo/core/file/blockfile/ABlockReader.java
new file mode 100644
index 0000000..af95060
--- /dev/null
+++ b/1.5/core/src/main/java/org/apache/accumulo/core/file/blockfile/ABlockReader.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file.blockfile;
+
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.IOException;
+
+/*
+ * Minimal interface to read a block from a 
+ * block based file
+ * 
+ */
+
+public interface ABlockReader extends DataInput {
+  
+  public long getRawSize();
+  
+  public DataInputStream getStream() throws IOException;
+  
+  public void close() throws IOException;
+  
+  /**
+   * An indexable block supports seeking, getting a position, and associating an arbitrary index with the block
+   * 
+   * @return true, if the block is indexable; otherwise false.
+   */
+  public boolean isIndexable();
+
+  public void seek(int position);
+
+  /** Get the file position.
+
+   * @return the file position.
+   */
+  public int getPosition();
+
+  <T> T getIndex(Class<T> clazz);
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/7bdbfccb/1.5/core/src/main/java/org/apache/accumulo/core/file/blockfile/ABlockWriter.java
----------------------------------------------------------------------
diff --git a/1.5/core/src/main/java/org/apache/accumulo/core/file/blockfile/ABlockWriter.java b/1.5/core/src/main/java/org/apache/accumulo/core/file/blockfile/ABlockWriter.java
new file mode 100644
index 0000000..1ab4668
--- /dev/null
+++ b/1.5/core/src/main/java/org/apache/accumulo/core/file/blockfile/ABlockWriter.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file.blockfile;
+
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+/*
+ * Minimal interface to write a block to a 
+ * block based file
+ * 
+ */
+
+public interface ABlockWriter extends DataOutput {
+  
+  public long getCompressedSize() throws IOException;
+  
+  public void close() throws IOException;
+  
+  public long getRawSize() throws IOException;
+  
+  public long getStartPos() throws IOException;
+  
+  public DataOutputStream getStream() throws IOException;
+  
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/7bdbfccb/1.5/core/src/main/java/org/apache/accumulo/core/file/blockfile/BlockFileReader.java
----------------------------------------------------------------------
diff --git a/1.5/core/src/main/java/org/apache/accumulo/core/file/blockfile/BlockFileReader.java b/1.5/core/src/main/java/org/apache/accumulo/core/file/blockfile/BlockFileReader.java
new file mode 100644
index 0000000..c5a1d5e
--- /dev/null
+++ b/1.5/core/src/main/java/org/apache/accumulo/core/file/blockfile/BlockFileReader.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file.blockfile;
+
+import java.io.IOException;
+
+/**
+ * 
+ * Provides a generic interface for a Reader for a BlockBaseFile format. Supports the minimal interface required.
+ * 
+ * Read a metaBlock and a dataBlock
+ * 
+ */
+
+public interface BlockFileReader {
+  
+  public ABlockReader getMetaBlock(String name) throws IOException;
+  
+  public ABlockReader getDataBlock(int blockIndex) throws IOException;
+  
+  public void close() throws IOException;
+  
+  public ABlockReader getMetaBlock(long offset, long compressedSize, long rawSize) throws IOException;
+  
+  public ABlockReader getDataBlock(long offset, long compressedSize, long rawSize) throws IOException;
+  
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/7bdbfccb/1.5/core/src/main/java/org/apache/accumulo/core/file/blockfile/BlockFileWriter.java
----------------------------------------------------------------------
diff --git a/1.5/core/src/main/java/org/apache/accumulo/core/file/blockfile/BlockFileWriter.java b/1.5/core/src/main/java/org/apache/accumulo/core/file/blockfile/BlockFileWriter.java
new file mode 100644
index 0000000..56c1797
--- /dev/null
+++ b/1.5/core/src/main/java/org/apache/accumulo/core/file/blockfile/BlockFileWriter.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file.blockfile;
+
+import java.io.IOException;
+
+/**
+ * 
+ * Provides a generic interface for a Writer for a BlockBaseFile format. Supports the minimal interface required.
+ * 
+ * Write a metaBlock and a dataBlock.
+ * 
+ */
+
+public interface BlockFileWriter {
+  
+  public ABlockWriter prepareMetaBlock(String name, String compressionName) throws IOException;
+  
+  public ABlockWriter prepareMetaBlock(String name) throws IOException;
+  
+  public ABlockWriter prepareDataBlock() throws IOException;
+  
+  public void close() throws IOException;
+}


Mime
View raw message