accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject svn commit: r1342397 - in /accumulo/branches/ACCUMULO-578: core/src/main/java/org/apache/accumulo/core/conf/ core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/ core/src/main/java/org/apache/accumulo/core/util/ core/src/main/thrift/ server...
Date Thu, 24 May 2012 19:14:06 GMT
Author: ecn
Date: Thu May 24 19:14:05 2012
New Revision: 1342397

URL: http://svn.apache.org/viewvc?rev=1342397&view=rev
Log:
ACCUMULO-578 added back gc

Modified:
    accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/conf/Property.java
    accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
    accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
    accumulo/branches/ACCUMULO-578/core/src/main/thrift/tabletserver.thrift
    accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java
    accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java
    accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
    accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
    accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
    accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/IRemoteLogger.java
    accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/TabletServerLogger.java
    accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/test/performance/thrift/NullTserver.java

Modified: accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/conf/Property.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/conf/Property.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/conf/Property.java (original)
+++ accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/conf/Property.java Thu May 24 19:14:05 2012
@@ -152,27 +152,12 @@ public enum Property {
       "The replication to use when writing the Write-Ahead log to HDFS. If zero, it will use the HDFS default replication setting."),
   TSERV_RECOVERY_MAX_CONCURRENT("tserver.recovery.concurrent.max", "2", PropertyType.COUNT, "The maximum number of threads to use to sort logs during recovery"),
   TSERV_SORT_BUFFER_SIZE("tserver.sort.buffer.size", "200M", PropertyType.MEMORY, "The amount of memory to use when sorting logs during recovery."),
+  TSERV_ARCHIVE_WALOGS("tserver.archive.walogs", "false", PropertyType.BOOLEAN, "Keep copies of the WALOGs for debugging purposes"),
   
   // properties that are specific to logger server behavior
   LOGGER_PREFIX("logger.", null, PropertyType.PREFIX, "Properties in this category affect the behavior of the write-ahead logger servers"),
-  LOGGER_PORT("logger.port.client", "11224", PropertyType.PORT, "The port used for write-ahead logger services"),
-  LOGGER_COPY_THREADPOOL_SIZE("logger.copy.threadpool.size", "2", PropertyType.COUNT,
-      "size of the thread pool used to copy files from the local log area to HDFS"),
   LOGGER_DIR("logger.dir.walog", "walogs", PropertyType.PATH,
       "The directory used to store write-ahead logs on the local filesystem. It is possible to specify a comma-separated list of directories."),
-  LOGGER_PORTSEARCH("logger.port.search", "false", PropertyType.BOOLEAN, "if the port above is in use, search higher ports until one is available"),
-  LOGGER_ARCHIVE("logger.archive", "false", PropertyType.BOOLEAN, "determines if logs are archived in hdfs"),
-  LOGGER_ARCHIVE_REPLICATION("logger.archive.replication", "0", PropertyType.COUNT,
-      "determines the replication factor for walogs archived in hdfs, set to zero to use default"),
-  LOGGER_MONITOR_FS(
-      "logger.monitor.fs",
-      "true",
-      PropertyType.BOOLEAN,
-      "When enabled the logger will monitor file systems and kill itself when one switches from rw to ro.  This is usually and indication that Linux has detected a bad disk."),
-  LOGGER_RECOVERY_FILE_REPLICATION("logger.recovery.file.replication", "1", PropertyType.COUNT,
-      "When a logger puts a WALOG into HDFS, it will use this as the replication factor."),
-  LOGGER_MINTHREADS("logger.server.threads.minimum", "2", PropertyType.COUNT, "The miniumum number of threads to use to handle incoming requests."),
-  LOGGER_THREADCHECK("logger.server.threadcheck.time", "1s", PropertyType.TIMEDURATION, "The time between adjustments of the server thread pool."),
   
   // accumulo garbage collector properties
   GC_PREFIX("gc.", null, PropertyType.PREFIX, "Properties in this category affect the behavior of the accumulo garbage collector."),
@@ -345,8 +330,7 @@ public enum Property {
   }
   
   private static final EnumSet<Property> fixedProperties = EnumSet.of(Property.TSERV_CLIENTPORT, Property.TSERV_NATIVEMAP_ENABLED,
-      Property.TSERV_SCAN_MAX_OPENFILES, Property.TSERV_LOGGER_COUNT,
-      Property.LOGGER_PORT, Property.MASTER_CLIENTPORT, Property.GC_PORT);
+      Property.TSERV_SCAN_MAX_OPENFILES, Property.TSERV_LOGGER_COUNT, Property.MASTER_CLIENTPORT, Property.GC_PORT);
   
   public static boolean isFixedZooPropertyKey(Property key) {
     return fixedProperties.contains(key);

Modified: accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java (original)
+++ accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java Thu May 24 19:14:05 2012
@@ -76,6 +76,8 @@ import org.slf4j.LoggerFactory;
 
     public double sortLog(org.apache.accumulo.cloudtrace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.AuthInfo credentials, String lock, String path) throws org.apache.accumulo.core.security.thrift.ThriftSecurityException, org.apache.thrift.TException;
 
+    public void removeLogs(org.apache.accumulo.cloudtrace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.AuthInfo credentials, List<String> filenames) throws org.apache.accumulo.core.security.thrift.ThriftSecurityException, org.apache.thrift.TException;
+
   }
 
   public interface AsyncIface extends org.apache.accumulo.core.client.impl.thrift.ClientService .AsyncIface {
@@ -132,6 +134,8 @@ import org.slf4j.LoggerFactory;
 
     public void sortLog(org.apache.accumulo.cloudtrace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.AuthInfo credentials, String lock, String path, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.sortLog_call> resultHandler) throws org.apache.thrift.TException;
 
+    public void removeLogs(org.apache.accumulo.cloudtrace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.AuthInfo credentials, List<String> filenames, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.removeLogs_call> resultHandler) throws org.apache.thrift.TException;
+
   }
 
   public static class Client extends org.apache.accumulo.core.client.impl.thrift.ClientService.Client implements org.apache.thrift.TServiceClient, Iface {
@@ -1011,6 +1015,44 @@ import org.slf4j.LoggerFactory;
       throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "sortLog failed: unknown result");
     }
 
+    public void removeLogs(org.apache.accumulo.cloudtrace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.AuthInfo credentials, List<String> filenames) throws org.apache.accumulo.core.security.thrift.ThriftSecurityException, org.apache.thrift.TException
+    {
+      send_removeLogs(tinfo, credentials, filenames);
+      recv_removeLogs();
+    }
+
+    public void send_removeLogs(org.apache.accumulo.cloudtrace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.AuthInfo credentials, List<String> filenames) throws org.apache.thrift.TException
+    {
+      oprot_.writeMessageBegin(new org.apache.thrift.protocol.TMessage("removeLogs", org.apache.thrift.protocol.TMessageType.CALL, ++seqid_));
+      removeLogs_args args = new removeLogs_args();
+      args.setTinfo(tinfo);
+      args.setCredentials(credentials);
+      args.setFilenames(filenames);
+      args.write(oprot_);
+      oprot_.writeMessageEnd();
+      oprot_.getTransport().flush();
+    }
+
+    public void recv_removeLogs() throws org.apache.accumulo.core.security.thrift.ThriftSecurityException, org.apache.thrift.TException
+    {
+      org.apache.thrift.protocol.TMessage msg = iprot_.readMessageBegin();
+      if (msg.type == org.apache.thrift.protocol.TMessageType.EXCEPTION) {
+        org.apache.thrift.TApplicationException x = org.apache.thrift.TApplicationException.read(iprot_);
+        iprot_.readMessageEnd();
+        throw x;
+      }
+      if (msg.seqid != seqid_) {
+        throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.BAD_SEQUENCE_ID, "removeLogs failed: out of sequence response");
+      }
+      removeLogs_result result = new removeLogs_result();
+      result.read(iprot_);
+      iprot_.readMessageEnd();
+      if (result.sec != null) {
+        throw result.sec;
+      }
+      return;
+    }
+
   }
   public static class AsyncClient extends org.apache.accumulo.core.client.impl.thrift.ClientService.AsyncClient implements AsyncIface {
     public static class Factory implements org.apache.thrift.async.TAsyncClientFactory<AsyncClient> {
@@ -2070,6 +2112,44 @@ import org.slf4j.LoggerFactory;
       }
     }
 
+    public void removeLogs(org.apache.accumulo.cloudtrace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.AuthInfo credentials, List<String> filenames, org.apache.thrift.async.AsyncMethodCallback<removeLogs_call> resultHandler) throws org.apache.thrift.TException {
+      checkReady();
+      removeLogs_call method_call = new removeLogs_call(tinfo, credentials, filenames, resultHandler, this, protocolFactory, transport);
+      this.currentMethod = method_call;
+      manager.call(method_call);
+    }
+
+    public static class removeLogs_call extends org.apache.thrift.async.TAsyncMethodCall {
+      private org.apache.accumulo.cloudtrace.thrift.TInfo tinfo;
+      private org.apache.accumulo.core.security.thrift.AuthInfo credentials;
+      private List<String> filenames;
+      public removeLogs_call(org.apache.accumulo.cloudtrace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.AuthInfo credentials, List<String> filenames, org.apache.thrift.async.AsyncMethodCallback<removeLogs_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+        super(client, protocolFactory, transport, resultHandler, false);
+        this.tinfo = tinfo;
+        this.credentials = credentials;
+        this.filenames = filenames;
+      }
+
+      public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+        prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("removeLogs", org.apache.thrift.protocol.TMessageType.CALL, 0));
+        removeLogs_args args = new removeLogs_args();
+        args.setTinfo(tinfo);
+        args.setCredentials(credentials);
+        args.setFilenames(filenames);
+        args.write(prot);
+        prot.writeMessageEnd();
+      }
+
+      public void getResult() throws org.apache.accumulo.core.security.thrift.ThriftSecurityException, org.apache.thrift.TException {
+        if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+          throw new IllegalStateException("Method call not finished!");
+        }
+        org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+        org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+        (new Client(prot)).recv_removeLogs();
+      }
+    }
+
   }
 
   public static class Processor extends org.apache.accumulo.core.client.impl.thrift.ClientService.Processor implements org.apache.thrift.TProcessor {
@@ -2104,6 +2184,7 @@ import org.slf4j.LoggerFactory;
       processMap_.put("fastHalt", new fastHalt());
       processMap_.put("getActiveScans", new getActiveScans());
       processMap_.put("sortLog", new sortLog());
+      processMap_.put("removeLogs", new removeLogs());
     }
 
     private Iface iface_;
@@ -2960,6 +3041,44 @@ import org.slf4j.LoggerFactory;
 
     }
 
+    private class removeLogs implements ProcessFunction {
+      public void process(int seqid, org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException
+      {
+        removeLogs_args args = new removeLogs_args();
+        try {
+          args.read(iprot);
+        } catch (org.apache.thrift.protocol.TProtocolException e) {
+          iprot.readMessageEnd();
+          org.apache.thrift.TApplicationException x = new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.PROTOCOL_ERROR, e.getMessage());
+          oprot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("removeLogs", org.apache.thrift.protocol.TMessageType.EXCEPTION, seqid));
+          x.write(oprot);
+          oprot.writeMessageEnd();
+          oprot.getTransport().flush();
+          return;
+        }
+        iprot.readMessageEnd();
+        removeLogs_result result = new removeLogs_result();
+        try {
+          iface_.removeLogs(args.tinfo, args.credentials, args.filenames);
+        } catch (org.apache.accumulo.core.security.thrift.ThriftSecurityException sec) {
+          result.sec = sec;
+        } catch (Throwable th) {
+          LOGGER.error("Internal error processing removeLogs", th);
+          org.apache.thrift.TApplicationException x = new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, "Internal error processing removeLogs");
+          oprot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("removeLogs", org.apache.thrift.protocol.TMessageType.EXCEPTION, seqid));
+          x.write(oprot);
+          oprot.writeMessageEnd();
+          oprot.getTransport().flush();
+          return;
+        }
+        oprot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("removeLogs", org.apache.thrift.protocol.TMessageType.REPLY, seqid));
+        result.write(oprot);
+        oprot.writeMessageEnd();
+        oprot.getTransport().flush();
+      }
+
+    }
+
   }
 
   public static class startScan_args implements org.apache.thrift.TBase<startScan_args, startScan_args._Fields>, java.io.Serializable, Cloneable   {
@@ -24393,4 +24512,815 @@ import org.slf4j.LoggerFactory;
 
   }
 
+  public static class removeLogs_args implements org.apache.thrift.TBase<removeLogs_args, removeLogs_args._Fields>, java.io.Serializable, Cloneable   {
+    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("removeLogs_args");
+
+    private static final org.apache.thrift.protocol.TField TINFO_FIELD_DESC = new org.apache.thrift.protocol.TField("tinfo", org.apache.thrift.protocol.TType.STRUCT, (short)1);
+    private static final org.apache.thrift.protocol.TField CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials", org.apache.thrift.protocol.TType.STRUCT, (short)2);
+    private static final org.apache.thrift.protocol.TField FILENAMES_FIELD_DESC = new org.apache.thrift.protocol.TField("filenames", org.apache.thrift.protocol.TType.LIST, (short)3);
+
+    public org.apache.accumulo.cloudtrace.thrift.TInfo tinfo;
+    public org.apache.accumulo.core.security.thrift.AuthInfo credentials;
+    public List<String> filenames;
+
+    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+      TINFO((short)1, "tinfo"),
+      CREDENTIALS((short)2, "credentials"),
+      FILENAMES((short)3, "filenames");
+
+      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+      static {
+        for (_Fields field : EnumSet.allOf(_Fields.class)) {
+          byName.put(field.getFieldName(), field);
+        }
+      }
+
+      /**
+       * Find the _Fields constant that matches fieldId, or null if its not found.
+       */
+      public static _Fields findByThriftId(int fieldId) {
+        switch(fieldId) {
+          case 1: // TINFO
+            return TINFO;
+          case 2: // CREDENTIALS
+            return CREDENTIALS;
+          case 3: // FILENAMES
+            return FILENAMES;
+          default:
+            return null;
+        }
+      }
+
+      /**
+       * Find the _Fields constant that matches fieldId, throwing an exception
+       * if it is not found.
+       */
+      public static _Fields findByThriftIdOrThrow(int fieldId) {
+        _Fields fields = findByThriftId(fieldId);
+        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+        return fields;
+      }
+
+      /**
+       * Find the _Fields constant that matches name, or null if its not found.
+       */
+      public static _Fields findByName(String name) {
+        return byName.get(name);
+      }
+
+      private final short _thriftId;
+      private final String _fieldName;
+
+      _Fields(short thriftId, String fieldName) {
+        _thriftId = thriftId;
+        _fieldName = fieldName;
+      }
+
+      public short getThriftFieldId() {
+        return _thriftId;
+      }
+
+      public String getFieldName() {
+        return _fieldName;
+      }
+    }
+
+    // isset id assignments
+
+    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+    static {
+      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+      tmpMap.put(_Fields.TINFO, new org.apache.thrift.meta_data.FieldMetaData("tinfo", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.cloudtrace.thrift.TInfo.class)));
+      tmpMap.put(_Fields.CREDENTIALS, new org.apache.thrift.meta_data.FieldMetaData("credentials", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.security.thrift.AuthInfo.class)));
+      tmpMap.put(_Fields.FILENAMES, new org.apache.thrift.meta_data.FieldMetaData("filenames", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+          new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, 
+              new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
+      metaDataMap = Collections.unmodifiableMap(tmpMap);
+      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(removeLogs_args.class, metaDataMap);
+    }
+
+    public removeLogs_args() {
+    }
+
+    public removeLogs_args(
+      org.apache.accumulo.cloudtrace.thrift.TInfo tinfo,
+      org.apache.accumulo.core.security.thrift.AuthInfo credentials,
+      List<String> filenames)
+    {
+      this();
+      this.tinfo = tinfo;
+      this.credentials = credentials;
+      this.filenames = filenames;
+    }
+
+    /**
+     * Performs a deep copy on <i>other</i>.
+     */
+    public removeLogs_args(removeLogs_args other) {
+      if (other.isSetTinfo()) {
+        this.tinfo = new org.apache.accumulo.cloudtrace.thrift.TInfo(other.tinfo);
+      }
+      if (other.isSetCredentials()) {
+        this.credentials = new org.apache.accumulo.core.security.thrift.AuthInfo(other.credentials);
+      }
+      if (other.isSetFilenames()) {
+        List<String> __this__filenames = new ArrayList<String>();
+        for (String other_element : other.filenames) {
+          __this__filenames.add(other_element);
+        }
+        this.filenames = __this__filenames;
+      }
+    }
+
+    public removeLogs_args deepCopy() {
+      return new removeLogs_args(this);
+    }
+
+    @Override
+    public void clear() {
+      this.tinfo = null;
+      this.credentials = null;
+      this.filenames = null;
+    }
+
+    public org.apache.accumulo.cloudtrace.thrift.TInfo getTinfo() {
+      return this.tinfo;
+    }
+
+    public removeLogs_args setTinfo(org.apache.accumulo.cloudtrace.thrift.TInfo tinfo) {
+      this.tinfo = tinfo;
+      return this;
+    }
+
+    public void unsetTinfo() {
+      this.tinfo = null;
+    }
+
+    /** Returns true if field tinfo is set (has been assigned a value) and false otherwise */
+    public boolean isSetTinfo() {
+      return this.tinfo != null;
+    }
+
+    public void setTinfoIsSet(boolean value) {
+      if (!value) {
+        this.tinfo = null;
+      }
+    }
+
+    public org.apache.accumulo.core.security.thrift.AuthInfo getCredentials() {
+      return this.credentials;
+    }
+
+    public removeLogs_args setCredentials(org.apache.accumulo.core.security.thrift.AuthInfo credentials) {
+      this.credentials = credentials;
+      return this;
+    }
+
+    public void unsetCredentials() {
+      this.credentials = null;
+    }
+
+    /** Returns true if field credentials is set (has been assigned a value) and false otherwise */
+    public boolean isSetCredentials() {
+      return this.credentials != null;
+    }
+
+    public void setCredentialsIsSet(boolean value) {
+      if (!value) {
+        this.credentials = null;
+      }
+    }
+
+    public int getFilenamesSize() {
+      return (this.filenames == null) ? 0 : this.filenames.size();
+    }
+
+    public java.util.Iterator<String> getFilenamesIterator() {
+      return (this.filenames == null) ? null : this.filenames.iterator();
+    }
+
+    public void addToFilenames(String elem) {
+      if (this.filenames == null) {
+        this.filenames = new ArrayList<String>();
+      }
+      this.filenames.add(elem);
+    }
+
+    public List<String> getFilenames() {
+      return this.filenames;
+    }
+
+    public removeLogs_args setFilenames(List<String> filenames) {
+      this.filenames = filenames;
+      return this;
+    }
+
+    public void unsetFilenames() {
+      this.filenames = null;
+    }
+
+    /** Returns true if field filenames is set (has been assigned a value) and false otherwise */
+    public boolean isSetFilenames() {
+      return this.filenames != null;
+    }
+
+    public void setFilenamesIsSet(boolean value) {
+      if (!value) {
+        this.filenames = null;
+      }
+    }
+
+    public void setFieldValue(_Fields field, Object value) {
+      switch (field) {
+      case TINFO:
+        if (value == null) {
+          unsetTinfo();
+        } else {
+          setTinfo((org.apache.accumulo.cloudtrace.thrift.TInfo)value);
+        }
+        break;
+
+      case CREDENTIALS:
+        if (value == null) {
+          unsetCredentials();
+        } else {
+          setCredentials((org.apache.accumulo.core.security.thrift.AuthInfo)value);
+        }
+        break;
+
+      case FILENAMES:
+        if (value == null) {
+          unsetFilenames();
+        } else {
+          setFilenames((List<String>)value);
+        }
+        break;
+
+      }
+    }
+
+    public Object getFieldValue(_Fields field) {
+      switch (field) {
+      case TINFO:
+        return getTinfo();
+
+      case CREDENTIALS:
+        return getCredentials();
+
+      case FILENAMES:
+        return getFilenames();
+
+      }
+      throw new IllegalStateException();
+    }
+
+    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+    public boolean isSet(_Fields field) {
+      if (field == null) {
+        throw new IllegalArgumentException();
+      }
+
+      switch (field) {
+      case TINFO:
+        return isSetTinfo();
+      case CREDENTIALS:
+        return isSetCredentials();
+      case FILENAMES:
+        return isSetFilenames();
+      }
+      throw new IllegalStateException();
+    }
+
+    @Override
+    public boolean equals(Object that) {
+      if (that == null)
+        return false;
+      if (that instanceof removeLogs_args)
+        return this.equals((removeLogs_args)that);
+      return false;
+    }
+
+    public boolean equals(removeLogs_args that) {
+      if (that == null)
+        return false;
+
+      boolean this_present_tinfo = true && this.isSetTinfo();
+      boolean that_present_tinfo = true && that.isSetTinfo();
+      if (this_present_tinfo || that_present_tinfo) {
+        if (!(this_present_tinfo && that_present_tinfo))
+          return false;
+        if (!this.tinfo.equals(that.tinfo))
+          return false;
+      }
+
+      boolean this_present_credentials = true && this.isSetCredentials();
+      boolean that_present_credentials = true && that.isSetCredentials();
+      if (this_present_credentials || that_present_credentials) {
+        if (!(this_present_credentials && that_present_credentials))
+          return false;
+        if (!this.credentials.equals(that.credentials))
+          return false;
+      }
+
+      boolean this_present_filenames = true && this.isSetFilenames();
+      boolean that_present_filenames = true && that.isSetFilenames();
+      if (this_present_filenames || that_present_filenames) {
+        if (!(this_present_filenames && that_present_filenames))
+          return false;
+        if (!this.filenames.equals(that.filenames))
+          return false;
+      }
+
+      return true;
+    }
+
+    @Override
+    public int hashCode() {
+      return 0;
+    }
+
+    public int compareTo(removeLogs_args other) {
+      if (!getClass().equals(other.getClass())) {
+        return getClass().getName().compareTo(other.getClass().getName());
+      }
+
+      int lastComparison = 0;
+      removeLogs_args typedOther = (removeLogs_args)other;
+
+      lastComparison = Boolean.valueOf(isSetTinfo()).compareTo(typedOther.isSetTinfo());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+      if (isSetTinfo()) {
+        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tinfo, typedOther.tinfo);
+        if (lastComparison != 0) {
+          return lastComparison;
+        }
+      }
+      lastComparison = Boolean.valueOf(isSetCredentials()).compareTo(typedOther.isSetCredentials());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+      if (isSetCredentials()) {
+        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.credentials, typedOther.credentials);
+        if (lastComparison != 0) {
+          return lastComparison;
+        }
+      }
+      lastComparison = Boolean.valueOf(isSetFilenames()).compareTo(typedOther.isSetFilenames());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+      if (isSetFilenames()) {
+        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.filenames, typedOther.filenames);
+        if (lastComparison != 0) {
+          return lastComparison;
+        }
+      }
+      return 0;
+    }
+
+    public _Fields fieldForId(int fieldId) {
+      return _Fields.findByThriftId(fieldId);
+    }
+
+    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+      org.apache.thrift.protocol.TField field;
+      iprot.readStructBegin();
+      while (true)
+      {
+        field = iprot.readFieldBegin();
+        if (field.type == org.apache.thrift.protocol.TType.STOP) { 
+          break;
+        }
+        switch (field.id) {
+          case 1: // TINFO
+            if (field.type == org.apache.thrift.protocol.TType.STRUCT) {
+              this.tinfo = new org.apache.accumulo.cloudtrace.thrift.TInfo();
+              this.tinfo.read(iprot);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
+            }
+            break;
+          case 2: // CREDENTIALS
+            if (field.type == org.apache.thrift.protocol.TType.STRUCT) {
+              this.credentials = new org.apache.accumulo.core.security.thrift.AuthInfo();
+              this.credentials.read(iprot);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
+            }
+            break;
+          case 3: // FILENAMES
+            if (field.type == org.apache.thrift.protocol.TType.LIST) {
+              {
+                org.apache.thrift.protocol.TList _list109 = iprot.readListBegin();
+                this.filenames = new ArrayList<String>(_list109.size);
+                for (int _i110 = 0; _i110 < _list109.size; ++_i110)
+                {
+                  String _elem111;
+                  _elem111 = iprot.readString();
+                  this.filenames.add(_elem111);
+                }
+                iprot.readListEnd();
+              }
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
+            }
+            break;
+          default:
+            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
+        }
+        iprot.readFieldEnd();
+      }
+      iprot.readStructEnd();
+
+      // check for required fields of primitive type, which can't be checked in the validate method
+      validate();
+    }
+
+    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+      validate();
+
+      oprot.writeStructBegin(STRUCT_DESC);
+      if (this.tinfo != null) {
+        oprot.writeFieldBegin(TINFO_FIELD_DESC);
+        this.tinfo.write(oprot);
+        oprot.writeFieldEnd();
+      }
+      if (this.credentials != null) {
+        oprot.writeFieldBegin(CREDENTIALS_FIELD_DESC);
+        this.credentials.write(oprot);
+        oprot.writeFieldEnd();
+      }
+      if (this.filenames != null) {
+        oprot.writeFieldBegin(FILENAMES_FIELD_DESC);
+        {
+          oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, this.filenames.size()));
+          for (String _iter112 : this.filenames)
+          {
+            oprot.writeString(_iter112);
+          }
+          oprot.writeListEnd();
+        }
+        oprot.writeFieldEnd();
+      }
+      oprot.writeFieldStop();
+      oprot.writeStructEnd();
+    }
+
+    @Override
+    public String toString() {
+      StringBuilder sb = new StringBuilder("removeLogs_args(");
+      boolean first = true;
+
+      sb.append("tinfo:");
+      if (this.tinfo == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.tinfo);
+      }
+      first = false;
+      if (!first) sb.append(", ");
+      sb.append("credentials:");
+      if (this.credentials == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.credentials);
+      }
+      first = false;
+      if (!first) sb.append(", ");
+      sb.append("filenames:");
+      if (this.filenames == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.filenames);
+      }
+      first = false;
+      sb.append(")");
+      return sb.toString();
+    }
+
+    public void validate() throws org.apache.thrift.TException {
+      // check for required fields
+    }
+
+    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+      try {
+        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+      } catch (org.apache.thrift.TException te) {
+        throw new java.io.IOException(te);
+      }
+    }
+
+    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+      try {
+        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+      } catch (org.apache.thrift.TException te) {
+        throw new java.io.IOException(te);
+      }
+    }
+
+  }
+
+  public static class removeLogs_result implements org.apache.thrift.TBase<removeLogs_result, removeLogs_result._Fields>, java.io.Serializable, Cloneable   {
+    private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("removeLogs_result");
+
+    private static final org.apache.thrift.protocol.TField SEC_FIELD_DESC = new org.apache.thrift.protocol.TField("sec", org.apache.thrift.protocol.TType.STRUCT, (short)1);
+
+    public org.apache.accumulo.core.security.thrift.ThriftSecurityException sec;
+
+    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+      SEC((short)1, "sec");
+
+      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+      static {
+        for (_Fields field : EnumSet.allOf(_Fields.class)) {
+          byName.put(field.getFieldName(), field);
+        }
+      }
+
+      /**
+       * Find the _Fields constant that matches fieldId, or null if its not found.
+       */
+      public static _Fields findByThriftId(int fieldId) {
+        switch(fieldId) {
+          case 1: // SEC
+            return SEC;
+          default:
+            return null;
+        }
+      }
+
+      /**
+       * Find the _Fields constant that matches fieldId, throwing an exception
+       * if it is not found.
+       */
+      public static _Fields findByThriftIdOrThrow(int fieldId) {
+        _Fields fields = findByThriftId(fieldId);
+        if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+        return fields;
+      }
+
+      /**
+       * Find the _Fields constant that matches name, or null if its not found.
+       */
+      public static _Fields findByName(String name) {
+        return byName.get(name);
+      }
+
+      private final short _thriftId;
+      private final String _fieldName;
+
+      _Fields(short thriftId, String fieldName) {
+        _thriftId = thriftId;
+        _fieldName = fieldName;
+      }
+
+      public short getThriftFieldId() {
+        return _thriftId;
+      }
+
+      public String getFieldName() {
+        return _fieldName;
+      }
+    }
+
+    // isset id assignments
+
+    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+    static {
+      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+      tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT, 
+          new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
+      metaDataMap = Collections.unmodifiableMap(tmpMap);
+      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(removeLogs_result.class, metaDataMap);
+    }
+
+    public removeLogs_result() {
+    }
+
+    public removeLogs_result(
+      org.apache.accumulo.core.security.thrift.ThriftSecurityException sec)
+    {
+      this();
+      this.sec = sec;
+    }
+
+    /**
+     * Performs a deep copy on <i>other</i>.
+     */
+    public removeLogs_result(removeLogs_result other) {
+      if (other.isSetSec()) {
+        this.sec = new org.apache.accumulo.core.security.thrift.ThriftSecurityException(other.sec);
+      }
+    }
+
+    public removeLogs_result deepCopy() {
+      return new removeLogs_result(this);
+    }
+
+    @Override
+    public void clear() {
+      this.sec = null;
+    }
+
+    public org.apache.accumulo.core.security.thrift.ThriftSecurityException getSec() {
+      return this.sec;
+    }
+
+    public removeLogs_result setSec(org.apache.accumulo.core.security.thrift.ThriftSecurityException sec) {
+      this.sec = sec;
+      return this;
+    }
+
+    public void unsetSec() {
+      this.sec = null;
+    }
+
+    /** Returns true if field sec is set (has been assigned a value) and false otherwise */
+    public boolean isSetSec() {
+      return this.sec != null;
+    }
+
+    public void setSecIsSet(boolean value) {
+      if (!value) {
+        this.sec = null;
+      }
+    }
+
+    public void setFieldValue(_Fields field, Object value) {
+      switch (field) {
+      case SEC:
+        if (value == null) {
+          unsetSec();
+        } else {
+          setSec((org.apache.accumulo.core.security.thrift.ThriftSecurityException)value);
+        }
+        break;
+
+      }
+    }
+
+    public Object getFieldValue(_Fields field) {
+      switch (field) {
+      case SEC:
+        return getSec();
+
+      }
+      throw new IllegalStateException();
+    }
+
+    /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+    public boolean isSet(_Fields field) {
+      if (field == null) {
+        throw new IllegalArgumentException();
+      }
+
+      switch (field) {
+      case SEC:
+        return isSetSec();
+      }
+      throw new IllegalStateException();
+    }
+
+    @Override
+    public boolean equals(Object that) {
+      if (that == null)
+        return false;
+      if (that instanceof removeLogs_result)
+        return this.equals((removeLogs_result)that);
+      return false;
+    }
+
+    public boolean equals(removeLogs_result that) {
+      if (that == null)
+        return false;
+
+      boolean this_present_sec = true && this.isSetSec();
+      boolean that_present_sec = true && that.isSetSec();
+      if (this_present_sec || that_present_sec) {
+        if (!(this_present_sec && that_present_sec))
+          return false;
+        if (!this.sec.equals(that.sec))
+          return false;
+      }
+
+      return true;
+    }
+
+    @Override
+    public int hashCode() {
+      return 0;
+    }
+
+    public int compareTo(removeLogs_result other) {
+      if (!getClass().equals(other.getClass())) {
+        return getClass().getName().compareTo(other.getClass().getName());
+      }
+
+      int lastComparison = 0;
+      removeLogs_result typedOther = (removeLogs_result)other;
+
+      lastComparison = Boolean.valueOf(isSetSec()).compareTo(typedOther.isSetSec());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+      if (isSetSec()) {
+        lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sec, typedOther.sec);
+        if (lastComparison != 0) {
+          return lastComparison;
+        }
+      }
+      return 0;
+    }
+
+    public _Fields fieldForId(int fieldId) {
+      return _Fields.findByThriftId(fieldId);
+    }
+
+    public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+      org.apache.thrift.protocol.TField field;
+      iprot.readStructBegin();
+      while (true)
+      {
+        field = iprot.readFieldBegin();
+        if (field.type == org.apache.thrift.protocol.TType.STOP) { 
+          break;
+        }
+        switch (field.id) {
+          case 1: // SEC
+            if (field.type == org.apache.thrift.protocol.TType.STRUCT) {
+              this.sec = new org.apache.accumulo.core.security.thrift.ThriftSecurityException();
+              this.sec.read(iprot);
+            } else { 
+              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
+            }
+            break;
+          default:
+            org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
+        }
+        iprot.readFieldEnd();
+      }
+      iprot.readStructEnd();
+
+      // check for required fields of primitive type, which can't be checked in the validate method
+      validate();
+    }
+
+    public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+      oprot.writeStructBegin(STRUCT_DESC);
+
+      if (this.isSetSec()) {
+        oprot.writeFieldBegin(SEC_FIELD_DESC);
+        this.sec.write(oprot);
+        oprot.writeFieldEnd();
+      }
+      oprot.writeFieldStop();
+      oprot.writeStructEnd();
+    }
+
+    @Override
+    public String toString() {
+      StringBuilder sb = new StringBuilder("removeLogs_result(");
+      boolean first = true;
+
+      sb.append("sec:");
+      if (this.sec == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.sec);
+      }
+      first = false;
+      sb.append(")");
+      return sb.toString();
+    }
+
+    public void validate() throws org.apache.thrift.TException {
+      // check for required fields
+    }
+
+    private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+      try {
+        write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+      } catch (org.apache.thrift.TException te) {
+        throw new java.io.IOException(te);
+      }
+    }
+
+    private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+      try {
+        read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+      } catch (org.apache.thrift.TException te) {
+        throw new java.io.IOException(te);
+      }
+    }
+
+  }
+
 }

Modified: accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java (original)
+++ accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java Thu May 24 19:14:05 2012
@@ -23,7 +23,9 @@ import org.apache.thrift.transport.TSock
 
 public class AddressUtil {
   static public InetSocketAddress parseAddress(String address, int defaultPort) throws NumberFormatException {
-    final String[] parts = address.split(":", 2);
+    String[] parts = address.split(":", 2);
+    if (address.contains("+"))
+      parts = address.split(":", 2);
     if (parts.length == 2) {
       if (parts[1].isEmpty())
         return new InetSocketAddress(parts[0], defaultPort);

Modified: accumulo/branches/ACCUMULO-578/core/src/main/thrift/tabletserver.thrift
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/core/src/main/thrift/tabletserver.thrift?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/core/src/main/thrift/tabletserver.thrift (original)
+++ accumulo/branches/ACCUMULO-578/core/src/main/thrift/tabletserver.thrift Thu May 24 19:14:05 2012
@@ -147,6 +147,7 @@ service TabletClientService extends clie
   
   list<ActiveScan> getActiveScans(2:cloudtrace.TInfo tinfo, 1:security.AuthInfo credentials) throws (1:security.ThriftSecurityException sec)
   double sortLog(1:cloudtrace.TInfo tinfo, 2:security.AuthInfo credentials, 3:string lock, 4:string path) throws (1:security.ThriftSecurityException sec);
+  oneway void removeLogs(1:cloudtrace.TInfo tinfo, 2:security.AuthInfo credentials, 3:list<string> filenames) throws (1:security.ThriftSecurityException sec);
 }
 
 typedef i32 TabletID

Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java (original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java Thu May 24 19:14:05 2012
@@ -17,22 +17,34 @@
 package org.apache.accumulo.server.gc;
 
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.UUID;
 
 import org.apache.accumulo.cloudtrace.instrument.Span;
 import org.apache.accumulo.cloudtrace.instrument.Trace;
+import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.gc.thrift.GCStatus;
 import org.apache.accumulo.core.gc.thrift.GcCycleStats;
+import org.apache.accumulo.core.security.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface;
+import org.apache.accumulo.core.util.ThriftUtil;
 import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.accumulo.server.util.AddressUtil;
 import org.apache.accumulo.server.util.MetadataTable;
 import org.apache.accumulo.server.util.MetadataTable.LogEntry;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
 import org.apache.zookeeper.KeeperException;
 
 
@@ -93,6 +105,33 @@ public class GarbageCollectWriteAheadLog
   }
   
   private int removeFiles(Map<String,ArrayList<String>> serverToFileMap, final GCStatus status) {
+    for (Entry<String,ArrayList<String>> entry : serverToFileMap.entrySet()) {
+      if (entry.getKey().length() == 0) {
+        // old-style log entry, just remove it
+        for (String filename : entry.getValue()) {
+          log.debug("Removing old-style WAL " + entry.getValue());
+          try {
+            fs.delete(new Path(Constants.getWalDirectory(conf), filename), true);
+          } catch (IOException ex) {
+            log.error("Unable to delete wal " + filename + ": " + ex);
+          }
+        }
+      } else {
+        InetSocketAddress address = AddressUtil.parseAddress(entry.getKey(), Property.TSERV_CLIENTPORT);
+        Iface tserver = null;
+        try {
+          tserver = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+          tserver.removeLogs(null, SecurityConstants.getSystemCredentials(), entry.getValue());
+        } catch (ThriftSecurityException e) {
+          log.warn("Unexpected security exception during log file cleanup while talking to " + address + ". Is your configuration consistent over all nodes?");
+        } catch (TException e) {
+          log.warn("Error talking to " + address + ": " + e);
+        } finally {
+          if (tserver != null)
+            ThriftUtil.returnClient(tserver);
+        }
+      }
+    }
     return 0;
   }
   
@@ -124,8 +163,40 @@ public class GarbageCollectWriteAheadLog
   }
   
   private int scanServers(Map<String,String> fileToServerMap) throws Exception {
+    Path walRoot = new Path(Constants.getWalDirectory(conf));
+    for (FileStatus status : fs.listStatus(walRoot)) {
+      String name = status.getPath().getName();
+      if (status.isDir()) {
+        for (FileStatus file : fs.listStatus(new Path(walRoot, name))) {
+          if (isUUID(file.getPath().getName()))
+            fileToServerMap.put(file.getPath().getName(), name);
+          else {
+            log.info("Ignoring file " + file.getPath() + " because it doesn't look like a uuid");
+          }
+        }
+      } else if (isUUID(name)) {
+        // old-style WAL are not under a directory
+        fileToServerMap.put(name, "");
+      } else {
+        log.info("Ignoring file " + name + " because it doesn't look like a uuid");
+      }
+    }
+
     int count = 0;
     return count;
   }
   
+  /**
+   * @param name
+   * @return
+   */
+  static private boolean isUUID(String name) {
+    try {
+      UUID.fromString(name);
+      return true;
+    } catch (IllegalArgumentException ex) {
+      return false;
+    }
+  }
+  
 }

Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java (original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java Thu May 24 19:14:05 2012
@@ -78,10 +78,6 @@ public class CoordinateRecoveryTask impl
     String failedFileName() {
       return fullName(file + "/failed");
     }
-    
-    public String unsortedFileName() {
-      return fullName(file);
-    }
   }
   
   interface JobComplete {
@@ -98,7 +94,6 @@ public class CoordinateRecoveryTask impl
   private class RecoveryJob {
     final LogFile logFile;
     final long copyStartTime;
-    double copySize = 0;
     JobComplete notify = null;
     final AccumuloConfiguration config;
     
@@ -114,9 +109,11 @@ public class CoordinateRecoveryTask impl
       try {
         // Ask the logging server to put the file in HDFS
         IRemoteLogger logger = new DfsLogger(conf);
-        String source = Constants.getWalDirectory(config) + "/" + logFile.file;
+        String source = Constants.getWalDirectory(config) + "/" + logFile.server + "/" + logFile.file;
+        if (logFile.server.contains(":"))
+          source = Constants.getWalDirectory(config) + "/" + logFile.file;
         String dest = Constants.getRecoveryDir(config) + "/" + logFile.file;
-        copySize = logger.startCopy(source, dest);
+        logger.startCopy(source, dest);
       } catch (Throwable t) {
         log.warn("Unable to recover " + logFile + "(" + t + ")", t);
         fail();
@@ -166,14 +163,13 @@ public class CoordinateRecoveryTask impl
     }
     
     synchronized public String toString() {
-      return String.format("Copying %s from %s (for %f seconds) %2.1f", logFile.file, logFile.server, elapsedMillis() / 1000., copiedSoFar() * 100. / copySize);
+      return String.format("Copying %s from %s (for %f seconds) %2.1f", logFile.file, logFile.server, elapsedMillis() / 1000., copiedSoFar() * 100.);
     }
     
-    synchronized long copiedSoFar() {
+    synchronized double copiedSoFar() {
       try {
         ContentSummary contentSummary = fs.getContentSummary(new Path(logFile.recoveryFileName()));
-        // map files are bigger than sequence files
-        return (long) (contentSummary.getSpaceConsumed() * .8);
+        return contentSummary.getSpaceConsumed() / (double) config.getMemoryInBytes(Property.TSERV_WALOG_MAX_SIZE);
       } catch (Exception ex) {
         return 0;
       }
@@ -181,7 +177,7 @@ public class CoordinateRecoveryTask impl
     
     synchronized public RecoveryStatus getStatus() throws IOException {
       try {
-        return new RecoveryStatus(logFile.server, logFile.file, 0., 0., (int) (System.currentTimeMillis() - copyStartTime), (copiedSoFar() / (double) copySize));
+        return new RecoveryStatus(logFile.server, logFile.file, 0., 0., (int) (System.currentTimeMillis() - copyStartTime), copiedSoFar());
       } catch (Exception e) {
         return new RecoveryStatus(logFile.server, logFile.file, 1.0, 1.0, (int) (System.currentTimeMillis() - copyStartTime), 1.0);
       }

Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java (original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java Thu May 24 19:14:05 2012
@@ -110,6 +110,7 @@ import org.apache.accumulo.server.tablet
 import org.apache.accumulo.server.tabletserver.TabletServer.TservConstraintEnv;
 import org.apache.accumulo.server.tabletserver.TabletServerResourceManager.TabletResourceManager;
 import org.apache.accumulo.server.tabletserver.TabletStatsKeeper.Operation;
+import org.apache.accumulo.server.tabletserver.log.DfsLogger;
 import org.apache.accumulo.server.tabletserver.log.IRemoteLogger;
 import org.apache.accumulo.server.tabletserver.log.MutationReceiver;
 import org.apache.accumulo.server.tabletserver.mastermessage.TabletStatusMessage;
@@ -1456,6 +1457,15 @@ public class Tablet {
           throw new RuntimeException(t);
         }
       }
+      // make some closed references that represent the recovered logs
+      currentLogs = new HashSet<IRemoteLogger>();
+      for (LogEntry logEntry : logEntries) {
+        for (String log : logEntry.logSet) {
+          String[] parts = log.split("/", 2);
+          currentLogs.add(new DfsLogger(tabletServer.getServerConfig(), parts[1]));
+        }
+      }
+      
       log.info("Write-Ahead Log recovery complete for " + this.extent + " (" + count[0] + " mutations applied, " + tabletMemory.getNumEntries()
           + " entries created)");
     }
@@ -3606,6 +3616,14 @@ public class Tablet {
   
   private Set<IRemoteLogger> currentLogs = new HashSet<IRemoteLogger>();
   
+  synchronized public Set<String> getCurrentLogs() {
+    Set<String> result = new HashSet<String>();
+    for (IRemoteLogger log : currentLogs) {
+      result.add(log.toString());
+    }
+    return result;
+  }
+
   private Set<String> beginClearingUnusedLogs() {
     Set<String> doomed = new HashSet<String>();
     

Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java (original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java Thu May 24 19:14:05 2012
@@ -2074,6 +2074,49 @@ public class TabletServer extends Abstra
       }
     }
     
+    /*
+     * (non-Javadoc)
+     * 
+     * @see org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface#removeLogs(org.apache.accumulo.cloudtrace.thrift.TInfo,
+     * org.apache.accumulo.core.security.thrift.AuthInfo, java.util.List)
+     */
+    @Override
+    public void removeLogs(TInfo tinfo, AuthInfo credentials, List<String> filenames) throws ThriftSecurityException, TException {
+      Path logDir = new Path(Constants.getWalDirectory(acuConf), getClientAddressString());
+      Set<String> loggers = new HashSet<String>();
+      logger.getLoggers(loggers);
+      nextFile:
+      for (String filename : filenames) {
+        for (String logger : loggers) {
+          if (logger.contains(filename))
+            continue nextFile;
+        }
+        // this check is not strictly necessary
+        synchronized (onlineTablets) {
+          for (Entry<KeyExtent,Tablet> entry : onlineTablets.entrySet()) {
+            for (String current : entry.getValue().getCurrentLogs()) {
+              if (current.contains(filename)) {
+                log.error("Attempting to remove a write-ahead log that is in use.  This should never happen!");
+                log.info("Attempted to delete " + filename + " from tablet " + entry.getKey());
+                continue nextFile;
+              }
+            }
+          }
+        }
+        try {
+          if (acuConf.getBoolean(Property.TSERV_ARCHIVE_WALOGS)) {
+            log.info("Archiving walog " + filename);
+            fs.rename(new Path(logDir, filename), new Path(Constants.getBaseDir(acuConf) + "/walogArchive", filename));
+          } else {
+            log.info("Deleting walog " + filename);
+            fs.delete(new Path(logDir, filename), true);
+          }
+        } catch (IOException e) {
+          log.warn("Error attempting to delete write-ahead log " + filename + ": " + e);
+        }
+      }
+    }
+    
   }
   
   private class SplitRunner implements Runnable {

Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java (original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java Thu May 24 19:14:05 2012
@@ -24,6 +24,7 @@ import static org.apache.accumulo.server
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Random;
@@ -40,6 +41,7 @@ import org.apache.accumulo.core.data.Mut
 import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface;
 import org.apache.accumulo.core.tabletserver.thrift.TabletMutations;
 import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.StringUtil;
 import org.apache.accumulo.core.util.ThriftUtil;
 import org.apache.accumulo.server.logger.LogFileKey;
 import org.apache.accumulo.server.logger.LogFileValue;
@@ -186,6 +188,7 @@ public class DfsLogger implements IRemot
   private ServerConfig conf;
   private FSDataOutputStream logFile;
   private Path logPath;
+  private String logger;
   
   public DfsLogger(ServerConfig conf) throws IOException {
     this.conf = conf;
@@ -196,10 +199,11 @@ public class DfsLogger implements IRemot
     this.logPath = new Path(Constants.getWalDirectory(conf.getConfiguration()), filename);
   }
 
-  public synchronized void open() throws IOException {
+  public synchronized void open(String address) throws IOException {
     String filename = UUID.randomUUID().toString();
+    logger = StringUtil.join(Arrays.asList(address.split(":")), "+");
 
-    logPath = new Path(Constants.getWalDirectory(conf.getConfiguration()), filename);
+    logPath = new Path(Constants.getWalDirectory(conf.getConfiguration()) + "/" + logger + "/" + filename);
     try {
       FileSystem fs = conf.getFileSystem();
       short replication = (short) conf.getConfiguration().getCount(Property.TSERV_WAL_REPLICATION);
@@ -243,7 +247,7 @@ public class DfsLogger implements IRemot
    */
   @Override
   public String getLogger() {
-    return "";
+    return logger;
   }
   
   /* (non-Javadoc)

Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/IRemoteLogger.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/IRemoteLogger.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/IRemoteLogger.java (original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/IRemoteLogger.java Thu May 24 19:14:05 2012
@@ -57,6 +57,6 @@ public interface IRemoteLogger {
   
   public abstract void removeFile(List<String> files) throws IOException;
   
-  public abstract void open() throws IOException;
+  public abstract void open(String address) throws IOException;
 
 }
\ No newline at end of file

Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/TabletServerLogger.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/TabletServerLogger.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/TabletServerLogger.java (original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/TabletServerLogger.java Thu May 24 19:14:05 2012
@@ -182,7 +182,7 @@ public class TabletServerLogger {
     
     try {
       DfsLogger alog = new DfsLogger(tserver.getServerConfig());
-      alog.open();
+      alog.open(tserver.getClientAddressString());
       loggers.add(alog);
       logSetId.incrementAndGet();
       return;

Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/test/performance/thrift/NullTserver.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/test/performance/thrift/NullTserver.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/test/performance/thrift/NullTserver.java (original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/test/performance/thrift/NullTserver.java Thu May 24 19:14:05 2012
@@ -197,9 +197,18 @@ public class NullTserver {
      */
     @Override
     public double sortLog(TInfo tinfo, AuthInfo credentials, String lock, String path) throws ThriftSecurityException, TException {
-      // TODO Auto-generated method stub
       return 0;
     }
+    
+    /*
+     * (non-Javadoc)
+     * 
+     * @see org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface#removeLogs(org.apache.accumulo.cloudtrace.thrift.TInfo,
+     * org.apache.accumulo.core.security.thrift.AuthInfo, java.util.List)
+     */
+    @Override
+    public void removeLogs(TInfo tinfo, AuthInfo credentials, List<String> filenames) throws ThriftSecurityException, TException {
+    }
   }
   
   public static void main(String[] args) throws Exception {



Mime
View raw message