accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject [35/59] [abbrv] ACCUMULO-656,ACCUMULO-658 Update package names
Date Sat, 07 Sep 2013 03:28:38 GMT
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e0533561/server/gc/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
deleted file mode 100644
index 609a137..0000000
--- a/server/gc/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
+++ /dev/null
@@ -1,768 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.server.gc;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.accumulo.core.Constants;
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.client.IsolatedScanner;
-import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.client.impl.Tables;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.KeyExtent;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.file.FileOperations;
-import org.apache.accumulo.core.gc.thrift.GCMonitorService.Iface;
-import org.apache.accumulo.core.gc.thrift.GCMonitorService.Processor;
-import org.apache.accumulo.core.gc.thrift.GCStatus;
-import org.apache.accumulo.core.gc.thrift.GcCycleStats;
-import org.apache.accumulo.core.master.state.tables.TableState;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.RootTable;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.security.Credentials;
-import org.apache.accumulo.core.security.SecurityUtil;
-import org.apache.accumulo.core.security.thrift.TCredentials;
-import org.apache.accumulo.core.util.NamingThreadFactory;
-import org.apache.accumulo.core.util.ServerServices;
-import org.apache.accumulo.core.util.ServerServices.Service;
-import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.core.zookeeper.ZooUtil;
-import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
-import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher;
-import org.apache.accumulo.server.Accumulo;
-import org.apache.accumulo.server.ServerConstants;
-import org.apache.accumulo.server.ServerOpts;
-import org.apache.accumulo.server.client.HdfsZooInstance;
-import org.apache.accumulo.server.conf.ServerConfiguration;
-import org.apache.accumulo.server.fs.VolumeManager;
-import org.apache.accumulo.server.fs.VolumeManagerImpl;
-import org.apache.accumulo.server.security.SystemCredentials;
-import org.apache.accumulo.server.tables.TableManager;
-import org.apache.accumulo.server.util.Halt;
-import org.apache.accumulo.server.util.TServerUtils;
-import org.apache.accumulo.server.util.TabletIterator;
-import org.apache.accumulo.server.zookeeper.ZooLock;
-import org.apache.accumulo.trace.instrument.CountSampler;
-import org.apache.accumulo.trace.instrument.Sampler;
-import org.apache.accumulo.trace.instrument.Span;
-import org.apache.accumulo.trace.instrument.Trace;
-import org.apache.accumulo.trace.instrument.thrift.TraceWrap;
-import org.apache.accumulo.trace.thrift.TInfo;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.log4j.Logger;
-import org.apache.zookeeper.KeeperException;
-
-import com.beust.jcommander.Parameter;
-
-public class SimpleGarbageCollector implements Iface {
-  private static final Text EMPTY_TEXT = new Text();
-  
-  static class Opts extends ServerOpts {
-    @Parameter(names = {"-v", "--verbose"}, description = "extra information will get printed to stdout also")
-    boolean verbose = false;
-    @Parameter(names = {"-s", "--safemode"}, description = "safe mode will not delete files")
-    boolean safeMode = false;
-    @Parameter(names = {"-o", "--offline"},
-        description = "offline mode will run once and check data files directly; this is dangerous if accumulo is running or not shut down properly")
-    boolean offline = false;
-  }
-  
-  // how much of the JVM's available memory should it use gathering candidates
-  private static final float CANDIDATE_MEMORY_PERCENTAGE = 0.75f;
-  private boolean candidateMemExceeded;
-  
-  private static final Logger log = Logger.getLogger(SimpleGarbageCollector.class);
-  
-  private Credentials credentials;
-  private long gcStartDelay;
-  private boolean checkForBulkProcessingFiles;
-  private VolumeManager fs;
-  private boolean useTrash = true;
-  private Opts opts = new Opts();
-  private ZooLock lock;
-  private Key continueKey = null;
-  
-  private GCStatus status = new GCStatus(new GcCycleStats(), new GcCycleStats(), new GcCycleStats(), new GcCycleStats());
-  
-  private int numDeleteThreads;
-  
-  private Instance instance;
-  
-  public static void main(String[] args) throws UnknownHostException, IOException {
-    SecurityUtil.serverLogin();
-    Instance instance = HdfsZooInstance.getInstance();
-    ServerConfiguration serverConf = new ServerConfiguration(instance);
-    final VolumeManager fs = VolumeManagerImpl.get();
-    Accumulo.init(fs, serverConf, "gc");
-    Opts opts = new Opts();
-    opts.parseArgs("gc", args);
-    SimpleGarbageCollector gc = new SimpleGarbageCollector(opts);
-    
-    gc.init(fs, instance, SystemCredentials.get(), serverConf.getConfiguration().getBoolean(Property.GC_TRASH_IGNORE));
-    Accumulo.enableTracing(opts.getAddress(), "gc");
-    gc.run();
-  }
-  
-  public SimpleGarbageCollector(Opts opts) {
-    this.opts = opts;
-  }
-  
-  public void init(VolumeManager fs, Instance instance, Credentials credentials, boolean noTrash) throws IOException {
-    this.fs = fs;
-    this.credentials = credentials;
-    this.instance = instance;
-    
-    gcStartDelay = instance.getConfiguration().getTimeInMillis(Property.GC_CYCLE_START);
-    long gcDelay = instance.getConfiguration().getTimeInMillis(Property.GC_CYCLE_DELAY);
-    numDeleteThreads = instance.getConfiguration().getCount(Property.GC_DELETE_THREADS);
-    log.info("start delay: " + (opts.offline ? 0 + " sec (offline)" : gcStartDelay + " milliseconds"));
-    log.info("time delay: " + gcDelay + " milliseconds");
-    log.info("safemode: " + opts.safeMode);
-    log.info("offline: " + opts.offline);
-    log.info("verbose: " + opts.verbose);
-    log.info("memory threshold: " + CANDIDATE_MEMORY_PERCENTAGE + " of " + Runtime.getRuntime().maxMemory() + " bytes");
-    log.info("delete threads: " + numDeleteThreads);
-    useTrash = !noTrash;
-  }
-  
-  private void run() {
-    long tStart, tStop;
-    
-    // Sleep for an initial period, giving the master time to start up and
-    // old data files to be unused
-    if (!opts.offline) {
-      try {
-        getZooLock(startStatsService());
-      } catch (Exception ex) {
-        log.error(ex, ex);
-        System.exit(1);
-      }
-      
-      try {
-        log.debug("Sleeping for " + gcStartDelay + " milliseconds before beginning garbage collection cycles");
-        Thread.sleep(gcStartDelay);
-      } catch (InterruptedException e) {
-        log.warn(e, e);
-        return;
-      }
-    }
-    
-    Sampler sampler = new CountSampler(100);
-    
-    while (true) {
-      if (sampler.next())
-        Trace.on("gc");
-      
-      Span gcSpan = Trace.start("loop");
-      tStart = System.currentTimeMillis();
-      try {
-        // STEP 1: gather candidates
-        System.gc(); // make room
-        candidateMemExceeded = false;
-        checkForBulkProcessingFiles = false;
-        
-        Span candidatesSpan = Trace.start("getCandidates");
-        status.current.started = System.currentTimeMillis();
-        SortedSet<String> candidates;
-        try {
-          candidates = getCandidates();
-          status.current.candidates = candidates.size();
-        } finally {
-          candidatesSpan.stop();
-        }
-        
-        // STEP 2: confirm deletes
-        // WARNING: This line is EXTREMELY IMPORTANT.
-        // You MUST confirm candidates are okay to delete
-        Span confirmDeletesSpan = Trace.start("confirmDeletes");
-        try {
-          confirmDeletes(candidates);
-        status.current.inUse = status.current.candidates - candidates.size();
-        } finally {
-          confirmDeletesSpan.stop();
-        }
-        
-        // STEP 3: delete files
-        if (opts.safeMode) {
-          if (opts.verbose)
-            System.out.println("SAFEMODE: There are " + candidates.size() + " data file candidates marked for deletion.%n"
-                + "          Examine the log files to identify them.%n" + "          They can be removed by executing: bin/accumulo gc --offline%n"
-                + "WARNING:  Do not run the garbage collector in offline mode unless you are positive%n"
-                + "          that the accumulo METADATA table is in a clean state, or that accumulo%n"
-                + "          has not yet been run, in the case of an upgrade.");
-          log.info("SAFEMODE: Listing all data file candidates for deletion");
-          for (String s : candidates)
-            log.info("SAFEMODE: " + s);
-          log.info("SAFEMODE: End candidates for deletion");
-        } else {
-          Span deleteSpan = Trace.start("deleteFiles");
-          try {
-            deleteFiles(candidates);
-            log.info("Number of data file candidates for deletion: " + status.current.candidates);
-            log.info("Number of data file candidates still in use: " + status.current.inUse);
-            log.info("Number of successfully deleted data files: " + status.current.deleted);
-            log.info("Number of data files delete failures: " + status.current.errors);
-          } finally {
-            deleteSpan.stop();
-          }
-          
-          // delete empty dirs of deleted tables
-          // this can occur as a result of cloning
-          cleanUpDeletedTableDirs(candidates);
-        }
-        
-        status.current.finished = System.currentTimeMillis();
-        status.last = status.current;
-        status.current = new GcCycleStats();
-        
-      } catch (Exception e) {
-        log.error(e, e);
-      }
-      tStop = System.currentTimeMillis();
-      log.info(String.format("Collect cycle took %.2f seconds", ((tStop - tStart) / 1000.0)));
-      
-      if (opts.offline)
-        break;
-      
-      if (candidateMemExceeded) {
-        log.info("Gathering of candidates was interrupted due to memory shortage. Bypassing cycle delay to collect the remaining candidates.");
-        continue;
-      }
-      
-      // Clean up any unused write-ahead logs
-      Span waLogs = Trace.start("walogs");
-      try {
-        GarbageCollectWriteAheadLogs walogCollector = new GarbageCollectWriteAheadLogs(instance, fs, useTrash);
-        log.info("Beginning garbage collection of write-ahead logs");
-        walogCollector.collect(status);
-      } catch (Exception e) {
-        log.error(e, e);
-      } finally {
-        waLogs.stop();
-      }
-      gcSpan.stop();
-      
-      // we just made a lot of changes to the !METADATA table: flush them out
-      try {
-        Connector connector = instance.getConnector(credentials.getPrincipal(), credentials.getToken());
-        connector.tableOperations().compact(MetadataTable.NAME, null, null, true, true);
-        connector.tableOperations().compact(RootTable.NAME, null, null, true, true);
-      } catch (Exception e) {
-        log.warn(e, e);
-      }
-      
-      Trace.offNoFlush();
-      try {
-        long gcDelay = instance.getConfiguration().getTimeInMillis(Property.GC_CYCLE_DELAY);
-        log.debug("Sleeping for " + gcDelay + " milliseconds");
-        Thread.sleep(gcDelay);
-      } catch (InterruptedException e) {
-        log.warn(e, e);
-        return;
-      }
-    }
-  }
-  
-  private boolean moveToTrash(Path path) throws IOException {
-    if (!useTrash)
-      return false;
-    try {
-      return fs.moveToTrash(path);
-    } catch (FileNotFoundException ex) {
-      return false;
-    }
-  }
-  
-  /*
-   * this method removes deleted table dirs that are empty
-   */
-  private void cleanUpDeletedTableDirs(SortedSet<String> candidates) throws Exception {
-    
-    HashSet<String> tableIdsWithDeletes = new HashSet<String>();
-    
-    // find the table ids that had dirs deleted
-    for (String delete : candidates) {
-      if (isDir(delete)) {
-        String tableId = delete.split("/")[1];
-        tableIdsWithDeletes.add(tableId);
-      }
-    }
-    
-    Tables.clearCache(instance);
-    Set<String> tableIdsInZookeeper = Tables.getIdToNameMap(instance).keySet();
-    
-    tableIdsWithDeletes.removeAll(tableIdsInZookeeper);
-    
-    // tableIdsWithDeletes should now contain the set of deleted tables that had dirs deleted
-    
-    for (String delTableId : tableIdsWithDeletes) {
-      // if dir exist and is empty, then empty list is returned...
-      // hadoop 1.0 will return null if the file doesn't exist
-      // hadoop 2.0 will throw an exception if the file does not exist
-      for (String dir : ServerConstants.getTablesDirs()) {
-        FileStatus[] tabletDirs = null;
-        try {
-          tabletDirs = fs.listStatus(new Path(dir + "/" + delTableId));
-        } catch (FileNotFoundException ex) {
-          // ignored
-        }
-        if (tabletDirs == null)
-          continue;
-        
-        if (tabletDirs.length == 0) {
-          Path p = new Path(dir + "/" + delTableId);
-          if (!moveToTrash(p))
-            fs.delete(p);
-        }
-      }
-    }
-  }
-  
-  private void getZooLock(InetSocketAddress addr) throws KeeperException, InterruptedException {
-    String address = addr.getHostName() + ":" + addr.getPort();
-    String path = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZGC_LOCK;
-    
-    LockWatcher lockWatcher = new LockWatcher() {
-      @Override
-      public void lostLock(LockLossReason reason) {
-        Halt.halt("GC lock in zookeeper lost (reason = " + reason + "), exiting!");
-      }
-      
-      @Override
-      public void unableToMonitorLockNode(final Throwable e) {
-        Halt.halt(-1, new Runnable() {
-          
-          @Override
-          public void run() {
-            log.fatal("No longer able to monitor lock node ", e);
-          }
-        });
-        
-      }
-    };
-    
-    while (true) {
-      lock = new ZooLock(path);
-      if (lock.tryLock(lockWatcher, new ServerServices(address, Service.GC_CLIENT).toString().getBytes())) {
-        break;
-      }
-      UtilWaitThread.sleep(1000);
-    }
-  }
-  
-  private InetSocketAddress startStatsService() throws UnknownHostException {
-    Processor<Iface> processor = new Processor<Iface>(TraceWrap.service(this));
-    int port = instance.getConfiguration().getPort(Property.GC_PORT);
-    long maxMessageSize = instance.getConfiguration().getMemoryInBytes(Property.GENERAL_MAX_MESSAGE_SIZE);
-    InetSocketAddress result = new InetSocketAddress(opts.getAddress(), port);
-    try {
-      TServerUtils.startTServer(result, processor, this.getClass().getSimpleName(), "GC Monitor Service", 2, 1000, maxMessageSize);
-    } catch (Exception ex) {
-      log.fatal(ex, ex);
-      throw new RuntimeException(ex);
-    }
-    return result;
-  }
-  
-  /**
-   * This method gets a set of candidates for deletion by scanning the METADATA table deleted flag keyspace
-   */
-  SortedSet<String> getCandidates() throws Exception {
-    TreeSet<String> candidates = new TreeSet<String>();
-    
-    if (opts.offline) {
-      checkForBulkProcessingFiles = true;
-      try {
-        for (String validExtension : FileOperations.getValidExtensions()) {
-          for (String dir : ServerConstants.getTablesDirs()) {
-            for (FileStatus stat : fs.globStatus(new Path(dir + "/*/*/*." + validExtension))) {
-              String cand = stat.getPath().toUri().getPath();
-              if (cand.contains(ServerConstants.getRootTabletDir()))
-                continue;
-              candidates.add(cand.substring(dir.length()));
-              log.debug("Offline candidate: " + cand);
-            }
-          }
-        }
-      } catch (IOException e) {
-        log.error("Unable to check the filesystem for offline candidates. Removing all candidates for deletion to be safe.", e);
-        candidates.clear();
-      }
-      return candidates;
-    }
-    
-    checkForBulkProcessingFiles = false;
-    candidates.addAll(getBatch(RootTable.NAME));
-    if (candidateMemExceeded)
-      return candidates;
-    
-    candidates.addAll(getBatch(MetadataTable.NAME));
-    return candidates;
-  }
-  
-  /**
-   * Gets a batch of delete markers from the specified table
-   * 
-   * @param tableName
-   *          the name of the system table to scan (either {@link RootTable.NAME} or {@link MetadataTable.NAME})
-   */
-  private Collection<String> getBatch(String tableName) throws Exception {
-    // want to ensure GC makes progress... if the 1st N deletes are stable and we keep processing them,
-    // then will never inspect deletes after N
-    Range range = MetadataSchema.DeletesSection.getRange();
-    if (continueKey != null) {
-      if (!range.contains(continueKey)) {
-        // continue key is for some other range
-        return Collections.emptyList();
-      }
-      range = new Range(continueKey, true, range.getEndKey(), range.isEndKeyInclusive());
-      continueKey = null;
-    }
-    
-    Scanner scanner = instance.getConnector(credentials.getPrincipal(), credentials.getToken()).createScanner(tableName, Authorizations.EMPTY);
-    scanner.setRange(range);
-    List<String> result = new ArrayList<String>();
-    // find candidates for deletion; chop off the prefix
-    for (Entry<Key,Value> entry : scanner) {
-      String cand = entry.getKey().getRow().toString().substring(MetadataSchema.DeletesSection.getRowPrefix().length());
-      result.add(cand);
-      checkForBulkProcessingFiles |= cand.toLowerCase(Locale.ENGLISH).contains(Constants.BULK_PREFIX);
-      if (almostOutOfMemory()) {
-        candidateMemExceeded = true;
-        log.info("List of delete candidates has exceeded the memory threshold. Attempting to delete what has been gathered so far.");
-        continueKey = entry.getKey();
-        break;
-      }
-    }
-    
-    return result;
-  }
-  
-  static public boolean almostOutOfMemory() {
-    Runtime runtime = Runtime.getRuntime();
-    return runtime.totalMemory() - runtime.freeMemory() > CANDIDATE_MEMORY_PERCENTAGE * runtime.maxMemory();
-  }
-  
-  /**
-   * This method removes candidates from the candidate list under two conditions: 1. They are in the same folder as a bulk processing file, if that option is
-   * selected 2. They are still in use in the file column family in the METADATA table
-   */
-  public void confirmDeletes(SortedSet<String> candidates) throws AccumuloException {
-    confirmDeletes(RootTable.NAME, candidates);
-    confirmDeletes(MetadataTable.NAME, candidates);
-  }
-  
-  private void confirmDeletes(String tableName, SortedSet<String> candidates) throws AccumuloException {
-    Scanner scanner;
-    if (opts.offline) {
-      // TODO
-      throw new RuntimeException("Offline scanner no longer supported");
-      // try {
-      // scanner = new OfflineMetadataScanner(instance.getConfiguration(), fs);
-      // } catch (IOException e) {
-      // throw new IllegalStateException("Unable to create offline metadata scanner", e);
-      // }
-    } else {
-      try {
-        scanner = new IsolatedScanner(instance.getConnector(credentials.getPrincipal(), credentials.getToken()).createScanner(tableName, Authorizations.EMPTY));
-      } catch (AccumuloSecurityException ex) {
-        throw new AccumuloException(ex);
-      } catch (TableNotFoundException ex) {
-        throw new AccumuloException(ex);
-      }
-    }
-    
-    // skip candidates that are in a bulk processing folder
-    if (checkForBulkProcessingFiles) {
-      
-      log.debug("Checking for bulk processing flags");
-      
-      scanner.setRange(MetadataSchema.BlipSection.getRange());
-      
-      // WARNING: This block is IMPORTANT
-      // You MUST REMOVE candidates that are in the same folder as a bulk
-      // processing flag!
-      
-      for (Entry<Key,Value> entry : scanner) {
-        String blipPath = entry.getKey().getRow().toString().substring(MetadataSchema.BlipSection.getRowPrefix().length());
-        Iterator<String> tailIter = candidates.tailSet(blipPath).iterator();
-        int count = 0;
-        while (tailIter.hasNext()) {
-          if (tailIter.next().startsWith(blipPath)) {
-            count++;
-            tailIter.remove();
-          } else {
-            break;
-          }
-        }
-        
-        if (count > 0)
-          log.debug("Folder has bulk processing flag: " + blipPath);
-        
-      }
-    }
-    
-    // skip candidates that are still in use in the file column family in
-    // the metadata table
-    scanner.clearColumns();
-    scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
-    scanner.fetchColumnFamily(ScanFileColumnFamily.NAME);
-    TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner);
-    TabletIterator tabletIterator = new TabletIterator(scanner, MetadataSchema.TabletsSection.getRange(), false, true);
-    
-    while (tabletIterator.hasNext()) {
-      Map<Key,Value> tabletKeyValues = tabletIterator.next();
-      
-      for (Entry<Key,Value> entry : tabletKeyValues.entrySet()) {
-        if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME) || entry.getKey().getColumnFamily().equals(ScanFileColumnFamily.NAME)) {
-          
-          String cf = entry.getKey().getColumnQualifier().toString();
-          String delete = cf;
-          if (!cf.contains(":")) {
-            if (cf.startsWith("../")) {
-              delete = cf.substring(2);
-            } else {
-              String table = new String(KeyExtent.tableOfMetadataRow(entry.getKey().getRow()));
-              if (cf.startsWith("/"))
-                delete = "/" + table + cf;
-              else
-                delete = "/" + table + "/" + cf;
-            }
-          }
-          // WARNING: This line is EXTREMELY IMPORTANT.
-          // You MUST REMOVE candidates that are still in use
-          if (candidates.remove(delete))
-            log.debug("Candidate was still in use in the " + tableName + " table: " + delete);
-          
-          String path = delete.substring(0, delete.lastIndexOf('/'));
-          if (candidates.remove(path))
-            log.debug("Candidate was still in use in the " + tableName + " table: " + path);
-        } else if (TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(entry.getKey())) {
-          String table = new String(KeyExtent.tableOfMetadataRow(entry.getKey().getRow()));
-          String delete = "/" + table + entry.getValue().toString();
-          if (candidates.remove(delete))
-            log.debug("Candidate was still in use in the " + tableName + " table: " + delete);
-        } else
-          throw new AccumuloException("Scanner over metadata table returned unexpected column : " + entry.getKey());
-      }
-    }
-  }
-  
-  final static String METADATA_TABLE_DIR = "/" + MetadataTable.ID;
-  
-  private static void putMarkerDeleteMutation(final String delete, final BatchWriter metadataWriter, final BatchWriter rootWriter)
-      throws MutationsRejectedException {
-    BatchWriter writer = delete.contains(METADATA_TABLE_DIR) ? rootWriter : metadataWriter;
-    Mutation m = new Mutation(MetadataSchema.DeletesSection.getRowPrefix() + delete);
-    m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
-    writer.addMutation(m);
-  }
-  
-  /**
-   * This method attempts to do its best to remove files from the filesystem that have been confirmed for deletion.
-   */
-  private void deleteFiles(SortedSet<String> confirmedDeletes) {
-    // create a batchwriter to remove the delete flags for successful
-    // deletes; Need separate writer for the root tablet.
-    BatchWriter writer = null;
-    BatchWriter rootWriter = null;
-    if (!opts.offline) {
-      Connector c;
-      try {
-        c = instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
-        writer = c.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
-        rootWriter = c.createBatchWriter(RootTable.NAME, new BatchWriterConfig());
-      } catch (AccumuloException e) {
-        log.error("Unable to connect to Accumulo to write deletes", e);
-      } catch (AccumuloSecurityException e) {
-        log.error("Unable to connect to Accumulo to write deletes", e);
-      } catch (TableNotFoundException e) {
-        log.error("Unable to create writer to remove file from the " + e.getTableName() + " table", e);
-      }
-    }
-    // when deleting a dir and all files in that dir, only need to delete the dir
-    // the dir will sort right before the files... so remove the files in this case
-    // to minimize namenode ops
-    Iterator<String> cdIter = confirmedDeletes.iterator();
-    String lastDir = null;
-    while (cdIter.hasNext()) {
-      String delete = cdIter.next();
-      if (isDir(delete)) {
-        lastDir = delete;
-      } else if (lastDir != null) {
-        if (delete.startsWith(lastDir)) {
-          log.debug("Ignoring " + delete + " because " + lastDir + " exist");
-          try {
-            putMarkerDeleteMutation(delete, writer, rootWriter);
-          } catch (MutationsRejectedException e) {
-            throw new RuntimeException(e);
-          }
-          cdIter.remove();
-        } else {
-          lastDir = null;
-        }
-        
-      }
-    }
-    
-    final BatchWriter finalWriter = writer;
-    final BatchWriter finalRootWriter = rootWriter;
-    
-    ExecutorService deleteThreadPool = Executors.newFixedThreadPool(numDeleteThreads, new NamingThreadFactory("deleting"));
-    
-    for (final String delete : confirmedDeletes) {
-      
-      Runnable deleteTask = new Runnable() {
-        @Override
-        public void run() {
-          boolean removeFlag;
-          
-          try {
-            Path fullPath;
-            
-            if (delete.contains(":"))
-              fullPath = new Path(delete);
-            else
-              fullPath = fs.getFullPath(ServerConstants.getTablesDirs(), delete);
-            log.debug("Deleting " + fullPath);
-            
-            if (moveToTrash(fullPath) || fs.deleteRecursively(fullPath)) {
-              // delete succeeded, still want to delete
-              removeFlag = true;
-              synchronized (SimpleGarbageCollector.this) {
-                ++status.current.deleted;
-              }
-            } else if (fs.exists(fullPath)) {
-              // leave the entry in the METADATA table; we'll try again
-              // later
-              removeFlag = false;
-              synchronized (SimpleGarbageCollector.this) {
-                ++status.current.errors;
-              }
-              log.warn("File exists, but was not deleted for an unknown reason: " + fullPath);
-            } else {
-              // this failure, we still want to remove the METADATA table
-              // entry
-              removeFlag = true;
-              synchronized (SimpleGarbageCollector.this) {
-                ++status.current.errors;
-              }
-              String parts[] = delete.split("/");
-              if (parts.length > 2) {
-                String tableId = parts[parts.length - 3];
-                String tabletDir = parts[parts.length - 2];
-                TableManager.getInstance().updateTableStateCache(tableId);
-                TableState tableState = TableManager.getInstance().getTableState(tableId);
-                if (tableState != null && tableState != TableState.DELETING) {
-                  // clone directories don't always exist
-                  if (!tabletDir.startsWith("c-"))
-                    log.warn("File doesn't exist: " + fullPath);
-                }
-              } else {
-                log.warn("Very strange path name: " + delete);
-              }
-            }
-            
-            // proceed to clearing out the flags for successful deletes and
-            // non-existent files
-            if (removeFlag && finalWriter != null) {
-              putMarkerDeleteMutation(delete, finalWriter, finalRootWriter);
-            }
-          } catch (Exception e) {
-            log.error(e, e);
-          }
-          
-        }
-        
-      };
-      
-      deleteThreadPool.execute(deleteTask);
-    }
-    
-    deleteThreadPool.shutdown();
-    
-    try {
-      while (!deleteThreadPool.awaitTermination(1000, TimeUnit.MILLISECONDS)) {}
-    } catch (InterruptedException e1) {
-      log.error(e1, e1);
-    }
-    
-    if (writer != null) {
-      try {
-        writer.close();
-      } catch (MutationsRejectedException e) {
-        log.error("Problem removing entries from the metadata table: ", e);
-      }
-    }
-    if (rootWriter != null) {
-      try {
-        rootWriter.close();
-      } catch (MutationsRejectedException e) {
-        log.error("Problem removing entries from the metadata table: ", e);
-      }
-    }
-  }
-  
-  private boolean isDir(String delete) {
-    int slashCount = 0;
-    for (int i = 0; i < delete.length(); i++)
-      if (delete.charAt(i) == '/')
-        slashCount++;
-    return slashCount == 2;
-  }
-  
-  @Override
-  public GCStatus getStatus(TInfo info, TCredentials credentials) {
-    return status;
-  }
-}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e0533561/server/gc/src/test/java/org/apache/accumulo/gc/TestConfirmDeletes.java
----------------------------------------------------------------------
diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/TestConfirmDeletes.java b/server/gc/src/test/java/org/apache/accumulo/gc/TestConfirmDeletes.java
new file mode 100644
index 0000000..01628ad
--- /dev/null
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/TestConfirmDeletes.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.gc;
+
+import java.util.Arrays;
+import java.util.Map.Entry;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.gc.SimpleGarbageCollector.Opts;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * 
+ */
+public class TestConfirmDeletes {
+  
+  SortedSet<String> newSet(String... s) {
+    SortedSet<String> result = new TreeSet<String>(Arrays.asList(s));
+    return result;
+  }
+  
+  @Test
+  public void test() throws Exception {
+    
+    // have a directory reference
+    String metadata[] = {"1636< last:3353986642a66eb 192.168.117.9:9997", "1636< srv:dir /default_tablet", "1636< srv:flush 2",
+        "1636< srv:lock tservers/192.168.117.9:9997/zlock-0000000000$3353986642a66eb", "1636< srv:time M1328505870023", "1636< ~tab:~pr \0"};
+    String deletes[] = {"~del/1636/default_tablet"};
+    
+    test1(metadata, deletes, 1, 0);
+    
+    // have no file reference
+    deletes = new String[] {"~del/1636/default_tablet/someFile"};
+    test1(metadata, deletes, 1, 1);
+    
+    // have a file reference
+    metadata = new String[] {"1636< file:/default_tablet/someFile 10,100", "1636< last:3353986642a66eb 192.168.117.9:9997", "1636< srv:dir /default_tablet",
+        "1636< srv:flush 2", "1636< srv:lock tservers/192.168.117.9:9997/zlock-0000000000$3353986642a66eb", "1636< srv:time M1328505870023",
+        "1636< ~tab:~pr \0"};
+    test1(metadata, deletes, 1, 0);
+    
+    // have an indirect file reference
+    deletes = new String[] {"~del/9/default_tablet/someFile"};
+    metadata = new String[] {"1636< file:../9/default_tablet/someFile 10,100", "1636< last:3353986642a66eb 192.168.117.9:9997",
+        "1636< srv:dir /default_tablet", "1636< srv:flush 2", "1636< srv:lock tservers/192.168.117.9:9997/zlock-0000000000$3353986642a66eb",
+        "1636< srv:time M1328505870023", "1636< ~tab:~pr \0"};
+    
+    test1(metadata, deletes, 1, 0);
+    
+    // have an indirect file reference and a directory candidate
+    deletes = new String[] {"~del/9/default_tablet"};
+    test1(metadata, deletes, 1, 0);
+    
+    deletes = new String[] {"~del/9/default_tablet", "~del/9/default_tablet/someFile"};
+    test1(metadata, deletes, 2, 0);
+    
+    deletes = new String[] {"~blip/1636/b-0001", "~del/1636/b-0001/I0000"};
+    test1(metadata, deletes, 1, 0);
+  }
+  
+  private void test1(String[] metadata, String[] deletes, int expectedInitial, int expected) throws Exception {
+    Credentials credentials = new Credentials("root", new PasswordToken(new byte[0]));
+    
+    Instance instance = new MockInstance();
+    VolumeManager fs = VolumeManagerImpl.getLocal();
+    
+    load(instance, metadata, deletes);
+    
+    SimpleGarbageCollector gc = new SimpleGarbageCollector(new Opts());
+    gc.init(fs, instance, credentials, false);
+    SortedSet<String> candidates = gc.getCandidates();
+    Assert.assertEquals(expectedInitial, candidates.size());
+    gc.confirmDeletes(candidates);
+    Assert.assertEquals(expected, candidates.size());
+  }
+  
+  private void load(Instance instance, String[] metadata, String[] deletes) throws Exception {
+    Credentials credentials = new Credentials("root", new PasswordToken(new byte[0]));
+    
+    Scanner scanner = instance.getConnector(credentials.getPrincipal(), credentials.getToken()).createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+    int count = 0;
+    for (@SuppressWarnings("unused")
+    Entry<Key,Value> entry : scanner) {
+      count++;
+    }
+    
+    // ensure there is no data from previous test
+    Assert.assertEquals(0, count);
+    
+    Connector conn = instance.getConnector(credentials.getPrincipal(), credentials.getToken());
+    BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+    for (String line : metadata) {
+      String[] parts = line.split(" ");
+      String[] columnParts = parts[1].split(":");
+      Mutation m = new Mutation(parts[0]);
+      m.put(new Text(columnParts[0]), new Text(columnParts[1]), new Value(parts[2].getBytes()));
+      bw.addMutation(m);
+    }
+    
+    for (String line : deletes) {
+      Mutation m = new Mutation(line);
+      m.put("", "", "");
+      bw.addMutation(m);
+    }
+    bw.close();
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e0533561/server/gc/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java
----------------------------------------------------------------------
diff --git a/server/gc/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java b/server/gc/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java
deleted file mode 100644
index 36938d7..0000000
--- a/server/gc/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.server.gc;
-
-import java.util.Arrays;
-import java.util.Map.Entry;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.mock.MockInstance;
-import org.apache.accumulo.core.client.security.tokens.PasswordToken;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.security.Credentials;
-import org.apache.accumulo.server.fs.VolumeManager;
-import org.apache.accumulo.server.fs.VolumeManagerImpl;
-import org.apache.accumulo.server.gc.SimpleGarbageCollector.Opts;
-import org.apache.hadoop.io.Text;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * 
- */
-public class TestConfirmDeletes {
-  
-  SortedSet<String> newSet(String... s) {
-    SortedSet<String> result = new TreeSet<String>(Arrays.asList(s));
-    return result;
-  }
-  
-  @Test
-  public void test() throws Exception {
-    
-    // have a directory reference
-    String metadata[] = {"1636< last:3353986642a66eb 192.168.117.9:9997", "1636< srv:dir /default_tablet", "1636< srv:flush 2",
-        "1636< srv:lock tservers/192.168.117.9:9997/zlock-0000000000$3353986642a66eb", "1636< srv:time M1328505870023", "1636< ~tab:~pr \0"};
-    String deletes[] = {"~del/1636/default_tablet"};
-    
-    test1(metadata, deletes, 1, 0);
-    
-    // have no file reference
-    deletes = new String[] {"~del/1636/default_tablet/someFile"};
-    test1(metadata, deletes, 1, 1);
-    
-    // have a file reference
-    metadata = new String[] {"1636< file:/default_tablet/someFile 10,100", "1636< last:3353986642a66eb 192.168.117.9:9997", "1636< srv:dir /default_tablet",
-        "1636< srv:flush 2", "1636< srv:lock tservers/192.168.117.9:9997/zlock-0000000000$3353986642a66eb", "1636< srv:time M1328505870023",
-        "1636< ~tab:~pr \0"};
-    test1(metadata, deletes, 1, 0);
-    
-    // have an indirect file reference
-    deletes = new String[] {"~del/9/default_tablet/someFile"};
-    metadata = new String[] {"1636< file:../9/default_tablet/someFile 10,100", "1636< last:3353986642a66eb 192.168.117.9:9997",
-        "1636< srv:dir /default_tablet", "1636< srv:flush 2", "1636< srv:lock tservers/192.168.117.9:9997/zlock-0000000000$3353986642a66eb",
-        "1636< srv:time M1328505870023", "1636< ~tab:~pr \0"};
-    
-    test1(metadata, deletes, 1, 0);
-    
-    // have an indirect file reference and a directory candidate
-    deletes = new String[] {"~del/9/default_tablet"};
-    test1(metadata, deletes, 1, 0);
-    
-    deletes = new String[] {"~del/9/default_tablet", "~del/9/default_tablet/someFile"};
-    test1(metadata, deletes, 2, 0);
-    
-    deletes = new String[] {"~blip/1636/b-0001", "~del/1636/b-0001/I0000"};
-    test1(metadata, deletes, 1, 0);
-  }
-  
-  private void test1(String[] metadata, String[] deletes, int expectedInitial, int expected) throws Exception {
-    Credentials credentials = new Credentials("root", new PasswordToken(new byte[0]));
-    
-    Instance instance = new MockInstance();
-    VolumeManager fs = VolumeManagerImpl.getLocal();
-    
-    load(instance, metadata, deletes);
-    
-    SimpleGarbageCollector gc = new SimpleGarbageCollector(new Opts());
-    gc.init(fs, instance, credentials, false);
-    SortedSet<String> candidates = gc.getCandidates();
-    Assert.assertEquals(expectedInitial, candidates.size());
-    gc.confirmDeletes(candidates);
-    Assert.assertEquals(expected, candidates.size());
-  }
-  
-  private void load(Instance instance, String[] metadata, String[] deletes) throws Exception {
-    Credentials credentials = new Credentials("root", new PasswordToken(new byte[0]));
-    
-    Scanner scanner = instance.getConnector(credentials.getPrincipal(), credentials.getToken()).createScanner(MetadataTable.NAME, Authorizations.EMPTY);
-    int count = 0;
-    for (@SuppressWarnings("unused")
-    Entry<Key,Value> entry : scanner) {
-      count++;
-    }
-    
-    // ensure there is no data from previous test
-    Assert.assertEquals(0, count);
-    
-    Connector conn = instance.getConnector(credentials.getPrincipal(), credentials.getToken());
-    BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
-    for (String line : metadata) {
-      String[] parts = line.split(" ");
-      String[] columnParts = parts[1].split(":");
-      Mutation m = new Mutation(parts[0]);
-      m.put(new Text(columnParts[0]), new Text(columnParts[1]), new Value(parts[2].getBytes()));
-      bw.addMutation(m);
-    }
-    
-    for (String line : deletes) {
-      Mutation m = new Mutation(line);
-      m.put("", "", "");
-      bw.addMutation(m);
-    }
-    bw.close();
-  }
-}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e0533561/server/monitor/src/main/java/org/apache/accumulo/monitor/DedupedLogEvent.java
----------------------------------------------------------------------
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/DedupedLogEvent.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/DedupedLogEvent.java
new file mode 100644
index 0000000..f5afd0a
--- /dev/null
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/DedupedLogEvent.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor;
+
+import org.apache.log4j.spi.LoggingEvent;
+
+public class DedupedLogEvent {
+  
+  private LoggingEvent event;
+  private int count = 0;
+  private int hash = -1;
+  
+  public DedupedLogEvent(LoggingEvent event) {
+    this(event, 1);
+  }
+  
+  public DedupedLogEvent(LoggingEvent event, int count) {
+    this.event = event;
+    this.count = count;
+  }
+  
+  public LoggingEvent getEvent() {
+    return event;
+  }
+  
+  public int getCount() {
+    return count;
+  }
+  
+  public void setCount(int count) {
+    this.count = count;
+  }
+  
+  @Override
+  public int hashCode() {
+    if (hash == -1) {
+      String eventId = event.getMDC("application").toString() + ":" + event.getLevel().toString() + ":" + event.getMessage().toString();
+      hash = eventId.hashCode();
+    }
+    return hash;
+  }
+  
+  @Override
+  public boolean equals(Object obj) {
+    if (obj instanceof DedupedLogEvent)
+      return this.event.equals(((DedupedLogEvent) obj).event);
+    return false;
+  }
+  
+  @Override
+  public String toString() {
+    return event.getMDC("application").toString() + ":" + event.getLevel().toString() + ":" + event.getMessage().toString();
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e0533561/server/monitor/src/main/java/org/apache/accumulo/monitor/EmbeddedWebServer.java
----------------------------------------------------------------------
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/EmbeddedWebServer.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/EmbeddedWebServer.java
new file mode 100644
index 0000000..e16b598
--- /dev/null
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/EmbeddedWebServer.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor;
+
+import javax.servlet.http.HttpServlet;
+
+import org.apache.accumulo.core.conf.Property;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.bio.SocketConnector;
+import org.mortbay.jetty.handler.ContextHandlerCollection;
+import org.mortbay.jetty.security.SslSocketConnector;
+import org.mortbay.jetty.servlet.Context;
+import org.mortbay.jetty.servlet.SessionHandler;
+
+public class EmbeddedWebServer {
+  
+  Server server = null;
+  SocketConnector sock;
+  ContextHandlerCollection handler;
+  Context root;
+  boolean usingSsl;
+  
+  public EmbeddedWebServer() {
+    this("0.0.0.0", 0);
+  }
+  
+  public EmbeddedWebServer(String host, int port) {
+    server = new Server();
+    handler = new ContextHandlerCollection();
+    root = new Context(handler, "/", new SessionHandler(), null, null, null);
+    
+    if (Monitor.getSystemConfiguration().get(Property.MONITOR_SSL_KEYSTORE) == ""
+        || Monitor.getSystemConfiguration().get(Property.MONITOR_SSL_KEYSTOREPASS) == ""
+        || Monitor.getSystemConfiguration().get(Property.MONITOR_SSL_TRUSTSTORE) == ""
+        || Monitor.getSystemConfiguration().get(Property.MONITOR_SSL_TRUSTSTOREPASS) == "") {
+      sock = new SocketConnector();
+      usingSsl = false;
+    } else {
+      sock = new SslSocketConnector();
+      ((SslSocketConnector) sock).setKeystore(Monitor.getSystemConfiguration().get(Property.MONITOR_SSL_KEYSTORE));
+      ((SslSocketConnector) sock).setKeyPassword(Monitor.getSystemConfiguration().get(Property.MONITOR_SSL_KEYSTOREPASS));
+      ((SslSocketConnector) sock).setTruststore(Monitor.getSystemConfiguration().get(Property.MONITOR_SSL_TRUSTSTORE));
+      ((SslSocketConnector) sock).setTrustPassword(Monitor.getSystemConfiguration().get(Property.MONITOR_SSL_TRUSTSTOREPASS));
+      usingSsl = true;
+    }
+    sock.setHost(host);
+    sock.setPort(port);
+  }
+  
+  public void addServlet(Class<? extends HttpServlet> klass, String where) {
+    root.addServlet(klass, where);
+  }
+  
+  public int getPort() {
+    return sock.getLocalPort();
+  }
+  
+  public void start() {
+    try {
+      server.addConnector(sock);
+      server.setHandler(handler);
+      server.start();
+    } catch (Exception e) {
+      stop();
+      throw new RuntimeException(e);
+    }
+  }
+  
+  public void stop() {
+    try {
+      server.stop();
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+  
+  public boolean isUsingSsl() {
+    return usingSsl;
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e0533561/server/monitor/src/main/java/org/apache/accumulo/monitor/LogService.java
----------------------------------------------------------------------
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/LogService.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/LogService.java
new file mode 100644
index 0000000..7bb847c
--- /dev/null
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/LogService.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor;
+
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.apache.log4j.net.SocketNode;
+import org.apache.log4j.spi.LoggingEvent;
+
+/**
+ * Hijack log4j and capture log events for display.
+ * 
+ */
+public class LogService extends org.apache.log4j.AppenderSkeleton {
+  
+  private static final Logger log = Logger.getLogger(LogService.class);
+  
+  /**
+   * Read logging events forward to us over the net.
+   * 
+   */
+  static class SocketServer implements Runnable {
+    private ServerSocket server = null;
+    
+    public SocketServer(int port) {
+      try {
+        server = new ServerSocket(port);
+      } catch (IOException io) {
+        throw new RuntimeException(io);
+      }
+    }
+    
+    @Override
+    public void run() {
+      try {
+        while (true) {
+          log.debug("Waiting for log message senders");
+          Socket socket = server.accept();
+          log.debug("Got a new connection");
+          Thread t = new Daemon(new SocketNode(socket, LogManager.getLoggerRepository()));
+          t.start();
+        }
+      } catch (IOException io) {
+        log.error(io, io);
+      }
+    }
+  }
+  
+  static void startLogListener(AccumuloConfiguration conf) {
+    try {
+      new Daemon(new SocketServer(conf.getPort(Property.MONITOR_LOG4J_PORT))).start();
+    } catch (Throwable t) {
+      log.info("Unable to listen to cluster-wide ports", t);
+    }
+  }
+  
+  static private LogService instance = null;
+  
+  synchronized public static LogService getInstance() {
+    if (instance == null)
+      return new LogService();
+    return instance;
+  }
+  
+  private static final int MAX_LOGS = 50;
+  
+  private LinkedHashMap<String,DedupedLogEvent> events = new LinkedHashMap<String,DedupedLogEvent>(MAX_LOGS + 1, (float) .75, true) {
+    
+    private static final long serialVersionUID = 1L;
+    
+    @Override
+    @SuppressWarnings("rawtypes")
+    protected boolean removeEldestEntry(Map.Entry eldest) {
+      return size() > MAX_LOGS;
+    }
+  };
+  
+  public LogService() {
+    synchronized (LogService.class) {
+      instance = this;
+    }
+  }
+  
+  @Override
+  synchronized protected void append(LoggingEvent ev) {
+    Object application = ev.getMDC("application");
+    if (application == null || application.toString().isEmpty())
+      return;
+    
+    DedupedLogEvent dev = new DedupedLogEvent(ev);
+    
+    // if event is present, increase the count
+    if (events.containsKey(dev.toString())) {
+      DedupedLogEvent oldDev = events.remove(dev.toString());
+      dev.setCount(oldDev.getCount() + 1);
+    }
+    events.put(dev.toString(), dev);
+  }
+  
+  @Override
+  public void close() {
+    events = null;
+  }
+  
+  @Override
+  public synchronized void doAppend(LoggingEvent event) {
+    super.doAppend(event);
+  }
+  
+  @Override
+  public boolean requiresLayout() {
+    return false;
+  }
+  
+  synchronized public List<DedupedLogEvent> getEvents() {
+    return new ArrayList<DedupedLogEvent>(events.values());
+  }
+  
+  synchronized public void clear() {
+    events.clear();
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e0533561/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
----------------------------------------------------------------------
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
new file mode 100644
index 0000000..c82d4d4
--- /dev/null
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@ -0,0 +1,668 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor;
+
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.MasterClient;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.gc.thrift.GCMonitorService;
+import org.apache.accumulo.core.gc.thrift.GCStatus;
+import org.apache.accumulo.core.master.thrift.Compacting;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.LoggingRunnable;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.ServerServices;
+import org.apache.accumulo.core.util.ServerServices.Service;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.monitor.servlets.DefaultServlet;
+import org.apache.accumulo.monitor.servlets.GcStatusServlet;
+import org.apache.accumulo.monitor.servlets.JSONServlet;
+import org.apache.accumulo.monitor.servlets.LogServlet;
+import org.apache.accumulo.monitor.servlets.MasterServlet;
+import org.apache.accumulo.monitor.servlets.OperationServlet;
+import org.apache.accumulo.monitor.servlets.ProblemServlet;
+import org.apache.accumulo.monitor.servlets.ShellServlet;
+import org.apache.accumulo.monitor.servlets.TServersServlet;
+import org.apache.accumulo.monitor.servlets.TablesServlet;
+import org.apache.accumulo.monitor.servlets.VisServlet;
+import org.apache.accumulo.monitor.servlets.XMLServlet;
+import org.apache.accumulo.monitor.servlets.trace.ListType;
+import org.apache.accumulo.monitor.servlets.trace.ShowTrace;
+import org.apache.accumulo.monitor.servlets.trace.Summary;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.ServerOpts;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.problems.ProblemReports;
+import org.apache.accumulo.server.problems.ProblemType;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+
+/**
+ * Serve master statistics with an embedded web server.
+ */
+public class Monitor {
+  private static final Logger log = Logger.getLogger(Monitor.class);
+  
+  public static final int REFRESH_TIME = 5;
+  private static long lastRecalc = 0L;
+  private static double totalIngestRate = 0.0;
+  private static double totalIngestByteRate = 0.0;
+  private static double totalQueryRate = 0.0;
+  private static double totalScanRate = 0.0;
+  private static double totalQueryByteRate = 0.0;
+  private static long totalEntries = 0L;
+  private static int totalTabletCount = 0;
+  private static int onlineTabletCount = 0;
+  private static long totalHoldTime = 0;
+  private static long totalLookups = 0;
+  private static int totalTables = 0;
+  
+  private static class MaxList<T> extends LinkedList<Pair<Long,T>> {
+    private static final long serialVersionUID = 1L;
+    
+    private long maxDelta;
+    
+    public MaxList(long maxDelta) {
+      this.maxDelta = maxDelta;
+    }
+    
+    @Override
+    public boolean add(Pair<Long,T> obj) {
+      boolean result = super.add(obj);
+      
+      if (obj.getFirst() - get(0).getFirst() > maxDelta)
+        remove(0);
+      
+      return result;
+    }
+    
+  }
+  
+  private static final int MAX_TIME_PERIOD = 60 * 60 * 1000;
+  private static final List<Pair<Long,Double>> loadOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+  private static final List<Pair<Long,Double>> ingestRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+  private static final List<Pair<Long,Double>> ingestByteRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+  private static final List<Pair<Long,Integer>> recoveriesOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
+  private static final List<Pair<Long,Integer>> minorCompactionsOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
+  private static final List<Pair<Long,Integer>> majorCompactionsOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
+  private static final List<Pair<Long,Double>> lookupsOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+  private static final List<Pair<Long,Integer>> queryRateOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
+  private static final List<Pair<Long,Integer>> scanRateOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
+  private static final List<Pair<Long,Double>> queryByteRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+  private static final List<Pair<Long,Double>> indexCacheHitRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+  private static final List<Pair<Long,Double>> dataCacheHitRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+  private static EventCounter lookupRateTracker = new EventCounter();
+  private static EventCounter indexCacheHitTracker = new EventCounter();
+  private static EventCounter indexCacheRequestTracker = new EventCounter();
+  private static EventCounter dataCacheHitTracker = new EventCounter();
+  private static EventCounter dataCacheRequestTracker = new EventCounter();
+  
+  private static volatile boolean fetching = false;
+  private static MasterMonitorInfo mmi;
+  private static Map<String,Map<ProblemType,Integer>> problemSummary = Collections.emptyMap();
+  private static Exception problemException;
+  private static GCStatus gcStatus;
+  
+  private static Instance instance;
+  
+  private static ServerConfiguration config;
+  
+  private static EmbeddedWebServer server;
+  
+  public static Map<String,Double> summarizeTableStats(MasterMonitorInfo mmi) {
+    Map<String,Double> compactingByTable = new HashMap<String,Double>();
+    if (mmi != null && mmi.tServerInfo != null) {
+      for (TabletServerStatus status : mmi.tServerInfo) {
+        if (status != null && status.tableMap != null) {
+          for (String table : status.tableMap.keySet()) {
+            Double holdTime = compactingByTable.get(table);
+            compactingByTable.put(table, Math.max(holdTime == null ? 0. : holdTime.doubleValue(), status.holdTime));
+          }
+        }
+      }
+    }
+    return compactingByTable;
+  }
+  
+  public static void add(TableInfo total, TableInfo more) {
+    if (total.minors == null)
+      total.minors = new Compacting();
+    if (total.majors == null)
+      total.majors = new Compacting();
+    if (total.scans == null)
+      total.scans = new Compacting();
+    if (more.minors != null) {
+      total.minors.running += more.minors.running;
+      total.minors.queued += more.minors.queued;
+    }
+    if (more.majors != null) {
+      total.majors.running += more.majors.running;
+      total.majors.queued += more.majors.queued;
+    }
+    if (more.scans != null) {
+      total.scans.running += more.scans.running;
+      total.scans.queued += more.scans.queued;
+    }
+    total.onlineTablets += more.onlineTablets;
+    total.recs += more.recs;
+    total.recsInMemory += more.recsInMemory;
+    total.tablets += more.tablets;
+    total.ingestRate += more.ingestRate;
+    total.ingestByteRate += more.ingestByteRate;
+    total.queryRate += more.queryRate;
+    total.queryByteRate += more.queryByteRate;
+    total.scanRate += more.scanRate;
+  }
+  
+  public static TableInfo summarizeTableStats(TabletServerStatus status) {
+    TableInfo summary = new TableInfo();
+    summary.majors = new Compacting();
+    summary.minors = new Compacting();
+    summary.scans = new Compacting();
+    for (TableInfo rates : status.tableMap.values()) {
+      add(summary, rates);
+    }
+    return summary;
+  }
+  
+  private static class EventCounter {
+    
+    Map<String,Pair<Long,Long>> prevSamples = new HashMap<String,Pair<Long,Long>>();
+    Map<String,Pair<Long,Long>> samples = new HashMap<String,Pair<Long,Long>>();
+    Set<String> serversUpdated = new HashSet<String>();
+    
+    void startingUpdates() {
+      serversUpdated.clear();
+    }
+    
+    void updateTabletServer(String name, long sampleTime, long numEvents) {
+      Pair<Long,Long> newSample = new Pair<Long,Long>(sampleTime, numEvents);
+      Pair<Long,Long> lastSample = samples.get(name);
+      
+      if (lastSample == null || !lastSample.equals(newSample)) {
+        samples.put(name, newSample);
+        if (lastSample != null) {
+          prevSamples.put(name, lastSample);
+        }
+      }
+      serversUpdated.add(name);
+    }
+    
+    void finishedUpdating() {
+      // remove any tablet servers not updated
+      samples.keySet().retainAll(serversUpdated);
+      prevSamples.keySet().retainAll(serversUpdated);
+    }
+    
+    double calculateRate() {
+      double totalRate = 0;
+      
+      for (Entry<String,Pair<Long,Long>> entry : prevSamples.entrySet()) {
+        Pair<Long,Long> prevSample = entry.getValue();
+        Pair<Long,Long> sample = samples.get(entry.getKey());
+        
+        totalRate += (sample.getSecond() - prevSample.getSecond()) / ((sample.getFirst() - prevSample.getFirst()) / (double) 1000);
+      }
+      
+      return totalRate;
+    }
+    
+    long calculateCount() {
+      long count = 0;
+      
+      for (Entry<String,Pair<Long,Long>> entry : prevSamples.entrySet()) {
+        Pair<Long,Long> prevSample = entry.getValue();
+        Pair<Long,Long> sample = samples.get(entry.getKey());
+        
+        count += sample.getSecond() - prevSample.getSecond();
+      }
+      
+      return count;
+    }
+  }
+  
+  public static void fetchData() {
+    double totalIngestRate = 0.;
+    double totalIngestByteRate = 0.;
+    double totalQueryRate = 0.;
+    double totalQueryByteRate = 0.;
+    double totalScanRate = 0.;
+    long totalEntries = 0;
+    int totalTabletCount = 0;
+    int onlineTabletCount = 0;
+    long totalHoldTime = 0;
+    long totalLookups = 0;
+    boolean retry = true;
+    
+    // only recalc every so often
+    long currentTime = System.currentTimeMillis();
+    if (currentTime - lastRecalc < REFRESH_TIME * 1000)
+      return;
+    
+    synchronized (Monitor.class) {
+      if (fetching)
+        return;
+      fetching = true;
+    }
+    
+    try {
+      while (retry) {
+        MasterClientService.Iface client = null;
+        try {
+          client = MasterClient.getConnection(HdfsZooInstance.getInstance());
+          if (client != null) {
+            mmi = client.getMasterStats(Tracer.traceInfo(), SystemCredentials.get().toThrift(HdfsZooInstance.getInstance()));
+            retry = false;
+          } else {
+            mmi = null;
+          }
+          Monitor.gcStatus = fetchGcStatus();
+        } catch (Exception e) {
+          mmi = null;
+          log.info("Error fetching stats: " + e);
+        } finally {
+          if (client != null) {
+            MasterClient.close(client);
+          }
+        }
+        if (mmi == null)
+          UtilWaitThread.sleep(1000);
+      }
+      if (mmi != null) {
+        int majorCompactions = 0;
+        int minorCompactions = 0;
+        
+        lookupRateTracker.startingUpdates();
+        indexCacheHitTracker.startingUpdates();
+        indexCacheRequestTracker.startingUpdates();
+        dataCacheHitTracker.startingUpdates();
+        dataCacheRequestTracker.startingUpdates();
+        
+        for (TabletServerStatus server : mmi.tServerInfo) {
+          TableInfo summary = Monitor.summarizeTableStats(server);
+          totalIngestRate += summary.ingestRate;
+          totalIngestByteRate += summary.ingestByteRate;
+          totalQueryRate += summary.queryRate;
+          totalScanRate += summary.scanRate;
+          totalQueryByteRate += summary.queryByteRate;
+          totalEntries += summary.recs;
+          totalHoldTime += server.holdTime;
+          totalLookups += server.lookups;
+          majorCompactions += summary.majors.running;
+          minorCompactions += summary.minors.running;
+          lookupRateTracker.updateTabletServer(server.name, server.lastContact, server.lookups);
+          indexCacheHitTracker.updateTabletServer(server.name, server.lastContact, server.indexCacheHits);
+          indexCacheRequestTracker.updateTabletServer(server.name, server.lastContact, server.indexCacheRequest);
+          dataCacheHitTracker.updateTabletServer(server.name, server.lastContact, server.dataCacheHits);
+          dataCacheRequestTracker.updateTabletServer(server.name, server.lastContact, server.dataCacheRequest);
+        }
+        
+        lookupRateTracker.finishedUpdating();
+        indexCacheHitTracker.finishedUpdating();
+        indexCacheRequestTracker.finishedUpdating();
+        dataCacheHitTracker.finishedUpdating();
+        dataCacheRequestTracker.finishedUpdating();
+        
+        int totalTables = 0;
+        for (TableInfo tInfo : mmi.tableMap.values()) {
+          totalTabletCount += tInfo.tablets;
+          onlineTabletCount += tInfo.onlineTablets;
+          totalTables++;
+        }
+        Monitor.totalIngestRate = totalIngestRate;
+        Monitor.totalTables = totalTables;
+        totalIngestByteRate = totalIngestByteRate / 1000000.0;
+        Monitor.totalIngestByteRate = totalIngestByteRate;
+        Monitor.totalQueryRate = totalQueryRate;
+        Monitor.totalScanRate = totalScanRate;
+        totalQueryByteRate = totalQueryByteRate / 1000000.0;
+        Monitor.totalQueryByteRate = totalQueryByteRate;
+        Monitor.totalEntries = totalEntries;
+        Monitor.totalTabletCount = totalTabletCount;
+        Monitor.onlineTabletCount = onlineTabletCount;
+        Monitor.totalHoldTime = totalHoldTime;
+        Monitor.totalLookups = totalLookups;
+        
+        ingestRateOverTime.add(new Pair<Long,Double>(currentTime, totalIngestRate));
+        ingestByteRateOverTime.add(new Pair<Long,Double>(currentTime, totalIngestByteRate));
+        
+        double totalLoad = 0.;
+        for (TabletServerStatus status : mmi.tServerInfo) {
+          if (status != null)
+            totalLoad += status.osLoad;
+        }
+        loadOverTime.add(new Pair<Long,Double>(currentTime, totalLoad));
+        
+        minorCompactionsOverTime.add(new Pair<Long,Integer>(currentTime, minorCompactions));
+        majorCompactionsOverTime.add(new Pair<Long,Integer>(currentTime, majorCompactions));
+        
+        lookupsOverTime.add(new Pair<Long,Double>(currentTime, lookupRateTracker.calculateRate()));
+        
+        queryRateOverTime.add(new Pair<Long,Integer>(currentTime, (int) totalQueryRate));
+        queryByteRateOverTime.add(new Pair<Long,Double>(currentTime, totalQueryByteRate));
+        
+        scanRateOverTime.add(new Pair<Long,Integer>(currentTime, (int) totalScanRate));
+        
+        calcCacheHitRate(indexCacheHitRateOverTime, currentTime, indexCacheHitTracker, indexCacheRequestTracker);
+        calcCacheHitRate(dataCacheHitRateOverTime, currentTime, dataCacheHitTracker, dataCacheRequestTracker);
+      }
+      try {
+        Monitor.problemSummary = ProblemReports.getInstance().summarize();
+        Monitor.problemException = null;
+      } catch (Exception e) {
+        log.info("Failed to obtain problem reports ", e);
+        Monitor.problemSummary = Collections.emptyMap();
+        Monitor.problemException = e;
+      }
+      
+    } finally {
+      synchronized (Monitor.class) {
+        fetching = false;
+        lastRecalc = currentTime;
+      }
+    }
+  }
+  
+  private static void calcCacheHitRate(List<Pair<Long,Double>> hitRate, long currentTime, EventCounter cacheHits, EventCounter cacheReq) {
+    long req = cacheReq.calculateCount();
+    if (req > 0)
+      hitRate.add(new Pair<Long,Double>(currentTime, cacheHits.calculateCount() / (double) cacheReq.calculateCount()));
+    else
+      hitRate.add(new Pair<Long,Double>(currentTime, null));
+  }
+  
+  private static GCStatus fetchGcStatus() {
+    GCStatus result = null;
+    InetSocketAddress address = null;
+    try {
+      // Read the gc location from its lock
+      Instance instance = HdfsZooInstance.getInstance();
+      String zooKeepers = instance.getZooKeepers();
+      log.debug("connecting to zookeepers " + zooKeepers);
+      ZooKeeper zk = new ZooKeeper(zooKeepers, (int) config.getConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT), new Watcher() {
+        @Override
+        public void process(WatchedEvent event) {}
+      });
+      try {
+        String path = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZGC_LOCK;
+        List<String> locks = zk.getChildren(path, null);
+        if (locks != null && locks.size() > 0) {
+          Collections.sort(locks);
+          address = new ServerServices(new String(zk.getData(path + "/" + locks.get(0), null, null))).getAddress(Service.GC_CLIENT);
+          GCMonitorService.Client client = ThriftUtil.getClient(new GCMonitorService.Client.Factory(), address, config.getConfiguration());
+          try {
+            result = client.getStatus(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance));
+          } finally {
+            ThriftUtil.returnClient(client);
+          }
+        }
+      } finally {
+        zk.close();
+      }
+    } catch (Exception ex) {
+      log.warn("Unable to contact the garbage collector at " + address, ex);
+    }
+    return result;
+  }
+  
+  public static void main(String[] args) throws Exception {
+    SecurityUtil.serverLogin();
+    
+    VolumeManager fs = VolumeManagerImpl.get();
+    ServerOpts opts = new ServerOpts();
+    opts.parseArgs("monitor", args);
+    String hostname = opts.getAddress();
+    instance = HdfsZooInstance.getInstance();
+    config = new ServerConfiguration(instance);
+    Accumulo.init(fs, config, "monitor");
+    Monitor monitor = new Monitor();
+    Accumulo.enableTracing(hostname, "monitor");
+    monitor.run(hostname);
+  }
+  
+  private static long START_TIME;
+  
+  public void run(String hostname) {
+    Monitor.START_TIME = System.currentTimeMillis();
+    int port = config.getConfiguration().getPort(Property.MONITOR_PORT);
+    try {
+      log.debug("Creating monitor on port " + port);
+      server = new EmbeddedWebServer(hostname, port);
+    } catch (Throwable ex) {
+      log.error("Unable to start embedded web server", ex);
+      throw new RuntimeException(ex);
+    }
+    
+    server.addServlet(DefaultServlet.class, "/");
+    server.addServlet(OperationServlet.class, "/op");
+    server.addServlet(MasterServlet.class, "/master");
+    server.addServlet(TablesServlet.class, "/tables");
+    server.addServlet(TServersServlet.class, "/tservers");
+    server.addServlet(ProblemServlet.class, "/problems");
+    server.addServlet(GcStatusServlet.class, "/gc");
+    server.addServlet(LogServlet.class, "/log");
+    server.addServlet(XMLServlet.class, "/xml");
+    server.addServlet(JSONServlet.class, "/json");
+    server.addServlet(VisServlet.class, "/vis");
+    server.addServlet(Summary.class, "/trace/summary");
+    server.addServlet(ListType.class, "/trace/listType");
+    server.addServlet(ShowTrace.class, "/trace/show");
+    if (server.isUsingSsl())
+      server.addServlet(ShellServlet.class, "/shell");
+    LogService.startLogListener(Monitor.getSystemConfiguration());
+    server.start();
+    
+    new Daemon(new LoggingRunnable(log, new ZooKeeperStatus()), "ZooKeeperStatus").start();
+    
+    // need to regularly fetch data so plot data is updated
+    new Daemon(new LoggingRunnable(log, new Runnable() {
+      
+      @Override
+      public void run() {
+        while (true) {
+          try {
+            Monitor.fetchData();
+          } catch (Exception e) {
+            log.warn(e.getMessage(), e);
+          }
+          
+          UtilWaitThread.sleep(333);
+        }
+        
+      }
+    }), "Data fetcher").start();
+  }
+  
+  public static MasterMonitorInfo getMmi() {
+    return mmi;
+  }
+  
+  public static int getTotalTables() {
+    return totalTables;
+  }
+  
+  public static int getTotalTabletCount() {
+    return totalTabletCount;
+  }
+  
+  public static int getOnlineTabletCount() {
+    return onlineTabletCount;
+  }
+  
+  public static long getTotalEntries() {
+    return totalEntries;
+  }
+  
+  public static double getTotalIngestRate() {
+    return totalIngestRate;
+  }
+  
+  public static double getTotalIngestByteRate() {
+    return totalIngestByteRate;
+  }
+  
+  public static double getTotalQueryRate() {
+    return totalQueryRate;
+  }
+  
+  public static double getTotalScanRate() {
+    return totalScanRate;
+  }
+  
+  public static double getTotalQueryByteRate() {
+    return totalQueryByteRate;
+  }
+  
+  public static long getTotalHoldTime() {
+    return totalHoldTime;
+  }
+  
+  public static Exception getProblemException() {
+    return problemException;
+  }
+  
+  public static Map<String,Map<ProblemType,Integer>> getProblemSummary() {
+    return problemSummary;
+  }
+  
+  public static GCStatus getGcStatus() {
+    return gcStatus;
+  }
+  
+  public static long getTotalLookups() {
+    return totalLookups;
+  }
+  
+  public static long getStartTime() {
+    return START_TIME;
+  }
+  
+  public static List<Pair<Long,Double>> getLoadOverTime() {
+    synchronized (loadOverTime) {
+      return new ArrayList<Pair<Long,Double>>(loadOverTime);
+    }
+  }
+  
+  public static List<Pair<Long,Double>> getIngestRateOverTime() {
+    synchronized (ingestRateOverTime) {
+      return new ArrayList<Pair<Long,Double>>(ingestRateOverTime);
+    }
+  }
+  
+  public static List<Pair<Long,Double>> getIngestByteRateOverTime() {
+    synchronized (ingestByteRateOverTime) {
+      return new ArrayList<Pair<Long,Double>>(ingestByteRateOverTime);
+    }
+  }
+  
+  public static List<Pair<Long,Integer>> getRecoveriesOverTime() {
+    synchronized (recoveriesOverTime) {
+      return new ArrayList<Pair<Long,Integer>>(recoveriesOverTime);
+    }
+  }
+  
+  public static List<Pair<Long,Integer>> getMinorCompactionsOverTime() {
+    synchronized (minorCompactionsOverTime) {
+      return new ArrayList<Pair<Long,Integer>>(minorCompactionsOverTime);
+    }
+  }
+  
+  public static List<Pair<Long,Integer>> getMajorCompactionsOverTime() {
+    synchronized (majorCompactionsOverTime) {
+      return new ArrayList<Pair<Long,Integer>>(majorCompactionsOverTime);
+    }
+  }
+  
+  public static List<Pair<Long,Double>> getLookupsOverTime() {
+    synchronized (lookupsOverTime) {
+      return new ArrayList<Pair<Long,Double>>(lookupsOverTime);
+    }
+  }
+  
+  public static double getLookupRate() {
+    return lookupRateTracker.calculateRate();
+  }
+  
+  public static List<Pair<Long,Integer>> getQueryRateOverTime() {
+    synchronized (queryRateOverTime) {
+      return new ArrayList<Pair<Long,Integer>>(queryRateOverTime);
+    }
+  }
+  
+  public static List<Pair<Long,Integer>> getScanRateOverTime() {
+    synchronized (scanRateOverTime) {
+      return new ArrayList<Pair<Long,Integer>>(scanRateOverTime);
+    }
+  }
+  
+  public static List<Pair<Long,Double>> getQueryByteRateOverTime() {
+    synchronized (queryByteRateOverTime) {
+      return new ArrayList<Pair<Long,Double>>(queryByteRateOverTime);
+    }
+  }
+  
+  public static List<Pair<Long,Double>> getIndexCacheHitRateOverTime() {
+    synchronized (indexCacheHitRateOverTime) {
+      return new ArrayList<Pair<Long,Double>>(indexCacheHitRateOverTime);
+    }
+  }
+  
+  public static List<Pair<Long,Double>> getDataCacheHitRateOverTime() {
+    synchronized (dataCacheHitRateOverTime) {
+      return new ArrayList<Pair<Long,Double>>(dataCacheHitRateOverTime);
+    }
+  }
+  
+  public static AccumuloConfiguration getSystemConfiguration() {
+    return config.getConfiguration();
+  }
+  
+  public static Instance getInstance() {
+    return instance;
+  }
+  
+  public static boolean isUsingSsl() {
+    return server.isUsingSsl();
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/e0533561/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
----------------------------------------------------------------------
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
new file mode 100644
index 0000000..4b56828
--- /dev/null
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor;
+
+import java.net.InetSocketAddress;
+import java.util.Collection;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.TTimeoutTransport;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.log4j.Logger;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+
+public class ZooKeeperStatus implements Runnable {
+  
+  private static final Logger log = Logger.getLogger(ZooKeeperStatus.class);
+  
+  private volatile boolean stop = false;
+  
+  public static class ZooKeeperState implements Comparable<ZooKeeperState> {
+    public final String keeper;
+    public final String mode;
+    public final int clients;
+    
+    public ZooKeeperState(String keeper, String mode, int clients) {
+      this.keeper = keeper;
+      this.mode = mode;
+      this.clients = clients;
+    }
+    
+    @Override
+    public int compareTo(ZooKeeperState other) {
+      if (this == other) {
+        return 0;
+      } else if (other == null) {
+        return 1;
+      } else {
+        if (this.keeper == other.keeper) {
+          return 0;
+        } else if (null == this.keeper) {
+          return -1;
+        } else if (null == other.keeper) {
+          return 1;
+        } else {
+          return this.keeper.compareTo(other.keeper);
+        }
+      }
+    }
+  }
+  
+  private static SortedSet<ZooKeeperState> status = new TreeSet<ZooKeeperState>();
+  
+  public static Collection<ZooKeeperState> getZooKeeperStatus() {
+    return status;
+  }
+  
+  public void stop() {
+    this.stop = true;
+  }
+  
+  @Override
+  public void run() {
+    
+    while (!stop) {
+      
+      TreeSet<ZooKeeperState> update = new TreeSet<ZooKeeperState>();
+      
+      String zookeepers[] = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST).split(",");
+      for (String keeper : zookeepers) {
+        int clients = 0;
+        String mode = "unknown";
+        
+        String[] parts = keeper.split(":");
+        TTransport transport = null;
+        try {
+          InetSocketAddress addr;
+          if (parts.length > 1)
+            addr = new InetSocketAddress(parts[0], Integer.parseInt(parts[1]));
+          else
+            addr = new InetSocketAddress(parts[0], 2181);
+          
+          transport = TTimeoutTransport.create(addr, 10 * 1000l);
+          transport.write("stat\n".getBytes(), 0, 5);
+          StringBuilder response = new StringBuilder();
+          try {
+            transport.flush();
+            byte[] buffer = new byte[1024 * 100];
+            int n = 0;
+            while ((n = transport.read(buffer, 0, buffer.length)) > 0) {
+              response.append(new String(buffer, 0, n));
+            }
+          } catch (TTransportException ex) {
+            // happens at EOF
+          }
+          for (String line : response.toString().split("\n")) {
+            if (line.startsWith(" "))
+              clients++;
+            if (line.startsWith("Mode"))
+              mode = line.split(":")[1];
+          }
+          update.add(new ZooKeeperState(keeper, mode, clients));
+        } catch (Exception ex) {
+          log.info("Exception talking to zookeeper " + keeper, ex);
+          update.add(new ZooKeeperState(keeper, "Down", -1));
+        } finally {
+          if (transport != null) {
+            try {
+              transport.close();
+            } catch (Exception ex) {
+              log.error(ex, ex);
+            }
+          }
+        }
+      }
+      status = update;
+      UtilWaitThread.sleep(1000);
+    }
+  }
+  
+}


Mime
View raw message