accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [40/48] Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Date Tue, 04 Feb 2014 17:55:09 GMT
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7688eaf0/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index 3a3692e,0000000..05ce1b2
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@@ -1,619 -1,0 +1,619 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.monitor;
 +
 +import java.net.InetAddress;
 +import java.util.ArrayList;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.LinkedList;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.impl.MasterClient;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.gc.thrift.GCMonitorService;
 +import org.apache.accumulo.core.gc.thrift.GCStatus;
 +import org.apache.accumulo.core.master.thrift.MasterClientService;
 +import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
 +import org.apache.accumulo.core.master.thrift.TableInfo;
 +import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 +import org.apache.accumulo.core.security.SecurityUtil;
 +import org.apache.accumulo.core.util.Daemon;
 +import org.apache.accumulo.core.util.LoggingRunnable;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.core.util.ServerServices;
 +import org.apache.accumulo.core.util.ServerServices.Service;
 +import org.apache.accumulo.core.util.ThriftUtil;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.core.zookeeper.ZooUtil;
 +import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
 +import org.apache.accumulo.monitor.servlets.DefaultServlet;
 +import org.apache.accumulo.monitor.servlets.GcStatusServlet;
 +import org.apache.accumulo.monitor.servlets.JSONServlet;
 +import org.apache.accumulo.monitor.servlets.LogServlet;
 +import org.apache.accumulo.monitor.servlets.MasterServlet;
 +import org.apache.accumulo.monitor.servlets.OperationServlet;
 +import org.apache.accumulo.monitor.servlets.ProblemServlet;
 +import org.apache.accumulo.monitor.servlets.ShellServlet;
 +import org.apache.accumulo.monitor.servlets.TServersServlet;
 +import org.apache.accumulo.monitor.servlets.TablesServlet;
 +import org.apache.accumulo.monitor.servlets.VisServlet;
 +import org.apache.accumulo.monitor.servlets.XMLServlet;
 +import org.apache.accumulo.monitor.servlets.trace.ListType;
 +import org.apache.accumulo.monitor.servlets.trace.ShowTrace;
 +import org.apache.accumulo.monitor.servlets.trace.Summary;
 +import org.apache.accumulo.server.Accumulo;
 +import org.apache.accumulo.server.ServerOpts;
 +import org.apache.accumulo.server.client.HdfsZooInstance;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.fs.VolumeManager;
 +import org.apache.accumulo.server.fs.VolumeManagerImpl;
 +import org.apache.accumulo.server.monitor.LogService;
 +import org.apache.accumulo.server.problems.ProblemReports;
 +import org.apache.accumulo.server.problems.ProblemType;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.util.TableInfoUtil;
 +import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 +import org.apache.accumulo.trace.instrument.Tracer;
 +import org.apache.log4j.Logger;
 +
 +import com.google.common.net.HostAndPort;
 +
 +/**
 + * Serve master statistics with an embedded web server.
 + */
 +public class Monitor {
 +  private static final Logger log = Logger.getLogger(Monitor.class);
 +  
 +  private static final int REFRESH_TIME = 5;
 +  private static long lastRecalc = 0L;
 +  private static double totalIngestRate = 0.0;
 +  private static double totalIngestByteRate = 0.0;
 +  private static double totalQueryRate = 0.0;
 +  private static double totalScanRate = 0.0;
 +  private static double totalQueryByteRate = 0.0;
 +  private static long totalEntries = 0L;
 +  private static int totalTabletCount = 0;
 +  private static int onlineTabletCount = 0;
 +  private static long totalHoldTime = 0;
 +  private static long totalLookups = 0;
 +  private static int totalTables = 0;
 +  
 +  private static class MaxList<T> extends LinkedList<Pair<Long,T>> {
 +    private static final long serialVersionUID = 1L;
 +    
 +    private long maxDelta;
 +    
 +    public MaxList(long maxDelta) {
 +      this.maxDelta = maxDelta;
 +    }
 +    
 +    @Override
 +    public boolean add(Pair<Long,T> obj) {
 +      boolean result = super.add(obj);
 +      
 +      if (obj.getFirst() - get(0).getFirst() > maxDelta)
 +        remove(0);
 +      
 +      return result;
 +    }
 +    
 +  }
 +  
 +  private static final int MAX_TIME_PERIOD = 60 * 60 * 1000;
 +  private static final List<Pair<Long,Double>> loadOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Double>> ingestRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Double>> ingestByteRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Integer>> recoveriesOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Integer>> minorCompactionsOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Integer>> majorCompactionsOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Double>> lookupsOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Integer>> queryRateOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Integer>> scanRateOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Double>> queryByteRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Double>> indexCacheHitRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
 +  private static final List<Pair<Long,Double>> dataCacheHitRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
 +  private static EventCounter lookupRateTracker = new EventCounter();
 +  private static EventCounter indexCacheHitTracker = new EventCounter();
 +  private static EventCounter indexCacheRequestTracker = new EventCounter();
 +  private static EventCounter dataCacheHitTracker = new EventCounter();
 +  private static EventCounter dataCacheRequestTracker = new EventCounter();
 +  
 +  private static volatile boolean fetching = false;
 +  private static MasterMonitorInfo mmi;
 +  private static Map<String,Map<ProblemType,Integer>> problemSummary = Collections.emptyMap();
 +  private static Exception problemException;
 +  private static GCStatus gcStatus;
 +  
 +  private static Instance instance;
 +  
 +  private static ServerConfiguration config;
 +  
 +  private static EmbeddedWebServer server;
 +  
 +  private static class EventCounter {
 +    
 +    Map<String,Pair<Long,Long>> prevSamples = new HashMap<String,Pair<Long,Long>>();
 +    Map<String,Pair<Long,Long>> samples = new HashMap<String,Pair<Long,Long>>();
 +    Set<String> serversUpdated = new HashSet<String>();
 +    
 +    void startingUpdates() {
 +      serversUpdated.clear();
 +    }
 +    
 +    void updateTabletServer(String name, long sampleTime, long numEvents) {
 +      Pair<Long,Long> newSample = new Pair<Long,Long>(sampleTime, numEvents);
 +      Pair<Long,Long> lastSample = samples.get(name);
 +      
 +      if (lastSample == null || !lastSample.equals(newSample)) {
 +        samples.put(name, newSample);
 +        if (lastSample != null) {
 +          prevSamples.put(name, lastSample);
 +        }
 +      }
 +      serversUpdated.add(name);
 +    }
 +    
 +    void finishedUpdating() {
 +      // remove any tablet servers not updated
 +      samples.keySet().retainAll(serversUpdated);
 +      prevSamples.keySet().retainAll(serversUpdated);
 +    }
 +    
 +    double calculateRate() {
 +      double totalRate = 0;
 +      
 +      for (Entry<String,Pair<Long,Long>> entry : prevSamples.entrySet()) {
 +        Pair<Long,Long> prevSample = entry.getValue();
 +        Pair<Long,Long> sample = samples.get(entry.getKey());
 +        
 +        totalRate += (sample.getSecond() - prevSample.getSecond()) / ((sample.getFirst() - prevSample.getFirst()) / (double) 1000);
 +      }
 +      
 +      return totalRate;
 +    }
 +    
 +    long calculateCount() {
 +      long count = 0;
 +      
 +      for (Entry<String,Pair<Long,Long>> entry : prevSamples.entrySet()) {
 +        Pair<Long,Long> prevSample = entry.getValue();
 +        Pair<Long,Long> sample = samples.get(entry.getKey());
 +        
 +        count += sample.getSecond() - prevSample.getSecond();
 +      }
 +      
 +      return count;
 +    }
 +  }
 +  
 +  public static void fetchData() {
 +    double totalIngestRate = 0.;
 +    double totalIngestByteRate = 0.;
 +    double totalQueryRate = 0.;
 +    double totalQueryByteRate = 0.;
 +    double totalScanRate = 0.;
 +    long totalEntries = 0;
 +    int totalTabletCount = 0;
 +    int onlineTabletCount = 0;
 +    long totalHoldTime = 0;
 +    long totalLookups = 0;
 +    boolean retry = true;
 +    
 +    // only recalc every so often
 +    long currentTime = System.currentTimeMillis();
 +    if (currentTime - lastRecalc < REFRESH_TIME * 1000)
 +      return;
 +    
 +    synchronized (Monitor.class) {
 +      if (fetching)
 +        return;
 +      fetching = true;
 +    }
 +    
 +    try {
 +      while (retry) {
 +        MasterClientService.Iface client = null;
 +        try {
 +          client = MasterClient.getConnection(HdfsZooInstance.getInstance());
 +          if (client != null) {
 +            mmi = client.getMasterStats(Tracer.traceInfo(), SystemCredentials.get().toThrift(HdfsZooInstance.getInstance()));
 +            retry = false;
 +          } else {
 +            mmi = null;
 +          }
 +          Monitor.gcStatus = fetchGcStatus();
 +        } catch (Exception e) {
 +          mmi = null;
 +          log.info("Error fetching stats: " + e);
 +        } finally {
 +          if (client != null) {
 +            MasterClient.close(client);
 +          }
 +        }
 +        if (mmi == null)
 +          UtilWaitThread.sleep(1000);
 +      }
 +      if (mmi != null) {
 +        int majorCompactions = 0;
 +        int minorCompactions = 0;
 +        
 +        lookupRateTracker.startingUpdates();
 +        indexCacheHitTracker.startingUpdates();
 +        indexCacheRequestTracker.startingUpdates();
 +        dataCacheHitTracker.startingUpdates();
 +        dataCacheRequestTracker.startingUpdates();
 +        
 +        for (TabletServerStatus server : mmi.tServerInfo) {
 +          TableInfo summary = TableInfoUtil.summarizeTableStats(server);
 +          totalIngestRate += summary.ingestRate;
 +          totalIngestByteRate += summary.ingestByteRate;
 +          totalQueryRate += summary.queryRate;
 +          totalScanRate += summary.scanRate;
 +          totalQueryByteRate += summary.queryByteRate;
 +          totalEntries += summary.recs;
 +          totalHoldTime += server.holdTime;
 +          totalLookups += server.lookups;
 +          majorCompactions += summary.majors.running;
 +          minorCompactions += summary.minors.running;
 +          lookupRateTracker.updateTabletServer(server.name, server.lastContact, server.lookups);
 +          indexCacheHitTracker.updateTabletServer(server.name, server.lastContact, server.indexCacheHits);
 +          indexCacheRequestTracker.updateTabletServer(server.name, server.lastContact, server.indexCacheRequest);
 +          dataCacheHitTracker.updateTabletServer(server.name, server.lastContact, server.dataCacheHits);
 +          dataCacheRequestTracker.updateTabletServer(server.name, server.lastContact, server.dataCacheRequest);
 +        }
 +        
 +        lookupRateTracker.finishedUpdating();
 +        indexCacheHitTracker.finishedUpdating();
 +        indexCacheRequestTracker.finishedUpdating();
 +        dataCacheHitTracker.finishedUpdating();
 +        dataCacheRequestTracker.finishedUpdating();
 +        
 +        int totalTables = 0;
 +        for (TableInfo tInfo : mmi.tableMap.values()) {
 +          totalTabletCount += tInfo.tablets;
 +          onlineTabletCount += tInfo.onlineTablets;
 +          totalTables++;
 +        }
 +        Monitor.totalIngestRate = totalIngestRate;
 +        Monitor.totalTables = totalTables;
 +        totalIngestByteRate = totalIngestByteRate / 1000000.0;
 +        Monitor.totalIngestByteRate = totalIngestByteRate;
 +        Monitor.totalQueryRate = totalQueryRate;
 +        Monitor.totalScanRate = totalScanRate;
 +        totalQueryByteRate = totalQueryByteRate / 1000000.0;
 +        Monitor.totalQueryByteRate = totalQueryByteRate;
 +        Monitor.totalEntries = totalEntries;
 +        Monitor.totalTabletCount = totalTabletCount;
 +        Monitor.onlineTabletCount = onlineTabletCount;
 +        Monitor.totalHoldTime = totalHoldTime;
 +        Monitor.totalLookups = totalLookups;
 +        
 +        ingestRateOverTime.add(new Pair<Long,Double>(currentTime, totalIngestRate));
 +        ingestByteRateOverTime.add(new Pair<Long,Double>(currentTime, totalIngestByteRate));
 +        
 +        double totalLoad = 0.;
 +        for (TabletServerStatus status : mmi.tServerInfo) {
 +          if (status != null)
 +            totalLoad += status.osLoad;
 +        }
 +        loadOverTime.add(new Pair<Long,Double>(currentTime, totalLoad));
 +        
 +        minorCompactionsOverTime.add(new Pair<Long,Integer>(currentTime, minorCompactions));
 +        majorCompactionsOverTime.add(new Pair<Long,Integer>(currentTime, majorCompactions));
 +        
 +        lookupsOverTime.add(new Pair<Long,Double>(currentTime, lookupRateTracker.calculateRate()));
 +        
 +        queryRateOverTime.add(new Pair<Long,Integer>(currentTime, (int) totalQueryRate));
 +        queryByteRateOverTime.add(new Pair<Long,Double>(currentTime, totalQueryByteRate));
 +        
 +        scanRateOverTime.add(new Pair<Long,Integer>(currentTime, (int) totalScanRate));
 +        
 +        calcCacheHitRate(indexCacheHitRateOverTime, currentTime, indexCacheHitTracker, indexCacheRequestTracker);
 +        calcCacheHitRate(dataCacheHitRateOverTime, currentTime, dataCacheHitTracker, dataCacheRequestTracker);
 +      }
 +      try {
 +        Monitor.problemSummary = ProblemReports.getInstance().summarize();
 +        Monitor.problemException = null;
 +      } catch (Exception e) {
 +        log.info("Failed to obtain problem reports ", e);
 +        Monitor.problemSummary = Collections.emptyMap();
 +        Monitor.problemException = e;
 +      }
 +      
 +    } finally {
 +      synchronized (Monitor.class) {
 +        fetching = false;
 +        lastRecalc = currentTime;
 +      }
 +    }
 +  }
 +  
 +  private static void calcCacheHitRate(List<Pair<Long,Double>> hitRate, long currentTime, EventCounter cacheHits, EventCounter cacheReq) {
 +    long req = cacheReq.calculateCount();
 +    if (req > 0)
 +      hitRate.add(new Pair<Long,Double>(currentTime, cacheHits.calculateCount() / (double) cacheReq.calculateCount()));
 +    else
 +      hitRate.add(new Pair<Long,Double>(currentTime, null));
 +  }
 +  
 +  private static GCStatus fetchGcStatus() {
 +    GCStatus result = null;
 +    HostAndPort address = null;
 +    try {
 +      // Read the gc location from its lock
 +      ZooReaderWriter zk = ZooReaderWriter.getInstance();
 +      String path = ZooUtil.getRoot(instance) + Constants.ZGC_LOCK;
 +      List<String> locks = zk.getChildren(path, null);
 +      if (locks != null && locks.size() > 0) {
 +        Collections.sort(locks);
-         address = new ServerServices(new String(zk.getData(path + "/" + locks.get(0), null))).getAddress(Service.GC_CLIENT);
++        address = new ServerServices(new String(zk.getData(path + "/" + locks.get(0), null), Constants.UTF8)).getAddress(Service.GC_CLIENT);
 +        GCMonitorService.Client client = ThriftUtil.getClient(new GCMonitorService.Client.Factory(), address, config.getConfiguration());
 +        try {
 +          result = client.getStatus(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance));
 +        } finally {
 +          ThriftUtil.returnClient(client);
 +        }
 +      }
 +    } catch (Exception ex) {
 +      log.warn("Unable to contact the garbage collector at " + address, ex);
 +    }
 +    return result;
 +  }
 +  
 +  public static void main(String[] args) throws Exception {
 +    SecurityUtil.serverLogin();
 +    
 +    VolumeManager fs = VolumeManagerImpl.get();
 +    ServerOpts opts = new ServerOpts();
 +    opts.parseArgs("monitor", args);
 +    String hostname = opts.getAddress();
 +    instance = HdfsZooInstance.getInstance();
 +    config = new ServerConfiguration(instance);
 +    Accumulo.init(fs, config, "monitor");
 +    Monitor monitor = new Monitor();
 +    Accumulo.enableTracing(hostname, "monitor");
 +    monitor.run(hostname);
 +  }
 +  
 +  private static long START_TIME;
 +  
 +  public void run(String hostname) {
 +    Monitor.START_TIME = System.currentTimeMillis();
 +    int port = config.getConfiguration().getPort(Property.MONITOR_PORT);
 +    try {
 +      log.debug("Creating monitor on port " + port);
 +      server = new EmbeddedWebServer(hostname, port);
 +    } catch (Throwable ex) {
 +      log.error("Unable to start embedded web server", ex);
 +      throw new RuntimeException(ex);
 +    }
 +    
 +    server.addServlet(DefaultServlet.class, "/");
 +    server.addServlet(OperationServlet.class, "/op");
 +    server.addServlet(MasterServlet.class, "/master");
 +    server.addServlet(TablesServlet.class, "/tables");
 +    server.addServlet(TServersServlet.class, "/tservers");
 +    server.addServlet(ProblemServlet.class, "/problems");
 +    server.addServlet(GcStatusServlet.class, "/gc");
 +    server.addServlet(LogServlet.class, "/log");
 +    server.addServlet(XMLServlet.class, "/xml");
 +    server.addServlet(JSONServlet.class, "/json");
 +    server.addServlet(VisServlet.class, "/vis");
 +    server.addServlet(Summary.class, "/trace/summary");
 +    server.addServlet(ListType.class, "/trace/listType");
 +    server.addServlet(ShowTrace.class, "/trace/show");
 +    if (server.isUsingSsl())
 +      server.addServlet(ShellServlet.class, "/shell");
 +    server.start();
 +    
 +    
 +    try {
 +      hostname = InetAddress.getLocalHost().getHostName();
 +      
 +      log.debug("Using " + hostname + " to advertise monitor location in ZooKeeper");
 +
 +      String monitorAddress = HostAndPort.fromParts(hostname, server.getPort()).toString();
 +
-       ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(instance) + Constants.ZMONITOR, monitorAddress.getBytes(),
++      ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(instance) + Constants.ZMONITOR, monitorAddress.getBytes(Constants.UTF8),
 +          NodeExistsPolicy.OVERWRITE);
 +      log.info("Set monitor address in zookeeper to " + monitorAddress);
 +    } catch (Exception ex) {
 +      log.error("Unable to set monitor address in zookeeper");
 +    }
 +    LogService.startLogListener(Monitor.getSystemConfiguration(), instance.getInstanceID());
 +    
 +    new Daemon(new LoggingRunnable(log, new ZooKeeperStatus()), "ZooKeeperStatus").start();
 +    
 +    // need to regularly fetch data so plot data is updated
 +    new Daemon(new LoggingRunnable(log, new Runnable() {
 +      
 +      @Override
 +      public void run() {
 +        while (true) {
 +          try {
 +            Monitor.fetchData();
 +          } catch (Exception e) {
 +            log.warn(e.getMessage(), e);
 +          }
 +          
 +          UtilWaitThread.sleep(333);
 +        }
 +        
 +      }
 +    }), "Data fetcher").start();
 +  }
 +  
 +  public static MasterMonitorInfo getMmi() {
 +    return mmi;
 +  }
 +  
 +  public static int getTotalTables() {
 +    return totalTables;
 +  }
 +  
 +  public static int getTotalTabletCount() {
 +    return totalTabletCount;
 +  }
 +  
 +  public static int getOnlineTabletCount() {
 +    return onlineTabletCount;
 +  }
 +  
 +  public static long getTotalEntries() {
 +    return totalEntries;
 +  }
 +  
 +  public static double getTotalIngestRate() {
 +    return totalIngestRate;
 +  }
 +  
 +  public static double getTotalIngestByteRate() {
 +    return totalIngestByteRate;
 +  }
 +  
 +  public static double getTotalQueryRate() {
 +    return totalQueryRate;
 +  }
 +  
 +  public static double getTotalScanRate() {
 +    return totalScanRate;
 +  }
 +  
 +  public static double getTotalQueryByteRate() {
 +    return totalQueryByteRate;
 +  }
 +  
 +  public static long getTotalHoldTime() {
 +    return totalHoldTime;
 +  }
 +  
 +  public static Exception getProblemException() {
 +    return problemException;
 +  }
 +  
 +  public static Map<String,Map<ProblemType,Integer>> getProblemSummary() {
 +    return problemSummary;
 +  }
 +  
 +  public static GCStatus getGcStatus() {
 +    return gcStatus;
 +  }
 +  
 +  public static long getTotalLookups() {
 +    return totalLookups;
 +  }
 +  
 +  public static long getStartTime() {
 +    return START_TIME;
 +  }
 +  
 +  public static List<Pair<Long,Double>> getLoadOverTime() {
 +    synchronized (loadOverTime) {
 +      return new ArrayList<Pair<Long,Double>>(loadOverTime);
 +    }
 +  }
 +  
 +  public static List<Pair<Long,Double>> getIngestRateOverTime() {
 +    synchronized (ingestRateOverTime) {
 +      return new ArrayList<Pair<Long,Double>>(ingestRateOverTime);
 +    }
 +  }
 +  
 +  public static List<Pair<Long,Double>> getIngestByteRateOverTime() {
 +    synchronized (ingestByteRateOverTime) {
 +      return new ArrayList<Pair<Long,Double>>(ingestByteRateOverTime);
 +    }
 +  }
 +  
 +  public static List<Pair<Long,Integer>> getRecoveriesOverTime() {
 +    synchronized (recoveriesOverTime) {
 +      return new ArrayList<Pair<Long,Integer>>(recoveriesOverTime);
 +    }
 +  }
 +  
 +  public static List<Pair<Long,Integer>> getMinorCompactionsOverTime() {
 +    synchronized (minorCompactionsOverTime) {
 +      return new ArrayList<Pair<Long,Integer>>(minorCompactionsOverTime);
 +    }
 +  }
 +  
 +  public static List<Pair<Long,Integer>> getMajorCompactionsOverTime() {
 +    synchronized (majorCompactionsOverTime) {
 +      return new ArrayList<Pair<Long,Integer>>(majorCompactionsOverTime);
 +    }
 +  }
 +  
 +  public static List<Pair<Long,Double>> getLookupsOverTime() {
 +    synchronized (lookupsOverTime) {
 +      return new ArrayList<Pair<Long,Double>>(lookupsOverTime);
 +    }
 +  }
 +  
 +  public static double getLookupRate() {
 +    return lookupRateTracker.calculateRate();
 +  }
 +  
 +  public static List<Pair<Long,Integer>> getQueryRateOverTime() {
 +    synchronized (queryRateOverTime) {
 +      return new ArrayList<Pair<Long,Integer>>(queryRateOverTime);
 +    }
 +  }
 +  
 +  public static List<Pair<Long,Integer>> getScanRateOverTime() {
 +    synchronized (scanRateOverTime) {
 +      return new ArrayList<Pair<Long,Integer>>(scanRateOverTime);
 +    }
 +  }
 +  
 +  public static List<Pair<Long,Double>> getQueryByteRateOverTime() {
 +    synchronized (queryByteRateOverTime) {
 +      return new ArrayList<Pair<Long,Double>>(queryByteRateOverTime);
 +    }
 +  }
 +  
 +  public static List<Pair<Long,Double>> getIndexCacheHitRateOverTime() {
 +    synchronized (indexCacheHitRateOverTime) {
 +      return new ArrayList<Pair<Long,Double>>(indexCacheHitRateOverTime);
 +    }
 +  }
 +  
 +  public static List<Pair<Long,Double>> getDataCacheHitRateOverTime() {
 +    synchronized (dataCacheHitRateOverTime) {
 +      return new ArrayList<Pair<Long,Double>>(dataCacheHitRateOverTime);
 +    }
 +  }
 +  
 +  public static AccumuloConfiguration getSystemConfiguration() {
 +    return config.getConfiguration();
 +  }
 +  
 +  public static Instance getInstance() {
 +    return instance;
 +  }
 +  
 +  public static boolean isUsingSsl() {
 +    return server.isUsingSsl();
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/7688eaf0/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
index 3cac98d,0000000..5e3e475
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/ZooKeeperStatus.java
@@@ -1,139 -1,0 +1,140 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.monitor;
 +
 +import java.util.Collection;
 +import java.util.SortedSet;
 +import java.util.TreeSet;
 +
++import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.util.TTimeoutTransport;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.log4j.Logger;
 +import org.apache.thrift.transport.TTransport;
 +import org.apache.thrift.transport.TTransportException;
 +
 +import com.google.common.net.HostAndPort;
 +
 +public class ZooKeeperStatus implements Runnable {
 +  
 +  private static final Logger log = Logger.getLogger(ZooKeeperStatus.class);
 +  
 +  private volatile boolean stop = false;
 +  
 +  public static class ZooKeeperState implements Comparable<ZooKeeperState> {
 +    public final String keeper;
 +    public final String mode;
 +    public final int clients;
 +    
 +    public ZooKeeperState(String keeper, String mode, int clients) {
 +      this.keeper = keeper;
 +      this.mode = mode;
 +      this.clients = clients;
 +    }
 +    
 +    @Override
 +    public int compareTo(ZooKeeperState other) {
 +      if (this == other) {
 +        return 0;
 +      } else if (other == null) {
 +        return 1;
 +      } else {
 +        if (this.keeper == other.keeper) {
 +          return 0;
 +        } else if (null == this.keeper) {
 +          return -1;
 +        } else if (null == other.keeper) {
 +          return 1;
 +        } else {
 +          return this.keeper.compareTo(other.keeper);
 +        }
 +      }
 +    }
 +  }
 +  
 +  private static SortedSet<ZooKeeperState> status = new TreeSet<ZooKeeperState>();
 +  
 +  public static Collection<ZooKeeperState> getZooKeeperStatus() {
 +    return status;
 +  }
 +  
 +  public void stop() {
 +    this.stop = true;
 +  }
 +  
 +  @Override
 +  public void run() {
 +    
 +    while (!stop) {
 +      
 +      TreeSet<ZooKeeperState> update = new TreeSet<ZooKeeperState>();
 +      
 +      String zookeepers[] = ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST).split(",");
 +      for (String keeper : zookeepers) {
 +        int clients = 0;
 +        String mode = "unknown";
 +        
 +        String[] parts = keeper.split(":");
 +        TTransport transport = null;
 +        try {
 +          HostAndPort addr;
 +          if (parts.length > 1)
 +            addr = HostAndPort.fromParts(parts[0], Integer.parseInt(parts[1]));
 +          else
 +            addr = HostAndPort.fromParts(parts[0], 2181);
 +          
 +          transport = TTimeoutTransport.create(addr, 10 * 1000l);
-           transport.write("stat\n".getBytes(), 0, 5);
++          transport.write("stat\n".getBytes(Constants.UTF8), 0, 5);
 +          StringBuilder response = new StringBuilder();
 +          try {
 +            transport.flush();
 +            byte[] buffer = new byte[1024 * 100];
 +            int n = 0;
 +            while ((n = transport.read(buffer, 0, buffer.length)) > 0) {
-               response.append(new String(buffer, 0, n));
++              response.append(new String(buffer, 0, n, Constants.UTF8));
 +            }
 +          } catch (TTransportException ex) {
 +            // happens at EOF
 +          }
 +          for (String line : response.toString().split("\n")) {
 +            if (line.startsWith(" "))
 +              clients++;
 +            if (line.startsWith("Mode"))
 +              mode = line.split(":")[1];
 +          }
 +          update.add(new ZooKeeperState(keeper, mode, clients));
 +        } catch (Exception ex) {
 +          log.info("Exception talking to zookeeper " + keeper, ex);
 +          update.add(new ZooKeeperState(keeper, "Down", -1));
 +        } finally {
 +          if (transport != null) {
 +            try {
 +              transport.close();
 +            } catch (Exception ex) {
 +              log.error(ex, ex);
 +            }
 +          }
 +        }
 +      }
 +      status = update;
 +      UtilWaitThread.sleep(5 * 1000);
 +    }
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/7688eaf0/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
index f6dd461,0000000..4a7838a
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
@@@ -1,331 -1,0 +1,332 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.monitor.servlets;
 +
 +import java.io.IOException;
 +import java.io.InputStream;
 +import java.io.PrintStream;
 +import java.util.ArrayList;
 +import java.util.Arrays;
 +import java.util.Calendar;
 +import java.util.Date;
 +import java.util.List;
 +import java.util.TimeZone;
 +
 +import javax.servlet.ServletException;
 +import javax.servlet.ServletOutputStream;
 +import javax.servlet.http.HttpServletRequest;
 +import javax.servlet.http.HttpServletResponse;
 +
++import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.conf.DefaultConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.file.FileUtil;
 +import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
 +import org.apache.accumulo.core.util.CachedConfiguration;
 +import org.apache.accumulo.core.util.Duration;
 +import org.apache.accumulo.core.util.NumUtil;
 +import org.apache.accumulo.core.util.Pair;
 +import org.apache.accumulo.monitor.Monitor;
 +import org.apache.accumulo.monitor.ZooKeeperStatus;
 +import org.apache.accumulo.monitor.ZooKeeperStatus.ZooKeeperState;
 +import org.apache.accumulo.monitor.util.celltypes.NumberType;
 +import org.apache.accumulo.server.conf.ServerConfiguration;
 +import org.apache.accumulo.server.trace.TraceFileSystem;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.ContentSummary;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +
 +public class DefaultServlet extends BasicServlet {
 +
 +  private static final long serialVersionUID = 1L;
 +
 +  @Override
 +  protected String getTitle(HttpServletRequest req) {
 +    return req.getRequestURI().startsWith("/docs") ? "Documentation" : "Accumulo Overview";
 +  }
 +
 +  private void getResource(HttpServletRequest req, HttpServletResponse resp) throws IOException {
 +    try {
 +      String path = req.getRequestURI();
 +
 +      if (path.endsWith(".jpg"))
 +        resp.setContentType("image/jpeg");
 +
 +      if (path.endsWith(".html"))
 +        resp.setContentType("text/html");
 +
 +      path = path.substring(1);
 +      InputStream data = BasicServlet.class.getClassLoader().getResourceAsStream(path);
 +      ServletOutputStream out = resp.getOutputStream();
 +      try {
 +        if (data != null) {
 +          byte[] buffer = new byte[1024];
 +          int n;
 +          while ((n = data.read(buffer)) > 0)
 +            out.write(buffer, 0, n);
 +        } else {
-           out.write(("could not get resource " + path + "").getBytes());
++          out.write(("could not get resource " + path + "").getBytes(Constants.UTF8));
 +        }
 +      } finally {
 +        if (data != null)
 +          data.close();
 +      }
 +    } catch (Throwable t) {
 +      log.error(t, t);
 +      throw new IOException(t);
 +    }
 +  }
 +
 +  @Override
 +  public void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
 +    if (req.getRequestURI().startsWith("/web"))
 +      getResource(req, resp);
 +    else if (req.getRequestURI().equals("/docs") || req.getRequestURI().equals("/docs/apidocs"))
 +      super.doGet(req, resp);
 +    else if (req.getRequestURI().equals("/docs/config.html"))
 +      new DefaultConfiguration() {
 +
 +        public void generate(HttpServletResponse resp) throws IOException {
 +          generateDocumentation(new PrintStream(resp.getOutputStream()));
 +
 +        }
 +      }.generate(resp);
 +    else if (req.getRequestURI().startsWith("/docs"))
 +      getResource(req, resp);
 +    else if (req.getRequestURI().startsWith("/monitor"))
 +      resp.sendRedirect("/master");
 +    else if (req.getRequestURI().startsWith("/errors"))
 +      resp.sendRedirect("/problems");
 +    else
 +      super.doGet(req, resp);
 +  }
 +
 +  public static final int GRAPH_WIDTH = 450;
 +  public static final int GRAPH_HEIGHT = 150;
 +
 +  private static void plotData(StringBuilder sb, String title, @SuppressWarnings("rawtypes") List data, boolean points) {
 +    plotData(sb, title, points, new ArrayList<String>(), data);
 +  }
 +
 +  @SuppressWarnings("rawtypes")
 +  private static void plotData(StringBuilder sb, String title, boolean points, List<String> labels, List... series) {
 +    sb.append("<div class=\"plotHeading\">");
 +    sb.append(title);
 +    sb.append("</div>");
 +    sb.append("</br>");
 +    String id = "c" + title.hashCode();
 +    sb.append("<div id=\"" + id + "\" style=\"width:" + GRAPH_WIDTH + "px;height:" + GRAPH_HEIGHT + "px;\"></div>\n");
 +
 +    sb.append("<script type=\"text/javascript\">\n");
 +    sb.append("$(function () {\n");
 +
 +    for (int i = 0; i < series.length; i++) {
 +
 +      @SuppressWarnings("unchecked")
 +      List<Pair<Long,? extends Number>> data = series[i];
 +      sb.append("    var d" + i + " = [");
 +
 +      String sep = "";
 +      for (Pair<Long,? extends Number> point : data) {
 +        if (point.getSecond() == null)
 +          continue;
 +
 +        String y;
 +        if (point.getSecond() instanceof Double)
 +          y = String.format("%1.2f", point.getSecond());
 +        else
 +          y = point.getSecond().toString();
 +
 +        sb.append(sep);
 +        sep = ",";
 +        sb.append("[" + utc2local(point.getFirst()) + "," + y + "]");
 +      }
 +      sb.append("    ];\n");
 +    }
 +
 +    String opts = "lines: { show: true }";
 +    if (points)
 +      opts = "points: { show: true, radius: 1 }";
 +
 +    sb.append("    $.plot($(\"#" + id + "\"),");
 +    String sep = "";
 +
 +    String colors[] = new String[] {"red", "blue", "green", "black"};
 +
 +    sb.append("[");
 +    for (int i = 0; i < series.length; i++) {
 +      sb.append(sep);
 +      sep = ",";
 +      sb.append("{ ");
 +      if (labels.size() > 0) {
 +        sb.append("label: \"" + labels.get(i) + "\", ");
 +      }
 +      sb.append("data: d" + i + ", " + opts + ", color:\"" + colors[i] + "\" }");
 +    }
 +    sb.append("], ");
 +    sb.append("{yaxis:{}, xaxis:{mode:\"time\",minTickSize: [1, \"minute\"],timeformat: \"%H:%M<br />" + getShortTZName() + "\", ticks:3}});");
 +    sb.append("   });\n");
 +    sb.append("</script>\n");
 +  }
 +
 +  /**
 +   * Shows the current time zone (based on the current time) short name
 +   */
 +  private static String getShortTZName() {
 +    TimeZone tz = TimeZone.getDefault();
 +    return tz.getDisplayName(tz.inDaylightTime(new Date()), TimeZone.SHORT);
 +  }
 +
 +  /**
 +   * Converts a unix timestamp in UTC to one that is relative to the local timezone
 +   */
 +  private static Long utc2local(Long utcMillis) {
 +    Calendar currentCalendar = Calendar.getInstance(); // default timezone
 +    currentCalendar.setTimeInMillis(utcMillis + currentCalendar.getTimeZone().getOffset(utcMillis));
 +    return currentCalendar.getTime().getTime();
 +  }
 +
 +  @Override
 +  protected void pageBody(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) throws IOException {
 +    if (req.getRequestURI().equals("/docs") || req.getRequestURI().equals("/docs/apidocs")) {
 +      sb.append("<object data='").append(req.getRequestURI()).append("/index.html' type='text/html' width='100%' height='100%'></object>");
 +      return;
 +    }
 +
 +    sb.append("<table class='noborder'>\n");
 +    sb.append("<tr>\n");
 +
 +    sb.append("<td class='noborder'>\n");
 +    doAccumuloTable(sb);
 +    sb.append("</td>\n");
 +
 +    sb.append("<td class='noborder'>\n");
 +    doZooKeeperTable(sb);
 +    sb.append("</td>\n");
 +
 +    sb.append("</tr></table>\n");
 +    sb.append("<br/>\n");
 +
 +    sb.append("<p/><table class=\"noborder\">\n");
 +
 +    sb.append("<tr><td>\n");
 +    plotData(sb, "Ingest (Entries/s)", Monitor.getIngestRateOverTime(), false);
 +    sb.append("</td><td>\n");
 +    plotData(sb, "Scan (Entries/s)", false, Arrays.asList("Read", "Returned"), Monitor.getScanRateOverTime(), Monitor.getQueryRateOverTime());
 +    sb.append("</td></tr>\n");
 +
 +    sb.append("<tr><td>\n");
 +    plotData(sb, "Ingest (MB/s)", Monitor.getIngestByteRateOverTime(), false);
 +    sb.append("</td><td>\n");
 +    plotData(sb, "Scan (MB/s)", Monitor.getQueryByteRateOverTime(), false);
 +    sb.append("</td></tr>\n");
 +
 +    sb.append("<tr><td>\n");
 +    plotData(sb, "Load Average", Monitor.getLoadOverTime(), false);
 +    sb.append("</td><td>\n");
 +    plotData(sb, "Seeks", Monitor.getLookupsOverTime(), false);
 +    sb.append("</td></tr>\n");
 +
 +    sb.append("<tr><td>\n");
 +    plotData(sb, "Minor Compactions", Monitor.getMinorCompactionsOverTime(), false);
 +    sb.append("</td><td>\n");
 +    plotData(sb, "Major Compactions", Monitor.getMajorCompactionsOverTime(), false);
 +    sb.append("</td></tr>\n");
 +
 +    sb.append("<tr><td>\n");
 +    plotData(sb, "Index Cache Hit Rate", Monitor.getIndexCacheHitRateOverTime(), true);
 +    sb.append("</td><td>\n");
 +    plotData(sb, "Data Cache Hit Rate", Monitor.getDataCacheHitRateOverTime(), true);
 +    sb.append("</td></tr>\n");
 +
 +    sb.append("</table>\n");
 +  }
 +
 +  private void doAccumuloTable(StringBuilder sb) throws IOException {
 +    // Accumulo
 +    Configuration conf = CachedConfiguration.getInstance();
 +    FileSystem fs = TraceFileSystem.wrap(FileUtil.getFileSystem(conf, ServerConfiguration.getSiteConfiguration()));
 +    MasterMonitorInfo info = Monitor.getMmi();
 +    sb.append("<table>\n");
 +    sb.append("<tr><th colspan='2'><a href='/master'>Accumulo Master</a></th></tr>\n");
 +    if (info == null) {
 +      sb.append("<tr><td colspan='2'><span class='error'>Master is Down</span></td></tr>\n");
 +    } else {
 +      String consumed = "Unknown";
 +      String diskUsed = "Unknown";
 +      try {
 +        Path path = new Path(Monitor.getSystemConfiguration().get(Property.INSTANCE_DFS_DIR));
 +        log.debug("Reading the content summary for " + path);
 +        try {
 +          ContentSummary acu = fs.getContentSummary(path);
 +          diskUsed = bytes(acu.getSpaceConsumed());
 +          ContentSummary rootSummary = fs.getContentSummary(new Path("/"));
 +          consumed = String.format("%.2f%%", acu.getSpaceConsumed() * 100. / rootSummary.getSpaceConsumed());
 +        } catch (Exception ex) {
 +          log.trace("Unable to get disk usage information from hdfs", ex);
 +        }
 +
 +        boolean highlight = false;
 +        tableRow(sb, (highlight = !highlight), "Disk&nbsp;Used", diskUsed);
 +        if (fs.getUsed() != 0)
 +          tableRow(sb, (highlight = !highlight), "%&nbsp;of&nbsp;Used&nbsp;DFS", consumed);
 +        tableRow(sb, (highlight = !highlight), "<a href='/tables'>Tables</a>", NumberType.commas(Monitor.getTotalTables()));
 +        tableRow(sb, (highlight = !highlight), "<a href='/tservers'>Tablet&nbsp;Servers</a>", NumberType.commas(info.tServerInfo.size(), 1, Long.MAX_VALUE));
 +        tableRow(sb, (highlight = !highlight), "<a href='/tservers'>Dead&nbsp;Tablet&nbsp;Servers</a>", NumberType.commas(info.deadTabletServers.size(), 0, 0));
 +        tableRow(sb, (highlight = !highlight), "Tablets", NumberType.commas(Monitor.getTotalTabletCount(), 1, Long.MAX_VALUE));
 +        tableRow(sb, (highlight = !highlight), "Entries", NumberType.commas(Monitor.getTotalEntries()));
 +        tableRow(sb, (highlight = !highlight), "Lookups", NumberType.commas(Monitor.getTotalLookups()));
 +        tableRow(sb, (highlight = !highlight), "Uptime", Duration.format(System.currentTimeMillis() - Monitor.getStartTime()));
 +      } catch (Exception e) {
 +        log.debug(e, e);
 +      }
 +    }
 +    sb.append("</table>\n");
 +  }
 +
 +  private void doZooKeeperTable(StringBuilder sb) throws IOException {
 +    // Zookeepers
 +    sb.append("<table>\n");
 +    sb.append("<tr><th colspan='3'>Zookeeper</th></tr>\n");
 +    sb.append("<tr><th>Server</th><th>Mode</th><th>Clients</th></tr>\n");
 +
 +    boolean highlight = false;
 +    for (ZooKeeperState k : ZooKeeperStatus.getZooKeeperStatus()) {
 +      if (k.clients >= 0) {
 +        tableRow(sb, (highlight = !highlight), k.keeper, k.mode, k.clients);
 +      } else {
 +        tableRow(sb, false, k.keeper, "<span class='error'>Down</span>", "");
 +      }
 +    }
 +    sb.append("</table>\n");
 +  }
 +
 +  private static String bytes(long big) {
 +    return NumUtil.bigNumberForSize(big);
 +  }
 +
 +  public static void tableRow(StringBuilder sb, boolean highlight, Object... cells) {
 +    sb.append(highlight ? "<tr class='highlight'>" : "<tr>");
 +    for (int i = 0; i < cells.length; ++i) {
 +      Object cell = cells[i];
 +      String cellValue = cell == null ? "" : String.valueOf(cell).trim();
 +      sb.append("<td class='").append(i < cells.length - 1 ? "left" : "right").append("'>").append(cellValue.isEmpty() ? "-" : cellValue).append("</td>");
 +    }
 +    sb.append("</tr>\n");
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/7688eaf0/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/LogServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/LogServlet.java
index 122f154,0000000..2ea7881
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/LogServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/LogServlet.java
@@@ -1,106 -1,0 +1,106 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.monitor.servlets;
 +
 +import java.text.SimpleDateFormat;
 +
 +import javax.servlet.http.HttpServletRequest;
 +import javax.servlet.http.HttpServletResponse;
 +
 +import org.apache.accumulo.monitor.util.Table;
 +import org.apache.accumulo.monitor.util.celltypes.DateTimeType;
 +import org.apache.accumulo.monitor.util.celltypes.StringType;
 +import org.apache.accumulo.server.monitor.DedupedLogEvent;
 +import org.apache.accumulo.server.monitor.LogService;
 +import org.apache.log4j.Level;
 +import org.apache.log4j.spi.LoggingEvent;
 +
 +public class LogServlet extends BasicServlet {
 +  
 +  private static final long serialVersionUID = 1L;
 +  
 +  @Override
 +  protected String getTitle(HttpServletRequest req) {
 +    return "Recent Logs";
 +  }
 +  
 +  @Override
 +  protected void pageBody(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb) {
 +    boolean clear = true;
 +    SimpleDateFormat fmt = new SimpleDateFormat("dd HH:mm:ss,SSSS");
 +    Table logTable = new Table("logTable", "Recent&nbsp;Logs");
 +    logTable.addSortableColumn("Time", new DateTimeType(fmt), null);
 +    logTable.addSortableColumn("Application");
 +    logTable.addSortableColumn("Count");
 +    logTable.addSortableColumn("Level", new LogLevelType(), null);
 +    logTable.addSortableColumn("Message");
 +    for (DedupedLogEvent dev : LogService.getInstance().getEvents()) {
 +      clear = false;
 +      LoggingEvent ev = dev.getEvent();
 +      Object application = ev.getMDC("application");
 +      if (application == null)
 +        application = "";
 +      String msg = ev.getMessage().toString();
 +      StringBuilder text = new StringBuilder();
 +      for (int i = 0; i < msg.length(); i++) {
 +        char c = msg.charAt(i);
 +        switch (Character.getType(c)) {
 +          case Character.UNASSIGNED:
 +          case Character.LINE_SEPARATOR:
 +          case Character.NON_SPACING_MARK:
 +          case Character.PRIVATE_USE:
 +            c = '?';
 +          default:
 +            text.append(c);
 +        }
 +        
 +      }
-       msg = text.toString();
++      StringBuilder builder = new StringBuilder(text.toString());
 +      if (ev.getThrowableStrRep() != null)
 +        for (String line : ev.getThrowableStrRep())
-           msg += "\n\t" + line;
-       msg = sanitize(msg.trim());
++          builder.append("\n\t").append(line);
++      msg = sanitize(builder.toString().trim());
 +      msg = "<pre class='logevent'>" + msg + "</pre>";
 +      logTable.addRow(ev.getTimeStamp(), application, dev.getCount(), ev.getLevel(), msg);
 +    }
 +    if (!clear)
 +      logTable.setSubCaption("<a href='/op?action=clearLog&redir=" + currentPage(req) + "'>Clear&nbsp;All&nbsp;Events</a>");
 +    logTable.generate(req, sb);
 +    if (!clear)
 +      sb.append("<div class='center'><a href='/op?action=clearLog&redir=").append(currentPage(req)).append("'>Clear&nbsp;All&nbsp;Events</a></div>\n");
 +  }
 +  
 +  private static class LogLevelType extends StringType<Level> {
 +    @Override
 +    public String alignment() {
 +      return "center";
 +    }
 +    
 +    @Override
 +    public String format(Object obj) {
 +      if (obj == null)
 +        return "-";
 +      Level l = (Level) obj;
 +      if (l.equals(Level.ERROR) || l.equals(Level.FATAL))
 +        return "<div class='error'>" + l.toString() + "</div>";
 +      else if (l.equals(Level.WARN))
 +        return "<div class='warning'>" + l.toString() + "</div>";
 +      else
 +        return l.toString();
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/7688eaf0/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
index 881b294,0000000..03f6831
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
@@@ -1,331 -1,0 +1,333 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.monitor.servlets;
 +
 +import java.io.IOException;
 +import java.io.InputStream;
 +import java.io.OutputStream;
++import java.io.OutputStreamWriter;
 +import java.io.PrintWriter;
 +import java.util.HashMap;
 +import java.util.Map;
 +import java.util.concurrent.ExecutorService;
 +import java.util.concurrent.Executors;
 +
 +import javax.servlet.ServletException;
 +import javax.servlet.http.HttpServletRequest;
 +import javax.servlet.http.HttpServletResponse;
 +import javax.servlet.http.HttpSession;
 +
 +import jline.console.ConsoleReader;
 +
++import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.util.shell.Shell;
 +
 +public class ShellServlet extends BasicServlet {
 +  private static final long serialVersionUID = 1L;
 +  private Map<String,ShellExecutionThread> userShells = new HashMap<String,ShellExecutionThread>();
 +  private ExecutorService service = Executors.newCachedThreadPool();
 +  
 +  @Override
 +  protected String getTitle(HttpServletRequest req) {
 +    return "Shell";
 +  }
 +  
 +  @Override
 +  protected void pageBody(HttpServletRequest req, HttpServletResponse response, StringBuilder sb) throws IOException {
 +    HttpSession session = req.getSession(true);
 +    String user = (String) session.getAttribute("user");
 +    if (user == null) {
 +      // user attribute is null, check to see if username and password are passed as parameters
 +      user = req.getParameter("user");
 +      String pass = req.getParameter("pass");
 +      String mock = req.getParameter("mock");
 +      if (user == null || pass == null) {
 +        // username or password are null, re-authenticate
 +        sb.append(authenticationForm(req.getRequestURI()));
 +        return;
 +      }
 +      try {
 +        // get a new shell for this user
 +        ShellExecutionThread shellThread = new ShellExecutionThread(user, pass, mock);
 +        service.submit(shellThread);
 +        userShells.put(session.getId(), shellThread);
 +      } catch (IOException e) {
 +        // error validating user, reauthenticate
 +        sb.append("<div id='loginError'>Invalid user/password</div>" + authenticationForm(req.getRequestURI()));
 +        return;
 +      }
 +      session.setAttribute("user", user);
 +    }
 +    if (!userShells.containsKey(session.getId())) {
 +      // no existing shell for this user, re-authenticate
 +      sb.append(authenticationForm(req.getRequestURI()));
 +      return;
 +    }
 +    ShellExecutionThread shellThread = userShells.get(session.getId());
 +    shellThread.getOutput();
 +    shellThread.printInfo();
 +    sb.append("<div id='shell'>\n");
 +    sb.append("<pre id='shellResponse'>").append(shellThread.getOutput()).append("</pre>\n");
 +    sb.append("<form><span id='shellPrompt'>").append(shellThread.getPrompt());
 +    sb.append("</span><input type='text' name='cmd' id='cmd' onkeydown='return handleKeyDown(event.keyCode);'>\n");
 +    sb.append("</form>\n</div>\n");
 +    sb.append("<script type='text/javascript'>\n");
 +    sb.append("var url = '").append(req.getRequestURL().toString()).append("';\n");
 +    sb.append("var xmlhttp = new XMLHttpRequest();\n");
 +    sb.append("var hsize = 1000;\n");
 +    sb.append("var hindex = 0;\n");
 +    sb.append("var history = new Array();\n");
 +    sb.append("\n");
 +    sb.append("function handleKeyDown(keyCode) {\n");
 +    sb.append("  if (keyCode==13) {\n");
 +    sb.append("    submitCmd(document.getElementById('cmd').value);\n");
 +    sb.append("    hindex = history.length;\n");
 +    sb.append("    return false;\n");
 +    sb.append("  } else if (keyCode==38) {\n");
 +    sb.append("    hindex = hindex==0 ? history.length : hindex - 1;\n");
 +    sb.append("    if (hindex == history.length)\n");
 +    sb.append("      document.getElementById('cmd').value = '';\n");
 +    sb.append("    else\n");
 +    sb.append("      document.getElementById('cmd').value = history[hindex];\n");
 +    sb.append("    return false;\n");
 +    sb.append("  } else if (keyCode==40) {\n");
 +    sb.append("    hindex = hindex==history.length ? history.length : hindex + 1;\n");
 +    sb.append("    if (hindex == history.length)\n");
 +    sb.append("      document.getElementById('cmd').value = '';\n");
 +    sb.append("    else\n");
 +    sb.append("      document.getElementById('cmd').value = history[hindex];\n");
 +    sb.append("    return false;\n");
 +    sb.append("  }\n");
 +    sb.append("  return true;\n");
 +    sb.append("}\n");
 +    sb.append("\n");
 +    sb.append("function submitCmd(cmd) {\n");
 +    sb.append("  if (cmd=='history') {\n");
 +    sb.append("    document.getElementById('shellResponse').innerHTML += document.getElementById('shellPrompt').innerHTML+cmd+'\\n';\n");
 +    sb.append("    document.getElementById('shellResponse').innerHTML += history.join('\\n');\n");
 +    sb.append("    return\n");
 +    sb.append("  }\n");
 +    sb.append("  xmlhttp.open('POST',url+'?cmd='+cmd,false);\n");
 +    sb.append("  xmlhttp.send();\n");
 +    sb.append("  var text = xmlhttp.responseText;\n");
 +    sb.append("  var index = text.lastIndexOf('\\n');\n");
 +    sb.append("  if (index >= 0) {\n");
 +    sb.append("    if (index > 0 && document.getElementById('cmd').type == 'text') {\n");
 +    sb.append("      if (history.length == hsize)\n");
 +    sb.append("        history.shift()\n");
 +    sb.append("      history.push(cmd)\n");
 +    sb.append("    }\n");
 +    sb.append("    if (text.charAt(text.length-1)=='*') {\n");
 +    sb.append("      document.getElementById('cmd').type = 'password';\n");
 +    sb.append("      text = text.substring(0,xmlhttp.responseText.length-2);\n");
 +    sb.append("    } else {\n");
 +    sb.append("      document.getElementById('cmd').type = 'text';\n");
 +    sb.append("    }\n");
 +    sb.append("    document.getElementById('shellResponse').innerHTML += text.substring(0,index+1);\n");
 +    sb.append("    document.getElementById('shellPrompt').innerHTML = text.substring(index+1);\n");
 +    sb.append("    document.getElementById('cmd').value = '';\n");
 +    sb.append("    document.getElementById('shell').scrollTop = document.getElementById('cmd').offsetTop;\n");
 +    sb.append("  } else {\n");
 +    sb.append("    window.location = url;\n");
 +    sb.append("  }\n");
 +    sb.append("}\n");
 +    sb.append("</script>\n");
 +    sb.append("<script type='text/javascript'>window.onload = function() { document.getElementById('cmd').select(); }</script>\n");
 +  }
 +  
 +  @Override
 +  protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
 +    HttpSession session = req.getSession(true);
 +    String user = (String) session.getAttribute("user");
 +    if (user == null || !userShells.containsKey(session.getId())) {
 +      // no existing shell for user, re-authenticate
 +      doGet(req, resp);
 +      return;
 +    }
 +    ShellExecutionThread shellThread = userShells.get(session.getId());
 +    String cmd = req.getParameter("cmd");
 +    if (cmd == null) {
 +      // the command is null, just print prompt
 +      resp.getWriter().append(shellThread.getPrompt());
 +      resp.getWriter().flush();
 +      return;
 +    }
 +    shellThread.addInputString(cmd);
 +    shellThread.waitUntilReady();
 +    if (shellThread.isDone()) {
 +      // the command was exit, invalidate session
 +      userShells.remove(session.getId());
 +      session.invalidate();
 +      return;
 +    }
 +    // get the shell's output
 +    StringBuilder sb = new StringBuilder();
 +    sb.append(shellThread.getOutput().replace("<", "&lt;").replace(">", "&gt;"));
 +    if (sb.length() == 0 || !(sb.charAt(sb.length() - 1) == '\n'))
 +      sb.append("\n");
 +    // check if shell is waiting for input
 +    if (!shellThread.isWaitingForInput())
 +      sb.append(shellThread.getPrompt());
 +    // check if shell is waiting for password input
 +    if (shellThread.isMasking())
 +      sb.append("*");
 +    resp.getWriter().append(sb.toString());
 +    resp.getWriter().flush();
 +  }
 +  
 +  private String authenticationForm(String requestURI) {
 +    return "<div id='login'><form method=POST action='" + requestURI + "'>"
 +        + "<table><tr><td>Mock:&nbsp</td><td><input type='checkbox' name='mock' value='mock'></td></tr>"
 +        + "<tr><td>Username:&nbsp;</td><td><input type='text' name='user'></td></tr>"
 +        + "<tr><td>Password:&nbsp;</td><td><input type='password' name='pass'></td><td><input type='submit' value='Enter'></td></tr></table></form></div>";
 +  }
 +  
 +  private static class StringBuilderOutputStream extends OutputStream {
 +    StringBuilder sb = new StringBuilder();
 +    
 +    @Override
 +    public void write(int b) throws IOException {
 +      sb.append((char) (0xff & b));
 +    }
 +    
 +    public String get() {
 +      return sb.toString();
 +    }
 +    
 +    public void clear() {
 +      sb.setLength(0);
 +    }
 +  }
 +  
 +  private static class ShellExecutionThread extends InputStream implements Runnable {
 +    private Shell shell;
 +    StringBuilderOutputStream output;
 +    private String cmd;
 +    private int cmdIndex;
 +    private boolean done;
 +    private boolean readWait;
 +    
 +    private ShellExecutionThread(String username, String password, String mock) throws IOException {
 +      this.done = false;
 +      this.cmd = null;
 +      this.cmdIndex = 0;
 +      this.readWait = false;
 +      this.output = new StringBuilderOutputStream();
 +      ConsoleReader reader = new ConsoleReader(this, output);
-       this.shell = new Shell(reader, new PrintWriter(output));
++      this.shell = new Shell(reader, new PrintWriter(new OutputStreamWriter(output, Constants.UTF8)));
 +      shell.setLogErrorsToConsole();
 +      if (mock != null) {
 +        if (shell.config("--fake", "-u", username, "-p", password))
 +          throw new IOException("mock shell config error");
 +      } else if (shell.config("-u", username, "-p", password)) {
 +        throw new IOException("shell config error");
 +      }
 +    }
 +    
 +    @Override
 +    public synchronized int read() throws IOException {
 +      if (cmd == null) {
 +        readWait = true;
 +        this.notifyAll();
 +      }
 +      while (cmd == null) {
 +        try {
 +          this.wait();
 +        } catch (InterruptedException e) {}
 +      }
 +      readWait = false;
 +      int c;
 +      if (cmdIndex == cmd.length())
 +        c = '\n';
 +      else
 +        c = cmd.charAt(cmdIndex);
 +      cmdIndex++;
 +      if (cmdIndex > cmd.length()) {
 +        cmd = null;
 +        cmdIndex = 0;
 +        this.notifyAll();
 +      }
 +      return c;
 +    }
 +    
 +    @Override
 +    public synchronized void run() {
 +      Thread.currentThread().setName("shell thread");
 +      while (!shell.hasExited()) {
 +        while (cmd == null) {
 +          try {
 +            this.wait();
 +          } catch (InterruptedException e) {}
 +        }
 +        String tcmd = cmd;
 +        cmd = null;
 +        cmdIndex = 0;
 +        try {
 +          shell.execCommand(tcmd, false, true);
 +        } catch (IOException e) {}
 +        this.notifyAll();
 +      }
 +      done = true;
 +      this.notifyAll();
 +    }
 +    
 +    public synchronized void addInputString(String s) {
 +      if (done)
 +        throw new IllegalStateException("adding string to exited shell");
 +      if (cmd == null) {
 +        cmd = s;
 +      } else {
 +        throw new IllegalStateException("adding string to shell not waiting for input");
 +      }
 +      this.notifyAll();
 +    }
 +    
 +    public synchronized void waitUntilReady() {
 +      while (cmd != null) {
 +        try {
 +          this.wait();
 +        } catch (InterruptedException e) {}
 +      }
 +    }
 +    
 +    public synchronized String getOutput() {
 +      String s = output.get();
 +      output.clear();
 +      return s;
 +    }
 +    
 +    public String getPrompt() {
 +      return shell.getDefaultPrompt();
 +    }
 +    
 +    public void printInfo() throws IOException {
 +      shell.printInfo();
 +    }
 +    
 +    public boolean isMasking() {
 +      return shell.isMasking();
 +    }
 +    
 +    public synchronized boolean isWaitingForInput() {
 +      return readWait;
 +    }
 +    
 +    public boolean isDone() {
 +      return done;
 +    }
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/7688eaf0/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/TServersServlet.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/TServersServlet.java
index 681e696,0000000..9f1bd1f
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/TServersServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/TServersServlet.java
@@@ -1,369 -1,0 +1,370 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.monitor.servlets;
 +
 +import java.lang.management.ManagementFactory;
 +import java.security.MessageDigest;
 +import java.text.DateFormat;
 +import java.util.ArrayList;
 +import java.util.List;
 +import java.util.Map.Entry;
 +
 +import javax.servlet.http.HttpServletRequest;
 +import javax.servlet.http.HttpServletResponse;
 +
++import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.data.KeyExtent;
 +import org.apache.accumulo.core.master.thrift.DeadServer;
 +import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
 +import org.apache.accumulo.core.master.thrift.TableInfo;
 +import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 +import org.apache.accumulo.core.tabletserver.thrift.ActionStats;
 +import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
 +import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
 +import org.apache.accumulo.core.util.Duration;
 +import org.apache.accumulo.core.util.ThriftUtil;
 +import org.apache.accumulo.monitor.Monitor;
 +import org.apache.accumulo.monitor.util.Table;
 +import org.apache.accumulo.monitor.util.TableRow;
 +import org.apache.accumulo.monitor.util.celltypes.CompactionsType;
 +import org.apache.accumulo.monitor.util.celltypes.DateTimeType;
 +import org.apache.accumulo.monitor.util.celltypes.DurationType;
 +import org.apache.accumulo.monitor.util.celltypes.NumberType;
 +import org.apache.accumulo.monitor.util.celltypes.PercentageType;
 +import org.apache.accumulo.monitor.util.celltypes.ProgressChartType;
 +import org.apache.accumulo.monitor.util.celltypes.TServerLinkType;
 +import org.apache.accumulo.monitor.util.celltypes.TableLinkType;
 +import org.apache.accumulo.server.master.state.TabletServerState;
 +import org.apache.accumulo.server.security.SystemCredentials;
 +import org.apache.accumulo.server.util.ActionStatsUpdator;
 +import org.apache.accumulo.server.util.TableInfoUtil;
 +import org.apache.accumulo.trace.instrument.Tracer;
 +import org.apache.commons.codec.binary.Base64;
 +
 +import com.google.common.net.HostAndPort;
 +
 +public class TServersServlet extends BasicServlet {
 +  
 +  private static final long serialVersionUID = 1L;
 +  private static final TabletServerStatus NO_STATUS = new TabletServerStatus();
 +  
 +  static class SecondType extends NumberType<Double> {
 +    
 +    @Override
 +    public String format(Object obj) {
 +      if (obj == null)
 +        return "&mdash;";
 +      return Duration.format((long) (1000.0 * (Double) obj));
 +    }
 +    
 +  }
 +  
 +  @Override
 +  protected String getTitle(HttpServletRequest req) {
 +    return "Tablet Server Status";
 +  }
 +  
 +  @Override
 +  protected void pageBody(HttpServletRequest req, HttpServletResponse response, StringBuilder sb) throws Exception {
 +    String tserverAddress = req.getParameter("s");
 +    
 +    // Check to make sure tserver is a known address
 +    boolean tserverExists = false;
 +    if (tserverAddress != null && tserverAddress.isEmpty() == false) {
 +      for (TabletServerStatus ts : Monitor.getMmi().getTServerInfo()) {
 +        if (tserverAddress.equals(ts.getName())) {
 +          tserverExists = true;
 +          break;
 +        }
 +      }
 +    }
 +    
 +    if (tserverAddress == null || tserverAddress.isEmpty() || tserverExists == false) {
 +      doBadTserverList(req, sb);
 +      
 +      doDeadTserverList(req, sb);
 +      
 +      ArrayList<TabletServerStatus> tservers = new ArrayList<TabletServerStatus>();
 +      if (Monitor.getMmi() != null)
 +        tservers.addAll(Monitor.getMmi().tServerInfo);
 +      
 +      Table tServerList = new Table("tservers", "Tablet&nbsp;Servers");
 +      tServerList.setSubCaption("Click on the <span style='color: #0000ff;'>server address</span> to view detailed performance statistics for that server.");
 +      
 +      doTserverList(req, sb, tservers, null, tServerList);
 +      return;
 +    }
 +    
 +    double totalElapsedForAll = 0;
 +    double splitStdDev = 0;
 +    double minorStdDev = 0;
 +    double minorQueueStdDev = 0;
 +    double majorStdDev = 0;
 +    double majorQueueStdDev = 0;
 +    double currentMinorAvg = 0;
 +    double currentMajorAvg = 0;
 +    double currentMinorStdDev = 0;
 +    double currentMajorStdDev = 0;
 +    TabletStats total = new TabletStats(null, new ActionStats(), new ActionStats(), new ActionStats(), 0, 0, 0, 0);
 +    
 +    HostAndPort address = HostAndPort.fromString(tserverAddress);
 +    TabletStats historical = new TabletStats(null, new ActionStats(), new ActionStats(), new ActionStats(), 0, 0, 0, 0);
 +    List<TabletStats> tsStats = new ArrayList<TabletStats>();
 +    try {
 +      TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, Monitor.getSystemConfiguration());
 +      try {
 +        for (String tableId : Monitor.getMmi().tableMap.keySet()) {
 +          tsStats.addAll(client.getTabletStats(Tracer.traceInfo(), SystemCredentials.get().toThrift(Monitor.getInstance()), tableId));
 +        }
 +        historical = client.getHistoricalStats(Tracer.traceInfo(), SystemCredentials.get().toThrift(Monitor.getInstance()));
 +      } finally {
 +        ThriftUtil.returnClient(client);
 +      }
 +    } catch (Exception e) {
 +      banner(sb, "error", "No Such Tablet ServerAvailable");
 +      log.error(e, e);
 +      return;
 +    }
 +    
 +    Table perTabletResults = new Table("perTabletResults", "Detailed&nbsp;Current&nbsp;Operations");
 +    perTabletResults.setSubCaption("Per-tablet&nbsp;Details");
 +    perTabletResults.addSortableColumn("Table", new TableLinkType(), null);
 +    perTabletResults.addSortableColumn("Tablet");
 +    perTabletResults.addSortableColumn("Entries", new NumberType<Long>(), null);
 +    perTabletResults.addSortableColumn("Ingest", new NumberType<Long>(), null);
 +    perTabletResults.addSortableColumn("Query", new NumberType<Long>(), null);
 +    perTabletResults.addSortableColumn("Minor&nbsp;Avg", new SecondType(), null);
 +    perTabletResults.addSortableColumn("Minor&nbsp;Std&nbsp;Dev", new SecondType(), null);
 +    perTabletResults.addSortableColumn("Minor&nbsp;Avg&nbsp;e/s", new NumberType<Double>(), null);
 +    perTabletResults.addSortableColumn("Major&nbsp;Avg", new SecondType(), null);
 +    perTabletResults.addSortableColumn("Major&nbsp;Std&nbsp;Dev", new SecondType(), null);
 +    perTabletResults.addSortableColumn("Major&nbsp;Avg&nbsp;e/s", new NumberType<Double>(), null);
 +    
 +    for (TabletStats info : tsStats) {
 +      if (info.extent == null) {
 +        historical = info;
 +        continue;
 +      }
 +      total.numEntries += info.numEntries;
 +      ActionStatsUpdator.update(total.minors, info.minors);
 +      ActionStatsUpdator.update(total.majors, info.majors);
 +      
 +      KeyExtent extent = new KeyExtent(info.extent);
 +      String tableId = extent.getTableId().toString();
 +      MessageDigest digester = MessageDigest.getInstance("MD5");
 +      if (extent.getEndRow() != null && extent.getEndRow().getLength() > 0) {
 +        digester.update(extent.getEndRow().getBytes(), 0, extent.getEndRow().getLength());
 +      }
-       String obscuredExtent = new String(Base64.encodeBase64(digester.digest()));
++      String obscuredExtent = new String(Base64.encodeBase64(digester.digest()), Constants.UTF8);
 +      String displayExtent = String.format("<code>[%s]</code>", obscuredExtent);
 +      
 +      TableRow row = perTabletResults.prepareRow();
 +      row.add(tableId);
 +      row.add(displayExtent);
 +      row.add(info.numEntries);
 +      row.add(info.ingestRate);
 +      row.add(info.queryRate);
 +      row.add(info.minors.num != 0 ? info.minors.elapsed / info.minors.num : null);
 +      row.add(stddev(info.minors.elapsed, info.minors.num, info.minors.sumDev));
 +      row.add(info.minors.elapsed != 0 ? info.minors.count / info.minors.elapsed : null);
 +      row.add(info.majors.num != 0 ? info.majors.elapsed / info.majors.num : null);
 +      row.add(stddev(info.majors.elapsed, info.majors.num, info.majors.sumDev));
 +      row.add(info.majors.elapsed != 0 ? info.majors.count / info.majors.elapsed : null);
 +      perTabletResults.addRow(row);
 +    }
 +    
 +    // Calculate current averages oldServer adding in historical data
 +    if (total.minors.num != 0)
 +      currentMinorAvg = (long) (total.minors.elapsed / total.minors.num);
 +    if (total.minors.elapsed != 0 && total.minors.num != 0)
 +      currentMinorStdDev = stddev(total.minors.elapsed, total.minors.num, total.minors.sumDev);
 +    if (total.majors.num != 0)
 +      currentMajorAvg = total.majors.elapsed / total.majors.num;
 +    if (total.majors.elapsed != 0 && total.majors.num != 0 && total.majors.elapsed > total.majors.num)
 +      currentMajorStdDev = stddev(total.majors.elapsed, total.majors.num, total.majors.sumDev);
 +    
 +    // After these += operations, these variables are now total for current
 +    // tablets and historical tablets
 +    ActionStatsUpdator.update(total.minors, historical.minors);
 +    ActionStatsUpdator.update(total.majors, historical.majors);
 +    totalElapsedForAll += total.majors.elapsed + historical.splits.elapsed + total.minors.elapsed;
 +    
 +    minorStdDev = stddev(total.minors.elapsed, total.minors.num, total.minors.sumDev);
 +    minorQueueStdDev = stddev(total.minors.queueTime, total.minors.num, total.minors.queueSumDev);
 +    majorStdDev = stddev(total.majors.elapsed, total.majors.num, total.majors.sumDev);
 +    majorQueueStdDev = stddev(total.majors.queueTime, total.majors.num, total.majors.queueSumDev);
 +    splitStdDev = stddev(historical.splits.num, historical.splits.elapsed, historical.splits.sumDev);
 +    
 +    doDetailTable(req, sb, address, tsStats.size(), total, historical);
 +    doAllTimeTable(req, sb, total, historical, majorQueueStdDev, minorQueueStdDev, totalElapsedForAll, splitStdDev, majorStdDev, minorStdDev);
 +    doCurrentTabletOps(req, sb, currentMinorAvg, currentMinorStdDev, currentMajorAvg, currentMajorStdDev);
 +    perTabletResults.generate(req, sb);
 +  }
 +  
 +  private void doCurrentTabletOps(HttpServletRequest req, StringBuilder sb, double currentMinorAvg, double currentMinorStdDev, double currentMajorAvg,
 +      double currentMajorStdDev) {
 +    Table currentTabletOps = new Table("currentTabletOps", "Current&nbsp;Tablet&nbsp;Operation&nbsp;Results");
 +    currentTabletOps.addSortableColumn("Minor&nbsp;Average", new SecondType(), null);
 +    currentTabletOps.addSortableColumn("Minor&nbsp;Std&nbsp;Dev", new SecondType(), null);
 +    currentTabletOps.addSortableColumn("Major&nbsp;Avg", new SecondType(), null);
 +    currentTabletOps.addSortableColumn("Major&nbsp;Std&nbsp;Dev", new SecondType(), null);
 +    currentTabletOps.addRow(currentMinorAvg, currentMinorStdDev, currentMajorAvg, currentMajorStdDev);
 +    currentTabletOps.generate(req, sb);
 +  }
 +  
 +  private void doAllTimeTable(HttpServletRequest req, StringBuilder sb, TabletStats total, TabletStats historical, double majorQueueStdDev,
 +      double minorQueueStdDev, double totalElapsedForAll, double splitStdDev, double majorStdDev, double minorStdDev) {
 +    
 +    Table opHistoryDetails = new Table("opHistoryDetails", "All-Time&nbsp;Tablet&nbsp;Operation&nbsp;Results");
 +    opHistoryDetails.addSortableColumn("Operation");
 +    opHistoryDetails.addSortableColumn("Success", new NumberType<Integer>(), null);
 +    opHistoryDetails.addSortableColumn("Failure", new NumberType<Integer>(), null);
 +    opHistoryDetails.addSortableColumn("Average<br />Queue&nbsp;Time", new SecondType(), null);
 +    opHistoryDetails.addSortableColumn("Std.&nbsp;Dev.<br />Queue&nbsp;Time", new SecondType(), null);
 +    opHistoryDetails.addSortableColumn("Average<br />Time", new SecondType(), null);
 +    opHistoryDetails.addSortableColumn("Std.&nbsp;Dev.<br />Time", new SecondType(), null);
 +    opHistoryDetails.addSortableColumn("Percentage&nbsp;Time&nbsp;Spent", new ProgressChartType(totalElapsedForAll), null);
 +    
 +    opHistoryDetails.addRow("Split", historical.splits.num, historical.splits.fail, null, null,
 +        historical.splits.num != 0 ? (historical.splits.elapsed / historical.splits.num) : null, splitStdDev, historical.splits.elapsed);
 +    opHistoryDetails.addRow("Major&nbsp;Compaction", total.majors.num, total.majors.fail, total.majors.num != 0 ? (total.majors.queueTime / total.majors.num)
 +        : null, majorQueueStdDev, total.majors.num != 0 ? (total.majors.elapsed / total.majors.num) : null, majorStdDev, total.majors.elapsed);
 +    opHistoryDetails.addRow("Minor&nbsp;Compaction", total.minors.num, total.minors.fail, total.minors.num != 0 ? (total.minors.queueTime / total.minors.num)
 +        : null, minorQueueStdDev, total.minors.num != 0 ? (total.minors.elapsed / total.minors.num) : null, minorStdDev, total.minors.elapsed);
 +    opHistoryDetails.generate(req, sb);
 +  }
 +  
 +  private void doDetailTable(HttpServletRequest req, StringBuilder sb, HostAndPort address, int numTablets, TabletStats total, TabletStats historical) {
 +    Table detailTable = new Table("tServerDetail", "Details");
 +    detailTable.setSubCaption(address.getHostText() + ":" + address.getPort());
 +    detailTable.addSortableColumn("Hosted&nbsp;Tablets", new NumberType<Integer>(), null);
 +    detailTable.addSortableColumn("Entries", new NumberType<Long>(), null);
 +    detailTable.addSortableColumn("Minor&nbsp;Compacting", new NumberType<Integer>(), null);
 +    detailTable.addSortableColumn("Major&nbsp;Compacting", new NumberType<Integer>(), null);
 +    detailTable.addSortableColumn("Splitting", new NumberType<Integer>(), null);
 +    detailTable.addRow(numTablets, total.numEntries, total.minors.status, total.majors.status, historical.splits.status);
 +    detailTable.generate(req, sb);
 +  }
 +  
 +  /*
 +   * omg there's so much undocumented stuff going on here. First, sumDev is a partial standard deviation computation. It is the (clue 1) sum of the squares of
 +   * (clue 2) seconds of elapsed time.
 +   */
 +  private static double stddev(double elapsed, double num, double sumDev) {
 +    if (num != 0) {
 +      double average = elapsed / num;
 +      return Math.sqrt((sumDev / num) - (average * average));
 +    }
 +    return 0;
 +  }
 +  
 +  private void doBadTserverList(HttpServletRequest req, StringBuilder sb) {
 +    if (Monitor.getMmi() != null && !Monitor.getMmi().badTServers.isEmpty()) {
 +      Table badTServerList = new Table("badtservers", "Non-Functioning&nbsp;Tablet&nbsp;Servers", "error");
 +      badTServerList.setSubCaption("The following tablet servers reported a status other than Online.");
 +      badTServerList.addSortableColumn("Tablet&nbsp;Server");
 +      badTServerList.addSortableColumn("Tablet&nbsp;Server&nbsp;Status");
 +      for (Entry<String,Byte> badserver : Monitor.getMmi().badTServers.entrySet())
 +        badTServerList.addRow(badserver.getKey(), TabletServerState.getStateById(badserver.getValue()).name());
 +      badTServerList.generate(req, sb);
 +    }
 +  }
 +  
 +  private void doDeadTserverList(HttpServletRequest req, StringBuilder sb) {
 +    MasterMonitorInfo mmi = Monitor.getMmi();
 +    if (mmi != null) {
 +      List<DeadServer> obit = mmi.deadTabletServers;
 +      Table deadTServerList = new Table("deaddtservers", "Dead&nbsp;Tablet&nbsp;Servers", "error");
 +      deadTServerList.setSubCaption("The following tablet servers are no longer reachable.");
 +      doDeadServerTable(req, sb, deadTServerList, obit);
 +    }
 +  }
 +  
 +  public static void doDeadServerTable(HttpServletRequest req, StringBuilder sb, Table deadTServerList, List<DeadServer> obit) {
 +    if (obit != null && !obit.isEmpty()) {
 +      deadTServerList.addSortableColumn("Server");
 +      deadTServerList.addSortableColumn("Last&nbsp;Updated", new DateTimeType(DateFormat.MEDIUM, DateFormat.SHORT), null);
 +      deadTServerList.addSortableColumn("Event");
 +      deadTServerList.addUnsortableColumn("Clear");
 +      for (DeadServer dead : obit)
 +        deadTServerList.addRow(TServerLinkType.displayName(dead.server), dead.lastStatus, dead.status, "<a href='/op?action=clearDeadServer&redir="
 +            + currentPage(req) + "&server=" + encode(dead.server) + "'>clear</a>");
 +      deadTServerList.generate(req, sb);
 +    }
 +  }
 +  
 +  static void doTserverList(HttpServletRequest req, StringBuilder sb, List<TabletServerStatus> tservers, String tableId, Table tServerList) {
 +    int guessHighLoad = ManagementFactory.getOperatingSystemMXBean().getAvailableProcessors();
 +    long now = System.currentTimeMillis();
 +    
 +    double avgLastContact = 0.;
 +    for (TabletServerStatus status : tservers) {
 +      avgLastContact += (now - status.lastContact);
 +    }
 +    final long MINUTES = 3 * 60 * 1000;
 +    tServerList.addSortableColumn("Server", new TServerLinkType(), null);
 +    tServerList.addSortableColumn("Hosted&nbsp;Tablets", new NumberType<Integer>(0, Integer.MAX_VALUE), null);
 +    tServerList.addSortableColumn("Last&nbsp;Contact", new DurationType(0l, (long) Math.min(avgLastContact * 4, MINUTES)), null);
 +    tServerList.addSortableColumn("Entries", new NumberType<Long>(), "The number of key/value pairs.");
 +    tServerList.addSortableColumn("Ingest", new NumberType<Long>(), "The number of key/value pairs inserted. (Note that deletes are also 'inserted')");
 +    tServerList.addSortableColumn("Query", new NumberType<Long>(), "The number of key/value pairs returned to clients. (Not the number of scans)");
 +    tServerList.addSortableColumn("Hold&nbsp;Time", new DurationType(), "The amount of time ingest is suspended waiting for data to be written to disk.");
 +    tServerList.addSortableColumn("Running<br />Scans", new CompactionsType("scans"), "The number of scans running and queued on this tablet server.");
 +    tServerList
 +        .addSortableColumn(
 +            "Minor<br />Compactions",
 +            new CompactionsType("minor"),
 +            "The number of minor compactions running and (queued waiting for resources). Minor compactions are the operations where entries are flushed from memory to disk.");
 +    tServerList.addSortableColumn("Major<br />Compactions", new CompactionsType("major"),
 +        "The number of major compactions running and (queued waiting for resources). "
 +            + "Major compactions are the operations where many smaller files are grouped into a larger file, eliminating duplicates and cleaning up deletes.");
 +    tServerList.addSortableColumn("Index Cache<br />Hit Rate", new PercentageType(), "The recent index cache hit rate.");
 +    tServerList.addSortableColumn("Data Cache<br />Hit Rate", new PercentageType(), "The recent data cache hit rate.");
 +    tServerList.addSortableColumn("OS&nbsp;Load", new NumberType<Double>(0., guessHighLoad * 1., 0., guessHighLoad * 3.),
 +        "The Unix one minute load average. The average number of processes in the run queue over a one minute interval.");
 +    
 +    log.debug("tableId: " + tableId);
 +    for (TabletServerStatus status : tservers) {
 +      if (status == null)
 +        status = NO_STATUS;
 +      TableInfo summary = TableInfoUtil.summarizeTableStats(status);
 +      if (tableId != null)
 +        summary = status.tableMap.get(tableId);
 +      if (summary == null)
 +        continue;
 +      TableRow row = tServerList.prepareRow();
 +      row.add(status); // add for server name
 +      row.add(summary.tablets);
 +      row.add(now - status.lastContact);
 +      row.add(summary.recs);
 +      row.add(summary.ingestRate);
 +      row.add(summary.queryRate);
 +      row.add(status.holdTime);
 +      row.add(summary); // add for scans
 +      row.add(summary); // add for minor compactions
 +      row.add(summary); // add for major compactions
 +      double indexCacheHitRate = status.indexCacheHits / (double) Math.max(status.indexCacheRequest, 1);
 +      row.add(indexCacheHitRate);
 +      double dataCacheHitRate = status.dataCacheHits / (double) Math.max(status.dataCacheRequest, 1);
 +      row.add(dataCacheHitRate);
 +      row.add(status.osLoad);
 +      tServerList.addRow(row);
 +    }
 +    tServerList.generate(req, sb);
 +  }
 +  
 +}


Mime
View raw message