htrace-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cmcc...@apache.org
Subject incubator-htrace git commit: HTRACE-303. Add client-side htraceDropped log file to track dropped spans (cmccabe)
Date Sat, 21 Nov 2015 21:31:55 GMT
Repository: incubator-htrace
Updated Branches:
  refs/heads/master 699c8cf80 -> 26518585d


HTRACE-303. Add client-side htraceDropped log file to track dropped spans (cmccabe)


Project: http://git-wip-us.apache.org/repos/asf/incubator-htrace/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-htrace/commit/26518585
Tree: http://git-wip-us.apache.org/repos/asf/incubator-htrace/tree/26518585
Diff: http://git-wip-us.apache.org/repos/asf/incubator-htrace/diff/26518585

Branch: refs/heads/master
Commit: 26518585d52c3b89f5967fa3776a075c700b6831
Parents: 699c8cf
Author: Colin P. Mccabe <cmccabe@apache.org>
Authored: Sat Nov 21 13:27:23 2015 -0800
Committer: Colin P. Mccabe <cmccabe@apache.org>
Committed: Sat Nov 21 13:27:23 2015 -0800

----------------------------------------------------------------------
 .../go/src/org/apache/htrace/common/rpc.go      |  19 --
 .../src/org/apache/htrace/htraced/datastore.go  |   4 +-
 .../org/apache/htrace/htraced/datastore_test.go |   4 +-
 .../go/src/org/apache/htrace/htraced/hrpc.go    |   2 +-
 .../go/src/org/apache/htrace/htraced/metrics.go |  15 +-
 .../org/apache/htrace/htraced/reaper_test.go    |   2 +-
 .../go/src/org/apache/htrace/htraced/rest.go    |   2 +-
 .../go/src/org/apache/htrace/htracedTool/cmd.go |   6 +-
 .../main/java/org/apache/htrace/impl/Conf.java  |  24 +++
 .../apache/htrace/impl/HTracedSpanReceiver.java |  91 ++++++++--
 .../apache/htrace/impl/TestDroppedSpans.java    | 173 +++++++++++++++++++
 .../src/main/webapp/app/server_info_view.js     |   2 -
 .../src/main/webapp/app/server_stats.js         |   1 -
 htrace-webapp/src/main/webapp/index.html        |   4 -
 14 files changed, 291 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/go/src/org/apache/htrace/common/rpc.go
----------------------------------------------------------------------
diff --git a/htrace-htraced/go/src/org/apache/htrace/common/rpc.go b/htrace-htraced/go/src/org/apache/htrace/common/rpc.go
index 6375688..8028cc6 100644
--- a/htrace-htraced/go/src/org/apache/htrace/common/rpc.go
+++ b/htrace-htraced/go/src/org/apache/htrace/common/rpc.go
@@ -41,7 +41,6 @@ type WriteSpansReq struct {
 	Addr          string `json:",omitempty"` // This gets filled in by the RPC layer.
 	DefaultTrid   string `json:",omitempty"`
 	Spans         []*Span
-	ClientDropped uint64 `json:",omitempty"`
 }
 
 // Info returned by /server/version
@@ -97,20 +96,6 @@ type SpanMetrics struct {
 
 	// The total number of spans dropped by the server.
 	ServerDropped uint64
-
-	// The total number of spans dropped by the client.
-	//
-	// This number is just an estimate and may be incorrect for many reasons.
-	// If the client can't contact the server at all, then obviously the server
-	// will never increment ClientDropped... even though spans are being
-	// dropped.  The client may also tell the server about some new spans it
-	// has dropped, but then for some reason fail to get the acknowledgement
-	// from the server.  In that case, the client would re-send its client
-	// dropped estimate and it would be double-counted by the server
-	//
-	// The intention here is to provide a rough estimate of how overloaded
-	// htraced clients are, not to provide strongly consistent numbers.
-	ClientDroppedEstimate uint64
 }
 
 // A map from network address strings to SpanMetrics structures.
@@ -145,10 +130,6 @@ type ServerStats struct {
 	// The total number of spans dropped by the server since the server started.
 	ServerDroppedSpans uint64
 
-	// An estimate of the total number of spans dropped by the server since the server started.
-	// See SpanMetrics#ClientDroppedEstimate
-	ClientDroppedEstimate uint64
-
 	// The maximum latency of a writeSpans request, in milliseconds.
 	MaxWriteSpansLatencyMs uint32
 

http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go
----------------------------------------------------------------------
diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go b/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go
index 9310e6e..1dab5c8 100644
--- a/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go
+++ b/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go
@@ -792,7 +792,7 @@ func (ing *SpanIngestor) IngestSpan(span *common.Span) {
 	}
 }
 
-func (ing *SpanIngestor) Close(clientDropped int, startTime time.Time) {
+func (ing *SpanIngestor) Close(startTime time.Time) {
 	for shardIdx := range(ing.batches) {
 		batch := ing.batches[shardIdx]
 		if len(batch.incoming) > 0 {
@@ -809,7 +809,7 @@ func (ing *SpanIngestor) Close(clientDropped int, startTime time.Time)
{
 
 	endTime := time.Now()
 	ing.store.msink.UpdateIngested(ing.addr, ing.totalIngested,
-		ing.serverDropped, clientDropped, endTime.Sub(startTime))
+		ing.serverDropped, endTime.Sub(startTime))
 }
 
 func (store *dataStore) WriteSpans(shardIdx int, ispans []*IncomingSpan) {

http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/go/src/org/apache/htrace/htraced/datastore_test.go
----------------------------------------------------------------------
diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/datastore_test.go b/htrace-htraced/go/src/org/apache/htrace/htraced/datastore_test.go
index e6d1df7..d38c1b0 100644
--- a/htrace-htraced/go/src/org/apache/htrace/htraced/datastore_test.go
+++ b/htrace-htraced/go/src/org/apache/htrace/htraced/datastore_test.go
@@ -77,7 +77,7 @@ func createSpans(spans []common.Span, store *dataStore) {
 	for idx := range spans {
 		ing.IngestSpan(&spans[idx])
 	}
-	ing.Close(0, time.Now())
+	ing.Close(time.Now())
 	store.WrittenSpans.Waits(int64(len(spans)))
 }
 
@@ -364,7 +364,7 @@ func BenchmarkDatastoreWrites(b *testing.B) {
 	for n := 0; n < b.N; n++ {
 		ing.IngestSpan(allSpans[n])
 	}
-	ing.Close(0, time.Now())
+	ing.Close(time.Now())
 	// Wait for all the spans to be written.
 	ht.Store.WrittenSpans.Waits(int64(b.N))
 	assertNumWrittenEquals(b, ht.Store.msink, b.N)

http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/go/src/org/apache/htrace/htraced/hrpc.go
----------------------------------------------------------------------
diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/hrpc.go b/htrace-htraced/go/src/org/apache/htrace/htraced/hrpc.go
index a649420..0d569a0 100644
--- a/htrace-htraced/go/src/org/apache/htrace/htraced/hrpc.go
+++ b/htrace-htraced/go/src/org/apache/htrace/htraced/hrpc.go
@@ -270,7 +270,7 @@ func (hand *HrpcHandler) WriteSpans(req *common.WriteSpansReq,
 	for spanIdx := range req.Spans {
 		ing.IngestSpan(req.Spans[spanIdx])
 	}
-	ing.Close(int(req.ClientDropped), startTime)
+	ing.Close(startTime)
 	return nil
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/go/src/org/apache/htrace/htraced/metrics.go
----------------------------------------------------------------------
diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/metrics.go b/htrace-htraced/go/src/org/apache/htrace/htraced/metrics.go
index 5ce3339..7bf42fd 100644
--- a/htrace-htraced/go/src/org/apache/htrace/htraced/metrics.go
+++ b/htrace-htraced/go/src/org/apache/htrace/htraced/metrics.go
@@ -55,9 +55,6 @@ type MetricsSink struct {
 	// The total number of spans dropped by the server.
 	ServerDropped uint64
 
-	// The total number of spans dropped by the client (self-reported).
-	ClientDroppedEstimate uint64
-
 	// Per-host Span Metrics
 	HostSpanMetrics common.SpanMetricsMap
 
@@ -80,13 +77,12 @@ func NewMetricsSink(cnf *conf.Config) *MetricsSink {
 // Update the total number of spans which were ingested, as well as other
 // metrics that get updated during span ingest. 
 func (msink *MetricsSink) UpdateIngested(addr string, totalIngested int,
-		serverDropped int, clientDroppedEstimate int, wsLatency time.Duration) {
+		serverDropped int, wsLatency time.Duration) {
 	msink.lock.Lock()
 	defer msink.lock.Unlock()
 	msink.IngestedSpans += uint64(totalIngested)
 	msink.ServerDropped += uint64(serverDropped)
-	msink.ClientDroppedEstimate += uint64(clientDroppedEstimate)
-	msink.updateSpanMetrics(addr, 0, serverDropped, clientDroppedEstimate)
+	msink.updateSpanMetrics(addr, 0, serverDropped)
 	wsLatencyMs := wsLatency.Nanoseconds() / 1000000
 	var wsLatency32 uint32
 	if wsLatencyMs > math.MaxUint32 {
@@ -99,7 +95,7 @@ func (msink *MetricsSink) UpdateIngested(addr string, totalIngested int,
 
 // Update the per-host span metrics.  Must be called with the lock held.
 func (msink *MetricsSink) updateSpanMetrics(addr string, numWritten int,
-		serverDropped int, clientDroppedEstimate int) {
+		serverDropped int) {
 	mtx, found := msink.HostSpanMetrics[addr]
 	if !found {
 		// Ensure that the per-host span metrics map doesn't grow too large.
@@ -117,7 +113,6 @@ func (msink *MetricsSink) updateSpanMetrics(addr string, numWritten int,
 	}
 	mtx.Written += uint64(numWritten)
 	mtx.ServerDropped += uint64(serverDropped)
-	mtx.ClientDroppedEstimate += uint64(clientDroppedEstimate)
 }
 
 // Update the total number of spans which were persisted to disk.
@@ -127,7 +122,7 @@ func (msink *MetricsSink) UpdatePersisted(addr string, totalWritten int,
 	defer msink.lock.Unlock()
 	msink.WrittenSpans += uint64(totalWritten)
 	msink.ServerDropped += uint64(serverDropped)
-	msink.updateSpanMetrics(addr, totalWritten, serverDropped, 0)
+	msink.updateSpanMetrics(addr, totalWritten, serverDropped)
 }
 
 // Read the server stats.
@@ -137,7 +132,6 @@ func (msink *MetricsSink) PopulateServerStats(stats *common.ServerStats)
{
 	stats.IngestedSpans = msink.IngestedSpans
 	stats.WrittenSpans = msink.WrittenSpans
 	stats.ServerDroppedSpans = msink.ServerDropped
-	stats.ClientDroppedEstimate = msink.ClientDroppedEstimate
 	stats.MaxWriteSpansLatencyMs = msink.wsLatencyCircBuf.Max()
 	stats.AverageWriteSpansLatencyMs = msink.wsLatencyCircBuf.Average()
 	stats.HostSpanMetrics = make(common.SpanMetricsMap)
@@ -145,7 +139,6 @@ func (msink *MetricsSink) PopulateServerStats(stats *common.ServerStats)
{
 		stats.HostSpanMetrics[k] = &common.SpanMetrics {
 			Written: v.Written,
 			ServerDropped: v.ServerDropped,
-			ClientDroppedEstimate: v.ClientDroppedEstimate,
 		}
 	}
 }

http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/go/src/org/apache/htrace/htraced/reaper_test.go
----------------------------------------------------------------------
diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/reaper_test.go b/htrace-htraced/go/src/org/apache/htrace/htraced/reaper_test.go
index 0140dbb..b354a2c 100644
--- a/htrace-htraced/go/src/org/apache/htrace/htraced/reaper_test.go
+++ b/htrace-htraced/go/src/org/apache/htrace/htraced/reaper_test.go
@@ -56,7 +56,7 @@ func TestReapingOldSpans(t *testing.T) {
 	for spanIdx := range testSpans {
 		ing.IngestSpan(testSpans[spanIdx])
 	}
-	ing.Close(0, time.Now())
+	ing.Close(time.Now())
 	// Wait the spans to be created
 	ht.Store.WrittenSpans.Waits(NUM_TEST_SPANS)
 	// Set a reaper date that will remove all the spans except final one.

http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/go/src/org/apache/htrace/htraced/rest.go
----------------------------------------------------------------------
diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/rest.go b/htrace-htraced/go/src/org/apache/htrace/htraced/rest.go
index 1b90bd4..c327cdd 100644
--- a/htrace-htraced/go/src/org/apache/htrace/htraced/rest.go
+++ b/htrace-htraced/go/src/org/apache/htrace/htraced/rest.go
@@ -257,7 +257,7 @@ func (hand *writeSpansHandler) ServeHTTP(w http.ResponseWriter, req *http.Reques
 	for spanIdx := range msg.Spans {
 		ing.IngestSpan(msg.Spans[spanIdx])
 	}
-	ing.Close(int(msg.ClientDropped), startTime)
+	ing.Close(startTime)
 }
 
 type queryHandler struct {

http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/go/src/org/apache/htrace/htracedTool/cmd.go
----------------------------------------------------------------------
diff --git a/htrace-htraced/go/src/org/apache/htrace/htracedTool/cmd.go b/htrace-htraced/go/src/org/apache/htrace/htracedTool/cmd.go
index c81bbb7..9837e94 100644
--- a/htrace-htraced/go/src/org/apache/htrace/htracedTool/cmd.go
+++ b/htrace-htraced/go/src/org/apache/htrace/htracedTool/cmd.go
@@ -218,8 +218,6 @@ func printServerStats(hcl *htrace.Client) int {
 	fmt.Fprintf(w, "Spans ingested\t%d\n", stats.IngestedSpans)
 	fmt.Fprintf(w, "Spans written\t%d\n", stats.WrittenSpans)
 	fmt.Fprintf(w, "Spans dropped by server\t%d\n", stats.ServerDroppedSpans)
-	fmt.Fprintf(w, "Estimated spans dropped by clients\t%d\n",
-		stats.ClientDroppedEstimate)
 	dur := time.Millisecond * time.Duration(stats.AverageWriteSpansLatencyMs)
 	fmt.Fprintf(w, "Average WriteSpan Latency\t%s\n", dur.String())
 	dur = time.Millisecond * time.Duration(stats.MaxWriteSpansLatencyMs)
@@ -247,8 +245,8 @@ func printServerStats(hcl *htrace.Client) int {
 	sort.Sort(keys)
 	for k := range keys {
 		mtx := mtxMap[keys[k]]
-		fmt.Fprintf(w, "%s\twritten: %d\tserver dropped: %d\tclient dropped estimate: %d\n",
-			keys[k], mtx.Written, mtx.ServerDropped, mtx.ClientDroppedEstimate)
+		fmt.Fprintf(w, "%s\twritten: %d\tserver dropped: %d\n",
+			keys[k], mtx.Written, mtx.ServerDropped)
 	}
 	w.Flush()
 	return EXIT_SUCCESS

http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/src/main/java/org/apache/htrace/impl/Conf.java
----------------------------------------------------------------------
diff --git a/htrace-htraced/src/main/java/org/apache/htrace/impl/Conf.java b/htrace-htraced/src/main/java/org/apache/htrace/impl/Conf.java
index cdd176f..3206dd6 100644
--- a/htrace-htraced/src/main/java/org/apache/htrace/impl/Conf.java
+++ b/htrace-htraced/src/main/java/org/apache/htrace/impl/Conf.java
@@ -18,6 +18,7 @@
 package org.apache.htrace.impl;
 
 import java.io.IOException;
+import java.io.File;
 import java.net.InetSocketAddress;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -144,6 +145,18 @@ class Conf {
       "htraced.error.log.period.ms";
   final static long ERROR_LOG_PERIOD_MS_DEFAULT = 30000L;
 
+  final static String DROPPED_SPANS_LOG_PATH_KEY =
+      "htraced.dropped.spans.log.path";
+
+  final static String DROPPED_SPANS_LOG_PATH_DEFAULT =
+      new File(System.getProperty("java.io.tmpdir", "/tmp"), "htraceDropped").
+        getAbsolutePath();
+
+  final static String DROPPED_SPANS_LOG_MAX_SIZE_KEY =
+      "htraced.dropped.spans.log.max.size";
+
+  final static long DROPPED_SPANS_LOG_MAX_SIZE_DEFAULT = 1024L * 1024L;
+
   @JsonProperty("ioTimeoutMs")
   final int ioTimeoutMs;
 
@@ -180,6 +193,12 @@ class Conf {
   @JsonProperty("endpoint")
   final InetSocketAddress endpoint;
 
+  @JsonProperty("droppedSpansLogPath")
+  final String droppedSpansLogPath;
+
+  @JsonProperty("droppedSpansLogMaxSize")
+  final long droppedSpansLogMaxSize;
+
   private static int getBoundedInt(final HTraceConfiguration conf,
         String key, int defaultValue, int minValue, int maxValue) {
     int val = conf.getInt(key, defaultValue);
@@ -341,6 +360,11 @@ class Conf {
       throw new IOException("Error reading " + ADDRESS_KEY + ": " +
           e.getMessage());
     }
+    this.droppedSpansLogPath = conf.get(
+        DROPPED_SPANS_LOG_PATH_KEY, DROPPED_SPANS_LOG_PATH_DEFAULT);
+    this.droppedSpansLogMaxSize = getBoundedLong(conf,
+        DROPPED_SPANS_LOG_MAX_SIZE_KEY, DROPPED_SPANS_LOG_MAX_SIZE_DEFAULT,
+        0, Long.MAX_VALUE);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/src/main/java/org/apache/htrace/impl/HTracedSpanReceiver.java
----------------------------------------------------------------------
diff --git a/htrace-htraced/src/main/java/org/apache/htrace/impl/HTracedSpanReceiver.java
b/htrace-htraced/src/main/java/org/apache/htrace/impl/HTracedSpanReceiver.java
index f5f493c..22b64f6 100644
--- a/htrace-htraced/src/main/java/org/apache/htrace/impl/HTracedSpanReceiver.java
+++ b/htrace-htraced/src/main/java/org/apache/htrace/impl/HTracedSpanReceiver.java
@@ -17,7 +17,20 @@
  */
 package org.apache.htrace.impl;
 
+import static java.nio.file.StandardOpenOption.APPEND;
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.WRITE;
+
 import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+//import java.nio.file.attribute.FileAttribute;
+//import java.util.EnumSet;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.Condition;
 import java.util.concurrent.locks.ReentrantLock;
@@ -75,6 +88,8 @@ public class HTracedSpanReceiver extends SpanReceiver {
 
   private long lastBufferClearedTimeMs = 0;
 
+  private long unbufferableSpans = 0;
+
   static class FaultInjector {
     static FaultInjector NO_OP = new FaultInjector();
     public void handleContentLengthTrigger(int len) { }
@@ -140,10 +155,12 @@ public class HTracedSpanReceiver extends SpanReceiver {
         }
         long deltaMs = TimeUtil.deltaMs(startTimeMs, TimeUtil.nowMs());
         if (deltaMs > conf.spanDropTimeoutMs) {
+          StringBuilder bld = new StringBuilder();
           spanDropLog.error("Dropping a span after unsuccessfully " +
               "attempting to add it for " + deltaMs + " ms.  There is not " +
               "enough buffer space. Please increase " + Conf.BUFFER_SIZE_KEY +
               " or decrease the rate of spans being generated.");
+          unbufferableSpans++;
           return;
         } else if (LOG.isDebugEnabled()) {
           LOG.debug("Unable to write span to buffer #" + activeBuf +
@@ -296,6 +313,16 @@ public class HTracedSpanReceiver extends SpanReceiver {
         return;
       }
       int flushTries = 0;
+      if (unbufferableSpans > 0) {
+        try {
+          appendToDroppedSpansLog("Dropped " + unbufferableSpans +
+              " spans because of lack of local buffer space.\n");
+        } catch (IOException e) {
+          // Ignore.  We already logged a message about the dropped spans
+          // earlier.
+        }
+        unbufferableSpans = 0;
+      }
       while (true) {
         Throwable exc;
         try {
@@ -311,17 +338,22 @@ public class HTracedSpanReceiver extends SpanReceiver {
           return;
         }
         int numSpans = flushBufManager.getNumberOfSpans();
-        String excMessage = "Failed to flush " + numSpans  + " htrace " +
-            "spans to " + conf.endpointStr + " on try " + (flushTries + 1);
-        if (flushTries >= conf.flushRetryDelays.length) {
-          excMessage += ".  Discarding all spans.";
-        }
-        if (LOG.isDebugEnabled()) {
-          LOG.error(excMessage, exc);
-        } else {
-          flushErrorLog.error(excMessage, exc);
-        }
+        flushErrorLog.error("Failed to flush " + numSpans  + " htrace " +
+            "spans to " + conf.endpointStr + " on try " + (flushTries + 1),
+            exc);
         if (flushTries >= conf.flushRetryDelays.length) {
+          StringBuilder bld = new StringBuilder();
+          bld.append("Failed to flush ").append(numSpans).
+            append(" spans to htraced at").append(conf.endpointStr).
+            append(" after ").append(flushTries).append(" tries: ").
+            append(exc.getMessage());
+          try {
+            appendToDroppedSpansLog(bld.toString());
+          } catch (IOException e) {
+            bld.append(".  Failed to write to dropped spans log: ").
+              append(e.getMessage());
+          }
+          spanDropLog.error(bld.toString());
           return;
         }
         int delayMs = conf.flushRetryDelays[flushTries];
@@ -330,4 +362,43 @@ public class HTracedSpanReceiver extends SpanReceiver {
       }
     }
   }
+
+  void appendToDroppedSpansLog(String text) throws IOException {
+    // Is the dropped spans log is disabled?
+    if (conf.droppedSpansLogPath.isEmpty() ||
+        (conf.droppedSpansLogMaxSize == 0)) {
+      return;
+    }
+    FileLock lock = null;
+    ByteBuffer bb = ByteBuffer.wrap(
+        text.getBytes(StandardCharsets.UTF_8));
+    // FileChannel locking corresponds to advisory locking on UNIX.  It will
+    // protect multiple processes from attempting to write to the same dropped
+    // spans log at once.  However, within a single process, we need this
+    // synchronized block to ensure that multiple HTracedSpanReceiver objects
+    // don't try to write to the same log at once.  (It is unusal to configure
+    // multiple HTracedSpanReceiver objects, but possible.)
+    synchronized(HTracedSpanReceiver.class) {
+      FileChannel channel = FileChannel.open(
+          Paths.get(conf.droppedSpansLogPath), APPEND, CREATE, WRITE);
+      try {
+        lock = channel.lock();
+        long size = channel.size();
+        if (size > conf.droppedSpansLogMaxSize) {
+          throw new IOException("Dropped spans log " +
+              conf.droppedSpansLogPath + " is already " + size +
+              " bytes; will not add to it.");
+        }
+        channel.write(bb);
+      } finally {
+        try {
+          if (lock != null) {
+            lock.release();
+          }
+        } finally {
+          channel.close();
+        }
+      }
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-htraced/src/test/java/org/apache/htrace/impl/TestDroppedSpans.java
----------------------------------------------------------------------
diff --git a/htrace-htraced/src/test/java/org/apache/htrace/impl/TestDroppedSpans.java b/htrace-htraced/src/test/java/org/apache/htrace/impl/TestDroppedSpans.java
new file mode 100644
index 0000000..8947365
--- /dev/null
+++ b/htrace-htraced/src/test/java/org/apache/htrace/impl/TestDroppedSpans.java
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.htrace.impl;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.htrace.core.HTraceConfiguration;
+import org.apache.htrace.core.MilliSpan;
+import org.apache.htrace.core.TracerId;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestDroppedSpans {
+  private static final Log LOG = LogFactory.getLog(TestDroppedSpans.class);
+
+  private static Path tempDir;
+
+  @BeforeClass
+  public static void beforeClass() throws IOException {
+    // Allow setting really small buffer sizes for testing purposes.
+    // We do not allow setting such small sizes in production.
+    Conf.BUFFER_SIZE_MIN = 0;
+
+    // Create a temporary directory to hold the dropped spans logs.
+    String tmp = System.getProperty("java.io.tmpdir", "/tmp");
+    File dir = new File(tmp,
+        "TestDroppedSpans." + UUID.randomUUID().toString());
+    Files.createDirectory(dir.toPath());
+    tempDir = dir.toPath();
+  }
+
+  @BeforeClass
+  public static void afterClass() throws IOException {
+    if (tempDir != null) {
+      try (DirectoryStream<Path> stream = Files.newDirectoryStream(tempDir)) {
+        for (final Iterator<Path> it = stream.iterator(); it.hasNext();) {
+          Files.delete(it.next());
+        }
+      }
+      Files.delete(tempDir);
+      tempDir = null;
+    }
+  }
+
+  /**
+   * Test that we can disable the dropped spans log.
+   */
+  @Test(timeout = 60000)
+  public void testDisableDroppedSpansLog() throws Exception {
+    HTraceConfiguration conf = HTraceConfiguration.fromMap(
+        new HashMap<String, String>() {{
+          put(Conf.ADDRESS_KEY, "127.0.0.1:8080");
+          put(TracerId.TRACER_ID_KEY, "testAppendToDroppedSpansLog");
+          put(Conf.DROPPED_SPANS_LOG_PATH_KEY, "/");
+          put(Conf.DROPPED_SPANS_LOG_MAX_SIZE_KEY, "0");
+        }});
+    HTracedSpanReceiver rcvr = new HTracedSpanReceiver(conf);
+    try {
+      rcvr.appendToDroppedSpansLog("this won't get written");
+    } finally {
+      rcvr.close();
+    }
+  }
+
+  /**
+   * Test that we can write to the dropped spans log.
+   */
+  @Test(timeout = 60000)
+  public void testWriteToDroppedSpansLog() throws Exception {
+    final String logPath = new File(
+        tempDir.toFile(), "testWriteToDroppedSpansLog").getAbsolutePath();
+    HTraceConfiguration conf = HTraceConfiguration.fromMap(
+        new HashMap<String, String>() {{
+          put(Conf.ADDRESS_KEY, "127.0.0.1:8080");
+          put(TracerId.TRACER_ID_KEY, "testWriteToDroppedSpansLog");
+          put(Conf.DROPPED_SPANS_LOG_PATH_KEY, logPath);
+          put(Conf.DROPPED_SPANS_LOG_MAX_SIZE_KEY, "78");
+        }});
+    HTracedSpanReceiver rcvr = new HTracedSpanReceiver(conf);
+    try {
+      final String LINE1 = "This is a test of the dropped spans log.";
+      rcvr.appendToDroppedSpansLog(LINE1 + "\n");
+      final String LINE2 = "These lines should appear in the log.";
+      rcvr.appendToDroppedSpansLog(LINE2 + "\n");
+      try {
+        rcvr.appendToDroppedSpansLog("This line won't be written because we're " +
+            "out of space.");
+        Assert.fail("expected append to fail because of lack of space");
+      } catch (IOException e) {
+        // ignore
+      }
+      List<String> lines =
+          Files.readAllLines(Paths.get(logPath), StandardCharsets.UTF_8);
+      Assert.assertEquals(2, lines.size());
+      Assert.assertEquals(LINE1, lines.get(0));
+      Assert.assertEquals(LINE2, lines.get(1));
+    } finally {
+      rcvr.close();
+    }
+  }
+
+  /**
+   * Test that we write to the dropped spans log when htraced is unreachable.
+   */
+  @Test(timeout = 60000)
+  public void testSpansDroppedBecauseOfUnreachableHTraced() throws Exception {
+    final String logPath = new File(tempDir.toFile(),
+        "testSpansDroppedBecauseOfUnreachableHTraced").getAbsolutePath();
+    // Open a local socket.  We know that nobody is listening on this socket, so
+    // all attempts to send to it will fail.
+    final ServerSocket serverSocket = new ServerSocket(0);
+    HTracedSpanReceiver rcvr = null;
+    try {
+      HTraceConfiguration conf = HTraceConfiguration.fromMap(
+          new HashMap<String, String>() {{
+            put(Conf.ADDRESS_KEY, "127.0.0.1:" + serverSocket.getLocalPort());
+            put(TracerId.TRACER_ID_KEY,
+                "testSpansDroppedBecauseOfUnreachableHTraced");
+            put(Conf.DROPPED_SPANS_LOG_PATH_KEY, logPath);
+            put(Conf.DROPPED_SPANS_LOG_MAX_SIZE_KEY, "78");
+            put(Conf.CONNECT_TIMEOUT_MS_KEY, "1");
+            put(Conf.IO_TIMEOUT_MS_KEY, "1");
+            put(Conf.FLUSH_RETRY_DELAYS_KEY, "1,1");
+          }});
+      rcvr = new HTracedSpanReceiver(conf);
+      rcvr.receiveSpan(new MilliSpan.Builder().
+          begin(123).end(456).description("FooBar").build());
+      HTracedSpanReceiver tmpRcvr = rcvr;
+      rcvr = null;
+      tmpRcvr.close();
+      List<String> lines =
+          Files.readAllLines(Paths.get(logPath), StandardCharsets.UTF_8);
+      Assert.assertTrue(lines.size() >= 1);
+      Assert.assertTrue(lines.get(0).contains("Failed to flush "));
+    } finally {
+      serverSocket.close();
+      if (rcvr != null) {
+        rcvr.close();
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-webapp/src/main/webapp/app/server_info_view.js
----------------------------------------------------------------------
diff --git a/htrace-webapp/src/main/webapp/app/server_info_view.js b/htrace-webapp/src/main/webapp/app/server_info_view.js
index efb7545..43533d4 100644
--- a/htrace-webapp/src/main/webapp/app/server_info_view.js
+++ b/htrace-webapp/src/main/webapp/app/server_info_view.js
@@ -50,7 +50,6 @@ htrace.ServerInfoView = Backbone.View.extend({
             '<th>Remote</th>' +
             '<th>Written</th>' +
             '<th>ServerDropped</th>' +
-            '<th>ClientDroppedEstimate</th>' +
           '</tr>' +
         '</thead>';
     var remotes = [];
@@ -69,7 +68,6 @@ htrace.ServerInfoView = Backbone.View.extend({
         "<td>" + remote + "</td>" +
         "<td>" + smtx.Written + "</td>" +
         "<td>" + smtx.ServerDropped + "</td>" +
-        "<td>" + smtx.ClientDroppedEstimate + "</td>" +
         "</tr>";
     }
     out = out + '</table>';

http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-webapp/src/main/webapp/app/server_stats.js
----------------------------------------------------------------------
diff --git a/htrace-webapp/src/main/webapp/app/server_stats.js b/htrace-webapp/src/main/webapp/app/server_stats.js
index 4cfea92..65d8c60 100644
--- a/htrace-webapp/src/main/webapp/app/server_stats.js
+++ b/htrace-webapp/src/main/webapp/app/server_stats.js
@@ -26,7 +26,6 @@ htrace.ServerStats = Backbone.Model.extend({
     "IngestedSpans": "(unknown)",
     "WrittenSpans": "(unknown)",
     "ServerDroppedSpans": "(unknown)",
-    "ClientDroppedSpans": "(unknown)",
     "MaxWriteSpansLatencyMs": "(unknown)",
     "AverageWriteSpansLatencyMs": "(unknown)"
   },

http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/26518585/htrace-webapp/src/main/webapp/index.html
----------------------------------------------------------------------
diff --git a/htrace-webapp/src/main/webapp/index.html b/htrace-webapp/src/main/webapp/index.html
index 1e20ec0..edbfd26 100644
--- a/htrace-webapp/src/main/webapp/index.html
+++ b/htrace-webapp/src/main/webapp/index.html
@@ -92,10 +92,6 @@
               <td><%= model.stats.get("ServerDroppedSpans") %></td>
             </tr>
             <tr>
-              <td>Estimated Client Dropped Spans</td>
-              <td><%= model.stats.get("ClientDroppedEstimate") %></td>
-            </tr>
-            <tr>
               <td>Maximum WriteSpans Latency (ms)</td>
               <td><%= model.stats.get("MaxWriteSpansLatencyMs") %></td>
             </tr>


Mime
View raw message