incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: More trace updates and fixes.
Date Tue, 26 Nov 2013 14:57:13 GMT
Updated Branches:
  refs/heads/master bdf6957f7 -> 772facd97


More trace updates and fixes.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/772facd9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/772facd9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/772facd9

Branch: refs/heads/master
Commit: 772facd9783d2986e6cf82d33d7170335367f3e0
Parents: bdf6957
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Tue Nov 26 09:56:40 2013 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Tue Nov 26 09:57:05 2013 -0500

----------------------------------------------------------------------
 .../org/apache/blur/manager/IndexManager.java   |  61 +++++---
 .../apache/blur/server/BlurServerContext.java   |  24 +++-
 .../apache/blur/server/FilteredBlurServer.java  |  11 +-
 .../blur/thrift/BlurControllerServer.java       |  10 +-
 .../org/apache/blur/thrift/BlurShardServer.java |   5 +-
 .../blur/thrift/ThriftBlurControllerServer.java |  34 +++--
 .../blur/thrift/ThriftBlurShardServer.java      |   6 +-
 .../java/org/apache/blur/utils/BlurUtil.java    |   8 +-
 .../test/java/org/apache/blur/MiniCluster.java  |  19 ++-
 .../apache/blur/thrift/BlurClientManager.java   |   5 +-
 .../org/apache/blur/thrift/generated/Blur.java  | 144 +++++++++++++++++--
 .../org/apache/blur/trace/LogTraceReporter.java |   7 +
 .../main/java/org/apache/blur/trace/Trace.java  |  55 +++++--
 .../org/apache/blur/trace/TraceCollector.java   |  19 ++-
 .../org/apache/blur/trace/TraceReporter.java    |   4 +-
 .../blur/trace/ZooKeeperDumpContents.java       |  13 +-
 .../blur/trace/ZooKeeperTraceReporter.java      |  20 +--
 .../src/main/resources/blur-default.properties  |   2 +-
 .../java/org/apache/blur/trace/TraceTest.java   |  34 +++--
 .../src/main/scripts/interface/Blur.thrift      |   4 +-
 .../main/scripts/interface/gen-html/Blur.html   |   5 +-
 .../org/apache/blur/thrift/generated/Blur.java  | 144 +++++++++++++++++--
 .../src/main/scripts/interface/gen-js/Blur.js   |  24 +++-
 .../scripts/interface/gen-perl/Blur/Blur.pm     |  28 +++-
 .../src/main/scripts/interface/gen-rb/blur.rb   |  15 +-
 docs/Blur.html                                  |   5 +-
 26 files changed, 551 insertions(+), 155 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
index 115815b..25b1a8f 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
@@ -337,6 +337,7 @@ public class IndexManager {
   }
 
   private void populateSelector(String table, Selector selector) throws IOException, BlurException {
+    Tracer trace = Trace.trace("populate selector");
     String rowId = selector.rowId;
     String recordId = selector.recordId;
     String shardName = MutationHelper.getShardName(table, rowId, getNumberOfShards(table), _blurPartitioner);
@@ -372,6 +373,7 @@ public class IndexManager {
     } finally {
       // this will allow for closing of index
       searcher.close();
+      trace.done();
     }
   }
 
@@ -563,7 +565,9 @@ public class IndexManager {
       returnIdsOnly = true;
     }
 
+    Tracer t1 = Trace.trace("fetchRow - live docs");
     Bits liveDocs = MultiFields.getLiveDocs(reader);
+    t1.done();
     ResetableDocumentStoredFieldVisitor fieldVisitor = getFieldSelector(selector);
     if (selector.isRecordOnly()) {
       // select only the row for the given data or location id.
@@ -592,33 +596,44 @@ public class IndexManager {
         return;
       }
     } else {
-      if (liveDocs != null && !liveDocs.get(docId)) {
-        fetchResult.exists = false;
-        fetchResult.deleted = true;
-        return;
-      } else {
-        fetchResult.exists = true;
-        fetchResult.deleted = false;
-        String rowId = getRowId(reader, docId);
-        Term term = new Term(ROW_ID, rowId);
-        if (returnIdsOnly) {
-          int recordCount = BlurUtil.countDocuments(reader, term);
-          fetchResult.rowResult = new FetchRowResult();
-          fetchResult.rowResult.row = new Row(rowId, null, recordCount);
+      Tracer trace = Trace.trace("fetchRow - Row read");
+      try {
+        if (liveDocs != null && !liveDocs.get(docId)) {
+          fetchResult.exists = false;
+          fetchResult.deleted = true;
+          return;
         } else {
-          List<Document> docs;
-          if (highlightQuery != null && fieldManager != null) {
-            HighlightOptions highlightOptions = selector.getHighlightOptions();
-            String preTag = highlightOptions.getPreTag();
-            String postTag = highlightOptions.getPostTag();
-            docs = HighlightHelper.highlightDocuments(reader, term, fieldVisitor, selector, highlightQuery,
-                fieldManager, preTag, postTag);
+          fetchResult.exists = true;
+          fetchResult.deleted = false;
+          String rowId = getRowId(reader, docId);
+          Term term = new Term(ROW_ID, rowId);
+          if (returnIdsOnly) {
+            int recordCount = BlurUtil.countDocuments(reader, term);
+            fetchResult.rowResult = new FetchRowResult();
+            fetchResult.rowResult.row = new Row(rowId, null, recordCount);
           } else {
-            docs = BlurUtil.fetchDocuments(reader, term, fieldVisitor, selector, maxHeap, table + "/" + shard);
+            List<Document> docs;
+            if (highlightQuery != null && fieldManager != null) {
+              HighlightOptions highlightOptions = selector.getHighlightOptions();
+              String preTag = highlightOptions.getPreTag();
+              String postTag = highlightOptions.getPostTag();
+              Tracer docTrace = Trace.trace("fetchRow - Document w/Highlight read");
+              docs = HighlightHelper.highlightDocuments(reader, term, fieldVisitor, selector, highlightQuery,
+                  fieldManager, preTag, postTag);
+              docTrace.done();
+            } else {
+              Tracer docTrace = Trace.trace("fetchRow - Document read");
+              docs = BlurUtil.fetchDocuments(reader, term, fieldVisitor, selector, maxHeap, table + "/" + shard);
+              docTrace.done();
+            }
+            Tracer rowTrace = Trace.trace("fetchRow - Row create");
+            fetchResult.rowResult = new FetchRowResult(getRow(docs));
+            rowTrace.done();
           }
-          fetchResult.rowResult = new FetchRowResult(getRow(docs));
+          return;
         }
-        return;
+      } finally {
+        trace.done();
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/blur-core/src/main/java/org/apache/blur/server/BlurServerContext.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/BlurServerContext.java b/blur-core/src/main/java/org/apache/blur/server/BlurServerContext.java
index ca378eb..60c59eb 100644
--- a/blur-core/src/main/java/org/apache/blur/server/BlurServerContext.java
+++ b/blur-core/src/main/java/org/apache/blur/server/BlurServerContext.java
@@ -32,7 +32,8 @@ public class BlurServerContext implements ServerContext {
   private final SocketAddress _remoteSocketAddress;
   private final String _localConnectionString;
   private final String _remoteConnectionString;
-  private String _traceId;
+  private String _traceRootId;
+  private String _traceRequestId;
 
   public BlurServerContext(SocketAddress localSocketAddress, SocketAddress remoteSocketAddress) {
     _localSocketAddress = localSocketAddress;
@@ -50,12 +51,20 @@ public class BlurServerContext implements ServerContext {
     return _user;
   }
 
-  public String getTraceId() {
-    return _traceId;
+  public String getTraceRootId() {
+    return _traceRootId;
   }
 
-  public void setTraceId(String traceId) {
-    _traceId = traceId;
+  public String getTraceRequestId() {
+    return _traceRequestId;
+  }
+
+  public void setTraceRootId(String traceRootId) {
+    _traceRootId = traceRootId;
+  }
+
+  public void setTraceRequestId(String traceRequestId) {
+    _traceRequestId = traceRequestId;
   }
 
   public SocketAddress getRocalSocketAddress() {
@@ -70,4 +79,9 @@ public class BlurServerContext implements ServerContext {
     return _localConnectionString + sep + _remoteConnectionString;
   }
 
+  public void resetTraceIds() {
+    _traceRootId = null;
+    _traceRequestId = null;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/blur-core/src/main/java/org/apache/blur/server/FilteredBlurServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/FilteredBlurServer.java b/blur-core/src/main/java/org/apache/blur/server/FilteredBlurServer.java
index a610c9e..6056bb3 100644
--- a/blur-core/src/main/java/org/apache/blur/server/FilteredBlurServer.java
+++ b/blur-core/src/main/java/org/apache/blur/server/FilteredBlurServer.java
@@ -40,15 +40,16 @@ import org.apache.blur.thrift.generated.TableStats;
 import org.apache.blur.thrift.generated.User;
 
 public class FilteredBlurServer implements Iface {
-  
+
   protected final Iface _iface;
-  public void startTrace(String traceId) throws TException {
-    _iface.startTrace(traceId);
+
+  public void startTrace(String rootId, String requestId) throws TException {
+    _iface.startTrace(rootId, requestId);
   }
 
   protected final boolean _shard;
   protected final BlurConfiguration _configuration;
-  
+
   public FilteredBlurServer(BlurConfiguration configuration, Iface iface, boolean shard) {
     _iface = iface;
     _shard = shard;
@@ -58,7 +59,7 @@ public class FilteredBlurServer implements Iface {
   public void setUser(User user) throws TException {
     _iface.setUser(user);
   }
-  
+
   public final BlurServerContext getServerContext() {
     if (_shard) {
       return ShardServerContext.getShardServerContext();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
index 65dea65..03a3134 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
@@ -86,6 +86,7 @@ import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.blur.thrift.generated.TableStats;
 import org.apache.blur.thrift.generated.User;
 import org.apache.blur.trace.Trace;
+import org.apache.blur.trace.Trace.TraceId;
 import org.apache.blur.trace.Tracer;
 import org.apache.blur.utils.BlurConstants;
 import org.apache.blur.utils.BlurExecutorCompletionService;
@@ -487,9 +488,9 @@ public class BlurControllerServer extends TableAdmin implements Iface {
           public Boolean call() throws Exception {
             Tracer trace = Trace.trace("remote call - thrift", Trace.param("node", getNode(client)));
             try {
-              String traceId = Trace.getTraceId();
+              TraceId traceId = Trace.getTraceId();
               if (traceId != null) {
-                client.startTrace(traceId);
+                client.startTrace(traceId.getRootId(), traceId.getRequestId());
               }
               List<FetchResult> fetchRowBatch = client.fetchRowBatch(table, list);
               for (int i = 0; i < list.size(); i++) {
@@ -1220,9 +1221,10 @@ public class BlurControllerServer extends TableAdmin implements Iface {
   }
 
   @Override
-  public void startTrace(String traceId) throws TException {
+  public void startTrace(String rootId, String requestId) throws TException {
     ControllerServerContext context = ControllerServerContext.getControllerServerContext();
-    context.setTraceId(traceId);
+    context.setTraceRootId(rootId);
+    context.setTraceRequestId(requestId);
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
index 8f17dd1..2c24ad9 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
@@ -493,9 +493,10 @@ public class BlurShardServer extends TableAdmin implements Iface {
   }
 
   @Override
-  public void startTrace(String traceId) throws TException {
+  public void startTrace(String rootId, String requestId) throws TException {
     ShardServerContext context = ShardServerContext.getShardServerContext();
-    context.setTraceId(traceId);
+    context.setTraceRootId(rootId);
+    context.setTraceRequestId(requestId);
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
index e2b8d75..cb72594 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
@@ -59,12 +59,12 @@ import org.apache.blur.server.ControllerServerEventHandler;
 import org.apache.blur.thirdparty.thrift_0_9_0.transport.TNonblockingServerSocket;
 import org.apache.blur.thrift.generated.Blur.Iface;
 import org.apache.blur.trace.Trace;
+import org.apache.blur.trace.TraceReporter;
 import org.apache.blur.utils.BlurUtil;
 import org.apache.blur.utils.MemoryReporter;
 import org.apache.blur.zookeeper.ZkUtils;
 import org.apache.zookeeper.ZooKeeper;
 
-
 public class ThriftBlurControllerServer extends ThriftServer {
 
   private static final Log LOG = LogFactory.getLog(ThriftBlurControllerServer.class);
@@ -81,7 +81,8 @@ public class ThriftBlurControllerServer extends ThriftServer {
     server.start();
   }
 
-  public static ThriftServer createServer(int serverIndex, BlurConfiguration configuration, boolean randomPort) throws Exception {
+  public static ThriftServer createServer(int serverIndex, BlurConfiguration configuration, boolean randomPort)
+      throws Exception {
     Thread.setDefaultUncaughtExceptionHandler(new SimpleUncaughtExceptionHandler());
     String bindAddress = configuration.get(BLUR_CONTROLLER_BIND_ADDRESS);
     int bindPort = configuration.getInt(BLUR_CONTROLLER_BIND_PORT, -1);
@@ -94,14 +95,15 @@ public class ThriftBlurControllerServer extends ThriftServer {
       bindPort = tNonblockingServerSocket.getServerSocket().getLocalPort();
     }
 
-    LOG.info("Controller Server using index [{0}] bind address [{1}] random port assignment [{2}]", serverIndex, bindAddress + ":" + bindPort, randomPort);
+    LOG.info("Controller Server using index [{0}] bind address [{1}] random port assignment [{2}]", serverIndex,
+        bindAddress + ":" + bindPort, randomPort);
 
     String nodeName = getNodeName(configuration, BLUR_CONTROLLER_HOSTNAME);
     nodeName = nodeName + ":" + bindPort;
     String zkConnectionStr = isEmpty(configuration.get(BLUR_ZOOKEEPER_CONNECTION), BLUR_ZOOKEEPER_CONNECTION);
 
     BlurQueryChecker queryChecker = new BlurQueryChecker(configuration);
-    
+
     int sessionTimeout = configuration.getInt(BLUR_ZOOKEEPER_TIMEOUT, BLUR_ZOOKEEPER_TIMEOUT_DEFAULT);
 
     final ZooKeeper zooKeeper = ZkUtils.newZooKeeper(zkConnectionStr, sessionTimeout);
@@ -130,19 +132,21 @@ public class ThriftBlurControllerServer extends ThriftServer {
     controllerServer.setMaxFetchDelay(configuration.getInt(BLUR_CONTROLLER_RETRY_MAX_FETCH_DELAY, 2000));
     controllerServer.setMaxMutateDelay(configuration.getInt(BLUR_CONTROLLER_RETRY_MAX_MUTATE_DELAY, 2000));
     controllerServer.setMaxDefaultDelay(configuration.getInt(BLUR_CONTROLLER_RETRY_MAX_DEFAULT_DELAY, 2000));
-    controllerServer.setMaxRecordsPerRowFetchRequest(configuration.getInt(BLUR_MAX_RECORDS_PER_ROW_FETCH_REQUEST, 1000));
+    controllerServer
+        .setMaxRecordsPerRowFetchRequest(configuration.getInt(BLUR_MAX_RECORDS_PER_ROW_FETCH_REQUEST, 1000));
     controllerServer.setConfiguration(configuration);
 
     controllerServer.init();
-    
-    Trace.setReporter(setupTraceReporter(configuration));
+
+    final TraceReporter traceReporter = setupTraceReporter(configuration);
+    Trace.setReporter(traceReporter);
     Trace.setNodeName(nodeName);
-    
-    Iface iface= BlurUtil.wrapFilteredBlurServer(configuration, controllerServer, false);
+
+    Iface iface = BlurUtil.wrapFilteredBlurServer(configuration, controllerServer, false);
     iface = BlurUtil.recordMethodCallsAndAverageTimes(iface, Iface.class, true);
     iface = BlurUtil.runTrace(iface, true);
     int threadCount = configuration.getInt(BLUR_CONTROLLER_SERVER_THRIFT_THREAD_COUNT, 32);
-    
+
     ControllerServerEventHandler eventHandler = new ControllerServerEventHandler();
 
     final ThriftBlurControllerServer server = new ThriftBlurControllerServer();
@@ -159,10 +163,12 @@ public class ThriftBlurControllerServer extends ThriftServer {
     final HttpJettyServer httpServer;
     if (baseGuiPort > 0) {
       int webServerPort = baseGuiPort + serverIndex;
-      // TODO: this got ugly, there has to be a better way to handle all these params
+      // TODO: this got ugly, there has to be a better way to handle all these
+      // params
       // without reversing the mvn dependancy and making blur-gui on top.
-      httpServer = new HttpJettyServer(bindPort, webServerPort, configuration.getInt(BLUR_CONTROLLER_BIND_PORT, -1), configuration.getInt(BLUR_SHARD_BIND_PORT, -1),
-          configuration.getInt(BLUR_GUI_CONTROLLER_PORT, -1), configuration.getInt(BLUR_GUI_SHARD_PORT, -1), "controller");
+      httpServer = new HttpJettyServer(bindPort, webServerPort, configuration.getInt(BLUR_CONTROLLER_BIND_PORT, -1),
+          configuration.getInt(BLUR_SHARD_BIND_PORT, -1), configuration.getInt(BLUR_GUI_CONTROLLER_PORT, -1),
+          configuration.getInt(BLUR_GUI_SHARD_PORT, -1), "controller");
     } else {
       httpServer = null;
     }
@@ -172,7 +178,7 @@ public class ThriftBlurControllerServer extends ThriftServer {
       @Override
       public void shutdown() {
         ThreadWatcher threadWatcher = ThreadWatcher.instance();
-        quietClose(server, controllerServer, clusterStatus, zooKeeper, threadWatcher, httpServer);
+        quietClose(traceReporter, server, controllerServer, clusterStatus, zooKeeper, threadWatcher, httpServer);
       }
     };
     server.setShutdown(shutdown);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index 7a0290a..ed9ce21 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -87,6 +87,7 @@ import org.apache.blur.thirdparty.thrift_0_9_0.transport.TNonblockingServerSocke
 import org.apache.blur.thrift.generated.Blur;
 import org.apache.blur.thrift.generated.Blur.Iface;
 import org.apache.blur.trace.Trace;
+import org.apache.blur.trace.TraceReporter;
 import org.apache.blur.utils.BlurUtil;
 import org.apache.blur.utils.GCWatcher;
 import org.apache.blur.utils.MemoryReporter;
@@ -231,7 +232,8 @@ public class ThriftBlurShardServer extends ThriftServer {
     shardServer.setConfiguration(configuration);
     shardServer.init();
     
-    Trace.setReporter(setupTraceReporter(configuration));
+    final TraceReporter traceReporter = setupTraceReporter(configuration);
+    Trace.setReporter(traceReporter);
     Trace.setNodeName(nodeName);
 
     Iface iface = BlurUtil.wrapFilteredBlurServer(configuration, shardServer, true);
@@ -266,7 +268,7 @@ public class ThriftBlurShardServer extends ThriftServer {
       @Override
       public void shutdown() {
         ThreadWatcher threadWatcher = ThreadWatcher.instance();
-        quietClose(refresher, server, shardServer, indexManager, indexServer, threadWatcher, clusterStatus, zooKeeper,
+        quietClose(traceReporter, refresher, server, shardServer, indexManager, indexServer, threadWatcher, clusterStatus, zooKeeper,
             httpServer);
       }
     };

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
index 1817c5b..1b69d35 100644
--- a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
+++ b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
@@ -941,9 +941,9 @@ public class BlurUtil {
           }
         }
         BlurServerContext context = getServerContext(controller);
-        String traceId = context.getTraceId();
-        if (traceId != null) {
-          Trace.setupTrace(traceId);
+        String rootTraceId = context.getTraceRootId();
+        if (rootTraceId != null) {
+          Trace.setupTrace(rootTraceId, context.getTraceRequestId());
         }
         try {
           return method.invoke(iface, args);
@@ -951,7 +951,7 @@ public class BlurUtil {
           throw e.getTargetException();
         } finally {
           Trace.tearDownTrace();
-          context.setTraceId(null);
+          context.resetTraceIds();
         }
       }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/blur-core/src/test/java/org/apache/blur/MiniCluster.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/MiniCluster.java b/blur-core/src/test/java/org/apache/blur/MiniCluster.java
index e0092b2..50d290a 100644
--- a/blur-core/src/test/java/org/apache/blur/MiniCluster.java
+++ b/blur-core/src/test/java/org/apache/blur/MiniCluster.java
@@ -436,9 +436,9 @@ public class MiniCluster {
           if (thread.isAlive()) {
             thread.interrupt();
             LOG.info("Stopping ThreadPoolExecutor [" + thread.getName() + "]");
-            Object target = getField(thread, "target");
+            Object target = getField(Thread.class, thread, "target");
             if (target != null) {
-              ThreadPoolExecutor e = (ThreadPoolExecutor) getField(target, "this$0");
+              ThreadPoolExecutor e = (ThreadPoolExecutor) getField(ThreadPoolExecutor.class, target, "this$0");
               if (e != null) {
                 e.shutdownNow();
               }
@@ -455,14 +455,21 @@ public class MiniCluster {
     }
   }
 
-  private static Object getField(Object o, String fieldName) {
+  private static Object getField(Class<?> c, Object o, String fieldName) {
     try {
-      Field field = o.getClass().getDeclaredField(fieldName);
+      Field field = c.getDeclaredField(fieldName);
       field.setAccessible(true);
       return field.get(o);
+    } catch (NoSuchFieldException e) {
+      try {
+        Field field = o.getClass().getDeclaredField(fieldName);
+        field.setAccessible(true);
+        return field.get(o);
+      } catch (Exception ex) {
+        throw new RuntimeException(ex);
+      }
     } catch (Exception e) {
-      e.printStackTrace();
-      return null;
+      throw new RuntimeException(e);
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/blur-thrift/src/main/java/org/apache/blur/thrift/BlurClientManager.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thrift/BlurClientManager.java b/blur-thrift/src/main/java/org/apache/blur/thrift/BlurClientManager.java
index 6292846..4556414 100644
--- a/blur-thrift/src/main/java/org/apache/blur/thrift/BlurClientManager.java
+++ b/blur-thrift/src/main/java/org/apache/blur/thrift/BlurClientManager.java
@@ -46,6 +46,7 @@ import org.apache.blur.thrift.generated.Blur.Client;
 import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.ErrorType;
 import org.apache.blur.trace.Trace;
+import org.apache.blur.trace.Trace.TraceId;
 import org.apache.blur.trace.Tracer;
 
 public class BlurClientManager {
@@ -172,9 +173,9 @@ public class BlurClientManager {
         }
         Tracer trace = null;
         try {
-          String traceId = Trace.getTraceId();
+          TraceId traceId = Trace.getTraceId();
           if (traceId != null) {
-            client.get().startTrace(traceId);
+            client.get().startTrace(traceId.getRootId(), traceId.getRequestId());
             trace = Trace.trace("thrift client", Trace.param("connection", getConnectionStr(client.get())));
           }
           T result = command.call((CLIENT) client.get(), connection);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/blur-thrift/src/main/java/org/apache/blur/thrift/generated/Blur.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thrift/generated/Blur.java b/blur-thrift/src/main/java/org/apache/blur/thrift/generated/Blur.java
index 8182634..c1d5514 100644
--- a/blur-thrift/src/main/java/org/apache/blur/thrift/generated/Blur.java
+++ b/blur-thrift/src/main/java/org/apache/blur/thrift/generated/Blur.java
@@ -370,8 +370,10 @@ public class Blur {
      * Starts a trace with the given trace id.
      * 
      * @param traceId the trace id.
+     * 
+     * @param requestId the request id, used to connected remote calls together.  Client can pass null.
      */
-    public void startTrace(String traceId) throws org.apache.blur.thirdparty.thrift_0_9_0.TException;
+    public void startTrace(String traceId, String requestId) throws org.apache.blur.thirdparty.thrift_0_9_0.TException;
 
   }
 
@@ -445,7 +447,7 @@ public class Blur {
 
     public void metrics(Set<String> metrics, org.apache.blur.thirdparty.thrift_0_9_0.async.AsyncMethodCallback<AsyncClient.metrics_call> resultHandler) throws org.apache.blur.thirdparty.thrift_0_9_0.TException;
 
-    public void startTrace(String traceId, org.apache.blur.thirdparty.thrift_0_9_0.async.AsyncMethodCallback<AsyncClient.startTrace_call> resultHandler) throws org.apache.blur.thirdparty.thrift_0_9_0.TException;
+    public void startTrace(String traceId, String requestId, org.apache.blur.thirdparty.thrift_0_9_0.async.AsyncMethodCallback<AsyncClient.startTrace_call> resultHandler) throws org.apache.blur.thirdparty.thrift_0_9_0.TException;
 
   }
 
@@ -1323,15 +1325,16 @@ public class Blur {
       throw new org.apache.blur.thirdparty.thrift_0_9_0.TApplicationException(org.apache.blur.thirdparty.thrift_0_9_0.TApplicationException.MISSING_RESULT, "metrics failed: unknown result");
     }
 
-    public void startTrace(String traceId) throws org.apache.blur.thirdparty.thrift_0_9_0.TException
+    public void startTrace(String traceId, String requestId) throws org.apache.blur.thirdparty.thrift_0_9_0.TException
     {
-      send_startTrace(traceId);
+      send_startTrace(traceId, requestId);
     }
 
-    public void send_startTrace(String traceId) throws org.apache.blur.thirdparty.thrift_0_9_0.TException
+    public void send_startTrace(String traceId, String requestId) throws org.apache.blur.thirdparty.thrift_0_9_0.TException
     {
       startTrace_args args = new startTrace_args();
       args.setTraceId(traceId);
+      args.setRequestId(requestId);
       sendBase("startTrace", args);
     }
 
@@ -2482,24 +2485,27 @@ public class Blur {
       }
     }
 
-    public void startTrace(String traceId, org.apache.blur.thirdparty.thrift_0_9_0.async.AsyncMethodCallback<startTrace_call> resultHandler) throws org.apache.blur.thirdparty.thrift_0_9_0.TException {
+    public void startTrace(String traceId, String requestId, org.apache.blur.thirdparty.thrift_0_9_0.async.AsyncMethodCallback<startTrace_call> resultHandler) throws org.apache.blur.thirdparty.thrift_0_9_0.TException {
       checkReady();
-      startTrace_call method_call = new startTrace_call(traceId, resultHandler, this, ___protocolFactory, ___transport);
+      startTrace_call method_call = new startTrace_call(traceId, requestId, resultHandler, this, ___protocolFactory, ___transport);
       this.___currentMethod = method_call;
       ___manager.call(method_call);
     }
 
     public static class startTrace_call extends org.apache.blur.thirdparty.thrift_0_9_0.async.TAsyncMethodCall {
       private String traceId;
-      public startTrace_call(String traceId, org.apache.blur.thirdparty.thrift_0_9_0.async.AsyncMethodCallback<startTrace_call> resultHandler, org.apache.blur.thirdparty.thrift_0_9_0.async.TAsyncClient client, org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocolFactory protocolFactory, org.apache.blur.thirdparty.thrift_0_9_0.transport.TNonblockingTransport transport) throws org.apache.blur.thirdparty.thrift_0_9_0.TException {
+      private String requestId;
+      public startTrace_call(String traceId, String requestId, org.apache.blur.thirdparty.thrift_0_9_0.async.AsyncMethodCallback<startTrace_call> resultHandler, org.apache.blur.thirdparty.thrift_0_9_0.async.TAsyncClient client, org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocolFactory protocolFactory, org.apache.blur.thirdparty.thrift_0_9_0.transport.TNonblockingTransport transport) throws org.apache.blur.thirdparty.thrift_0_9_0.TException {
         super(client, protocolFactory, transport, resultHandler, true);
         this.traceId = traceId;
+        this.requestId = requestId;
       }
 
       public void write_args(org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocol prot) throws org.apache.blur.thirdparty.thrift_0_9_0.TException {
         prot.writeMessageBegin(new org.apache.blur.thirdparty.thrift_0_9_0.protocol.TMessage("startTrace", org.apache.blur.thirdparty.thrift_0_9_0.protocol.TMessageType.CALL, 0));
         startTrace_args args = new startTrace_args();
         args.setTraceId(traceId);
+        args.setRequestId(requestId);
         args.write(prot);
         prot.writeMessageEnd();
       }
@@ -3392,7 +3398,7 @@ public class Blur {
       }
 
       public org.apache.blur.thirdparty.thrift_0_9_0.TBase getResult(I iface, startTrace_args args) throws org.apache.blur.thirdparty.thrift_0_9_0.TException {
-        iface.startTrace(args.traceId);
+        iface.startTrace(args.traceId, args.requestId);
         return null;
       }
     }
@@ -32412,6 +32418,7 @@ public class Blur {
     private static final org.apache.blur.thirdparty.thrift_0_9_0.protocol.TStruct STRUCT_DESC = new org.apache.blur.thirdparty.thrift_0_9_0.protocol.TStruct("startTrace_args");
 
     private static final org.apache.blur.thirdparty.thrift_0_9_0.protocol.TField TRACE_ID_FIELD_DESC = new org.apache.blur.thirdparty.thrift_0_9_0.protocol.TField("traceId", org.apache.blur.thirdparty.thrift_0_9_0.protocol.TType.STRING, (short)1);
+    private static final org.apache.blur.thirdparty.thrift_0_9_0.protocol.TField REQUEST_ID_FIELD_DESC = new org.apache.blur.thirdparty.thrift_0_9_0.protocol.TField("requestId", org.apache.blur.thirdparty.thrift_0_9_0.protocol.TType.STRING, (short)2);
 
     private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
     static {
@@ -32423,13 +32430,21 @@ public class Blur {
      * the trace id.
      */
     public String traceId; // required
+    /**
+     * the request id, used to connected remote calls together.  Client can pass null.
+     */
+    public String requestId; // required
 
     /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
     public enum _Fields implements org.apache.blur.thirdparty.thrift_0_9_0.TFieldIdEnum {
       /**
        * the trace id.
        */
-      TRACE_ID((short)1, "traceId");
+      TRACE_ID((short)1, "traceId"),
+      /**
+       * the request id, used to connected remote calls together.  Client can pass null.
+       */
+      REQUEST_ID((short)2, "requestId");
 
       private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
 
@@ -32446,6 +32461,8 @@ public class Blur {
         switch(fieldId) {
           case 1: // TRACE_ID
             return TRACE_ID;
+          case 2: // REQUEST_ID
+            return REQUEST_ID;
           default:
             return null;
         }
@@ -32491,6 +32508,8 @@ public class Blur {
       Map<_Fields, org.apache.blur.thirdparty.thrift_0_9_0.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.blur.thirdparty.thrift_0_9_0.meta_data.FieldMetaData>(_Fields.class);
       tmpMap.put(_Fields.TRACE_ID, new org.apache.blur.thirdparty.thrift_0_9_0.meta_data.FieldMetaData("traceId", org.apache.blur.thirdparty.thrift_0_9_0.TFieldRequirementType.DEFAULT, 
           new org.apache.blur.thirdparty.thrift_0_9_0.meta_data.FieldValueMetaData(org.apache.blur.thirdparty.thrift_0_9_0.protocol.TType.STRING)));
+      tmpMap.put(_Fields.REQUEST_ID, new org.apache.blur.thirdparty.thrift_0_9_0.meta_data.FieldMetaData("requestId", org.apache.blur.thirdparty.thrift_0_9_0.TFieldRequirementType.DEFAULT, 
+          new org.apache.blur.thirdparty.thrift_0_9_0.meta_data.FieldValueMetaData(org.apache.blur.thirdparty.thrift_0_9_0.protocol.TType.STRING)));
       metaDataMap = Collections.unmodifiableMap(tmpMap);
       org.apache.blur.thirdparty.thrift_0_9_0.meta_data.FieldMetaData.addStructMetaDataMap(startTrace_args.class, metaDataMap);
     }
@@ -32499,10 +32518,12 @@ public class Blur {
     }
 
     public startTrace_args(
-      String traceId)
+      String traceId,
+      String requestId)
     {
       this();
       this.traceId = traceId;
+      this.requestId = requestId;
     }
 
     /**
@@ -32512,6 +32533,9 @@ public class Blur {
       if (other.isSetTraceId()) {
         this.traceId = other.traceId;
       }
+      if (other.isSetRequestId()) {
+        this.requestId = other.requestId;
+      }
     }
 
     public startTrace_args deepCopy() {
@@ -32521,6 +32545,7 @@ public class Blur {
     @Override
     public void clear() {
       this.traceId = null;
+      this.requestId = null;
     }
 
     /**
@@ -32553,6 +32578,36 @@ public class Blur {
       }
     }
 
+    /**
+     * the request id, used to connected remote calls together.  Client can pass null.
+     */
+    public String getRequestId() {
+      return this.requestId;
+    }
+
+    /**
+     * the request id, used to connected remote calls together.  Client can pass null.
+     */
+    public startTrace_args setRequestId(String requestId) {
+      this.requestId = requestId;
+      return this;
+    }
+
+    public void unsetRequestId() {
+      this.requestId = null;
+    }
+
+    /** Returns true if field requestId is set (has been assigned a value) and false otherwise */
+    public boolean isSetRequestId() {
+      return this.requestId != null;
+    }
+
+    public void setRequestIdIsSet(boolean value) {
+      if (!value) {
+        this.requestId = null;
+      }
+    }
+
     public void setFieldValue(_Fields field, Object value) {
       switch (field) {
       case TRACE_ID:
@@ -32563,6 +32618,14 @@ public class Blur {
         }
         break;
 
+      case REQUEST_ID:
+        if (value == null) {
+          unsetRequestId();
+        } else {
+          setRequestId((String)value);
+        }
+        break;
+
       }
     }
 
@@ -32571,6 +32634,9 @@ public class Blur {
       case TRACE_ID:
         return getTraceId();
 
+      case REQUEST_ID:
+        return getRequestId();
+
       }
       throw new IllegalStateException();
     }
@@ -32584,6 +32650,8 @@ public class Blur {
       switch (field) {
       case TRACE_ID:
         return isSetTraceId();
+      case REQUEST_ID:
+        return isSetRequestId();
       }
       throw new IllegalStateException();
     }
@@ -32610,6 +32678,15 @@ public class Blur {
           return false;
       }
 
+      boolean this_present_requestId = true && this.isSetRequestId();
+      boolean that_present_requestId = true && that.isSetRequestId();
+      if (this_present_requestId || that_present_requestId) {
+        if (!(this_present_requestId && that_present_requestId))
+          return false;
+        if (!this.requestId.equals(that.requestId))
+          return false;
+      }
+
       return true;
     }
 
@@ -32636,6 +32713,16 @@ public class Blur {
           return lastComparison;
         }
       }
+      lastComparison = Boolean.valueOf(isSetRequestId()).compareTo(typedOther.isSetRequestId());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+      if (isSetRequestId()) {
+        lastComparison = org.apache.blur.thirdparty.thrift_0_9_0.TBaseHelper.compareTo(this.requestId, typedOther.requestId);
+        if (lastComparison != 0) {
+          return lastComparison;
+        }
+      }
       return 0;
     }
 
@@ -32663,6 +32750,14 @@ public class Blur {
         sb.append(this.traceId);
       }
       first = false;
+      if (!first) sb.append(", ");
+      sb.append("requestId:");
+      if (this.requestId == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.requestId);
+      }
+      first = false;
       sb.append(")");
       return sb.toString();
     }
@@ -32714,6 +32809,14 @@ public class Blur {
                 org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocolUtil.skip(iprot, schemeField.type);
               }
               break;
+            case 2: // REQUEST_ID
+              if (schemeField.type == org.apache.blur.thirdparty.thrift_0_9_0.protocol.TType.STRING) {
+                struct.requestId = iprot.readString();
+                struct.setRequestIdIsSet(true);
+              } else { 
+                org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+              }
+              break;
             default:
               org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocolUtil.skip(iprot, schemeField.type);
           }
@@ -32734,6 +32837,11 @@ public class Blur {
           oprot.writeString(struct.traceId);
           oprot.writeFieldEnd();
         }
+        if (struct.requestId != null) {
+          oprot.writeFieldBegin(REQUEST_ID_FIELD_DESC);
+          oprot.writeString(struct.requestId);
+          oprot.writeFieldEnd();
+        }
         oprot.writeFieldStop();
         oprot.writeStructEnd();
       }
@@ -32755,20 +32863,30 @@ public class Blur {
         if (struct.isSetTraceId()) {
           optionals.set(0);
         }
-        oprot.writeBitSet(optionals, 1);
+        if (struct.isSetRequestId()) {
+          optionals.set(1);
+        }
+        oprot.writeBitSet(optionals, 2);
         if (struct.isSetTraceId()) {
           oprot.writeString(struct.traceId);
         }
+        if (struct.isSetRequestId()) {
+          oprot.writeString(struct.requestId);
+        }
       }
 
       @Override
       public void read(org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocol prot, startTrace_args struct) throws org.apache.blur.thirdparty.thrift_0_9_0.TException {
         TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
+        BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           struct.traceId = iprot.readString();
           struct.setTraceIdIsSet(true);
         }
+        if (incoming.get(1)) {
+          struct.requestId = iprot.readString();
+          struct.setRequestIdIsSet(true);
+        }
       }
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/blur-util/src/main/java/org/apache/blur/trace/LogTraceReporter.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/trace/LogTraceReporter.java b/blur-util/src/main/java/org/apache/blur/trace/LogTraceReporter.java
index 4c0f280..586f9d7 100644
--- a/blur-util/src/main/java/org/apache/blur/trace/LogTraceReporter.java
+++ b/blur-util/src/main/java/org/apache/blur/trace/LogTraceReporter.java
@@ -16,6 +16,8 @@
  */
 package org.apache.blur.trace;
 
+import java.io.IOException;
+
 import org.apache.blur.BlurConfiguration;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
@@ -34,4 +36,9 @@ public class LogTraceReporter extends TraceReporter {
     LOG.info("Trace Complete [{0}]", json);
   }
 
+  @Override
+  public void close() throws IOException {
+    
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/blur-util/src/main/java/org/apache/blur/trace/Trace.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/trace/Trace.java b/blur-util/src/main/java/org/apache/blur/trace/Trace.java
index d11a5e5..94ba2a6 100644
--- a/blur-util/src/main/java/org/apache/blur/trace/Trace.java
+++ b/blur-util/src/main/java/org/apache/blur/trace/Trace.java
@@ -29,6 +29,30 @@ import java.util.concurrent.TimeoutException;
 
 public class Trace {
 
+  private static final String REQUEST_ID = "requestId";
+
+  public static class TraceId {
+    final String _rootId;
+    final String _requestId;
+
+    public TraceId(String rootId, String requestId) {
+      _rootId = rootId;
+      _requestId = requestId;
+    }
+
+    public String getRootId() {
+      return _rootId;
+    }
+
+    public String getRequestId() {
+      return _requestId;
+    }
+
+    public String toJson() {
+      return "{\"rootId\":\"" + _rootId + "\",\"requestId\":\"" + _requestId + "\"}";
+    }
+  }
+
   public static class Parameter {
 
     final String _name;
@@ -65,8 +89,13 @@ public class Trace {
     _nodeName = nodeName;
   }
 
-  public static void setupTrace(String id) {
-    TraceCollector collector = new TraceCollector(_nodeName, id);
+  public static void setupTrace(String rootId) {
+    setupTrace(rootId, rootId);
+  }
+
+  public static void setupTrace(String rootId, String requestId) {
+    TraceId traceId = new TraceId(rootId, requestId);
+    TraceCollector collector = new TraceCollector(_nodeName, traceId);
     _tracer.set(collector);
   }
 
@@ -80,8 +109,8 @@ public class Trace {
     return new Parameter(name.toString(), value.toString());
   }
 
-  private static void setupTraceOnNewThread(TraceCollector parentCollector) {
-    _tracer.set(parentCollector);
+  private static void setupTraceOnNewThread(TraceCollector parentCollector, String requestId) {
+    _tracer.set(new TraceCollector(parentCollector, requestId));
   }
 
   private static void tearDownTraceOnNewThread() {
@@ -119,14 +148,14 @@ public class Trace {
     if (tc == null) {
       return runnable;
     }
-    final long threadId = _random.get().nextLong();
-    Tracer trace = Trace.trace("new runnable", Trace.param("thread_trace", threadId));
+    final long requestId = _random.get().nextLong();
+    Tracer trace = Trace.trace("new runnable", Trace.param(REQUEST_ID, requestId));
     try {
       return new Runnable() {
         @Override
         public void run() {
-          setupTraceOnNewThread(tc);
-          Tracer t = Trace.trace("executing runnable", Trace.param("thread_trace", threadId));
+          setupTraceOnNewThread(tc, Long.toString(requestId));
+          Tracer t = Trace.trace("executing runnable", Trace.param(REQUEST_ID, requestId));
           try {
             runnable.run();
           } finally {
@@ -145,14 +174,14 @@ public class Trace {
     if (tc == null) {
       return callable;
     }
-    final long threadId = _random.get().nextLong();
-    Tracer trace = Trace.trace("new callable", Trace.param("thread_trace", threadId));
+    final long requestId = _random.get().nextLong();
+    Tracer trace = Trace.trace("new callable", Trace.param(REQUEST_ID, requestId));
     try {
       return new Callable<V>() {
         @Override
         public V call() throws Exception {
-          setupTraceOnNewThread(tc);
-          Tracer t = Trace.trace("executing callable", Trace.param("thread_trace", threadId));
+          setupTraceOnNewThread(tc, Long.toString(requestId));
+          Tracer t = Trace.trace("executing callable", Trace.param(REQUEST_ID, requestId));
           try {
             return callable.call();
           } finally {
@@ -178,7 +207,7 @@ public class Trace {
     return list;
   }
 
-  public static String getTraceId() {
+  public static TraceId getTraceId() {
     TraceCollector collector = _tracer.get();
     if (collector == null) {
       return null;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/blur-util/src/main/java/org/apache/blur/trace/TraceCollector.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/trace/TraceCollector.java b/blur-util/src/main/java/org/apache/blur/trace/TraceCollector.java
index 47e01e7..68bc8b9 100644
--- a/blur-util/src/main/java/org/apache/blur/trace/TraceCollector.java
+++ b/blur-util/src/main/java/org/apache/blur/trace/TraceCollector.java
@@ -21,9 +21,11 @@ import java.util.List;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.atomic.AtomicLong;
 
+import org.apache.blur.trace.Trace.TraceId;
+
 public class TraceCollector {
 
-  protected final String _id;
+  protected final TraceId _id;
   protected final List<TracerImpl> _traces;
   protected final AtomicLong _traceCounter;
   protected final long _now = System.nanoTime();
@@ -31,7 +33,7 @@ public class TraceCollector {
   protected final String _threadName;
   protected final String _nodeName;
 
-  public TraceCollector(String nodeName, String id) {
+  public TraceCollector(String nodeName, TraceId id) {
     _nodeName = nodeName;
     _id = id;
     _traces = new CopyOnWriteArrayList<TracerImpl>();
@@ -40,6 +42,15 @@ public class TraceCollector {
     _threadName = Thread.currentThread().getName();
   }
 
+  public TraceCollector(TraceCollector parentCollector, String requestId) {
+    _nodeName = parentCollector._nodeName;
+    _id = new TraceId(parentCollector._id.getRootId(), requestId);
+    _traces = parentCollector._traces;
+    _traceCounter = parentCollector._traceCounter;
+    _pid = parentCollector._pid;
+    _threadName = parentCollector._threadName;
+  }
+
   public void add(TracerImpl tracer) {
     _traces.add(tracer);
   }
@@ -54,12 +65,12 @@ public class TraceCollector {
     for (TracerImpl t : _traces) {
       builder.append("    ").append(t.toJson()).append(",\n");
     }
-    return "{\n  \"id\":\"" + _id + "\",\n  \"nodeName\":\"" + (_nodeName == null ? "unknown" : _nodeName)
+    return "{\n  \"id\":" + _id.toJson() + ",\n  \"nodeName\":\"" + (_nodeName == null ? "unknown" : _nodeName)
         + "\",\n  \"pid\":\"" + _pid + "\",\n  \"thread\":\"" + _threadName + "\",\n  \"created\":" + _now
         + ",\n  \"traces\":[\n" + builder.toString() + "  ]\n}";
   }
 
-  public String getId() {
+  public TraceId getId() {
     return _id;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/blur-util/src/main/java/org/apache/blur/trace/TraceReporter.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/trace/TraceReporter.java b/blur-util/src/main/java/org/apache/blur/trace/TraceReporter.java
index fffb95f..c7a072c 100644
--- a/blur-util/src/main/java/org/apache/blur/trace/TraceReporter.java
+++ b/blur-util/src/main/java/org/apache/blur/trace/TraceReporter.java
@@ -16,9 +16,11 @@
  */
 package org.apache.blur.trace;
 
+import java.io.Closeable;
+
 import org.apache.blur.BlurConfiguration;
 
-public abstract class TraceReporter {
+public abstract class TraceReporter implements Closeable {
 
   protected final BlurConfiguration _configuration;
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/blur-util/src/main/java/org/apache/blur/trace/ZooKeeperDumpContents.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/trace/ZooKeeperDumpContents.java b/blur-util/src/main/java/org/apache/blur/trace/ZooKeeperDumpContents.java
index dcc4ef4..25db507 100644
--- a/blur-util/src/main/java/org/apache/blur/trace/ZooKeeperDumpContents.java
+++ b/blur-util/src/main/java/org/apache/blur/trace/ZooKeeperDumpContents.java
@@ -16,7 +16,7 @@
  */
 package org.apache.blur.trace;
 
-import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION;
+import static org.apache.blur.utils.BlurConstants.*;
 
 import java.io.IOException;
 import java.util.List;
@@ -40,17 +40,24 @@ public class ZooKeeperDumpContents {
       }
     });
 
-    String parentPath = args[0];
-    String id = args[1];
+    String parentPath = configuration.get(BLUR_ZOOKEEPER_TRACE_PATH);
+    String id = args[0];
 
     String path = parentPath + "/" + id;
     List<String> children = zooKeeper.getChildren(path, false);
+    System.out.println("[");
+    boolean first = true;
     for (String c : children) {
+      if (!first) {
+        System.out.println(",");
+      }
       Stat stat = zooKeeper.exists(path + "/" + c, false);
       byte[] data = zooKeeper.getData(path + "/" + c, false, stat);
       String string = new String(data);
       System.out.println(string);
+      first = false;
     }
+    System.out.println("]");
     zooKeeper.close();
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/blur-util/src/main/java/org/apache/blur/trace/ZooKeeperTraceReporter.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/trace/ZooKeeperTraceReporter.java b/blur-util/src/main/java/org/apache/blur/trace/ZooKeeperTraceReporter.java
index 32bb6b9..f39e686 100644
--- a/blur-util/src/main/java/org/apache/blur/trace/ZooKeeperTraceReporter.java
+++ b/blur-util/src/main/java/org/apache/blur/trace/ZooKeeperTraceReporter.java
@@ -27,6 +27,7 @@ import java.util.concurrent.LinkedBlockingQueue;
 import org.apache.blur.BlurConfiguration;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
+import org.apache.blur.trace.Trace.TraceId;
 import org.apache.blur.zookeeper.ZooKeeperClient;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
@@ -89,14 +90,8 @@ public class ZooKeeperTraceReporter extends TraceReporter {
   }
 
   private void writeCollector(TraceCollector collector) {
-    String id = collector.getId();
-    int indexOf = id.indexOf(":");
-    String storeId;
-    if (indexOf < 0) {
-      storeId = id;
-    } else {
-      storeId = id.substring(0, indexOf);
-    }
+    TraceId id = collector.getId();
+    String storeId = id.getRootId();
     String storePath = getStorePath(storeId);
     createIfMissing(storePath);
     String json = collector.toJson();
@@ -132,4 +127,13 @@ public class ZooKeeperTraceReporter extends TraceReporter {
     return _storePath + "/" + storeId;
   }
 
+  @Override
+  public void close() throws IOException {
+    try {
+      _zooKeeperClient.close();
+    } catch (InterruptedException e) {
+      throw new IOException(e);
+    }
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/blur-util/src/main/resources/blur-default.properties
----------------------------------------------------------------------
diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties
index 592bc65..393bfb2 100644
--- a/blur-util/src/main/resources/blur-default.properties
+++ b/blur-util/src/main/resources/blur-default.properties
@@ -17,7 +17,7 @@
 blur.zookeeper.timeout=90000
 
 # The path in ZooKeeper where the distributed traces will be stored, if blank trace output will be written to the log
-blur.zookeeper.trace.path=/blur_traces
+blur.zookeeper.trace.path=/blur/traces
 
 # The maximum number of results that can be fetched in a single request
 blur.query.max.results.fetch=1000

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/blur-util/src/test/java/org/apache/blur/trace/TraceTest.java
----------------------------------------------------------------------
diff --git a/blur-util/src/test/java/org/apache/blur/trace/TraceTest.java b/blur-util/src/test/java/org/apache/blur/trace/TraceTest.java
index b8074f4..222a41e 100644
--- a/blur-util/src/test/java/org/apache/blur/trace/TraceTest.java
+++ b/blur-util/src/test/java/org/apache/blur/trace/TraceTest.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.blur.BlurConfiguration;
+import org.apache.blur.trace.Trace.TraceId;
 import org.junit.Test;
 
 public class TraceTest {
@@ -31,11 +32,16 @@ public class TraceTest {
     Trace.setReporter(new TraceReporter(new BlurConfiguration()) {
       @Override
       public void report(TraceCollector collector) {
-        assertEquals("test", collector.getId());
+        assertEquals("test", collector.getId().getRootId());
         assertEquals(3, collector.getTraces().size());
       }
+
+      @Override
+      public void close() throws IOException {
+        
+      }
     });
-    Trace.setupTrace("test");
+    Trace.setupTrace("test", null);
     Tracer trace = Trace.trace("1");
     long meth1;
     try {
@@ -66,20 +72,20 @@ public class TraceTest {
       @Override
       public void report(TraceCollector collector) {
         System.out.println(collector.toJson());
-        String id = collector.getId();
-        int indexOf = id.indexOf(":");
-        if (indexOf < 0) {
-          assertEquals("test", id);
-        } else {
-          assertEquals("test", id.substring(0,indexOf));
-        }
-        assertEquals(7, collector.getTraces().size());
+        TraceId id = collector.getId();
+        assertEquals("test", id.getRootId());
+        assertEquals(8, collector.getTraces().size());
         count.addAndGet(collector.getTraces().size());
       }
+
+      @Override
+      public void close() throws IOException {
+        
+      }
     });
 
-    Trace.setupTrace("test");
-    
+    Trace.setupTrace("test", "1");
+
     final Runnable runnable = new Runnable() {
       @Override
       public void run() {
@@ -105,8 +111,8 @@ public class TraceTest {
     System.out.println(meth1);
     thread.join();
     Trace.tearDownTrace();
-    
-    assertEquals(7, count.get());
+
+    assertEquals(8, count.get());
   }
 
   private static long meth1() {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/distribution/src/main/scripts/interface/Blur.thrift
----------------------------------------------------------------------
diff --git a/distribution/src/main/scripts/interface/Blur.thrift b/distribution/src/main/scripts/interface/Blur.thrift
index 0b37090..0b24392 100644
--- a/distribution/src/main/scripts/interface/Blur.thrift
+++ b/distribution/src/main/scripts/interface/Blur.thrift
@@ -1089,7 +1089,9 @@ service Blur {
    */
   oneway void startTrace(
     /** the trace id. */
-    1:string traceId
+    1:string traceId,
+    /** the request id, used to connected remote calls together.  Client can pass null. */
+    2:string requestId
   )
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/distribution/src/main/scripts/interface/gen-html/Blur.html
----------------------------------------------------------------------
diff --git a/distribution/src/main/scripts/interface/gen-html/Blur.html b/distribution/src/main/scripts/interface/gen-html/Blur.html
index 4b6cd66..80e463b 100644
--- a/distribution/src/main/scripts/interface/gen-html/Blur.html
+++ b/distribution/src/main/scripts/interface/gen-html/Blur.html
@@ -832,9 +832,12 @@ the shard given the same situation.
 <table class="table-bordered table-striped table-condensed"><thead><th>Name</th><th>Description</th></thead><tr><td>metrics</td><td>the names of the metrics to return.  If null all are returned.
 </td></tr>
 </table></div><div class="definition"><h4 id="Fn_Blur_startTrace">Function: Blur.startTrace</h4>
-<pre><code>void</code> startTrace(<code>string</code> traceId)
+<pre><code>void</code> startTrace(<code>string</code> traceId,
+                <code>string</code> requestId)
 </pre>Starts a trace with the given trace id.
 <br/><br/><h4 id="Parameters_Blur_startTrace">Parameters</h4>
 <table class="table-bordered table-striped table-condensed"><thead><th>Name</th><th>Description</th></thead><tr><td>traceId</td><td>the trace id.
 </td></tr>
+<tr><td>requestId</td><td>the request id, used to connected remote calls together.  Client can pass null.
+</td></tr>
 </table></div></div></body></html>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/distribution/src/main/scripts/interface/gen-java/org/apache/blur/thrift/generated/Blur.java
----------------------------------------------------------------------
diff --git a/distribution/src/main/scripts/interface/gen-java/org/apache/blur/thrift/generated/Blur.java b/distribution/src/main/scripts/interface/gen-java/org/apache/blur/thrift/generated/Blur.java
index 8182634..c1d5514 100644
--- a/distribution/src/main/scripts/interface/gen-java/org/apache/blur/thrift/generated/Blur.java
+++ b/distribution/src/main/scripts/interface/gen-java/org/apache/blur/thrift/generated/Blur.java
@@ -370,8 +370,10 @@ public class Blur {
      * Starts a trace with the given trace id.
      * 
      * @param traceId the trace id.
+     * 
+     * @param requestId the request id, used to connected remote calls together.  Client can pass null.
      */
-    public void startTrace(String traceId) throws org.apache.blur.thirdparty.thrift_0_9_0.TException;
+    public void startTrace(String traceId, String requestId) throws org.apache.blur.thirdparty.thrift_0_9_0.TException;
 
   }
 
@@ -445,7 +447,7 @@ public class Blur {
 
     public void metrics(Set<String> metrics, org.apache.blur.thirdparty.thrift_0_9_0.async.AsyncMethodCallback<AsyncClient.metrics_call> resultHandler) throws org.apache.blur.thirdparty.thrift_0_9_0.TException;
 
-    public void startTrace(String traceId, org.apache.blur.thirdparty.thrift_0_9_0.async.AsyncMethodCallback<AsyncClient.startTrace_call> resultHandler) throws org.apache.blur.thirdparty.thrift_0_9_0.TException;
+    public void startTrace(String traceId, String requestId, org.apache.blur.thirdparty.thrift_0_9_0.async.AsyncMethodCallback<AsyncClient.startTrace_call> resultHandler) throws org.apache.blur.thirdparty.thrift_0_9_0.TException;
 
   }
 
@@ -1323,15 +1325,16 @@ public class Blur {
       throw new org.apache.blur.thirdparty.thrift_0_9_0.TApplicationException(org.apache.blur.thirdparty.thrift_0_9_0.TApplicationException.MISSING_RESULT, "metrics failed: unknown result");
     }
 
-    public void startTrace(String traceId) throws org.apache.blur.thirdparty.thrift_0_9_0.TException
+    public void startTrace(String traceId, String requestId) throws org.apache.blur.thirdparty.thrift_0_9_0.TException
     {
-      send_startTrace(traceId);
+      send_startTrace(traceId, requestId);
     }
 
-    public void send_startTrace(String traceId) throws org.apache.blur.thirdparty.thrift_0_9_0.TException
+    public void send_startTrace(String traceId, String requestId) throws org.apache.blur.thirdparty.thrift_0_9_0.TException
     {
       startTrace_args args = new startTrace_args();
       args.setTraceId(traceId);
+      args.setRequestId(requestId);
       sendBase("startTrace", args);
     }
 
@@ -2482,24 +2485,27 @@ public class Blur {
       }
     }
 
-    public void startTrace(String traceId, org.apache.blur.thirdparty.thrift_0_9_0.async.AsyncMethodCallback<startTrace_call> resultHandler) throws org.apache.blur.thirdparty.thrift_0_9_0.TException {
+    public void startTrace(String traceId, String requestId, org.apache.blur.thirdparty.thrift_0_9_0.async.AsyncMethodCallback<startTrace_call> resultHandler) throws org.apache.blur.thirdparty.thrift_0_9_0.TException {
       checkReady();
-      startTrace_call method_call = new startTrace_call(traceId, resultHandler, this, ___protocolFactory, ___transport);
+      startTrace_call method_call = new startTrace_call(traceId, requestId, resultHandler, this, ___protocolFactory, ___transport);
       this.___currentMethod = method_call;
       ___manager.call(method_call);
     }
 
     public static class startTrace_call extends org.apache.blur.thirdparty.thrift_0_9_0.async.TAsyncMethodCall {
       private String traceId;
-      public startTrace_call(String traceId, org.apache.blur.thirdparty.thrift_0_9_0.async.AsyncMethodCallback<startTrace_call> resultHandler, org.apache.blur.thirdparty.thrift_0_9_0.async.TAsyncClient client, org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocolFactory protocolFactory, org.apache.blur.thirdparty.thrift_0_9_0.transport.TNonblockingTransport transport) throws org.apache.blur.thirdparty.thrift_0_9_0.TException {
+      private String requestId;
+      public startTrace_call(String traceId, String requestId, org.apache.blur.thirdparty.thrift_0_9_0.async.AsyncMethodCallback<startTrace_call> resultHandler, org.apache.blur.thirdparty.thrift_0_9_0.async.TAsyncClient client, org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocolFactory protocolFactory, org.apache.blur.thirdparty.thrift_0_9_0.transport.TNonblockingTransport transport) throws org.apache.blur.thirdparty.thrift_0_9_0.TException {
         super(client, protocolFactory, transport, resultHandler, true);
         this.traceId = traceId;
+        this.requestId = requestId;
       }
 
       public void write_args(org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocol prot) throws org.apache.blur.thirdparty.thrift_0_9_0.TException {
         prot.writeMessageBegin(new org.apache.blur.thirdparty.thrift_0_9_0.protocol.TMessage("startTrace", org.apache.blur.thirdparty.thrift_0_9_0.protocol.TMessageType.CALL, 0));
         startTrace_args args = new startTrace_args();
         args.setTraceId(traceId);
+        args.setRequestId(requestId);
         args.write(prot);
         prot.writeMessageEnd();
       }
@@ -3392,7 +3398,7 @@ public class Blur {
       }
 
       public org.apache.blur.thirdparty.thrift_0_9_0.TBase getResult(I iface, startTrace_args args) throws org.apache.blur.thirdparty.thrift_0_9_0.TException {
-        iface.startTrace(args.traceId);
+        iface.startTrace(args.traceId, args.requestId);
         return null;
       }
     }
@@ -32412,6 +32418,7 @@ public class Blur {
     private static final org.apache.blur.thirdparty.thrift_0_9_0.protocol.TStruct STRUCT_DESC = new org.apache.blur.thirdparty.thrift_0_9_0.protocol.TStruct("startTrace_args");
 
     private static final org.apache.blur.thirdparty.thrift_0_9_0.protocol.TField TRACE_ID_FIELD_DESC = new org.apache.blur.thirdparty.thrift_0_9_0.protocol.TField("traceId", org.apache.blur.thirdparty.thrift_0_9_0.protocol.TType.STRING, (short)1);
+    private static final org.apache.blur.thirdparty.thrift_0_9_0.protocol.TField REQUEST_ID_FIELD_DESC = new org.apache.blur.thirdparty.thrift_0_9_0.protocol.TField("requestId", org.apache.blur.thirdparty.thrift_0_9_0.protocol.TType.STRING, (short)2);
 
     private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
     static {
@@ -32423,13 +32430,21 @@ public class Blur {
      * the trace id.
      */
     public String traceId; // required
+    /**
+     * the request id, used to connected remote calls together.  Client can pass null.
+     */
+    public String requestId; // required
 
     /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
     public enum _Fields implements org.apache.blur.thirdparty.thrift_0_9_0.TFieldIdEnum {
       /**
        * the trace id.
        */
-      TRACE_ID((short)1, "traceId");
+      TRACE_ID((short)1, "traceId"),
+      /**
+       * the request id, used to connected remote calls together.  Client can pass null.
+       */
+      REQUEST_ID((short)2, "requestId");
 
       private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
 
@@ -32446,6 +32461,8 @@ public class Blur {
         switch(fieldId) {
           case 1: // TRACE_ID
             return TRACE_ID;
+          case 2: // REQUEST_ID
+            return REQUEST_ID;
           default:
             return null;
         }
@@ -32491,6 +32508,8 @@ public class Blur {
       Map<_Fields, org.apache.blur.thirdparty.thrift_0_9_0.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.blur.thirdparty.thrift_0_9_0.meta_data.FieldMetaData>(_Fields.class);
       tmpMap.put(_Fields.TRACE_ID, new org.apache.blur.thirdparty.thrift_0_9_0.meta_data.FieldMetaData("traceId", org.apache.blur.thirdparty.thrift_0_9_0.TFieldRequirementType.DEFAULT, 
           new org.apache.blur.thirdparty.thrift_0_9_0.meta_data.FieldValueMetaData(org.apache.blur.thirdparty.thrift_0_9_0.protocol.TType.STRING)));
+      tmpMap.put(_Fields.REQUEST_ID, new org.apache.blur.thirdparty.thrift_0_9_0.meta_data.FieldMetaData("requestId", org.apache.blur.thirdparty.thrift_0_9_0.TFieldRequirementType.DEFAULT, 
+          new org.apache.blur.thirdparty.thrift_0_9_0.meta_data.FieldValueMetaData(org.apache.blur.thirdparty.thrift_0_9_0.protocol.TType.STRING)));
       metaDataMap = Collections.unmodifiableMap(tmpMap);
       org.apache.blur.thirdparty.thrift_0_9_0.meta_data.FieldMetaData.addStructMetaDataMap(startTrace_args.class, metaDataMap);
     }
@@ -32499,10 +32518,12 @@ public class Blur {
     }
 
     public startTrace_args(
-      String traceId)
+      String traceId,
+      String requestId)
     {
       this();
       this.traceId = traceId;
+      this.requestId = requestId;
     }
 
     /**
@@ -32512,6 +32533,9 @@ public class Blur {
       if (other.isSetTraceId()) {
         this.traceId = other.traceId;
       }
+      if (other.isSetRequestId()) {
+        this.requestId = other.requestId;
+      }
     }
 
     public startTrace_args deepCopy() {
@@ -32521,6 +32545,7 @@ public class Blur {
     @Override
     public void clear() {
       this.traceId = null;
+      this.requestId = null;
     }
 
     /**
@@ -32553,6 +32578,36 @@ public class Blur {
       }
     }
 
+    /**
+     * the request id, used to connected remote calls together.  Client can pass null.
+     */
+    public String getRequestId() {
+      return this.requestId;
+    }
+
+    /**
+     * the request id, used to connected remote calls together.  Client can pass null.
+     */
+    public startTrace_args setRequestId(String requestId) {
+      this.requestId = requestId;
+      return this;
+    }
+
+    public void unsetRequestId() {
+      this.requestId = null;
+    }
+
+    /** Returns true if field requestId is set (has been assigned a value) and false otherwise */
+    public boolean isSetRequestId() {
+      return this.requestId != null;
+    }
+
+    public void setRequestIdIsSet(boolean value) {
+      if (!value) {
+        this.requestId = null;
+      }
+    }
+
     public void setFieldValue(_Fields field, Object value) {
       switch (field) {
       case TRACE_ID:
@@ -32563,6 +32618,14 @@ public class Blur {
         }
         break;
 
+      case REQUEST_ID:
+        if (value == null) {
+          unsetRequestId();
+        } else {
+          setRequestId((String)value);
+        }
+        break;
+
       }
     }
 
@@ -32571,6 +32634,9 @@ public class Blur {
       case TRACE_ID:
         return getTraceId();
 
+      case REQUEST_ID:
+        return getRequestId();
+
       }
       throw new IllegalStateException();
     }
@@ -32584,6 +32650,8 @@ public class Blur {
       switch (field) {
       case TRACE_ID:
         return isSetTraceId();
+      case REQUEST_ID:
+        return isSetRequestId();
       }
       throw new IllegalStateException();
     }
@@ -32610,6 +32678,15 @@ public class Blur {
           return false;
       }
 
+      boolean this_present_requestId = true && this.isSetRequestId();
+      boolean that_present_requestId = true && that.isSetRequestId();
+      if (this_present_requestId || that_present_requestId) {
+        if (!(this_present_requestId && that_present_requestId))
+          return false;
+        if (!this.requestId.equals(that.requestId))
+          return false;
+      }
+
       return true;
     }
 
@@ -32636,6 +32713,16 @@ public class Blur {
           return lastComparison;
         }
       }
+      lastComparison = Boolean.valueOf(isSetRequestId()).compareTo(typedOther.isSetRequestId());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+      if (isSetRequestId()) {
+        lastComparison = org.apache.blur.thirdparty.thrift_0_9_0.TBaseHelper.compareTo(this.requestId, typedOther.requestId);
+        if (lastComparison != 0) {
+          return lastComparison;
+        }
+      }
       return 0;
     }
 
@@ -32663,6 +32750,14 @@ public class Blur {
         sb.append(this.traceId);
       }
       first = false;
+      if (!first) sb.append(", ");
+      sb.append("requestId:");
+      if (this.requestId == null) {
+        sb.append("null");
+      } else {
+        sb.append(this.requestId);
+      }
+      first = false;
       sb.append(")");
       return sb.toString();
     }
@@ -32714,6 +32809,14 @@ public class Blur {
                 org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocolUtil.skip(iprot, schemeField.type);
               }
               break;
+            case 2: // REQUEST_ID
+              if (schemeField.type == org.apache.blur.thirdparty.thrift_0_9_0.protocol.TType.STRING) {
+                struct.requestId = iprot.readString();
+                struct.setRequestIdIsSet(true);
+              } else { 
+                org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+              }
+              break;
             default:
               org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocolUtil.skip(iprot, schemeField.type);
           }
@@ -32734,6 +32837,11 @@ public class Blur {
           oprot.writeString(struct.traceId);
           oprot.writeFieldEnd();
         }
+        if (struct.requestId != null) {
+          oprot.writeFieldBegin(REQUEST_ID_FIELD_DESC);
+          oprot.writeString(struct.requestId);
+          oprot.writeFieldEnd();
+        }
         oprot.writeFieldStop();
         oprot.writeStructEnd();
       }
@@ -32755,20 +32863,30 @@ public class Blur {
         if (struct.isSetTraceId()) {
           optionals.set(0);
         }
-        oprot.writeBitSet(optionals, 1);
+        if (struct.isSetRequestId()) {
+          optionals.set(1);
+        }
+        oprot.writeBitSet(optionals, 2);
         if (struct.isSetTraceId()) {
           oprot.writeString(struct.traceId);
         }
+        if (struct.isSetRequestId()) {
+          oprot.writeString(struct.requestId);
+        }
       }
 
       @Override
       public void read(org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocol prot, startTrace_args struct) throws org.apache.blur.thirdparty.thrift_0_9_0.TException {
         TTupleProtocol iprot = (TTupleProtocol) prot;
-        BitSet incoming = iprot.readBitSet(1);
+        BitSet incoming = iprot.readBitSet(2);
         if (incoming.get(0)) {
           struct.traceId = iprot.readString();
           struct.setTraceIdIsSet(true);
         }
+        if (incoming.get(1)) {
+          struct.requestId = iprot.readString();
+          struct.setRequestIdIsSet(true);
+        }
       }
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/distribution/src/main/scripts/interface/gen-js/Blur.js
----------------------------------------------------------------------
diff --git a/distribution/src/main/scripts/interface/gen-js/Blur.js b/distribution/src/main/scripts/interface/gen-js/Blur.js
index b26e358..25aed89 100644
--- a/distribution/src/main/scripts/interface/gen-js/Blur.js
+++ b/distribution/src/main/scripts/interface/gen-js/Blur.js
@@ -4674,10 +4674,14 @@ Blur_metrics_result.prototype.write = function(output) {
 
 Blur_startTrace_args = function(args) {
   this.traceId = null;
+  this.requestId = null;
   if (args) {
     if (args.traceId !== undefined) {
       this.traceId = args.traceId;
     }
+    if (args.requestId !== undefined) {
+      this.requestId = args.requestId;
+    }
   }
 };
 Blur_startTrace_args.prototype = {};
@@ -4701,9 +4705,13 @@ Blur_startTrace_args.prototype.read = function(input) {
         input.skip(ftype);
       }
       break;
-      case 0:
+      case 2:
+      if (ftype == Thrift.Type.STRING) {
+        this.requestId = input.readString().value;
+      } else {
         input.skip(ftype);
-        break;
+      }
+      break;
       default:
         input.skip(ftype);
     }
@@ -4720,6 +4728,11 @@ Blur_startTrace_args.prototype.write = function(output) {
     output.writeString(this.traceId);
     output.writeFieldEnd();
   }
+  if (this.requestId !== null && this.requestId !== undefined) {
+    output.writeFieldBegin('requestId', Thrift.Type.STRING, 2);
+    output.writeString(this.requestId);
+    output.writeFieldEnd();
+  }
   output.writeFieldStop();
   output.writeStructEnd();
   return;
@@ -5976,14 +5989,15 @@ BlurClient.prototype.recv_metrics = function() {
   }
   throw 'metrics failed: unknown result';
 };
-BlurClient.prototype.startTrace = function(traceId) {
-  this.send_startTrace(traceId);
+BlurClient.prototype.startTrace = function(traceId, requestId) {
+  this.send_startTrace(traceId, requestId);
 };
 
-BlurClient.prototype.send_startTrace = function(traceId) {
+BlurClient.prototype.send_startTrace = function(traceId, requestId) {
   this.output.writeMessageBegin('startTrace', Thrift.MessageType.CALL, this.seqid);
   var args = new Blur_startTrace_args();
   args.traceId = traceId;
+  args.requestId = requestId;
   args.write(this.output);
   this.output.writeMessageEnd();
   return this.output.getTransport().flush();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/distribution/src/main/scripts/interface/gen-perl/Blur/Blur.pm
----------------------------------------------------------------------
diff --git a/distribution/src/main/scripts/interface/gen-perl/Blur/Blur.pm b/distribution/src/main/scripts/interface/gen-perl/Blur/Blur.pm
index 61a2877..335da5d 100644
--- a/distribution/src/main/scripts/interface/gen-perl/Blur/Blur.pm
+++ b/distribution/src/main/scripts/interface/gen-perl/Blur/Blur.pm
@@ -5343,17 +5343,21 @@ sub write {
 
 package Blur::Blur_startTrace_args;
 use base qw(Class::Accessor);
-Blur::Blur_startTrace_args->mk_accessors( qw( traceId ) );
+Blur::Blur_startTrace_args->mk_accessors( qw( traceId requestId ) );
 
 sub new {
   my $classname = shift;
   my $self      = {};
   my $vals      = shift || {};
   $self->{traceId} = undef;
+  $self->{requestId} = undef;
   if (UNIVERSAL::isa($vals,'HASH')) {
     if (defined $vals->{traceId}) {
       $self->{traceId} = $vals->{traceId};
     }
+    if (defined $vals->{requestId}) {
+      $self->{requestId} = $vals->{requestId};
+    }
   }
   return bless ($self, $classname);
 }
@@ -5383,6 +5387,12 @@ sub read {
         $xfer += $input->skip($ftype);
       }
       last; };
+      /^2$/ && do{      if ($ftype == TType::STRING) {
+        $xfer += $input->readString(\$self->{requestId});
+      } else {
+        $xfer += $input->skip($ftype);
+      }
+      last; };
         $xfer += $input->skip($ftype);
     }
     $xfer += $input->readFieldEnd();
@@ -5400,6 +5410,11 @@ sub write {
     $xfer += $output->writeString($self->{traceId});
     $xfer += $output->writeFieldEnd();
   }
+  if (defined $self->{requestId}) {
+    $xfer += $output->writeFieldBegin('requestId', TType::STRING, 2);
+    $xfer += $output->writeString($self->{requestId});
+    $xfer += $output->writeFieldEnd();
+  }
   $xfer += $output->writeFieldStop();
   $xfer += $output->writeStructEnd();
   return $xfer;
@@ -5711,6 +5726,7 @@ sub metrics{
 sub startTrace{
   my $self = shift;
   my $traceId = shift;
+  my $requestId = shift;
 
   die 'implement interface';
 }
@@ -5983,7 +5999,8 @@ sub startTrace{
   my ($self, $request) = @_;
 
   my $traceId = ($request->{'traceId'}) ? $request->{'traceId'} : undef;
-  return $self->{impl}->startTrace($traceId);
+  my $requestId = ($request->{'requestId'}) ? $request->{'requestId'} : undef;
+  return $self->{impl}->startTrace($traceId, $requestId);
 }
 
 package Blur::BlurClient;
@@ -7550,17 +7567,20 @@ sub recv_metrics{
 sub startTrace{
   my $self = shift;
   my $traceId = shift;
+  my $requestId = shift;
 
-    $self->send_startTrace($traceId);
+    $self->send_startTrace($traceId, $requestId);
 }
 
 sub send_startTrace{
   my $self = shift;
   my $traceId = shift;
+  my $requestId = shift;
 
   $self->{output}->writeMessageBegin('startTrace', TMessageType::CALL, $self->{seqid});
   my $args = new Blur::Blur_startTrace_args();
   $args->{traceId} = $traceId;
+  $args->{requestId} = $requestId;
   $args->write($self->{output});
   $self->{output}->writeMessageEnd();
   $self->{output}->getTransport()->flush();
@@ -8173,7 +8193,7 @@ sub process_startTrace {
     my $args = new Blur::Blur_startTrace_args();
     $args->read($input);
     $input->readMessageEnd();
-    $self->{handler}->startTrace($args->traceId);
+    $self->{handler}->startTrace($args->traceId, $args->requestId);
     return;
 }
 1;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/distribution/src/main/scripts/interface/gen-rb/blur.rb
----------------------------------------------------------------------
diff --git a/distribution/src/main/scripts/interface/gen-rb/blur.rb b/distribution/src/main/scripts/interface/gen-rb/blur.rb
index 00d0a63..6439b3b 100644
--- a/distribution/src/main/scripts/interface/gen-rb/blur.rb
+++ b/distribution/src/main/scripts/interface/gen-rb/blur.rb
@@ -537,12 +537,12 @@ module Blur
         raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'metrics failed: unknown result')
       end
 
-      def startTrace(traceId)
-        send_startTrace(traceId)
+      def startTrace(traceId, requestId)
+        send_startTrace(traceId, requestId)
       end
 
-      def send_startTrace(traceId)
-        send_message('startTrace', StartTrace_args, :traceId => traceId)
+      def send_startTrace(traceId, requestId)
+        send_message('startTrace', StartTrace_args, :traceId => traceId, :requestId => requestId)
       end
     end
 
@@ -920,7 +920,7 @@ module Blur
 
       def process_startTrace(seqid, iprot, oprot)
         args = read_args(iprot, StartTrace_args)
-        @handler.startTrace(args.traceId)
+        @handler.startTrace(args.traceId, args.requestId)
         return
       end
 
@@ -2139,10 +2139,13 @@ module Blur
     class StartTrace_args
       include ::Thrift::Struct, ::Thrift::Struct_Union
       TRACEID = 1
+      REQUESTID = 2
 
       FIELDS = {
         # the trace id.
-        TRACEID => {:type => ::Thrift::Types::STRING, :name => 'traceId'}
+        TRACEID => {:type => ::Thrift::Types::STRING, :name => 'traceId'},
+        # the request id, used to connected remote calls together.  Client can pass null.
+        REQUESTID => {:type => ::Thrift::Types::STRING, :name => 'requestId'}
       }
 
       def struct_fields; FIELDS; end

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/772facd9/docs/Blur.html
----------------------------------------------------------------------
diff --git a/docs/Blur.html b/docs/Blur.html
index 56349a9..2e17c2f 100644
--- a/docs/Blur.html
+++ b/docs/Blur.html
@@ -891,11 +891,14 @@ throws <code><a href="Blur.html#Struct_BlurException">BlurException</a></code>
 <table class="table-bordered table-striped table-condensed"><thead><th>Name</th><th>Description</th></thead><tr><td>metrics</td><td>the names of the metrics to return.  If null all are returned.
 </td></tr>
 </table></p></section><section><div class="page-header"><h4 id="Fn_Blur_startTrace">Function: Blur.startTrace</h4></div><p class="lead">
-<pre><code>void</code> startTrace(<code>string</code> traceId)
+<pre><code>void</code> startTrace(<code>string</code> traceId,
+<code>string</code> requestId)
 </pre>Starts a trace with the given trace id.
 <br/><br/><h4 id="Parameters_Blur_startTrace">Parameters</h4>
 <table class="table-bordered table-striped table-condensed"><thead><th>Name</th><th>Description</th></thead><tr><td>traceId</td><td>the trace id.
 </td></tr>
+<tr><td>requestId</td><td>the request id, used to connected remote calls together.  Client can pass null.
+</td></tr>
 </table></p></section>
 </div>
 </div>


Mime
View raw message