incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: A lot of changes to tracing, should be functional at this point
Date Mon, 25 Nov 2013 14:28:09 GMT
Updated Branches:
  refs/heads/master ea808c482 -> 53f9ecf53


A lot of changes to tracing, should be functional at this point


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/53f9ecf5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/53f9ecf5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/53f9ecf5

Branch: refs/heads/master
Commit: 53f9ecf531ab669c8fa0be094c46842e5b55cce8
Parents: ea808c4
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Mon Nov 25 09:27:30 2013 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Mon Nov 25 09:28:02 2013 -0500

----------------------------------------------------------------------
 .../org/apache/blur/manager/IndexManager.java   |   6 +-
 .../AbstractDistributedIndexServer.java         |   5 +
 .../apache/blur/server/BlurServerContext.java   |  11 +-
 .../blur/thrift/BlurControllerServer.java       |  10 +-
 .../blur/thrift/ThriftBlurControllerServer.java |   1 +
 .../blur/thrift/ThriftBlurShardServer.java      |   1 +
 .../java/org/apache/blur/utils/BlurUtil.java    |  12 +-
 .../test/java/org/apache/blur/MiniCluster.java  |   3 +-
 .../apache/blur/analysis/HdfsFieldManager.java  | 121 ++++++++++---------
 .../blur/index/OptimisticLockFactory.java       |  71 ++++++-----
 .../apache/blur/store/hdfs/BlurLockFactory.java |  82 ++++++++-----
 .../apache/blur/store/hdfs/HdfsDirectory.java   |  87 +++++++++----
 .../apache/blur/store/hdfs/HdfsIndexInput.java  |  83 +++++++------
 .../transport/TFramedTransport.java             |   8 ++
 .../apache/blur/thrift/BlurClientManager.java   |  27 +++++
 .../blur/thrift/util/SimpleQueryExample.java    |   6 +
 .../org/apache/blur/thrift/util/StatsTable.java |   3 +-
 .../main/java/org/apache/blur/trace/Trace.java  |  98 ++++++++++-----
 .../org/apache/blur/trace/TraceCollector.java   |  12 +-
 .../org/apache/blur/trace/TraceReporter.java    |   4 +-
 .../main/java/org/apache/blur/trace/Tracer.java |   1 +
 .../java/org/apache/blur/trace/TracerImpl.java  |  27 ++++-
 .../blur/trace/ZooKeeperDumpContents.java       |  57 +++++++++
 .../apache/blur/zookeeper/ZooKeeperClient.java  |   2 +-
 .../src/main/resources/blur-default.properties  |   2 +-
 .../java/org/apache/blur/trace/TraceTest.java   |   4 +-
 26 files changed, 518 insertions(+), 226 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
index 3a0ddcc..115815b 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
@@ -236,6 +236,7 @@ public class IndexManager {
     validSelector(selector);
     BlurIndex index;
     String shard;
+    Tracer trace = Trace.trace("manager fetch", Trace.param("table", table));
     try {
       if (selector.getLocationId() == null) {
         // Not looking up by location id so we should resetSearchers.
@@ -302,6 +303,7 @@ public class IndexManager {
       LOG.error("Unknown error while trying to fetch row.", e);
       throw new BException(e.getMessage(), e);
     } finally {
+      trace.done();
       timerContext.stop();
       if (!usedCache && searcher != null) {
         // if the cached search was not used, close the searcher.
@@ -432,7 +434,7 @@ public class IndexManager {
         LOG.error("Unknown error while trying to fetch index readers.", e);
         throw new BException(e.getMessage(), e);
       }
-      Tracer trace = Trace.trace("query setup");
+      Tracer trace = Trace.trace("query setup", Trace.param("table", table));
       ShardServerContext shardServerContext = ShardServerContext.getShardServerContext();
       ParallelCall<Entry<String, BlurIndex>, BlurResultIterable> call;
       TableContext context = getTableContext(table);
@@ -1138,7 +1140,7 @@ public class IndexManager {
           _shardServerContext.setIndexSearcherClosable(_table, shard, searcher);
         }
         searcher.setSimilarity(_similarity);
-        Tracer trace1 = Trace.trace("query rewrite");
+        Tracer trace1 = Trace.trace("query rewrite", Trace.param("table", _table));
         Query rewrite;
         try {
           rewrite = searcher.rewrite((Query) _query.clone());

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractDistributedIndexServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractDistributedIndexServer.java b/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractDistributedIndexServer.java
index aba7ca2..4b392ca 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractDistributedIndexServer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/AbstractDistributedIndexServer.java
@@ -35,6 +35,8 @@ import org.apache.blur.manager.clusterstatus.ClusterStatus;
 import org.apache.blur.metrics.AtomicLongGauge;
 import org.apache.blur.server.TableContext;
 import org.apache.blur.thrift.generated.TableDescriptor;
+import org.apache.blur.trace.Trace;
+import org.apache.blur.trace.Tracer;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.FileStatus;
@@ -93,6 +95,7 @@ public abstract class AbstractDistributedIndexServer extends AbstractIndexServer
   @Override
   public final List<String> getShardList(String table) {
     checkTable(table);
+    Tracer trace = Trace.trace("filesystem - getShardList", Trace.param("table", table));
     List<String> result = new ArrayList<String>();
     try {
       TableContext tableContext = getTableContext(table);
@@ -115,6 +118,8 @@ public abstract class AbstractDistributedIndexServer extends AbstractIndexServer
       return result;
     } catch (IOException e) {
       throw new RuntimeException(e);
+    } finally {
+      trace.done();
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-core/src/main/java/org/apache/blur/server/BlurServerContext.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/BlurServerContext.java b/blur-core/src/main/java/org/apache/blur/server/BlurServerContext.java
index b47a1dd..ca378eb 100644
--- a/blur-core/src/main/java/org/apache/blur/server/BlurServerContext.java
+++ b/blur-core/src/main/java/org/apache/blur/server/BlurServerContext.java
@@ -30,14 +30,15 @@ public class BlurServerContext implements ServerContext {
   private User _user;
   private final SocketAddress _localSocketAddress;
   private final SocketAddress _remoteSocketAddress;
-  private final String _connectionString;
-
+  private final String _localConnectionString;
+  private final String _remoteConnectionString;
   private String _traceId;
 
   public BlurServerContext(SocketAddress localSocketAddress, SocketAddress remoteSocketAddress) {
     _localSocketAddress = localSocketAddress;
     _remoteSocketAddress = remoteSocketAddress;
-    _connectionString = _localSocketAddress.toString() + "\t" + _remoteSocketAddress.toString();
+    _localConnectionString = _localSocketAddress.toString();
+    _remoteConnectionString = _remoteSocketAddress.toString();
   }
 
   public void setUser(User user) {
@@ -65,8 +66,8 @@ public class BlurServerContext implements ServerContext {
     return _remoteSocketAddress;
   }
 
-  public String getConnectionString() {
-    return _connectionString;
+  public String getConnectionString(String sep) {
+    return _localConnectionString + sep + _remoteConnectionString;
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
index 61b046e..6ad4228 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
@@ -113,7 +113,7 @@ public class BlurControllerServer extends TableAdmin implements Iface {
     @Override
     public <T> T execute(String node, BlurCommand<T> command, int maxRetries, long backOffTime, long maxBackOffTime)
         throws BlurException, TException, IOException {
-      Tracer trace = Trace.trace("remote call - " + node);
+      Tracer trace = Trace.trace("remote call - thrift", Trace.param("node", node));
       try {
         return BlurClientManager.execute(node + "#" + _timeout, command, maxRetries, backOffTime, maxBackOffTime);
       } finally {
@@ -448,6 +448,8 @@ public class BlurControllerServer extends TableAdmin implements Iface {
     }
     if (selector != null) {
 
+      Tracer trace = Trace.trace("fetch data", Trace.param("table", table));
+
       // Gather client objects and build batches for fetching.
       IdentityHashMap<Client, List<Selector>> map = new IdentityHashMap<Client, List<Selector>>();
 
@@ -477,6 +479,10 @@ public class BlurControllerServer extends TableAdmin implements Iface {
         futures.add(executor.submit(new Callable<Boolean>() {
           @Override
           public Boolean call() throws Exception {
+            String traceId = Trace.getTraceId();
+            if (traceId != null) {
+              client.startTrace(traceId);
+            }
             List<FetchResult> fetchRowBatch = client.fetchRowBatch(table, list);
             for (int i = 0; i < list.size(); i++) {
               int index = indexMap.get(list.get(i));
@@ -499,6 +505,8 @@ public class BlurControllerServer extends TableAdmin implements Iface {
         result.setFetchResult(fetchResult);
         result.setLocationId(null);
       }
+
+      trace.done();
     }
     results.query = query;
     results.query.selector = selector;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
index 3d44c4e..e2b8d75 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
@@ -136,6 +136,7 @@ public class ThriftBlurControllerServer extends ThriftServer {
     controllerServer.init();
     
     Trace.setReporter(setupTraceReporter(configuration));
+    Trace.setNodeName(nodeName);
     
     Iface iface= BlurUtil.wrapFilteredBlurServer(configuration, controllerServer, false);
     iface = BlurUtil.recordMethodCallsAndAverageTimes(iface, Iface.class, true);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index 06305f3..7a0290a 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -232,6 +232,7 @@ public class ThriftBlurShardServer extends ThriftServer {
     shardServer.init();
     
     Trace.setReporter(setupTraceReporter(configuration));
+    Trace.setNodeName(nodeName);
 
     Iface iface = BlurUtil.wrapFilteredBlurServer(configuration, shardServer, true);
     iface = BlurUtil.recordMethodCallsAndAverageTimes(iface, Iface.class, false);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
index d11c291..1817c5b 100644
--- a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
+++ b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
@@ -167,20 +167,25 @@ public class BlurUtil {
       public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
         long requestNumber = _requestCounter.incrementAndGet();
         String requestId = prefix + "-" + requestNumber;
+        String tracingConnectionString;
         String connectionString;
         if (controller) {
           ControllerServerContext controllerServerContext = ControllerServerContext.getControllerServerContext();
           if (controllerServerContext == null) {
             connectionString = "unknown";
+            tracingConnectionString = "unknown";
           } else {
-            connectionString = controllerServerContext.getConnectionString();
+            connectionString = controllerServerContext.getConnectionString("\t");
+            tracingConnectionString = controllerServerContext.getConnectionString(":");
           }
         } else {
           ShardServerContext shardServerContext = ShardServerContext.getShardServerContext();
           if (shardServerContext == null) {
             connectionString = "unknown";
+            tracingConnectionString = "unknown";
           } else {
-            connectionString = shardServerContext.getConnectionString();
+            connectionString = shardServerContext.getConnectionString("\t");
+            tracingConnectionString = shardServerContext.getConnectionString(":");
           }
         }
         String argsStr = null;
@@ -188,7 +193,8 @@ public class BlurUtil {
         String name = method.getName();
         boolean error = false;
         LoggerArgsState loggerArgsState = null;
-        Tracer trace = Trace.trace("thrift - "+method.getName());
+        Tracer trace = Trace.trace("thrift recv", Trace.param("method", method.getName()),
+            Trace.param("connection", tracingConnectionString));
         try {
           if (REQUEST_LOG.isInfoEnabled()) {
             if (argsStr == null) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-core/src/test/java/org/apache/blur/MiniCluster.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/MiniCluster.java b/blur-core/src/test/java/org/apache/blur/MiniCluster.java
index bd8af0c..e0092b2 100644
--- a/blur-core/src/test/java/org/apache/blur/MiniCluster.java
+++ b/blur-core/src/test/java/org/apache/blur/MiniCluster.java
@@ -461,7 +461,8 @@ public class MiniCluster {
       field.setAccessible(true);
       return field.get(o);
     } catch (Exception e) {
-      throw new RuntimeException(e);
+      e.printStackTrace();
+      return null;
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
index daf43ee..df77e3f 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/HdfsFieldManager.java
@@ -29,6 +29,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
+import org.apache.blur.trace.Trace;
+import org.apache.blur.trace.Tracer;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -93,72 +95,83 @@ public class HdfsFieldManager extends BaseFieldManager {
 
   @Override
   protected List<String> getFieldNamesToLoad() throws IOException {
-    if (!_fileSystem.exists(_storagePath)) {
-      return EMPTY_LIST;
-    }
-    FileStatus[] listStatus = _fileSystem.listStatus(_storagePath, new PathFilter() {
-      @Override
-      public boolean accept(Path path) {
-        if (path.getName().endsWith(".tmp")) {
-          return false;
+    Tracer trace = Trace.trace("filesystem - getFieldNamesToLoad",Trace.param("storagePath", _storagePath));
+    try {
+      if (!_fileSystem.exists(_storagePath)) {
+        return EMPTY_LIST;
+      }
+      FileStatus[] listStatus = _fileSystem.listStatus(_storagePath, new PathFilter() {
+        @Override
+        public boolean accept(Path path) {
+          if (path.getName().endsWith(".tmp")) {
+            return false;
+          }
+          return true;
         }
-        return true;
+      });
+      if (listStatus == null) {
+        return EMPTY_LIST;
       }
-    });
-    if (listStatus == null) {
-      return EMPTY_LIST;
-    }
-    List<String> fieldNames = new ArrayList<String>();
-    for (FileStatus fileStatus : listStatus) {
-      if (!fileStatus.isDir()) {
-        fieldNames.add(fileStatus.getPath().getName());
+      List<String> fieldNames = new ArrayList<String>();
+      for (FileStatus fileStatus : listStatus) {
+        if (!fileStatus.isDir()) {
+          fieldNames.add(fileStatus.getPath().getName());
+        }
       }
+      return fieldNames;
+    } finally {
+      trace.done();
     }
-    return fieldNames;
   }
 
   @Override
   protected boolean tryToStore(FieldTypeDefinition fieldTypeDefinition, String fieldName) throws IOException {
-    // Might want to make this a ZK lock
-    _lock.lock();
+    Tracer trace = Trace.trace("filesystem - tryToStore fieldName",Trace.param("fieldName", fieldName),Trace.param("storagePath", _storagePath));
     try {
-      String fieldType = fieldTypeDefinition.getFieldType();
-      boolean fieldLessIndexed = fieldTypeDefinition.isFieldLessIndexed();
-      LOG.info("Attempting to store new field [{0}] with fieldLessIndexing [{1}] with type [{2}] and properties [{3}]",
-          fieldName, fieldLessIndexed, fieldType, fieldTypeDefinition.getProperties());
-      Properties properties = new Properties();
-      setProperty(properties, FAMILY, fieldTypeDefinition.getFamily());
-      setProperty(properties, FAMILY, fieldTypeDefinition.getFamily());
-      setProperty(properties, COLUMN_NAME, fieldTypeDefinition.getColumnName());
-      setProperty(properties, SUB_COLUMN_NAME, fieldTypeDefinition.getSubColumnName());
-      setProperty(properties, FIELD_LESS_INDEXING, Boolean.toString(fieldLessIndexed));
-      setProperty(properties, FIELD_TYPE, fieldType);
-      Map<String, String> props = fieldTypeDefinition.getProperties();
-      if (props != null) {
-        for (Entry<String, String> e : props.entrySet()) {
-          properties.setProperty(e.getKey(), e.getValue());
+      // Might want to make this a ZK lock
+      _lock.lock();
+      try {
+        String fieldType = fieldTypeDefinition.getFieldType();
+        boolean fieldLessIndexed = fieldTypeDefinition.isFieldLessIndexed();
+        LOG.info(
+            "Attempting to store new field [{0}] with fieldLessIndexing [{1}] with type [{2}] and properties [{3}]",
+            fieldName, fieldLessIndexed, fieldType, fieldTypeDefinition.getProperties());
+        Properties properties = new Properties();
+        setProperty(properties, FAMILY, fieldTypeDefinition.getFamily());
+        setProperty(properties, FAMILY, fieldTypeDefinition.getFamily());
+        setProperty(properties, COLUMN_NAME, fieldTypeDefinition.getColumnName());
+        setProperty(properties, SUB_COLUMN_NAME, fieldTypeDefinition.getSubColumnName());
+        setProperty(properties, FIELD_LESS_INDEXING, Boolean.toString(fieldLessIndexed));
+        setProperty(properties, FIELD_TYPE, fieldType);
+        Map<String, String> props = fieldTypeDefinition.getProperties();
+        if (props != null) {
+          for (Entry<String, String> e : props.entrySet()) {
+            properties.setProperty(e.getKey(), e.getValue());
+          }
         }
-      }
-      Path path = getFieldPath(fieldName);
-      if (_fileSystem.exists(path)) {
-        LOG.info("Field [{0}] already exists.", fieldName);
-        return false;
-      }
-      Path tmpPath = new Path(path.getParent(), UUID.randomUUID().toString() + ".tmp");
-      FSDataOutputStream outputStream = _fileSystem.create(tmpPath, false);
-      properties.store(outputStream, getComments());
-      outputStream.close();
-      if (_fileSystem.rename(tmpPath, path)) {
-        // @TODO make this configurable
-        _fileSystem.setReplication(path, (short) 10);
-        return true;
-      } else {
-        _fileSystem.delete(tmpPath, false);
-        LOG.info("Field [{0}] already exists.", fieldName, fieldLessIndexed, fieldType, props);
-        return false;
+        Path path = getFieldPath(fieldName);
+        if (_fileSystem.exists(path)) {
+          LOG.info("Field [{0}] already exists.", fieldName);
+          return false;
+        }
+        Path tmpPath = new Path(path.getParent(), UUID.randomUUID().toString() + ".tmp");
+        FSDataOutputStream outputStream = _fileSystem.create(tmpPath, false);
+        properties.store(outputStream, getComments());
+        outputStream.close();
+        if (_fileSystem.rename(tmpPath, path)) {
+          // @TODO make this configurable
+          _fileSystem.setReplication(path, (short) 10);
+          return true;
+        } else {
+          _fileSystem.delete(tmpPath, false);
+          LOG.info("Field [{0}] already exists.", fieldName, fieldLessIndexed, fieldType, props);
+          return false;
+        }
+      } finally {
+        _lock.unlock();
       }
     } finally {
-      _lock.unlock();
+      trace.done();
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-store/src/main/java/org/apache/blur/index/OptimisticLockFactory.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/index/OptimisticLockFactory.java b/blur-store/src/main/java/org/apache/blur/index/OptimisticLockFactory.java
index f6347bc..0028edb 100644
--- a/blur-store/src/main/java/org/apache/blur/index/OptimisticLockFactory.java
+++ b/blur-store/src/main/java/org/apache/blur/index/OptimisticLockFactory.java
@@ -19,6 +19,8 @@ package org.apache.blur.index;
 import java.io.IOException;
 import java.util.Arrays;
 
+import org.apache.blur.trace.Trace;
+import org.apache.blur.trace.Tracer;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -51,46 +53,61 @@ public class OptimisticLockFactory extends LockFactory {
 
       @Override
       public boolean obtain() throws IOException {
-        if (_set) {
-          throw new IOException("Lock for [" + _baseLockKey + "] can only be set once.");
-        }
+        Tracer trace = Trace.trace("filesystem - release", Trace.param("lockPath", lockPath));
         try {
-          _lockKey = (_baseLockKey + "/" + System.currentTimeMillis()).getBytes();
-          FSDataOutputStream outputStream = _fileSystem.create(lockPath, true);
-          outputStream.write(_lockKey);
-          outputStream.close();
+          if (_set) {
+            throw new IOException("Lock for [" + _baseLockKey + "] can only be set once.");
+          }
+          try {
+            _lockKey = (_baseLockKey + "/" + System.currentTimeMillis()).getBytes();
+            FSDataOutputStream outputStream = _fileSystem.create(lockPath, true);
+            outputStream.write(_lockKey);
+            outputStream.close();
+          } finally {
+            _set = true;
+          }
+          return true;
         } finally {
-          _set = true;
+          trace.done();
         }
-        return true;
       }
 
       @Override
       public void release() throws IOException {
-        _fileSystem.delete(lockPath, false);
+        Tracer trace = Trace.trace("filesystem - release", Trace.param("lockPath", lockPath));
+        try {
+          _fileSystem.delete(lockPath, false);
+        } finally {
+          trace.done();
+        }
       }
 
       @Override
       public boolean isLocked() throws IOException {
-        if (!_set) {
-          return false;
-        }
-        if (!_fileSystem.exists(lockPath)) {
-          return false;
-        }
-        FileStatus fileStatus = _fileSystem.getFileStatus(lockPath);
-        long len = fileStatus.getLen();
-        if (len != _lockKey.length) {
+        Tracer trace = Trace.trace("filesystem - isLocked", Trace.param("lockPath", lockPath));
+        try {
+          if (!_set) {
+            return false;
+          }
+          if (!_fileSystem.exists(lockPath)) {
+            return false;
+          }
+          FileStatus fileStatus = _fileSystem.getFileStatus(lockPath);
+          long len = fileStatus.getLen();
+          if (len != _lockKey.length) {
+            return false;
+          }
+          byte[] buf = new byte[_lockKey.length];
+          FSDataInputStream inputStream = _fileSystem.open(lockPath);
+          inputStream.readFully(buf);
+          inputStream.close();
+          if (Arrays.equals(_lockKey, buf)) {
+            return true;
+          }
           return false;
+        } finally {
+          trace.done();
         }
-        byte[] buf = new byte[_lockKey.length];
-        FSDataInputStream inputStream = _fileSystem.open(lockPath);
-        inputStream.readFully(buf);
-        inputStream.close();
-        if (Arrays.equals(_lockKey, buf)) {
-          return true;
-        }
-        return false;
       }
     };
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-store/src/main/java/org/apache/blur/store/hdfs/BlurLockFactory.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/hdfs/BlurLockFactory.java b/blur-store/src/main/java/org/apache/blur/store/hdfs/BlurLockFactory.java
index f119446..c4f032c 100644
--- a/blur-store/src/main/java/org/apache/blur/store/hdfs/BlurLockFactory.java
+++ b/blur-store/src/main/java/org/apache/blur/store/hdfs/BlurLockFactory.java
@@ -21,6 +21,8 @@ import java.util.Arrays;
 
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
+import org.apache.blur.trace.Trace;
+import org.apache.blur.trace.Tracer;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -31,7 +33,7 @@ import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.LockFactory;
 
 public class BlurLockFactory extends LockFactory {
-  
+
   private static final Log LOG = LogFactory.getLog(BlurLockFactory.class);
 
   private final Configuration _configuration;
@@ -50,55 +52,71 @@ public class BlurLockFactory extends LockFactory {
   @Override
   public Lock makeLock(String lockName) {
     final Path lockPath = new Path(_dir, lockName);
+
     return new Lock() {
       private boolean _set;
 
       @Override
       public boolean obtain() throws IOException {
-        if (_set) {
-          throw new IOException("Lock for [" + _baseLockKey + "] can only be set once.");
-        }
+        Tracer trace = Trace.trace("filesystem - obtain", Trace.param("lockPath", lockPath));
         try {
-          _lockKey = (_baseLockKey + "/" + System.currentTimeMillis()).getBytes();
-          FSDataOutputStream outputStream = _fileSystem.create(lockPath, true);
-          outputStream.write(_lockKey);
-          outputStream.close();
+          if (_set) {
+            throw new IOException("Lock for [" + _baseLockKey + "] can only be set once.");
+          }
+          try {
+            _lockKey = (_baseLockKey + "/" + System.currentTimeMillis()).getBytes();
+            FSDataOutputStream outputStream = _fileSystem.create(lockPath, true);
+            outputStream.write(_lockKey);
+            outputStream.close();
+          } finally {
+            _set = true;
+          }
+          return true;
         } finally {
-          _set = true;
+          trace.done();
         }
-        return true;
       }
 
       @Override
       public void release() throws IOException {
-        _fileSystem.delete(lockPath, false);
+        Tracer trace = Trace.trace("filesystem - release",Trace.param("lockPath", lockPath));
+        try {
+          _fileSystem.delete(lockPath, false);
+        } finally {
+          trace.done();
+        }
       }
 
       @Override
       public boolean isLocked() throws IOException {
-        if (!_set) {
-          LOG.info("The lock has NOT been set.");
-          return false;
-        }
-        if (!_fileSystem.exists(lockPath)) {
-          LOG.info("The lock file has been removed.");
-          return false;
-        }
-        FileStatus fileStatus = _fileSystem.getFileStatus(lockPath);
-        long len = fileStatus.getLen();
-        if (len != _lockKey.length) {
-          LOG.info("The lock file length has changed.");
+        Tracer trace = Trace.trace("filesystem - isLocked", Trace.param("lockPath", lockPath));
+        try {
+          if (!_set) {
+            LOG.info("The lock has NOT been set.");
+            return false;
+          }
+          if (!_fileSystem.exists(lockPath)) {
+            LOG.info("The lock file has been removed.");
+            return false;
+          }
+          FileStatus fileStatus = _fileSystem.getFileStatus(lockPath);
+          long len = fileStatus.getLen();
+          if (len != _lockKey.length) {
+            LOG.info("The lock file length has changed.");
+            return false;
+          }
+          byte[] buf = new byte[_lockKey.length];
+          FSDataInputStream inputStream = _fileSystem.open(lockPath);
+          inputStream.readFully(buf);
+          inputStream.close();
+          if (Arrays.equals(_lockKey, buf)) {
+            return true;
+          }
+          LOG.info("The lock information has been changed.");
           return false;
+        } finally {
+          trace.done();
         }
-        byte[] buf = new byte[_lockKey.length];
-        FSDataInputStream inputStream = _fileSystem.open(lockPath);
-        inputStream.readFully(buf);
-        inputStream.close();
-        if (Arrays.equals(_lockKey, buf)) {
-          return true;
-        }
-        LOG.info("The lock information has been changed.");
-        return false;
       }
     };
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java b/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java
index 0b24ff0..9a5020d 100644
--- a/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java
+++ b/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java
@@ -32,6 +32,8 @@ import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.store.blockcache.LastModified;
+import org.apache.blur.trace.Trace;
+import org.apache.blur.trace.Tracer;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -95,7 +97,7 @@ public class HdfsDirectory extends Directory implements LastModified {
    * We keep the metrics separate per filesystem.
    */
   protected static Map<URI, MetricsGroup> _metricsGroupMap = new WeakHashMap<URI, MetricsGroup>();
-  
+
   protected final Path _path;
   protected final FileSystem _fileSystem;
   protected final MetricsGroup _metricsGroup;
@@ -122,7 +124,7 @@ public class HdfsDirectory extends Directory implements LastModified {
     MetricName writeAcccessName = new MetricName(ORG_APACHE_BLUR, HDFS, "Write Latency in \u00B5s", scope);
     MetricName readThroughputName = new MetricName(ORG_APACHE_BLUR, HDFS, "Read Throughput", scope);
     MetricName writeThroughputName = new MetricName(ORG_APACHE_BLUR, HDFS, "Write Throughput", scope);
-    
+
     Histogram readAccess = Metrics.newHistogram(readAccessName);
     Histogram writeAccess = Metrics.newHistogram(writeAcccessName);
     Meter readThroughput = Metrics.newMeter(readThroughputName, "Read Bytes", TimeUnit.SECONDS);
@@ -172,7 +174,13 @@ public class HdfsDirectory extends Directory implements LastModified {
   }
 
   protected FSDataOutputStream openForOutput(String name) throws IOException {
-    return _fileSystem.create(getPath(name));
+    Path path = getPath(name);
+    Tracer trace = Trace.trace("filesystem - create", Trace.param("path", path));
+    try {
+      return _fileSystem.create(path);
+    } finally {
+      trace.done();
+    }
   }
 
   @Override
@@ -183,31 +191,42 @@ public class HdfsDirectory extends Directory implements LastModified {
     }
     FSDataInputStream inputStream = openForInput(name);
     long fileLength = fileLength(name);
-    return new HdfsIndexInput(name, inputStream, fileLength, _metricsGroup, fetchImpl.get());
+    return new HdfsIndexInput(name, inputStream, fileLength, _metricsGroup, fetchImpl.get(), getPath(name));
   }
 
   protected FSDataInputStream openForInput(String name) throws IOException {
-    return _fileSystem.open(getPath(name));
+    Path path = getPath(name);
+    Tracer trace = Trace.trace("filesystem - open", Trace.param("path", path));
+    try {
+      return _fileSystem.open(path);
+    } finally {
+      trace.done();
+    }
   }
 
   @Override
   public String[] listAll() throws IOException {
     LOG.debug("listAll [{0}]", _path);
-    FileStatus[] files = _fileSystem.listStatus(_path, new PathFilter() {
-      @Override
-      public boolean accept(Path path) {
-        try {
-          return _fileSystem.isFile(path);
-        } catch (IOException e) {
-          throw new RuntimeException(e);
+    Tracer trace = Trace.trace("filesystem - list", Trace.param("path", _path));
+    try {
+      FileStatus[] files = _fileSystem.listStatus(_path, new PathFilter() {
+        @Override
+        public boolean accept(Path path) {
+          try {
+            return _fileSystem.isFile(path);
+          } catch (IOException e) {
+            throw new RuntimeException(e);
+          }
         }
+      });
+      String[] result = new String[files.length];
+      for (int i = 0; i < result.length; i++) {
+        result[i] = files[i].getPath().getName();
       }
-    });
-    String[] result = new String[files.length];
-    for (int i = 0; i < result.length; i++) {
-      result[i] = files[i].getPath().getName();
+      return result;
+    } finally {
+      trace.done();
     }
-    return result;
   }
 
   @Override
@@ -217,7 +236,13 @@ public class HdfsDirectory extends Directory implements LastModified {
   }
 
   protected boolean exists(String name) throws IOException {
-    return _fileSystem.exists(getPath(name));
+    Path path = getPath(name);
+    Tracer trace = Trace.trace("filesystem - exists", Trace.param("path", path));
+    try {
+      return _fileSystem.exists(path);
+    } finally {
+      trace.done();
+    }
   }
 
   @Override
@@ -231,7 +256,13 @@ public class HdfsDirectory extends Directory implements LastModified {
   }
 
   protected void delete(String name) throws IOException {
-    _fileSystem.delete(getPath(name), true);
+    Path path = getPath(name);
+    Tracer trace = Trace.trace("filesystem - delete", Trace.param("path", path));
+    try {
+      _fileSystem.delete(path, true);
+    } finally {
+      trace.done();
+    }
   }
 
   @Override
@@ -241,7 +272,13 @@ public class HdfsDirectory extends Directory implements LastModified {
   }
 
   protected long length(String name) throws IOException {
-    return _fileSystem.getFileStatus(getPath(name)).getLen();
+    Path path = getPath(name);
+    Tracer trace = Trace.trace("filesystem - length", Trace.param("path", path));
+    try {
+      return _fileSystem.getFileStatus(path).getLen();
+    } finally {
+      trace.done();
+    }
   }
 
   @Override
@@ -270,7 +307,13 @@ public class HdfsDirectory extends Directory implements LastModified {
   }
 
   protected long fileModified(String name) throws IOException {
-    return _fileSystem.getFileStatus(getPath(name)).getModificationTime();
+    Path path = getPath(name);
+    Tracer trace = Trace.trace("filesystem - fileModified", Trace.param("path", path));
+    try {
+      return _fileSystem.getFileStatus(path).getModificationTime();
+    } finally {
+      trace.done();
+    }
   }
 
   @Override
@@ -284,7 +327,7 @@ public class HdfsDirectory extends Directory implements LastModified {
       }
     } else {
       slowCopy(to, src, dest, context);
-      
+
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsIndexInput.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsIndexInput.java b/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsIndexInput.java
index 1c454a9..12aa5ee 100644
--- a/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsIndexInput.java
+++ b/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsIndexInput.java
@@ -19,7 +19,10 @@ package org.apache.blur.store.hdfs;
 import java.io.IOException;
 
 import org.apache.blur.store.buffer.ReusedBufferedIndexInput;
+import org.apache.blur.trace.Trace;
+import org.apache.blur.trace.Tracer;
 import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
 
 public class HdfsIndexInput extends ReusedBufferedIndexInput {
 
@@ -28,13 +31,16 @@ public class HdfsIndexInput extends ReusedBufferedIndexInput {
   private boolean _isClone;
   private final MetricsGroup _metricsGroup;
   private int _readVersion;
+  private final Path _path;
 
-  public HdfsIndexInput(String name, FSDataInputStream inputStream, long length, MetricsGroup metricsGroup, int readVersion) throws IOException {
+  public HdfsIndexInput(String name, FSDataInputStream inputStream, long length, MetricsGroup metricsGroup,
+      int readVersion, Path path) throws IOException {
     super(name);
     _inputStream = inputStream;
     _length = length;
     _metricsGroup = metricsGroup;
     _readVersion = readVersion;
+    _path = path;
   }
 
   @Override
@@ -49,45 +55,50 @@ public class HdfsIndexInput extends ReusedBufferedIndexInput {
 
   @Override
   protected void readInternal(byte[] b, int offset, int length) throws IOException {
-    long start = System.nanoTime();
-    long filePointer = getFilePointer();
-    switch (_readVersion) {
-    case 0:
-      synchronized (_inputStream) {
-        _inputStream.seek(getFilePointer());
-        _inputStream.readFully(b, offset, length);
-      }
-      break;
-    case 1:
-      while (length > 0) {
-        int amount;
+    Tracer trace = Trace.trace("filesystem - read", Trace.param("file", _path), Trace.param("location", getFilePointer()),Trace.param("length", length));
+    try {
+      long start = System.nanoTime();
+      long filePointer = getFilePointer();
+      switch (_readVersion) {
+      case 0:
         synchronized (_inputStream) {
-          _inputStream.seek(filePointer);
-          amount = _inputStream.read(b, offset, length);
+          _inputStream.seek(getFilePointer());
+          _inputStream.readFully(b, offset, length);
         }
-        length -= amount;
-        offset += amount;
-        filePointer += amount;
-      }
-      break;
-    case 2:
-      _inputStream.readFully(filePointer, b, offset, length);
-      break;
-    case 3:
-      while (length > 0) {
-        int amount;
-        amount = _inputStream.read(filePointer, b, offset, length);
-        length -= amount;
-        offset += amount;
-        filePointer += amount;
+        break;
+      case 1:
+        while (length > 0) {
+          int amount;
+          synchronized (_inputStream) {
+            _inputStream.seek(filePointer);
+            amount = _inputStream.read(b, offset, length);
+          }
+          length -= amount;
+          offset += amount;
+          filePointer += amount;
+        }
+        break;
+      case 2:
+        _inputStream.readFully(filePointer, b, offset, length);
+        break;
+      case 3:
+        while (length > 0) {
+          int amount;
+          amount = _inputStream.read(filePointer, b, offset, length);
+          length -= amount;
+          offset += amount;
+          filePointer += amount;
+        }
+        break;
+      default:
+        break;
       }
-      break;
-    default:
-      break;
+      long end = System.nanoTime();
+      _metricsGroup.readAccess.update((end - start) / 1000);
+      _metricsGroup.readThroughput.mark(length);
+    } finally {
+      trace.done();
     }
-    long end = System.nanoTime();
-    _metricsGroup.readAccess.update((end - start) / 1000);
-    _metricsGroup.readThroughput.mark(length);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/transport/TFramedTransport.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/transport/TFramedTransport.java b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/transport/TFramedTransport.java
index 9968b34..3950b9d 100644
--- a/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/transport/TFramedTransport.java
+++ b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/transport/TFramedTransport.java
@@ -172,4 +172,12 @@ public class TFramedTransport extends TTransport {
       ((buf[2] & 0xff) <<  8) |
       ((buf[3] & 0xff));
   }
+
+  public TTransport getTransport() {
+    return transport_;
+  }
+
+  public void setTransport(TTransport transport) {
+    transport_ = transport;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-thrift/src/main/java/org/apache/blur/thrift/BlurClientManager.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thrift/BlurClientManager.java b/blur-thrift/src/main/java/org/apache/blur/thrift/BlurClientManager.java
index 6c389fa..6292846 100644
--- a/blur-thrift/src/main/java/org/apache/blur/thrift/BlurClientManager.java
+++ b/blur-thrift/src/main/java/org/apache/blur/thrift/BlurClientManager.java
@@ -17,6 +17,8 @@ package org.apache.blur.thrift;
  * limitations under the License.
  */
 import java.io.IOException;
+import java.net.Socket;
+import java.net.SocketAddress;
 import java.net.SocketTimeoutException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -35,12 +37,16 @@ import java.util.concurrent.atomic.AtomicReference;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thirdparty.thrift_0_9_0.transport.TFramedTransport;
+import org.apache.blur.thirdparty.thrift_0_9_0.transport.TSocket;
+import org.apache.blur.thirdparty.thrift_0_9_0.transport.TTransport;
 import org.apache.blur.thirdparty.thrift_0_9_0.transport.TTransportException;
 import org.apache.blur.thrift.generated.Blur;
 import org.apache.blur.thrift.generated.Blur.Client;
 import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.ErrorType;
 import org.apache.blur.trace.Trace;
+import org.apache.blur.trace.Tracer;
 
 public class BlurClientManager {
 
@@ -164,10 +170,12 @@ public class BlurClientManager {
             continue;
           }
         }
+        Tracer trace = null;
         try {
           String traceId = Trace.getTraceId();
           if (traceId != null) {
             client.get().startTrace(traceId);
+            trace = Trace.trace("thrift client", Trace.param("connection", getConnectionStr(client.get())));
           }
           T result = command.call((CLIENT) client.get(), connection);
           allBad = false;
@@ -200,6 +208,9 @@ public class BlurClientManager {
             throw e;
           }
         } finally {
+          if (trace != null) {
+            trace.done();
+          }
           if (client.get() != null) {
             returnClient(connection, client);
           }
@@ -220,6 +231,22 @@ public class BlurClientManager {
     }
   }
 
+  private static String getConnectionStr(Client client) {
+    TTransport transport = client.getInputProtocol().getTransport();
+    if (transport instanceof TFramedTransport) {
+      TFramedTransport framedTransport = (TFramedTransport) transport;
+      transport = framedTransport.getTransport();
+    }
+    if (transport instanceof TSocket) {
+      TSocket tsocket = (TSocket) transport;
+      Socket socket = tsocket.getSocket();
+      SocketAddress localSocketAddress = socket.getLocalSocketAddress();
+      SocketAddress remoteSocketAddress = socket.getRemoteSocketAddress();
+      return localSocketAddress.toString() + ":" + remoteSocketAddress.toString();
+    }
+    return "unknown";
+  }
+
   private static void markBadConnection(Connection connection) {
     LOG.info("Marking bad connection [{0}]", connection);
     _badConnections.put(connection, NULL);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-thrift/src/main/java/org/apache/blur/thrift/util/SimpleQueryExample.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thrift/util/SimpleQueryExample.java b/blur-thrift/src/main/java/org/apache/blur/thrift/util/SimpleQueryExample.java
index 2e267ee..7bf3bf1 100644
--- a/blur-thrift/src/main/java/org/apache/blur/thrift/util/SimpleQueryExample.java
+++ b/blur-thrift/src/main/java/org/apache/blur/thrift/util/SimpleQueryExample.java
@@ -26,6 +26,8 @@ import org.apache.blur.thrift.generated.BlurQuery;
 import org.apache.blur.thrift.generated.BlurResult;
 import org.apache.blur.thrift.generated.BlurResults;
 import org.apache.blur.thrift.generated.Query;
+import org.apache.blur.thrift.generated.Selector;
+import org.apache.blur.trace.Trace;
 
 public class SimpleQueryExample {
 
@@ -36,15 +38,19 @@ public class SimpleQueryExample {
 
     Iface client = BlurClient.getClient(connectionStr);
 
+    String uuid = "123456";
+    Trace.setupTrace(uuid);
     final BlurQuery blurQuery = new BlurQuery();
     Query query = new Query();
     blurQuery.setQuery(query);
     query.setQuery(queryStr);
+    blurQuery.setSelector(new Selector());
     BlurResults results = client.query(tableName, blurQuery);
     System.out.println("Total Results: " + results.totalResults);
 
     for (BlurResult result : results.getResults()) {
       System.out.println(result);
     }
+    Trace.tearDownTrace();
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-thrift/src/main/java/org/apache/blur/thrift/util/StatsTable.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thrift/util/StatsTable.java b/blur-thrift/src/main/java/org/apache/blur/thrift/util/StatsTable.java
index 9d9c4e4..1d46638 100644
--- a/blur-thrift/src/main/java/org/apache/blur/thrift/util/StatsTable.java
+++ b/blur-thrift/src/main/java/org/apache/blur/thrift/util/StatsTable.java
@@ -31,7 +31,8 @@ public class StatsTable {
     String connectionStr = args[0];
     final String tableName = args[1];
 
-    String uuid = UUID.randomUUID().toString();
+    // String uuid = UUID.randomUUID().toString();
+    String uuid = "123456";
     Trace.setupTrace(uuid);
     System.out.println("Tracing on [" + uuid + "]");
     Iface client = BlurClient.getClient(connectionStr);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-util/src/main/java/org/apache/blur/trace/Trace.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/trace/Trace.java b/blur-util/src/main/java/org/apache/blur/trace/Trace.java
index a8b0b3f..4bd0a43 100644
--- a/blur-util/src/main/java/org/apache/blur/trace/Trace.java
+++ b/blur-util/src/main/java/org/apache/blur/trace/Trace.java
@@ -19,7 +19,6 @@ package org.apache.blur.trace;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
-import java.util.UUID;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
@@ -29,6 +28,18 @@ import java.util.concurrent.TimeoutException;
 
 public class Trace {
 
+  public static class Parameter {
+
+    final String _name;
+    final String _value;
+
+    public Parameter(String name, String value) {
+      _name = name;
+      _value = value;
+    }
+
+  }
+
   private static final Tracer DO_NOTHING = new Tracer() {
     @Override
     public void done() {
@@ -37,19 +48,37 @@ public class Trace {
   };
   private static ThreadLocal<TraceCollector> _tracer = new ThreadLocal<TraceCollector>();
   private static TraceReporter _reporter;
+  private static String _nodeName;
+
+  public static String getNodeName() {
+    return _nodeName;
+  }
+
+  public static void setNodeName(String nodeName) {
+    _nodeName = nodeName;
+  }
 
   public static void setupTrace(String id) {
-    TraceCollector collector = new TraceCollector(id);
+    TraceCollector collector = new TraceCollector(_nodeName, id);
     _tracer.set(collector);
   }
   
+  public static Parameter param(Object name, Object value) {
+    if (name == null) {
+      name = "null";
+    }
+    if (value == null) {
+      value = "null";
+    }
+    return new Parameter(name.toString(), value.toString());
+  }
+
   private static void setupTraceOnNewThread(TraceCollector parentCollector) {
-    TraceCollector collector = new TraceCollector(getNewThreadId(parentCollector));
-    _tracer.set(collector);
+    _tracer.set(parentCollector);
   }
 
-  private static String getNewThreadId(TraceCollector parentCollector) {
-    return parentCollector._id + ":" + Thread.currentThread().getName() + "/" + UUID.randomUUID().toString();
+  private static void tearDownTraceOnNewThread() {
+    _tracer.set(null);
   }
 
   public static void tearDownTrace() {
@@ -60,12 +89,12 @@ public class Trace {
     }
   }
 
-  public static Tracer trace(String name) {
+  public static Tracer trace(String desc, Parameter... parameters) {
     TraceCollector collector = _tracer.get();
     if (collector == null) {
       return DO_NOTHING;
     }
-    TracerImpl tracer = new TracerImpl(name, collector.getNextId());
+    TracerImpl tracer = new TracerImpl(desc, parameters, collector.getNextId());
     collector.add(tracer);
     return tracer;
   }
@@ -83,17 +112,22 @@ public class Trace {
     if (tc == null) {
       return runnable;
     }
-    return new Runnable() {
-      @Override
-      public void run() {
-        setupTraceOnNewThread(tc);
-        try {
-          runnable.run();
-        } finally {
-          tearDownTrace();
+    Tracer trace = Trace.trace("new runnable");
+    try {
+      return new Runnable() {
+        @Override
+        public void run() {
+          setupTraceOnNewThread(tc);
+          try {
+            runnable.run();
+          } finally {
+            tearDownTraceOnNewThread();
+          }
         }
-      }
-    };
+      };
+    } finally {
+      trace.done();
+    }
   }
 
   public static <V> Callable<V> getCallable(final Callable<V> callable) {
@@ -101,17 +135,22 @@ public class Trace {
     if (tc == null) {
       return callable;
     }
-    return new Callable<V>() {
-      @Override
-      public V call() throws Exception {
-        setupTraceOnNewThread(tc);
-        try {
-          return callable.call();
-        } finally {
-          tearDownTrace();
+    Tracer trace = Trace.trace("new callable");
+    try {
+      return new Callable<V>() {
+        @Override
+        public V call() throws Exception {
+          setupTraceOnNewThread(tc);
+          try {
+            return callable.call();
+          } finally {
+            tearDownTraceOnNewThread();
+          }
         }
-      }
-    };
+      };
+    } finally {
+      trace.done();
+    }
   }
 
   public static <V> Collection<Callable<V>> getCallables(final Collection<Callable<V>> callables) {
@@ -171,7 +210,7 @@ public class Trace {
           throws InterruptedException {
         return _service.invokeAll((Collection<? extends Callable<T>>) getCallable((Callable<T>) tasks), timeout, unit);
       }
-      
+
       public <T> T invokeAny(Collection<? extends Callable<T>> tasks) throws InterruptedException, ExecutionException {
         return _service.invokeAny((Collection<? extends Callable<T>>) getCallable((Callable<T>) tasks));
       }
@@ -205,4 +244,5 @@ public class Trace {
 
     };
   }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-util/src/main/java/org/apache/blur/trace/TraceCollector.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/trace/TraceCollector.java b/blur-util/src/main/java/org/apache/blur/trace/TraceCollector.java
index e3ab598..68ba2fd 100644
--- a/blur-util/src/main/java/org/apache/blur/trace/TraceCollector.java
+++ b/blur-util/src/main/java/org/apache/blur/trace/TraceCollector.java
@@ -28,12 +28,16 @@ public class TraceCollector {
   protected final AtomicLong _traceCounter;
   protected final long _now = System.nanoTime();
   protected final String _pid;
+  protected final String _threadName;
+  protected final String _nodeName;
 
-  public TraceCollector(String id) {
+  public TraceCollector(String nodeName, String id) {
+    _nodeName = nodeName;
     _id = id;
     _traces = new CopyOnWriteArrayList<TracerImpl>();
     _traceCounter = new AtomicLong();
     _pid = ManagementFactory.getRuntimeMXBean().getName();
+    _threadName = Thread.currentThread().getName();
   }
 
   public void add(TracerImpl tracer) {
@@ -49,10 +53,10 @@ public class TraceCollector {
     StringBuilder builder = new StringBuilder();
     for (TracerImpl t : _traces) {
       builder.append("    ").append(t.toJson()).append(",\n");
-
     }
-
-    return "{\n  \"id\"=\"" + _id + "\"\n  \"pid\"=\"" + _pid + "\"\n  \"created\"=" + _now + ",\n  \"traces\"=[\n" + builder.toString() + "  ]\n}";
+    return "{\n  \"id\":\"" + _id + "\"\n  \"nodeName\":\"" + (_nodeName == null ? "unknown" : _nodeName)
+        + "\"\n  \"pid\":\"" + _pid + "\"\n  \"thread\":\"" + _threadName + "\"\n  \"created\":" + _now
+        + ",\n  \"traces\":[\n" + builder.toString() + "  ]\n}";
   }
 
   public String getId() {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-util/src/main/java/org/apache/blur/trace/TraceReporter.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/trace/TraceReporter.java b/blur-util/src/main/java/org/apache/blur/trace/TraceReporter.java
index 96cf053..fffb95f 100644
--- a/blur-util/src/main/java/org/apache/blur/trace/TraceReporter.java
+++ b/blur-util/src/main/java/org/apache/blur/trace/TraceReporter.java
@@ -19,8 +19,8 @@ package org.apache.blur.trace;
 import org.apache.blur.BlurConfiguration;
 
 public abstract class TraceReporter {
-  
-  protected BlurConfiguration _configuration;
+
+  protected final BlurConfiguration _configuration;
 
   public TraceReporter(BlurConfiguration configuration) {
     _configuration = configuration;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-util/src/main/java/org/apache/blur/trace/Tracer.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/trace/Tracer.java b/blur-util/src/main/java/org/apache/blur/trace/Tracer.java
index 1251336..d4b2cd3 100644
--- a/blur-util/src/main/java/org/apache/blur/trace/Tracer.java
+++ b/blur-util/src/main/java/org/apache/blur/trace/Tracer.java
@@ -16,6 +16,7 @@
  */
 package org.apache.blur.trace;
 
+
 public interface Tracer {
   
   void done();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-util/src/main/java/org/apache/blur/trace/TracerImpl.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/trace/TracerImpl.java b/blur-util/src/main/java/org/apache/blur/trace/TracerImpl.java
index 0f4eb87..9e25fe6 100644
--- a/blur-util/src/main/java/org/apache/blur/trace/TracerImpl.java
+++ b/blur-util/src/main/java/org/apache/blur/trace/TracerImpl.java
@@ -16,6 +16,8 @@
  */
 package org.apache.blur.trace;
 
+import org.apache.blur.trace.Trace.Parameter;
+
 public class TracerImpl implements Tracer {
 
   protected final String _name;
@@ -23,12 +25,14 @@ public class TracerImpl implements Tracer {
   protected long _ended;
   protected final String _threadName;
   protected final long _id;
+  protected Parameter[] _parameters;
 
-  public TracerImpl(String name, long id) {
+  public TracerImpl(String name, Parameter[] parameters, long id) {
     _name = name;
     _start = System.nanoTime();
     _threadName = Thread.currentThread().getName();
     _id = id;
+    _parameters = parameters;
   }
 
   @Override
@@ -59,8 +63,25 @@ public class TracerImpl implements Tracer {
   }
 
   public String toJson() {
-    return "{\"id\"=" + _id + ", \"name\"=\"" + _name + "\", \"thread\"=\"" + _threadName + "\", \"took\"="
-        + (_ended - _start) + ", \"started\"=" + _start + ", \"ended\"=" + _ended + "}";
+    if (_parameters == null) {
+      return "{\"id\":" + _id + ", \"name\":\"" + _name + "\", \"thread\":\"" + _threadName + "\", \"took\":"
+          + (_ended - _start) + ", \"started\":" + _start + ", \"ended\":" + _ended + "}";
+    } else {
+      return "{\"id\":" + _id + ", \"name\":\"" + _name + "\", \"thread\":\"" + _threadName + "\", \"took\":"
+          + (_ended - _start) + ", \"started\":" + _start + ", \"ended\":" + _ended + ", \"parameters\":["
+          + getParametersJson() + "]}";
+    }
+  }
+
+  private String getParametersJson() {
+    StringBuilder builder = new StringBuilder();
+    for (Parameter parameter : _parameters) {
+      if (builder.length() != 0) {
+        builder.append(',');
+      }
+      builder.append("\"").append(parameter._name).append("\":\"").append(parameter._value).append("\"");
+    }
+    return builder.toString();
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-util/src/main/java/org/apache/blur/trace/ZooKeeperDumpContents.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/trace/ZooKeeperDumpContents.java b/blur-util/src/main/java/org/apache/blur/trace/ZooKeeperDumpContents.java
new file mode 100644
index 0000000..dcc4ef4
--- /dev/null
+++ b/blur-util/src/main/java/org/apache/blur/trace/ZooKeeperDumpContents.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.trace;
+
+import static org.apache.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.blur.BlurConfiguration;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.data.Stat;
+
+public class ZooKeeperDumpContents {
+
+  public static void main(String[] args) throws IOException, KeeperException, InterruptedException {
+    BlurConfiguration configuration = new BlurConfiguration();
+    String zkConnectionStr = configuration.get(BLUR_ZOOKEEPER_CONNECTION);
+    ZooKeeper zooKeeper = new ZooKeeper(zkConnectionStr, 30000, new Watcher() {
+      @Override
+      public void process(WatchedEvent event) {
+
+      }
+    });
+
+    String parentPath = args[0];
+    String id = args[1];
+
+    String path = parentPath + "/" + id;
+    List<String> children = zooKeeper.getChildren(path, false);
+    for (String c : children) {
+      Stat stat = zooKeeper.exists(path + "/" + c, false);
+      byte[] data = zooKeeper.getData(path + "/" + c, false, stat);
+      String string = new String(data);
+      System.out.println(string);
+    }
+    zooKeeper.close();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-util/src/main/java/org/apache/blur/zookeeper/ZooKeeperClient.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/zookeeper/ZooKeeperClient.java b/blur-util/src/main/java/org/apache/blur/zookeeper/ZooKeeperClient.java
index 9ea5a30..54a0a7f 100644
--- a/blur-util/src/main/java/org/apache/blur/zookeeper/ZooKeeperClient.java
+++ b/blur-util/src/main/java/org/apache/blur/zookeeper/ZooKeeperClient.java
@@ -78,7 +78,7 @@ public class ZooKeeperClient extends ZooKeeper {
       sessionTimeout = internalSessionTimeout;
     }
     while (true) {
-      Tracer trace = Trace.trace("remote call - zookeeper - " + executor._name);
+      Tracer trace = Trace.trace("remote call - zookeeper", Trace.param("method", executor._name));
       try {
         return executor.execute();
       } catch (KeeperException e) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-util/src/main/resources/blur-default.properties
----------------------------------------------------------------------
diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties
index d575daf..592bc65 100644
--- a/blur-util/src/main/resources/blur-default.properties
+++ b/blur-util/src/main/resources/blur-default.properties
@@ -17,7 +17,7 @@
 blur.zookeeper.timeout=90000
 
 # The path in ZooKeeper where the distributed traces will be stored, if blank trace output will be written to the log
-blur.zookeeper.trace.path=
+blur.zookeeper.trace.path=/blur_traces
 
 # The maximum number of results that can be fetched in a single request
 blur.query.max.results.fetch=1000

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/53f9ecf5/blur-util/src/test/java/org/apache/blur/trace/TraceTest.java
----------------------------------------------------------------------
diff --git a/blur-util/src/test/java/org/apache/blur/trace/TraceTest.java b/blur-util/src/test/java/org/apache/blur/trace/TraceTest.java
index dd5faaf..b8074f4 100644
--- a/blur-util/src/test/java/org/apache/blur/trace/TraceTest.java
+++ b/blur-util/src/test/java/org/apache/blur/trace/TraceTest.java
@@ -73,7 +73,7 @@ public class TraceTest {
         } else {
           assertEquals("test", id.substring(0,indexOf));
         }
-        assertEquals(3, collector.getTraces().size());
+        assertEquals(7, collector.getTraces().size());
         count.addAndGet(collector.getTraces().size());
       }
     });
@@ -106,7 +106,7 @@ public class TraceTest {
     thread.join();
     Trace.tearDownTrace();
     
-    assertEquals(6, count.get());
+    assertEquals(7, count.get());
   }
 
   private static long meth1() {


Mime
View raw message