incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [1/3] git commit: Adding tracing to the faceting code.
Date Fri, 10 Jan 2014 02:37:11 GMT
Updated Branches:
  refs/heads/apache-blur-0.2 4e5813433 -> 85ebe0a35


Adding tracing to the faceting code.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/c6486046
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/c6486046
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/c6486046

Branch: refs/heads/apache-blur-0.2
Commit: c6486046cf31a590bd18439cc817ea3d99aa1e2c
Parents: 4e58134
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Thu Jan 9 20:41:35 2014 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Thu Jan 9 20:41:35 2014 -0500

----------------------------------------------------------------------
 .../blur/lucene/search/FacetExecutor.java       | 48 ++++++++++++++------
 .../blur/lucene/search/FacetQueryTest.java      | 21 +++++++--
 2 files changed, 52 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c6486046/blur-query/src/main/java/org/apache/blur/lucene/search/FacetExecutor.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/lucene/search/FacetExecutor.java b/blur-query/src/main/java/org/apache/blur/lucene/search/FacetExecutor.java
index 10ae351..587695b 100644
--- a/blur-query/src/main/java/org/apache/blur/lucene/search/FacetExecutor.java
+++ b/blur-query/src/main/java/org/apache/blur/lucene/search/FacetExecutor.java
@@ -30,6 +30,8 @@ import java.util.concurrent.atomic.AtomicLongArray;
 
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
+import org.apache.blur.trace.Trace;
+import org.apache.blur.trace.Tracer;
 import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.search.Collector;
@@ -86,6 +88,8 @@ public class FacetExecutor {
     final OpenBitSet _bitSet;
     final Scorer[] _scorers;
     final AtomicReader _reader;
+    final String _readerStr;
+    final int _maxDoc;
 
     @Override
     public String toString() {
@@ -97,34 +101,47 @@ public class FacetExecutor {
       _bitSet = new OpenBitSet(reader.maxDoc());
       _scorers = scorers;
       _reader = reader;
+      _readerStr = _reader.toString();
+      _maxDoc = _reader.maxDoc();
     }
 
     void process(AtomicLongArray counts, long[] minimumsBeforeReturning) throws IOException
{
       SimpleCollector col = new SimpleCollector(_bitSet);
       if (minimumsBeforeReturning == null) {
-        for (int i = 0; i < _scorers.length; i++) {
-          Scorer scorer = _scorers[i];
-          if (scorer != null) {
-            scorer.score(col);
-            counts.addAndGet(i, col._hits);
+        Tracer trace = Trace.trace("processing facet - segment", Trace.param("reader", _readerStr),
+            Trace.param("maxDoc", _maxDoc), Trace.param("minimums", "NONE"), Trace.param("scorers",
_scorers.length));
+        try {
+          for (int i = 0; i < _scorers.length; i++) {
+            runFacet(counts, col, i);
           }
-          col._hits = 0;
+        } finally {
+          trace.done();
         }
       } else {
         for (int i = 0; i < _scorers.length; i++) {
           long min = minimumsBeforeReturning[i];
           long currentCount = counts.get(i);
           if (currentCount < min) {
-            Scorer scorer = _scorers[i];
-            if (scorer != null) {
-              scorer.score(col);
-              counts.addAndGet(i, col._hits);
-            }
-            col._hits = 0;
+            runFacet(counts, col, i);
           }
         }
       }
     }
+
+    private void runFacet(AtomicLongArray counts, SimpleCollector col, int i) throws IOException
{
+      Scorer scorer = _scorers[i];
+      if (scorer != null) {
+        Tracer traceInner = Trace.trace("processing facet - segment - scorer", Trace.param("scorer",
scorer),
+            Trace.param("scorer.cost", scorer.cost()));
+        try {
+          scorer.score(col);
+        } finally {
+          traceInner.done();
+        }
+        counts.addAndGet(i, col._hits);
+      }
+      col._hits = 0;
+    }
   }
 
   private final Map<Object, Info> _infoMap = new ConcurrentHashMap<Object, FacetExecutor.Info>();
@@ -188,7 +205,12 @@ public class FacetExecutor {
 
   public void processFacets(ExecutorService executor) throws IOException {
     if (!_processed) {
-      processInternal(executor);
+      Tracer trace = Trace.trace("processing facets");
+      try {
+        processInternal(executor);
+      } finally {
+        trace.done();
+      }
       _processed = true;
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c6486046/blur-query/src/test/java/org/apache/blur/lucene/search/FacetQueryTest.java
----------------------------------------------------------------------
diff --git a/blur-query/src/test/java/org/apache/blur/lucene/search/FacetQueryTest.java b/blur-query/src/test/java/org/apache/blur/lucene/search/FacetQueryTest.java
index 28e2c35..45d7354 100644
--- a/blur-query/src/test/java/org/apache/blur/lucene/search/FacetQueryTest.java
+++ b/blur-query/src/test/java/org/apache/blur/lucene/search/FacetQueryTest.java
@@ -24,9 +24,11 @@ import static org.junit.Assert.assertTrue;
 import java.io.File;
 import java.io.IOException;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.blur.concurrent.Executors;
+import org.apache.blur.trace.LogTraceStorage;
+import org.apache.blur.trace.Trace;
 import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
@@ -71,7 +73,7 @@ public class FacetQueryTest {
     IndexSearcher indexSearcher = new IndexSearcher(reader);
     indexSearcher.search(facetQuery, 10);
 
-    ExecutorService executor = Executors.newCachedThreadPool();
+    ExecutorService executor = getThreadPool();
     facetExecutor.processFacets(executor);
     executor.shutdown();
     executor.awaitTermination(10, TimeUnit.SECONDS);
@@ -83,6 +85,7 @@ public class FacetQueryTest {
 
   @Test
   public void testFacetQueryPerformance() throws IOException, InterruptedException {
+    Trace.setStorage(new LogTraceStorage(null));
     int facetCount = 200;
     int docCount = 1000000;
     IndexReader reader = createIndex(docCount, facetCount, false);
@@ -95,13 +98,19 @@ public class FacetQueryTest {
     ExecutorService executor = null;
     try {
       for (int t = 0; t < 5; t++) {
-        executor = Executors.newCachedThreadPool();
+        executor = getThreadPool();
         IndexSearcher indexSearcher = new IndexSearcher(reader, executor);
         FacetExecutor facetExecutor = new FacetExecutor(facets.length);
         FacetQuery facetQuery = new FacetQuery(new TermQuery(new Term("f1", "value")), facets,
facetExecutor);
         long t1 = System.nanoTime();
         indexSearcher.search(facetQuery, 10);
+        if (t == 4) {
+          Trace.setupTrace("unittest");
+        }
         facetExecutor.processFacets(executor);
+        if (t == 4) {
+          Trace.tearDownTrace();
+        }
         executor.shutdown();
         executor.awaitTermination(10, TimeUnit.SECONDS);
         long t2 = System.nanoTime();
@@ -136,7 +145,7 @@ public class FacetQueryTest {
     ExecutorService executor = null;
     try {
       for (int t = 0; t < 5; t++) {
-        executor = Executors.newCachedThreadPool();
+        executor = getThreadPool();
         IndexSearcher indexSearcher = new IndexSearcher(reader, executor);
         FacetExecutor facetExecutor = new FacetExecutor(facets.length, minimumsBeforeReturning);
         FacetQuery facetQuery = new FacetQuery(new TermQuery(new Term("f1", "value")), facets,
facetExecutor);
@@ -158,6 +167,10 @@ public class FacetQueryTest {
     }
   }
 
+  private ExecutorService getThreadPool() {
+    return Executors.newThreadPool("unittest-facets", 10);
+  }
+
   private IndexReader createIndex(int docCount, int facetFields, boolean ram) throws CorruptIndexException,
       LockObtainFailedException, IOException {
     Directory directory;


Mime
View raw message