incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [4/4] git commit: Adding more facting updates.
Date Sat, 22 Mar 2014 21:01:05 GMT
Adding more facting updates.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/831d22f3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/831d22f3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/831d22f3

Branch: refs/heads/apache-blur-0.2
Commit: 831d22f38e8531b8455d5d4677cc71ca96d3e827
Parents: 9b96e81
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Sat Mar 22 16:57:17 2014 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Sat Mar 22 16:57:17 2014 -0400

----------------------------------------------------------------------
 .../org/apache/blur/thrift/BlurClusterTest.java | 47 +++++++++++++++--
 .../blur/lucene/search/FacetExecutor.java       | 53 +++++++++++++++++---
 .../apache/blur/lucene/search/FacetQuery.java   |  6 ++-
 3 files changed, 93 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/831d22f3/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
index b332f5c..41672e2 100644
--- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
+++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
@@ -257,6 +257,10 @@ public class BlurClusterTest {
   }
 
   public void loadTable(String tableName) throws BlurException, TException, InterruptedException
{
+    loadTable(tableName, 1);
+  }
+
+  public void loadTable(String tableName, int pass) throws BlurException, TException, InterruptedException
{
     Iface client = getClient();
     int maxFacetValue = 100;
     List<RowMutation> mutations = new ArrayList<RowMutation>();
@@ -265,7 +269,8 @@ public class BlurClusterTest {
       String rowId = UUID.randomUUID().toString();
       RecordMutation mutation = BlurThriftHelper.newRecordMutation("test", rowId,
           BlurThriftHelper.newColumn("test", "value"),
-          BlurThriftHelper.newColumn("facet", Integer.toString(random.nextInt(maxFacetValue))));
+          BlurThriftHelper.newColumn("facet", Integer.toString(random.nextInt(maxFacetValue))),
+          BlurThriftHelper.newColumn("facetFixed", "test"));
       RowMutation rowMutation = BlurThriftHelper.newRowMutation(tableName, rowId, mutation);
       mutations.add(rowMutation);
     }
@@ -281,7 +286,7 @@ public class BlurClusterTest {
     blurQueryRow.setCacheResult(false);
     BlurResults resultsRow = client.query(tableName, blurQueryRow);
     assertRowResults(resultsRow);
-    assertEquals(numberOfDocs, resultsRow.getTotalResults());
+    assertEquals(numberOfDocs * pass, resultsRow.getTotalResults());
 
     BlurQuery blurQueryRecord = new BlurQuery();
     Query queryRecord = new Query();
@@ -290,7 +295,7 @@ public class BlurClusterTest {
     blurQueryRecord.setQuery(queryRecord);
     BlurResults resultsRecord = client.query(tableName, blurQueryRecord);
     assertRecordResults(resultsRecord);
-    assertEquals(numberOfDocs, resultsRecord.getTotalResults());
+    assertEquals(numberOfDocs * pass, resultsRecord.getTotalResults());
 
     Schema schema = client.schema(tableName);
     assertFalse(schema.getFamilies().isEmpty());
@@ -342,6 +347,11 @@ public class BlurClusterTest {
     // assertRowResults(resultsRow);
     assertEquals(numberOfDocs, resultsRow.getTotalResults());
 
+    List<Long> facetCounts = resultsRow.getFacetCounts();
+    for (Long l : facetCounts) {
+      assertTrue(l >= 50);
+    }
+
     System.out.println(resultsRow.getFacetCounts());
 
     System.out.println();
@@ -349,6 +359,37 @@ public class BlurClusterTest {
   }
 
   @Test
+  public void testQueryWithFacetsWithMins() throws BlurException, TException, IOException,
InterruptedException {
+    final String tableName = "testQueryWithFacetsWithMins";
+    createTable(tableName);
+    int pass = 1;
+    loadTable(tableName, pass);
+    Iface client = getClient();
+    BlurQuery blurQueryRow = new BlurQuery();
+    Query queryRow = new Query();
+    // queryRow.setQuery("test.test:value");
+    queryRow.setQuery("*");
+    blurQueryRow.setQuery(queryRow);
+    blurQueryRow.setUseCacheIfPresent(false);
+    blurQueryRow.setCacheResult(false);
+    blurQueryRow.setSelector(new Selector());
+    blurQueryRow.addToFacets(new Facet("test.facetFixed:test", 50));
+
+    BlurResults resultsRow = client.query(tableName, blurQueryRow);
+    // assertRowResults(resultsRow);
+    System.out.println("Pass [" + pass + "]");
+    assertEquals(numberOfDocs * pass, resultsRow.getTotalResults());
+
+    List<Long> facetCounts = resultsRow.getFacetCounts();
+    for (Long l : facetCounts) {
+      System.out.println("Count [" + l + "]");
+      assertTrue(l >= 50);
+    }
+    pass++;
+
+  }
+
+  @Test
   public void testBatchFetch() throws BlurException, TException, InterruptedException, IOException
{
     String tableName = "testBatchFetch";
     createTable(tableName);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/831d22f3/blur-query/src/main/java/org/apache/blur/lucene/search/FacetExecutor.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/lucene/search/FacetExecutor.java b/blur-query/src/main/java/org/apache/blur/lucene/search/FacetExecutor.java
index ed2abc7..913a318 100644
--- a/blur-query/src/main/java/org/apache/blur/lucene/search/FacetExecutor.java
+++ b/blur-query/src/main/java/org/apache/blur/lucene/search/FacetExecutor.java
@@ -39,13 +39,12 @@ import org.apache.blur.trace.Tracer;
 import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.search.Collector;
-import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.util.OpenBitSet;
 
 public class FacetExecutor {
 
-  private final Log LOG = LogFactory.getLog(FacetExecutor.class);
+  private static final Log LOG = LogFactory.getLog(FacetExecutor.class);
 
   static Comparator<Entry<Object, Info>> COMPARATOR = new Comparator<Entry<Object,
Info>>() {
     @Override
@@ -56,6 +55,10 @@ public class FacetExecutor {
     }
   };
 
+  static class Finished extends IOException {
+    private static final long serialVersionUID = 1L;
+  }
+
   static class SimpleCollector extends Collector {
 
     int _hits;
@@ -73,8 +76,7 @@ public class FacetExecutor {
       } else {
         int nextSetBit = _bitSet.nextSetBit(doc);
         if (nextSetBit < 0) {
-          // Move to the end of the scorer.
-          _scorer.advance(DocIdSetIterator.NO_MORE_DOCS);
+          throw new Finished();
         } else {
           int advance = _scorer.advance(nextSetBit);
           if (_bitSet.fastGet(advance)) {
@@ -101,6 +103,28 @@ public class FacetExecutor {
 
   }
 
+  static class SimpleCollectorExitEarly extends SimpleCollector {
+
+    private long _currentCount;
+    private final long _min;
+
+    SimpleCollectorExitEarly(OpenBitSet bitSet, long currentCount, long min) {
+      super(bitSet);
+      _currentCount = currentCount;
+      _min = min;
+    }
+
+    @Override
+    public void collect(int doc) throws IOException {
+      super.collect(doc);
+      _currentCount++;
+      if (_currentCount >= _min) {
+        throw new Finished();
+      }
+    }
+
+  }
+
   static class Info {
     final OpenBitSet _bitSet;
     final Scorer[] _scorers;
@@ -111,7 +135,7 @@ public class FacetExecutor {
 
     @Override
     public String toString() {
-      return "Info scorers length [" + _scorers.length + "] reader[" + _reader + "]";
+      return "Info scorers length [" + _scorers.length + "] reader [" + _reader + "]";
     }
 
     Info(AtomicReaderContext context, Scorer[] scorers, Lock[] locks) {
@@ -125,12 +149,12 @@ public class FacetExecutor {
     }
 
     void process(AtomicLongArray counts, long[] minimumsBeforeReturning) throws IOException
{
-      SimpleCollector col = new SimpleCollector(_bitSet);
       if (minimumsBeforeReturning == null) {
         Tracer trace = Trace.trace("processing facet - segment", Trace.param("reader", _readerStr),
             Trace.param("maxDoc", _maxDoc), Trace.param("minimums", "NONE"), Trace.param("scorers",
_scorers.length));
         try {
           for (int i = 0; i < _scorers.length; i++) {
+            SimpleCollector col = new SimpleCollector(_bitSet);
             runFacet(counts, col, i);
           }
         } finally {
@@ -148,6 +172,8 @@ public class FacetExecutor {
                 long min = minimumsBeforeReturning[id];
                 long currentCount = counts.get(id);
                 if (currentCount < min) {
+                  LOG.debug("Running facet, current count [{0}] min [{1}]", currentCount,
min);
+                  SimpleCollectorExitEarly col = new SimpleCollectorExitEarly(_bitSet, currentCount,
min);
                   runFacet(counts, col, id);
                 }
               } finally {
@@ -175,11 +201,16 @@ public class FacetExecutor {
         Tracer traceInner = Trace.trace("processing facet - segment - scorer", Trace.param("scorer",
scorer),
             Trace.param("scorer.cost", scorer.cost()));
         try {
+          // new ExitScorer(scorer).score(col);
           scorer.score(col);
+        } catch (Finished e) {
+          // Do nothing, exiting early.
         } finally {
           traceInner.done();
         }
-        counts.addAndGet(i, col._hits);
+        int hits = col._hits;
+        LOG.debug("Facet [{0}] result [{1}]", i, hits);
+        counts.addAndGet(i, hits);
       }
       col._hits = 0;
     }
@@ -221,11 +252,16 @@ public class FacetExecutor {
       _infoMap.put(key, info);
     } else {
       AtomicReader reader = context.reader();
-      LOG.info("Info about reader context [{0}] alread created, existing Info [{1}] current
reader [{2}].", context,
+      LOG.warn("Info about reader context [{0}] already created, existing Info [{1}] current
reader [{2}].", context,
           info, reader);
     }
   }
 
+  public boolean scorersAlreadyAdded(AtomicReaderContext context) {
+    Object key = getKey(context);
+    return _infoMap.containsKey(key);
+  }
+
   private Object getKey(AtomicReaderContext context) {
     return context.reader().getCoreCacheKey();
   }
@@ -297,4 +333,5 @@ public class FacetExecutor {
       }
     }
   }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/831d22f3/blur-query/src/main/java/org/apache/blur/lucene/search/FacetQuery.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/lucene/search/FacetQuery.java b/blur-query/src/main/java/org/apache/blur/lucene/search/FacetQuery.java
index b813a0e..fbb342c 100644
--- a/blur-query/src/main/java/org/apache/blur/lucene/search/FacetQuery.java
+++ b/blur-query/src/main/java/org/apache/blur/lucene/search/FacetQuery.java
@@ -123,8 +123,10 @@ public class FacetQuery extends AbstractWrapperQuery {
       if (scorer == null) {
         return null;
       }
-      Scorer[] scorers = getScorers(context, true, topScorer, acceptDocs);
-      _executor.addScorers(context, scorers);
+      if (!_executor.scorersAlreadyAdded(context)) {
+        Scorer[] scorers = getScorers(context, true, topScorer, acceptDocs);
+        _executor.addScorers(context, scorers);  
+      }
       return new FacetScorer(scorer, _executor, context);
     }
 


Mime
View raw message