lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jpou...@apache.org
Subject svn commit: r1645613 - in /lucene/dev/trunk/lucene: core/src/java/org/apache/lucene/search/ core/src/test/org/apache/lucene/search/ test-framework/src/java/org/apache/lucene/util/
Date Mon, 15 Dec 2014 10:32:09 GMT
Author: jpountz
Date: Mon Dec 15 10:32:09 2014
New Revision: 1645613

URL: http://svn.apache.org/r1645613
Log:
LUCENE-6106: Improve tracking of filter usage in LRUFilterCache.

Modified:
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FilterCachingPolicy.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/LRUFilterCache.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/UsageTrackingFilterCachingPolicy.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestLRUFilterCache.java
    lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FilterCachingPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FilterCachingPolicy.java?rev=1645613&r1=1645612&r2=1645613&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FilterCachingPolicy.java
(original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FilterCachingPolicy.java
Mon Dec 15 10:32:09 2014
@@ -38,7 +38,7 @@ public interface FilterCachingPolicy {
   public static final FilterCachingPolicy ALWAYS_CACHE = new FilterCachingPolicy() {
 
     @Override
-    public void onCache(Filter filter) {}
+    public void onUse(Filter filter) {}
 
     @Override
     public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws
IOException {
@@ -78,7 +78,7 @@ public interface FilterCachingPolicy {
     }
 
     @Override
-    public void onCache(Filter filter) {}
+    public void onUse(Filter filter) {}
 
     @Override
     public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws
IOException {
@@ -89,10 +89,10 @@ public interface FilterCachingPolicy {
 
   };
 
-  /** Callback that is called on every call to {@link FilterCache#doCache}.
+  /** Callback that is called every time that a cached filter is used.
    *  This is typically useful if the policy wants to track usage statistics
    *  in order to make decisions. */
-  void onCache(Filter filter);
+  void onUse(Filter filter);
 
   /** Whether the given {@link DocIdSet} should be cached on a given segment.
    *  This method will be called on each leaf context to know if the filter

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/LRUFilterCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/LRUFilterCache.java?rev=1645613&r1=1645612&r2=1645613&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/LRUFilterCache.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/LRUFilterCache.java Mon
Dec 15 10:32:09 2014
@@ -339,6 +339,10 @@ public class LRUFilterCache implements F
 
     @Override
     public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException
{
+      if (context.ord == 0) {
+        policy.onUse(in);
+      }
+
       DocIdSet set = get(in, context);
       if (set == null) {
         // do not apply acceptDocs yet, we want the cached filter to not take them into account

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/UsageTrackingFilterCachingPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/UsageTrackingFilterCachingPolicy.java?rev=1645613&r1=1645612&r2=1645613&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/UsageTrackingFilterCachingPolicy.java
(original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/UsageTrackingFilterCachingPolicy.java
Mon Dec 15 10:32:09 2014
@@ -103,7 +103,7 @@ public final class UsageTrackingFilterCa
   }
 
   @Override
-  public void onCache(Filter filter) {
+  public void onUse(Filter filter) {
     // Using the filter hash codes might help keep memory usage a bit lower
     // since some filters might have non-negligible memory usage?
     recentlyUsedFilters.add(filter.hashCode());

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestLRUFilterCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestLRUFilterCache.java?rev=1645613&r1=1645612&r2=1645613&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestLRUFilterCache.java
(original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestLRUFilterCache.java
Mon Dec 15 10:32:09 2014
@@ -22,6 +22,7 @@ import java.lang.reflect.Field;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -49,7 +50,7 @@ public class TestLRUFilterCache extends
   private static final FilterCachingPolicy NEVER_CACHE = new FilterCachingPolicy() {
 
     @Override
-    public void onCache(Filter filter) {}
+    public void onUse(Filter filter) {}
 
     @Override
     public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws
IOException {
@@ -398,6 +399,62 @@ public class TestLRUFilterCache extends
 
     reader.close();
     w.close();
+    dir.close();
+  }
+
+  public void testOnUse() throws IOException {
+    final LRUFilterCache filterCache = new LRUFilterCache(1 + random().nextInt(5), 1 + random().nextInt(1000));
+
+    Directory dir = newDirectory();
+    final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+
+    Document doc = new Document();
+    StringField f = new StringField("color", "", Store.NO);
+    doc.add(f);
+    final int numDocs = atLeast(10);
+    for (int i = 0; i < numDocs; ++i) {
+      f.setStringValue(RandomPicks.randomFrom(random(), Arrays.asList("red", "blue", "green",
"yellow")));
+      w.addDocument(doc);
+      if (random().nextBoolean()) {
+        w.getReader().close();
+      }
+    }
+    final DirectoryReader reader = w.getReader();
+    final IndexSearcher searcher = new IndexSearcher(reader);
+
+    final Map<Filter, Integer> actualCounts = new HashMap<>();
+    final Map<Filter, Integer> expectedCounts = new HashMap<>();
+
+    final FilterCachingPolicy countingPolicy = new FilterCachingPolicy() {
+
+      @Override
+      public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set)
throws IOException {
+        return random().nextBoolean();
+      }
+
+      @Override
+      public void onUse(Filter filter) {
+        expectedCounts.put(filter, expectedCounts.getOrDefault(filter, 0));
+      }
+    };
+
+    Filter[] filters = new Filter[10 + random().nextInt(10)];
+    Filter[] cachedFilters = new Filter[filters.length];
+    for (int i = 0; i < filters.length; ++i) {
+      filters[i] = new QueryWrapperFilter(new TermQuery(new Term("color", RandomPicks.randomFrom(random(),
Arrays.asList("red", "blue", "green", "yellow")))));
+      cachedFilters[i] = filterCache.doCache(filters[i], countingPolicy);
+    }
+
+    for (int i = 0; i < 20; ++i) {
+      final int idx = random().nextInt(filters.length);
+      searcher.search(new ConstantScoreQuery(cachedFilters[idx]), 1);
+      actualCounts.put(filters[idx], actualCounts.getOrDefault(filters[idx], 0));
+    }
+
+    assertEquals(actualCounts, expectedCounts);
+
+    reader.close();
+    w.close();
     dir.close();
   }
 

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1645613&r1=1645612&r2=1645613&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
(original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
Mon Dec 15 10:32:09 2014
@@ -474,7 +474,7 @@ public abstract class LuceneTestCase ext
   public static final FilterCachingPolicy MAYBE_CACHE_POLICY = new FilterCachingPolicy()
{
 
     @Override
-    public void onCache(Filter filter) {}
+    public void onUse(Filter filter) {}
 
     @Override
     public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws
IOException {



Mime
View raw message