lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jpou...@apache.org
Subject svn commit: r1686301 - in /lucene/dev/trunk/lucene: ./ core/src/java/org/apache/lucene/search/spans/ core/src/test/org/apache/lucene/search/spans/ test-framework/src/java/org/apache/lucene/search/spans/
Date Thu, 18 Jun 2015 19:37:06 GMT
Author: jpountz
Date: Thu Jun 18 19:37:05 2015
New Revision: 1686301

URL: http://svn.apache.org/r1686301
Log:
LUCENE-2880: Make span queries score more consistently with regular queries.

Modified:
    lucene/dev/trunk/lucene/CHANGES.txt
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/ContainSpans.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/FilterSpans.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanSearchEquivalence.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java
    lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/search/spans/AssertingSpans.java

Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1686301&r1=1686300&r2=1686301&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Thu Jun 18 19:37:05 2015
@@ -179,6 +179,9 @@ Changes in Runtime Behavior
 * LUCENE-6569: Optimize MultiFunction.anyExists and allExists to eliminate
   excessive array creation in common 2 argument usage (Jacob Graves, hossman)
 
+* LUCENE-2880: Span queries now score more consistently with regular queries.
+  (Robert Muir, Adrien Grand)
+
 
 Build
 

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/ContainSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/ContainSpans.java?rev=1686301&r1=1686300&r2=1686301&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/ContainSpans.java
(original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/ContainSpans.java
Thu Jun 18 19:37:05 2015
@@ -48,6 +48,11 @@ abstract class ContainSpans extends Conj
   }
 
   @Override
+  public int width() {
+    return sourceSpans.width();
+  }
+
+  @Override
   public void collect(SpanCollector collector) throws IOException {
     sourceSpans.collect(collector);
   }

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/FilterSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/FilterSpans.java?rev=1686301&r1=1686300&r2=1686301&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/FilterSpans.java
(original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/FilterSpans.java
Thu Jun 18 19:37:05 2015
@@ -111,6 +111,11 @@ public abstract class FilterSpans extend
   }
 
   @Override
+  public int width() {
+    return in.width();
+  }
+
+  @Override
   public void collect(SpanCollector collector) throws IOException {
     in.collect(collector);
   }

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java?rev=1686301&r1=1686300&r2=1686301&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java
(original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java
Thu Jun 18 19:37:05 2015
@@ -129,6 +129,11 @@ public class NearSpansOrdered extends Ne
   }
 
   @Override
+  public int width() {
+    return matchWidth;
+  }
+
+  @Override
   public void collect(SpanCollector collector) throws IOException {
     for (Spans span : subSpans) {
       span.collect(collector);

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java?rev=1686301&r1=1686300&r2=1686301&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java
(original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java
Thu Jun 18 19:37:05 2015
@@ -115,6 +115,11 @@ public class NearSpansUnordered extends
     }
 
     @Override
+    public int width() {
+      return in.width();
+    }
+
+    @Override
     public void collect(SpanCollector collector) throws IOException {
       in.collect(collector);
     }
@@ -242,6 +247,11 @@ public class NearSpansUnordered extends
   }
 
   @Override
+  public int width() {
+    return maxEndPositionCell.startPosition() - minPositionCell().startPosition();
+  }
+
+  @Override
   public void collect(SpanCollector collector) throws IOException {
     for (SpansCell cell : subSpanCells) {
       cell.collect(collector);

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java?rev=1686301&r1=1686300&r2=1686301&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java
(original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java
Thu Jun 18 19:37:05 2015
@@ -329,6 +329,11 @@ public class SpanOrQuery extends SpanQue
         }
 
         @Override
+        public int width() {
+          return topPositionSpans.width();
+        }
+
+        @Override
         public void collect(SpanCollector collector) throws IOException {
           if (topPositionSpans != null)
             topPositionSpans.collect(collector);

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java?rev=1686301&r1=1686300&r2=1686301&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java Thu
Jun 18 19:37:05 2015
@@ -99,8 +99,7 @@ public class SpanScorer extends Scorer {
         freq = 1;
         return;
       }
-      int matchLength = endPos - startPos;
-      freq += docScorer.computeSlopFactor(matchLength);
+      freq += docScorer.computeSlopFactor(spans.width());
       prevStartPos = startPos;
       prevEndPos = endPos;
       startPos = spans.nextStartPosition();

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java?rev=1686301&r1=1686300&r2=1686301&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java Thu Jun
18 19:37:05 2015
@@ -19,6 +19,7 @@ package org.apache.lucene.search.spans;
 
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.TwoPhaseIterator;
+import org.apache.lucene.search.similarities.Similarity.SimScorer;
 
 import java.io.IOException;
 
@@ -50,6 +51,15 @@ public abstract class Spans extends DocI
   public abstract int endPosition();
 
   /**
+   * Return the width of the match, which is typically used to compute
+   * the {@link SimScorer#computeSlopFactor(int) slop factor}. It is only legal
+   * to call this method when the iterator is on a valid doc ID and positioned.
+   * The return value must be positive, and lower values means that the match is
+   * better.
+   */
+  public abstract int width();
+
+  /**
    * Collect postings data from the leaves of the current Spans.
    *
    * This method should only be called after {@link #nextStartPosition()}, and before

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java?rev=1686301&r1=1686300&r2=1686301&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java Thu
Jun 18 19:37:05 2015
@@ -102,6 +102,11 @@ public class TermSpans extends Spans {
   }
 
   @Override
+  public int width() {
+    return 0;
+  }
+
+  @Override
   public long cost() {
     return postings.cost();
   }

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java?rev=1686301&r1=1686300&r2=1686301&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java
(original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java
Thu Jun 18 19:37:05 2015
@@ -61,6 +61,11 @@ final class JustCompileSearchSpans {
     }
 
     @Override
+    public int width() {
+      throw new UnsupportedOperationException(UNSUPPORTED_MSG);
+    }
+
+    @Override
     public void collect(SpanCollector collector) throws IOException {
 
     }
@@ -122,6 +127,11 @@ final class JustCompileSearchSpans {
       throw new UnsupportedOperationException(UNSUPPORTED_MSG);
     }
 
+    @Override
+    public int width() {
+      throw new UnsupportedOperationException(UNSUPPORTED_MSG);
+    }
+
     @Override
     public void collect(SpanCollector collector) throws IOException {
 

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanSearchEquivalence.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanSearchEquivalence.java?rev=1686301&r1=1686300&r2=1686301&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanSearchEquivalence.java
(original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanSearchEquivalence.java
Thu Jun 18 19:37:05 2015
@@ -39,7 +39,7 @@ public class TestSpanSearchEquivalence e
   /** SpanTermQuery(A) = TermQuery(A) */
   public void testSpanTermVersusTerm() throws Exception {
     Term t1 = randomTerm();
-    assertSameSet(new TermQuery(t1), spanQuery(new SpanTermQuery(t1)));
+    assertSameScores(new TermQuery(t1), spanQuery(new SpanTermQuery(t1)));
   }
   
   /** SpanOrQuery(A) = SpanTermQuery(A) */
@@ -154,7 +154,7 @@ public class TestSpanSearchEquivalence e
                            };
     SpanQuery q1 = spanQuery(new SpanNearQuery(subquery, 0, true));
     PhraseQuery q2 = new PhraseQuery(t1.field(), t1.bytes(), t2.bytes());
-    assertSameSet(q1, q2);
+    assertSameScores(q1, q2);
   }
   
   /** SpanNearQuery([A, B], ∞, false) = +A +B */

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java?rev=1686301&r1=1686300&r2=1686301&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java
(original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java
Thu Jun 18 19:37:05 2015
@@ -99,7 +99,7 @@ public class TestSpansAdvanced extends L
    */
   public void testBooleanQueryWithSpanQueries() throws IOException {
     
-    doTestBooleanQueryWithSpanQueries(searcher, 0.3884282f);
+    doTestBooleanQueryWithSpanQueries(searcher, 0.54932045f);
   }
   
   /**

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java?rev=1686301&r1=1686300&r2=1686301&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java
(original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java
Thu Jun 18 19:37:05 2015
@@ -86,8 +86,8 @@ public class TestSpansAdvanced2 extends
     final Query spanQuery = new SpanTermQuery(new Term(FIELD_TEXT, "should"));
     final String[] expectedIds = new String[] {"B", "D", "1", "2", "3", "4",
         "A"};
-    final float[] expectedScores = new float[] {0.625f, 0.45927936f,
-        0.35355338f, 0.35355338f, 0.35355338f, 0.35355338f, 0.26516503f,};
+    final float[] expectedScores = new float[] {0.8838834f, 0.64951903f,
+        0.5f, 0.5f, 0.5f, 0.5f, 0.375f,};
     assertHits(searcher2, spanQuery, "single span query", expectedIds,
         expectedScores);
   }
@@ -105,7 +105,7 @@ public class TestSpansAdvanced2 extends
     final String[] expectedIds = new String[] {"D", "A"};
     // these values were pre LUCENE-413
     // final float[] expectedScores = new float[] { 0.93163157f, 0.20698164f };
-    final float[] expectedScores = new float[] {1.0191123f, 0.93163157f};
+    final float[] expectedScores = new float[] {1.44124233f, 1.31752586f};
     assertHits(searcher2, query.build(), "multiple different span queries",
         expectedIds, expectedScores);
   }
@@ -116,6 +116,6 @@ public class TestSpansAdvanced2 extends
   @Override
   public void testBooleanQueryWithSpanQueries() throws IOException {
     
-    doTestBooleanQueryWithSpanQueries(searcher2, 0.73500174f);
+    doTestBooleanQueryWithSpanQueries(searcher2, 1.0394494f);
   }
 }

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/search/spans/AssertingSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/search/spans/AssertingSpans.java?rev=1686301&r1=1686300&r2=1686301&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/search/spans/AssertingSpans.java
(original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/search/spans/AssertingSpans.java
Thu Jun 18 19:37:05 2015
@@ -125,6 +125,14 @@ class AssertingSpans extends Spans {
   }
 
   @Override
+  public int width() {
+    assert state == State.ITERATING;
+    final int distance = in.width();
+    assert distance >= 0;
+    return distance;
+  }
+
+  @Override
   public void collect(SpanCollector collector) throws IOException {
     assert state == State.ITERATING : "collect() called in illegal state: " + state + ":
" + in;
     in.collect(collector);



Mime
View raw message