lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jpou...@apache.org
Subject svn commit: r1556483 - in /lucene/dev/trunk/lucene: ./ highlighter/src/java/org/apache/lucene/search/vectorhighlight/ highlighter/src/test/org/apache/lucene/search/vectorhighlight/
Date Wed, 08 Jan 2014 10:11:27 GMT
Author: jpountz
Date: Wed Jan  8 10:11:27 2014
New Revision: 1556483

URL: http://svn.apache.org/r1556483
Log:
LUCENE-5361: Fixed handling of query boosts in FastVectorHighlighter.

Modified:
    lucene/dev/trunk/lucene/CHANGES.txt
    lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java
    lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java
    lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java

Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1556483&r1=1556482&r2=1556483&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Wed Jan  8 10:11:27 2014
@@ -131,6 +131,9 @@ Bug fixes
   like when index / flush requests happen concurrently to the close or
   rollback call. (Simon Willnauer)
 
+* LUCENE-5361: Fixed handling of query boosts in FastVectorHighlighter.
+  (Nik Everett via Adrien Grand)
+
 API Changes
 
 * LUCENE-5339: The facet module was simplified/reworked to make the

Modified: lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java?rev=1556483&r1=1556482&r2=1556483&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java
(original)
+++ lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java
Wed Jan  8 10:11:27 2014
@@ -91,14 +91,15 @@ public class FieldQuery {
   void flatten( Query sourceQuery, IndexReader reader, Collection<Query> flatQueries
) throws IOException{
     if( sourceQuery instanceof BooleanQuery ){
       BooleanQuery bq = (BooleanQuery)sourceQuery;
-      for( BooleanClause clause : bq.getClauses() ){
-        if( !clause.isProhibited() )
-          flatten( clause.getQuery(), reader, flatQueries );
+      for( BooleanClause clause : bq ) {
+        if( !clause.isProhibited() ) {
+          flatten( applyParentBoost( clause.getQuery(), bq ), reader, flatQueries );
+        }
       }
     } else if( sourceQuery instanceof DisjunctionMaxQuery ){
       DisjunctionMaxQuery dmq = (DisjunctionMaxQuery)sourceQuery;
       for( Query query : dmq ){
-        flatten( query, reader, flatQueries );
+        flatten( applyParentBoost( query, dmq ), reader, flatQueries );
       }
     }
     else if( sourceQuery instanceof TermQuery ){
@@ -111,18 +112,20 @@ public class FieldQuery {
         if( pq.getTerms().length > 1 )
           flatQueries.add( pq );
         else if( pq.getTerms().length == 1 ){
-          flatQueries.add( new TermQuery( pq.getTerms()[0] ) );
+          Query flat = new TermQuery( pq.getTerms()[0] );
+          flat.setBoost( pq.getBoost() );
+          flatQueries.add( flat );
         }
       }
     } else if (sourceQuery instanceof ConstantScoreQuery) {
       final Query q = ((ConstantScoreQuery) sourceQuery).getQuery();
       if (q != null) {
-        flatten(q, reader, flatQueries);
+        flatten( applyParentBoost( q, sourceQuery ), reader, flatQueries);
       }
     } else if (sourceQuery instanceof FilteredQuery) {
       final Query q = ((FilteredQuery) sourceQuery).getQuery();
       if (q != null) {
-        flatten(q, reader, flatQueries);
+        flatten( applyParentBoost( q, sourceQuery ), reader, flatQueries);
       }
     } else if (reader != null){
       Query query = sourceQuery;
@@ -142,6 +145,18 @@ public class FieldQuery {
     }
     // else discard queries
   }
+
+  /**
+   * Push parent's boost into a clone of query if parent has a non 1 boost.
+   */
+  protected Query applyParentBoost( Query query, Query parent ) {
+    if ( parent.getBoost() == 1 ) {
+      return query;
+    }
+    Query cloned = query.clone();
+    cloned.setBoost( query.getBoost() * parent.getBoost() );
+    return cloned;
+  }
   
   /*
    * Create expandQueries from flatQueries.

Modified: lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java?rev=1556483&r1=1556482&r2=1556483&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java
(original)
+++ lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java
Wed Jan  8 10:11:27 2014
@@ -257,8 +257,53 @@ public class FastVectorHighlighterTest e
     writer.close();
     dir.close();
   }
-  
-  public void testCommonTermsQueryHighlightTest() throws IOException {
+
+  public void testBoostedPhraseHighlightTest() throws IOException {
+    Directory dir = newDirectory();
+    IndexWriter writer = new IndexWriter( dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer( random() ) ) );
+    Document doc = new Document();
+    FieldType type = new FieldType( TextField.TYPE_STORED  );
+    type.setStoreTermVectorOffsets( true );
+    type.setStoreTermVectorPositions( true );
+    type.setStoreTermVectors( true );
+    type.freeze();
+    StringBuilder text = new StringBuilder();
+    text.append("words words junk junk junk junk junk junk junk junk highlight junk junk
junk junk together junk ");
+    for ( int i = 0; i<10; i++ ) {
+      text.append("junk junk junk junk junk junk junk junk junk junk junk junk junk junk
junk junk junk junk junk junk ");
+    }
+    text.append("highlight words together ");
+    for ( int i = 0; i<10; i++ ) {
+      text.append("junk junk junk junk junk junk junk junk junk junk junk junk junk junk
junk junk junk junk junk junk ");
+    }
+    doc.add( new Field( "text", text.toString().trim(), type ) );
+    writer.addDocument(doc);
+    FastVectorHighlighter highlighter = new FastVectorHighlighter();
+    IndexReader reader = DirectoryReader.open(writer, true);
+
+    // This mimics what some query parsers do to <highlight words together>
+    BooleanQuery terms = new BooleanQuery();
+    terms.add( clause( "text", "highlight" ), Occur.MUST );
+    terms.add( clause( "text", "words" ), Occur.MUST );
+    terms.add( clause( "text", "together" ), Occur.MUST );
+    // This mimics what some query parsers do to <"highlight words together">
+    BooleanQuery phrase = new BooleanQuery();
+    phrase.add( clause( "text", "highlight", "words", "together" ), Occur.MUST );
+    phrase.setBoost( 100 );
+    // Now combine those results in a boolean query which should pull the phrases to the
front of the list of fragments 
+    BooleanQuery query = new BooleanQuery();
+    query.add( phrase, Occur.MUST );
+    query.add( phrase, Occur.SHOULD );
+    FieldQuery fieldQuery = new FieldQuery( query, reader, true, false );
+    String fragment = highlighter.getBestFragment( fieldQuery, reader, 0, "text", 100 );
+    assertEquals( "junk junk junk junk junk junk junk junk <b>highlight words together</b>
junk junk junk junk junk junk junk junk", fragment );
+
+    reader.close();
+    writer.close();
+    dir.close();
+  }
+
+  public void testCommonTermsQueryHighlight() throws IOException {
     Directory dir = newDirectory();
     IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
 new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET)));
     FieldType type = new FieldType(TextField.TYPE_STORED);

Modified: lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java?rev=1556483&r1=1556482&r2=1556483&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java
(original)
+++ lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java
Wed Jan  8 10:11:27 2014
@@ -44,44 +44,55 @@ import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 
 public class FieldQueryTest extends AbstractTestCase {
+  private float boost;
+
+  /**
+   * Set boost to a random value each time it is called.
+   */
+  private void initBoost() {
+    boost = usually() ? 1F : random().nextFloat() * 10000;
+  }
 
   public void testFlattenBoolean() throws Exception {
+    initBoost();
     BooleanQuery booleanQuery = new BooleanQuery();
-    booleanQuery.add(new TermQuery(new Term(F, "A")), Occur.MUST);
-    booleanQuery.add(new TermQuery(new Term(F, "B")), Occur.MUST);
-    booleanQuery.add(new TermQuery(new Term(F, "C")), Occur.SHOULD);
+    booleanQuery.setBoost( boost );
+    booleanQuery.add(tq("A"), Occur.MUST);
+    booleanQuery.add(tq("B"), Occur.MUST);
+    booleanQuery.add(tq("C"), Occur.SHOULD);
 
     BooleanQuery innerQuery = new BooleanQuery();
-    innerQuery.add(new TermQuery(new Term(F, "D")), Occur.MUST);
-    innerQuery.add(new TermQuery(new Term(F, "E")), Occur.MUST);
+    innerQuery.add(tq("D"), Occur.MUST);
+    innerQuery.add(tq("E"), Occur.MUST);
     booleanQuery.add(innerQuery, Occur.MUST_NOT);
 
     FieldQuery fq = new FieldQuery(booleanQuery, true, true );
     Set<Query> flatQueries = new HashSet<Query>();
     fq.flatten(booleanQuery, reader, flatQueries);
-    assertCollectionQueries( flatQueries, tq( "A" ), tq( "B" ), tq( "C" ) );
+    assertCollectionQueries( flatQueries, tq( boost, "A" ), tq( boost, "B" ), tq( boost,
"C" ) );
   }
 
   public void testFlattenDisjunctionMaxQuery() throws Exception {
+    initBoost();
     Query query = dmq( tq( "A" ), tq( "B" ), pqF( "C", "D" ) );
+    query.setBoost( boost );
     FieldQuery fq = new FieldQuery( query, true, true );
     Set<Query> flatQueries = new HashSet<Query>();
     fq.flatten( query, reader, flatQueries );
-    assertCollectionQueries( flatQueries, tq( "A" ), tq( "B" ), pqF( "C", "D" ) );
+    assertCollectionQueries( flatQueries, tq( boost, "A" ), tq( boost, "B" ), pqF( boost,
"C", "D" ) );
   }
 
   public void testFlattenTermAndPhrase() throws Exception {
+    initBoost();
     BooleanQuery booleanQuery = new BooleanQuery();
-    booleanQuery.add(new TermQuery(new Term(F, "A")), Occur.MUST);
-    PhraseQuery phraseQuery = new PhraseQuery();
-    phraseQuery.add(new Term(F, "B"));
-    phraseQuery.add(new Term(F, "C"));
-    booleanQuery.add(phraseQuery, Occur.MUST);
+    booleanQuery.setBoost( boost );
+    booleanQuery.add(tq("A"), Occur.MUST);
+    booleanQuery.add(pqF("B", "C"), Occur.MUST);
 
     FieldQuery fq = new FieldQuery(booleanQuery, true, true );
     Set<Query> flatQueries = new HashSet<Query>();
     fq.flatten(booleanQuery, reader, flatQueries);
-    assertCollectionQueries( flatQueries, tq( "A" ), pqF( "B", "C" ) );
+    assertCollectionQueries( flatQueries, tq( boost, "A" ), pqF( boost, "B", "C" ) );
   }
 
   public void testFlattenTermAndPhrase2gram() throws Exception {
@@ -926,6 +937,7 @@ public class FieldQueryTest extends Abst
   }
   
   public void testFlattenFilteredQuery() throws Exception {
+    initBoost();
     Query query = new FilteredQuery(pqF( "A" ), new Filter() {
       @Override
       public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs)
@@ -933,18 +945,21 @@ public class FieldQueryTest extends Abst
         return null;
       }
     });
+    query.setBoost(boost);
     FieldQuery fq = new FieldQuery( query, true, true );
     Set<Query> flatQueries = new HashSet<Query>();
     fq.flatten( query, reader, flatQueries );
-    assertCollectionQueries( flatQueries, tq( "A" ) );
+    assertCollectionQueries( flatQueries, tq( boost, "A" ) );
   }
   
   public void testFlattenConstantScoreQuery() throws Exception {
+    initBoost();
     Query query = new ConstantScoreQuery(pqF( "A" ));
+    query.setBoost(boost);
     FieldQuery fq = new FieldQuery( query, true, true );
     Set<Query> flatQueries = new HashSet<Query>();
     fq.flatten( query, reader, flatQueries );
-    assertCollectionQueries( flatQueries, tq( "A" ) );
+    assertCollectionQueries( flatQueries, tq( boost, "A" ) );
   }
   
 }



Mime
View raw message