lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1459817 - in /lucene/dev/branches/branch_4x: ./ lucene/ lucene/highlighter/ lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/ lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/ solr/ solr/core/ so...
Date Fri, 22 Mar 2013 14:51:44 GMT
Author: mikemccand
Date: Fri Mar 22 14:51:44 2013
New Revision: 1459817

URL: http://svn.apache.org/r1459817
Log:
LUCENE-4860: per-field control over scoring and formatting

Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/lucene/   (props changed)
    lucene/dev/branches/branch_4x/lucene/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/branch_4x/lucene/highlighter/   (props changed)
    lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/Passage.java
    lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java
    lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java
    lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java
    lucene/dev/branches/branch_4x/solr/   (props changed)
    lucene/dev/branches/branch_4x/solr/core/   (props changed)
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java

Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1459817&r1=1459816&r2=1459817&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Fri Mar 22 14:51:44 2013
@@ -103,6 +103,12 @@ New Features
 * LUCENE-4752: New SortingMergePolicy (in lucene/misc) that sorts documents
   before merging segments. (Adrien Grand, Shai Erera, David Smiley)
 
+* LUCENE-4860: Customize scoring and formatting per-field in
+  PosthingsHighlighter by subclassing and overriding the getFormatter
+  and/or getScorer methods.  This also changes Passage.getMatchTerms()
+  to return BytesRef[] instead of Term[].  (Robert Muir, Mike
+  McCandless)
+
 API Changes
 
 * LUCENE-4844: removed TaxonomyReader.getParent(), you should use

Modified: lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/Passage.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/Passage.java?rev=1459817&r1=1459816&r2=1459817&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/Passage.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/Passage.java
Fri Mar 22 14:51:44 2013
@@ -17,8 +17,8 @@ package org.apache.lucene.search.posting
  * limitations under the License.
  */
 
-import org.apache.lucene.index.Term;
 import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util.SorterTemplate;
 
@@ -36,15 +36,15 @@ public final class Passage {
 
   int matchStarts[] = new int[8];
   int matchEnds[] = new int[8];
-  Term matchTerms[] = new Term[8];
+  BytesRef matchTerms[] = new BytesRef[8];
   int numMatches = 0;
   
-  void addMatch(int startOffset, int endOffset, Term term) {
+  void addMatch(int startOffset, int endOffset, BytesRef term) {
     assert startOffset >= this.startOffset && startOffset <= this.endOffset;
     if (numMatches == matchStarts.length) {
       matchStarts = ArrayUtil.grow(matchStarts, numMatches+1);
       matchEnds = ArrayUtil.grow(matchEnds, numMatches+1);
-      Term newMatchTerms[] = new Term[ArrayUtil.oversize(numMatches+1, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+      BytesRef newMatchTerms[] = new BytesRef[ArrayUtil.oversize(numMatches+1, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
       System.arraycopy(matchTerms, 0, newMatchTerms, 0, numMatches);
       matchTerms = newMatchTerms;
     }
@@ -57,7 +57,7 @@ public final class Passage {
   void sort() {
     final int starts[] = matchStarts;
     final int ends[] = matchEnds;
-    final Term terms[] = matchTerms;
+    final BytesRef terms[] = matchTerms;
     new SorterTemplate() {
       @Override
       protected void swap(int i, int j) {
@@ -69,7 +69,7 @@ public final class Passage {
         ends[i] = ends[j];
         ends[j] = temp;
         
-        Term tempTerm = terms[i];
+        BytesRef tempTerm = terms[i];
         terms[i] = terms[j];
         terms[j] = tempTerm;
       }
@@ -157,11 +157,11 @@ public final class Passage {
   }
 
   /**
-   * Term of the matches, corresponding with {@link #getMatchStarts()}.
+   * BytesRef (term text) of the matches, corresponding with {@link #getMatchStarts()}.
    * <p>
    * Only {@link #getNumMatches()} are valid.
    */
-  public Term[] getMatchTerms() {
+  public BytesRef[] getMatchTerms() {
     return matchTerms;
   }
 }

Modified: lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java?rev=1459817&r1=1459816&r2=1459817&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java
Fri Mar 22 14:51:44 2013
@@ -97,8 +97,14 @@ public class PostingsHighlighter {
     
   private final int maxLength;
   private final BreakIterator breakIterator;
-  private final PassageScorer scorer;
-  private final PassageFormatter formatter;
+
+  /** Set the first time {@link #getFormatter} is called,
+   *  and then reused. */
+  private PassageFormatter defaultFormatter;
+
+  /** Set the first time {@link #getScorer} is called,
+   *  and then reused. */
+  private PassageScorer defaultScorer;
   
   /**
    * Creates a new highlighter with default parameters.
@@ -113,7 +119,7 @@ public class PostingsHighlighter {
    * @throws IllegalArgumentException if <code>maxLength</code> is negative or
<code>Integer.MAX_VALUE</code>
    */
   public PostingsHighlighter(int maxLength) {
-    this(maxLength, BreakIterator.getSentenceInstance(Locale.ROOT), new PassageScorer(),
new PassageFormatter());
+    this(maxLength, BreakIterator.getSentenceInstance(Locale.ROOT));
   }
   
   /**
@@ -122,11 +128,9 @@ public class PostingsHighlighter {
    * @param breakIterator used for finding passage
    *        boundaries; pass null to highlight the entire
    *        content as a single Passage.
-   * @param scorer used for ranking passages.
-   * @param formatter used for formatting passages into highlighted snippets.
    * @throws IllegalArgumentException if <code>maxLength</code> is negative or
<code>Integer.MAX_VALUE</code>
    */
-  public PostingsHighlighter(int maxLength, BreakIterator breakIterator, PassageScorer scorer,
PassageFormatter formatter) {
+  public PostingsHighlighter(int maxLength, BreakIterator breakIterator) {
     if (maxLength < 0 || maxLength == Integer.MAX_VALUE) {
       // two reasons: no overflow problems in BreakIterator.preceding(offset+1),
       // our sentinel in the offsets queue uses this value to terminate.
@@ -135,13 +139,30 @@ public class PostingsHighlighter {
     if (breakIterator == null) {
       breakIterator = new WholeBreakIterator();
     }
-    if (scorer == null || formatter == null) {
-      throw new NullPointerException();
-    }
     this.maxLength = maxLength;
     this.breakIterator = breakIterator;
-    this.scorer = scorer;
-    this.formatter = formatter;
+  }
+
+  /** Returns the {@link PassageFormatter} to use for
+   *  formatting passages into highlighted snippets.  This
+   *  returns a new {@code PassageFormatter} by default;
+   *  subclasses can override to customize. */
+  protected PassageFormatter getFormatter(String field) {
+    if (defaultFormatter == null) {
+      defaultFormatter = new PassageFormatter();
+    }
+    return defaultFormatter;
+  }
+
+  /** Returns the {@link PassageScorer} to use for
+   *  ranking passages.  This
+   *  returns a new {@code PassageScorer} by default;
+   *  subclasses can override to customize. */
+  protected PassageScorer getScorer(String field) {
+    if (defaultScorer == null) {
+      defaultScorer = new PassageScorer();
+    }
+    return defaultScorer;
   }
 
   /**
@@ -302,7 +323,13 @@ public class PostingsHighlighter {
       Term ceiling = new Term(field, UnicodeUtil.BIG_TERM);
       SortedSet<Term> fieldTerms = queryTerms.subSet(floor, ceiling);
       // TODO: should we have some reasonable defaults for term pruning? (e.g. stopwords)
-      Term terms[] = fieldTerms.toArray(new Term[fieldTerms.size()]);
+
+      // Strip off the redundant field:
+      BytesRef terms[] = new BytesRef[fieldTerms.size()];
+      int termUpto = 0;
+      for(Term term : fieldTerms) {
+        terms[termUpto++] = term.bytes();
+      }
       Map<Integer,String> fieldHighlights = highlightField(field, contents[i], bi,
terms, docids, leaves, maxPassages);
         
       String[] result = new String[docids.length];
@@ -333,7 +360,7 @@ public class PostingsHighlighter {
     return contents;
   }
     
-  private Map<Integer,String> highlightField(String field, String contents[], BreakIterator
bi, Term terms[], int[] docids, List<AtomicReaderContext> leaves, int maxPassages) throws
IOException {  
+  private Map<Integer,String> highlightField(String field, String contents[], BreakIterator
bi, BytesRef terms[], int[] docids, List<AtomicReaderContext> leaves, int maxPassages)
throws IOException {  
     Map<Integer,String> highlights = new HashMap<Integer,String>();
     
     // reuse in the real sense... for docs in same segment we just advance our old enum
@@ -341,6 +368,11 @@ public class PostingsHighlighter {
     TermsEnum termsEnum = null;
     int lastLeaf = -1;
 
+    PassageFormatter fieldFormatter = getFormatter(field);
+    if (fieldFormatter == null) {
+      throw new NullPointerException("PassageFormatter cannot be null");
+    }
+
     for (int i = 0; i < docids.length; i++) {
       String content = contents[i];
       if (content.length() == 0) {
@@ -366,7 +398,7 @@ public class PostingsHighlighter {
       if (passages.length > 0) {
         // otherwise a null snippet (eg if field is missing
         // entirely from the doc)
-        highlights.put(doc, formatter.format(passages, content));
+        highlights.put(doc, fieldFormatter.format(passages, content));
       }
       lastLeaf = leaf;
     }
@@ -377,8 +409,12 @@ public class PostingsHighlighter {
   // algorithm: treat sentence snippets as miniature documents
   // we can intersect these with the postings lists via BreakIterator.preceding(offset),s
   // score each sentence as norm(sentenceStartOffset) * sum(weight * tf(freq))
-  private Passage[] highlightDoc(String field, Term terms[], int contentLength, BreakIterator
bi, int doc, 
+  private Passage[] highlightDoc(String field, BytesRef terms[], int contentLength, BreakIterator
bi, int doc, 
       TermsEnum termsEnum, DocsAndPositionsEnum[] postings, int n) throws IOException {
+    PassageScorer scorer = getScorer(field);
+    if (scorer == null) {
+      throw new NullPointerException("PassageScorer cannot be null");
+    }
     PriorityQueue<OffsetsEnum> pq = new PriorityQueue<OffsetsEnum>();
     float weights[] = new float[terms.length];
     // initialize postings
@@ -389,7 +425,7 @@ public class PostingsHighlighter {
         continue;
       } else if (de == null) {
         postings[i] = EMPTY; // initially
-        if (!termsEnum.seekExact(terms[i].bytes(), true)) {
+        if (!termsEnum.seekExact(terms[i], true)) {
           continue; // term not found
         }
         de = postings[i] = termsEnum.docsAndPositions(null, null, DocsAndPositionsEnum.FLAG_OFFSETS);

Modified: lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java?rev=1459817&r1=1459816&r2=1459817&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java
Fri Mar 22 14:51:44 2013
@@ -456,7 +456,7 @@ public class TestPostingsHighlighter ext
     iw.close();
     
     IndexSearcher searcher = newSearcher(ir);
-    PostingsHighlighter highlighter = new PostingsHighlighter(10000, null, new PassageScorer(),
new PassageFormatter());
+    PostingsHighlighter highlighter = new PostingsHighlighter(10000, null);
     Query query = new TermQuery(new Term("body", "test"));
     TopDocs topDocs = searcher.search(query, null, 10, Sort.INDEXORDER);
     assertEquals(1, topDocs.totalHits);
@@ -526,7 +526,7 @@ public class TestPostingsHighlighter ext
     
     IndexSearcher searcher = newSearcher(ir);
 
-    PostingsHighlighter highlighter = new PostingsHighlighter(10000, null, new PassageScorer(),
new PassageFormatter()) {
+    PostingsHighlighter highlighter = new PostingsHighlighter(10000, null) {
         @Override
         protected String[][] loadFieldValues(IndexSearcher searcher, String[] fields, int[]
docids, int maxLength) throws IOException {
           assert fields.length == 1;
@@ -635,7 +635,7 @@ public class TestPostingsHighlighter ext
     iw.close();
     
     IndexSearcher searcher = newSearcher(ir);
-    PostingsHighlighter highlighter = new PostingsHighlighter(10000, null, new PassageScorer(),
new PassageFormatter());
+    PostingsHighlighter highlighter = new PostingsHighlighter(10000, null);
     Query query = new TermQuery(new Term("body", "highlighting"));
     int[] docIDs = new int[] {0};
     String snippets[] = highlighter.highlightFields(new String[] {"body"}, query, searcher,
docIDs, 2).get("body");

Modified: lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java?rev=1459817&r1=1459816&r2=1459817&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java
Fri Mar 22 14:51:44 2013
@@ -112,16 +112,26 @@ public class TestPostingsHighlighterRank
   
   private void checkQuery(IndexSearcher is, Query query, int doc, int maxTopN) throws IOException
{
     for (int n = 1; n < maxTopN; n++) {
-      FakePassageFormatter f1 = new FakePassageFormatter();
+      final FakePassageFormatter f1 = new FakePassageFormatter();
       PostingsHighlighter p1 = new PostingsHighlighter(Integer.MAX_VALUE-1, 
-                                                       BreakIterator.getSentenceInstance(Locale.ROOT),

-                                                       new PassageScorer(),
-                                                       f1);
-      FakePassageFormatter f2 = new FakePassageFormatter();
+                                                       BreakIterator.getSentenceInstance(Locale.ROOT))
{
+          @Override
+          protected PassageFormatter getFormatter(String field) {
+            assertEquals("body", field);
+            return f1;
+          }
+        };
+
+      final FakePassageFormatter f2 = new FakePassageFormatter();
       PostingsHighlighter p2 = new PostingsHighlighter(Integer.MAX_VALUE-1, 
-                                                       BreakIterator.getSentenceInstance(Locale.ROOT),

-                                                       new PassageScorer(),
-                                                       f2);
+                                                       BreakIterator.getSentenceInstance(Locale.ROOT))
{
+          @Override
+          protected PassageFormatter getFormatter(String field) {
+            assertEquals("body", field);
+            return f2;
+          }
+        };
+
       BooleanQuery bq = new BooleanQuery(false);
       bq.add(query, BooleanClause.Occur.MUST);
       bq.add(new TermQuery(new Term("id", Integer.toString(doc))), BooleanClause.Occur.MUST);
@@ -170,8 +180,7 @@ public class TestPostingsHighlighterRank
         // we use a very simple analyzer. so we can assert the matches are correct
         int lastMatchStart = -1;
         for (int i = 0; i < p.getNumMatches(); i++) {
-          Term term = p.getMatchTerms()[i];
-          assertEquals("body", term.field());
+          BytesRef term = p.getMatchTerms()[i];
           int matchStart = p.getMatchStarts()[i];
           assertTrue(matchStart >= 0);
           // must at least start within the passage
@@ -184,9 +193,8 @@ public class TestPostingsHighlighterRank
           // single character terms
           assertEquals(matchStart+1, matchEnd);
           // and the offsets must be correct...
-          BytesRef bytes = term.bytes();
-          assertEquals(1, bytes.length);
-          assertEquals((char)bytes.bytes[bytes.offset], Character.toLowerCase(content.charAt(matchStart)));
+          assertEquals(1, term.length);
+          assertEquals((char)term.bytes[term.offset], Character.toLowerCase(content.charAt(matchStart)));
         }
         // record just the start/end offset for simplicity
         seen.add(new Pair(p.getStartOffset(), p.getEndOffset()));
@@ -262,9 +270,12 @@ public class TestPostingsHighlighterRank
     
     IndexSearcher searcher = newSearcher(ir);
     PostingsHighlighter highlighter = new PostingsHighlighter(10000, 
-                                             BreakIterator.getSentenceInstance(Locale.ROOT),

-                                             new PassageScorer(1.2f, 0, 87), 
-                                             new PassageFormatter());
+                                                              BreakIterator.getSentenceInstance(Locale.ROOT))
{
+        @Override
+        protected PassageScorer getScorer(String field) {
+          return new PassageScorer(1.2f, 0, 87);
+        }
+      };
     Query query = new TermQuery(new Term("body", "test"));
     TopDocs topDocs = searcher.search(query, null, 10, Sort.INDEXORDER);
     assertEquals(1, topDocs.totalHits);
@@ -299,9 +310,12 @@ public class TestPostingsHighlighterRank
     
     IndexSearcher searcher = newSearcher(ir);
     PostingsHighlighter highlighter = new PostingsHighlighter(10000, 
-                                             BreakIterator.getSentenceInstance(Locale.ROOT),

-                                             new PassageScorer(0, 0.75f, 87), 
-                                             new PassageFormatter());
+                                                              BreakIterator.getSentenceInstance(Locale.ROOT))
{
+        @Override
+        protected PassageScorer getScorer(String field) {
+          return new PassageScorer(0, 0.75f, 87);
+        }
+      };
     BooleanQuery query = new BooleanQuery();
     query.add(new TermQuery(new Term("body", "foo")), BooleanClause.Occur.SHOULD);
     query.add(new TermQuery(new Term("body", "bar")), BooleanClause.Occur.SHOULD);

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java?rev=1459817&r1=1459816&r2=1459817&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java
Fri Mar 22 14:51:44 2013
@@ -97,7 +97,7 @@ public class PostingsSolrHighlighter ext
     if (pivot == null) {
       pivot = "87";
     }
-    PassageScorer scorer = new PassageScorer(Float.parseFloat(k1), Float.parseFloat(b), Float.parseFloat(pivot));
+    final PassageScorer scorer = new PassageScorer(Float.parseFloat(k1), Float.parseFloat(b),
Float.parseFloat(pivot));
     
     // formatter parameters: preTag/postTag/ellipsis
     String preTag = attributes.get("preTag");
@@ -112,7 +112,7 @@ public class PostingsSolrHighlighter ext
     if (ellipsis == null) {
       ellipsis = "... ";
     }
-    PassageFormatter formatter = new PassageFormatter(preTag, postTag, ellipsis);
+    final PassageFormatter formatter = new PassageFormatter(preTag, postTag, ellipsis);
 
     String summarizeEmpty = attributes.get("summarizeEmpty");
     final boolean summarizeEmptyBoolean;
@@ -127,7 +127,7 @@ public class PostingsSolrHighlighter ext
     if (attributes.containsKey("maxLength")) {
       maxLength = Integer.parseInt(attributes.get("maxLength"));
     }
-    highlighter = new PostingsHighlighter(maxLength, breakIterator, scorer, formatter) {
+    highlighter = new PostingsHighlighter(maxLength, breakIterator) {
         @Override
         protected Passage[] getEmptyHighlight(String fieldName, BreakIterator bi, int maxPassages)
{
           if (summarizeEmptyBoolean) {
@@ -136,6 +136,16 @@ public class PostingsSolrHighlighter ext
             return new Passage[0];
           }
         }
+
+        @Override
+        protected PassageFormatter getFormatter(String fieldName) {
+          return formatter;
+        }
+
+        @Override
+        protected PassageScorer getScorer(String fieldName) {
+          return scorer;
+        }
       };
   }
 



Mime
View raw message