Return-Path: 
 <solr-commits-return-6007-apmail-lucene-solr-commits-archive=lucene.apache.org@lucene.apache.org>
Delivered-To: apmail-lucene-solr-commits-archive@minotaur.apache.org
Received: (qmail 81463 invoked from network); 7 Dec 2009 19:24:53 -0000
Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3)
  by minotaur.apache.org with SMTP; 7 Dec 2009 19:24:53 -0000
Received: (qmail 38566 invoked by uid 500); 7 Dec 2009 19:24:53 -0000
Delivered-To: apmail-lucene-solr-commits-archive@lucene.apache.org
Received: (qmail 38519 invoked by uid 500); 7 Dec 2009 19:24:52 -0000
Mailing-List: contact solr-commits-help@lucene.apache.org; run by ezmlm
Precedence: bulk
List-Help: <mailto:solr-commits-help@lucene.apache.org>
List-Unsubscribe: <mailto:solr-commits-unsubscribe@lucene.apache.org>
List-Post: <mailto:solr-commits@lucene.apache.org>
List-Id: <solr-commits.lucene.apache.org>
Reply-To: solr-dev@lucene.apache.org
Delivered-To: mailing list solr-commits@lucene.apache.org
Received: (qmail 38510 invoked by uid 99); 7 Dec 2009 19:24:52 -0000
Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230)
    by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 07 Dec 2009 19:24:52 +0000
X-ASF-Spam-Status: No, hits=-2000.0 required=10.0
	tests=ALL_TRUSTED
X-Spam-Check-By: apache.org
Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4)
    by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 07 Dec 2009 19:24:49 +0000
Received: by eris.apache.org (Postfix, from userid 65534)
	id 89CEF23888DD; Mon,  7 Dec 2009 19:24:28 +0000 (UTC)
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r888096 - in /lucene/solr/trunk: CHANGES.txt
 src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
 src/test/org/apache/solr/highlight/HighlighterTest.java
Date: Mon, 07 Dec 2009 19:24:23 -0000
To: solr-commits@lucene.apache.org
From: yonik@apache.org
X-Mailer: svnmailer-1.0.8
Message-Id: <20091207192428.89CEF23888DD@eris.apache.org>
X-Virus-Checked: Checked by ClamAV on apache.org

Author: yonik
Date: Mon Dec  7 19:24:15 2009
New Revision: 888096

URL: http://svn.apache.org/viewvc?rev=888096&view=rev
Log:
SOLR-1624: Highlighter bug w/ term positons stored and multi valued field

Modified:
    lucene/solr/trunk/CHANGES.txt
    lucene/solr/trunk/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
    lucene/solr/trunk/src/test/org/apache/solr/highlight/HighlighterTest.java

Modified: lucene/solr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=888096&r1=888095&r2=888096&view=diff
==============================================================================
--- lucene/solr/trunk/CHANGES.txt (original)
+++ lucene/solr/trunk/CHANGES.txt Mon Dec  7 19:24:15 2009
@@ -110,6 +110,10 @@
 * SOLR-1628: log contains incorrect number of adds and deletes. 
   (Thijs Vonk via yonik)
 
+* SOLR-1624: Highlighter only highlights values from the first field  value
+  in a multivalued field when term positions (term vectors) are stored.
+  (Chris Harris via yonik)
+
 
 Other Changes
 ----------------------

Modified: lucene/solr/trunk/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java?rev=888096&r1=888095&r2=888096&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java Mon Dec  7 19:24:15 2009
@@ -284,23 +284,24 @@
 
           String[] summaries = null;
           List<TextFragment> frags = new ArrayList<TextFragment>();
-          TermOffsetsTokenStream tots = null;
-          for (int j = 0; j < docTexts.length; j++) {
-            // create TokenStream
-            try {
-              // attempt term vectors
-              if( tots == null ) {
-                TokenStream tvStream = TokenSources.getTokenStream(searcher.getReader(), docId, fieldName);
-                if (tvStream != null) {
-                  tots = new TermOffsetsTokenStream(tvStream);
-                  tstream = tots.getMultiValuedTokenStream( docTexts[j].length() );
-                } else {
-                  // fall back to analyzer
-                  tstream = createAnalyzerTStream(schema, fieldName, docTexts[j]);
-                }
+
+          TermOffsetsTokenStream tots = null; // to be non-null iff we're using TermOffsets optimization
+          try {
+              TokenStream tvStream = TokenSources.getTokenStream(searcher.getReader(), docId, fieldName);
+              if (tvStream != null) {
+                tots = new TermOffsetsTokenStream(tvStream);
               }
-            }
-            catch (IllegalArgumentException e) {
+          }
+          catch (IllegalArgumentException e) {
+            // No problem. But we can't use TermOffsets optimization.
+          }
+
+          for (int j = 0; j < docTexts.length; j++) {
+            if( tots != null ) {
+              // if we're using TermOffsets optimization, then get the next
+              // field value's TokenStream (i.e. get field j's TokenStream) from tots:
+              tstream = tots.getMultiValuedTokenStream( docTexts[j].length() );
+            } else {
               // fall back to analyzer
               tstream = createAnalyzerTStream(schema, fieldName, docTexts[j]);
             }

Modified: lucene/solr/trunk/src/test/org/apache/solr/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/highlight/HighlighterTest.java?rev=888096&r1=888095&r2=888096&view=diff
==============================================================================
--- lucene/solr/trunk/src/test/org/apache/solr/highlight/HighlighterTest.java (original)
+++ lucene/solr/trunk/src/test/org/apache/solr/highlight/HighlighterTest.java Mon Dec  7 19:24:15 2009
@@ -190,6 +190,33 @@
             );
   }
 
+  // Variant of testTermVecMultiValuedHighlight to make sure that
+  // more than just the first value of a multi-valued field is
+  // considered for highlighting.
+  public void testTermVecMultiValuedHighlight2() throws Exception {
+
+    // do summarization using term vectors on multivalued field
+    HashMap<String,String> args = new HashMap<String,String>();
+    args.put("hl", "true");
+    args.put("hl.fl", "tv_mv_text");
+    args.put("hl.snippets", "2");
+    TestHarness.LocalRequestFactory sumLRF = h.getRequestFactory(
+      "standard",0,200,args);
+
+    String shortText = "short";
+    assertU(adoc("tv_mv_text", shortText,
+                 "tv_mv_text", LONG_TEXT,
+                 "id", "1"));
+    assertU(commit());
+    assertU(optimize());
+    assertQ("Basic summarization",
+            sumLRF.makeRequest("tv_mv_text:long"),
+            "//lst[@name='highlighting']/lst[@name='1']",
+            "//lst[@name='1']/arr[@name='tv_mv_text']/str[.='a <em>long</em> days night this should be a piece of text which']",
+            "//arr[@name='tv_mv_text']/str[.=' <em>long</em> fragments.']"
+            );
+  }
+
   public void testDisMaxHighlight() {
 
     // same test run through dismax handler