Mailing-List: contact java-commits-help@lucene.apache.org; run by ezmlm
Precedence: bulk
Reply-To: java-dev@lucene.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r826390 - in /lucene/java/branches/lucene_2_9: ./
 CHANGES.txt
 src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java
 src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
Date: Sun, 18 Oct 2009 09:54:20 -0000
To: java-commits@lucene.apache.org
From: mikemccand@apache.org
Message-Id: <20091018095420.A01F4238889C@eris.apache.org>

Author: mikemccand
Date: Sun Oct 18 09:54:20 2009
New Revision: 826390

URL: http://svn.apache.org/viewvc?rev=826390&view=rev
Log:
LUCENE-1986 (on 2.9.x branch): Fix NPE when scoring PayloadNearQuery

Modified:
    lucene/java/branches/lucene_2_9/   (props changed)
    lucene/java/branches/lucene_2_9/CHANGES.txt
    lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java
    lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java

Propchange: lucene/java/branches/lucene_2_9/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sun Oct 18 09:54:20 2009
@@ -1,2 +1,2 @@
 /lucene/java/branches/lucene_2_4:748824
-/lucene/java/trunk:824125,826029
+/lucene/java/trunk:824125,826029,826385

Modified: lucene/java/branches/lucene_2_9/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/CHANGES.txt?rev=826390&r1=826389&r2=826390&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/CHANGES.txt (original)
+++ lucene/java/branches/lucene_2_9/CHANGES.txt Sun Oct 18 09:54:20 2009
@@ -16,6 +16,9 @@
    when the reader is a near real-time reader.  (Jake Mannix via Mike
    McCandless)
 
+ * LUCENE-1986: Fix NPE when scoring PayloadNearQuery (Peter Keegan,
+   Mark Miller via Mike McCandless)
+
 ======================= Release 2.9.0 2009-09-23 =======================
 
 Changes in backwards compatibility policy

Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java?rev=826390&r1=826389&r2=826390&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java (original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java Sun Oct 18 09:54:20 2009
@@ -207,6 +207,9 @@
 
     //
     protected boolean setFreqCurrentDoc() throws IOException {
+      if (!more) {
+        return false;
+      }
       Spans[] spansArr = new Spans[1];
       spansArr[0] = spans;
       payloadScore = 0;

Modified: lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java?rev=826390&r1=826389&r2=826390&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (original)
+++ lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java Sun Oct 18 09:54:20 2009
@@ -37,185 +37,221 @@
 import org.apache.lucene.search.Searcher;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.lucene.search.spans.SpanNearQuery;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.English;
 import org.apache.lucene.util.LuceneTestCase;
 
 
 public class TestPayloadNearQuery extends LuceneTestCase {
-	private IndexSearcher searcher;
-	private BoostingSimilarity similarity = new BoostingSimilarity();
-	private byte[] payload2 = new byte[]{2};
-	private byte[] payload4 = new byte[]{4};
-
-	public TestPayloadNearQuery(String s) {
-		super(s);
-	}
-
-	private class PayloadAnalyzer extends Analyzer {
-		public TokenStream tokenStream(String fieldName, Reader reader) {
-			TokenStream result = new LowerCaseTokenizer(reader);
-			result = new PayloadFilter(result, fieldName);
-			return result;
-		}
-	}
-
-	private class PayloadFilter extends TokenFilter {
-		String fieldName;
-		int numSeen = 0;
+  private IndexSearcher searcher;
+  private BoostingSimilarity similarity = new BoostingSimilarity();
+  private byte[] payload2 = new byte[]{2};
+  private byte[] payload4 = new byte[]{4};
+
+  public TestPayloadNearQuery(String s) {
+    super(s);
+  }
+
+  private class PayloadAnalyzer extends Analyzer {
+    public TokenStream tokenStream(String fieldName, Reader reader) {
+      TokenStream result = new LowerCaseTokenizer(reader);
+      result = new PayloadFilter(result, fieldName);
+      return result;
+    }
+  }
+
+  private class PayloadFilter extends TokenFilter {
+    String fieldName;
+    int numSeen = 0;
     protected PayloadAttribute payAtt;
 
-		public PayloadFilter(TokenStream input, String fieldName) {
-			super(input);
-			this.fieldName = fieldName;
+    public PayloadFilter(TokenStream input, String fieldName) {
+      super(input);
+      this.fieldName = fieldName;
       payAtt = (PayloadAttribute) addAttribute(PayloadAttribute.class);
-		}
+    }
 
     public boolean incrementToken() throws IOException {
       boolean result = false;
       if (input.incrementToken() == true){
         if (numSeen % 2 == 0) {
-					payAtt.setPayload(new Payload(payload2));
-				} else {
-					payAtt.setPayload(new Payload(payload4));
-				}
-				numSeen++;
+          payAtt.setPayload(new Payload(payload2));
+        } else {
+          payAtt.setPayload(new Payload(payload4));
+        }
+        numSeen++;
         result = true;
       }
       return result;
     }
   }
   
-	private PayloadNearQuery newPhraseQuery (String fieldName, String phrase, boolean inOrder) {
-		int n;
-		String[] words = phrase.split("[\\s]+");
-		SpanQuery clauses[] = new SpanQuery[words.length];
-		for (int i=0;i<clauses.length;i++) {
-			clauses[i] = new PayloadTermQuery(new Term(fieldName, words[i]), new AveragePayloadFunction());  
-		} 
-		return new PayloadNearQuery(clauses, 0, inOrder);
-	}
-
-	protected void setUp() throws Exception {
-		super.setUp();
-		RAMDirectory directory = new RAMDirectory();
-		PayloadAnalyzer analyzer = new PayloadAnalyzer();
-		IndexWriter writer
-		= new IndexWriter(directory, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
-		writer.setSimilarity(similarity);
-		//writer.infoStream = System.out;
-		for (int i = 0; i < 1000; i++) {
-			Document doc = new Document();
-			doc.add(new Field("field", English.intToEnglish(i), Field.Store.YES, Field.Index.ANALYZED));
-			writer.addDocument(doc);
-		}
-		writer.optimize();
-		writer.close();
-
-		searcher = new IndexSearcher(directory, true);
-		searcher.setSimilarity(similarity);
-	}
-
-	public void test() throws IOException {
-		PayloadNearQuery query;
-		TopDocs hits;
+  private PayloadNearQuery newPhraseQuery (String fieldName, String phrase, boolean inOrder) {
+    int n;
+    String[] words = phrase.split("[\\s]+");
+    SpanQuery clauses[] = new SpanQuery[words.length];
+    for (int i=0;i<clauses.length;i++) {
+      clauses[i] = new PayloadTermQuery(new Term(fieldName, words[i]), new AveragePayloadFunction());  
+    } 
+    return new PayloadNearQuery(clauses, 0, inOrder);
+  }
+
+  protected void setUp() throws Exception {
+    super.setUp();
+    RAMDirectory directory = new RAMDirectory();
+    PayloadAnalyzer analyzer = new PayloadAnalyzer();
+    IndexWriter writer
+      = new IndexWriter(directory, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
+    writer.setSimilarity(similarity);
+    //writer.infoStream = System.out;
+    for (int i = 0; i < 1000; i++) {
+      Document doc = new Document();
+      doc.add(new Field("field", English.intToEnglish(i), Field.Store.YES, Field.Index.ANALYZED));
+      String txt = English.intToEnglish(i) +' '+English.intToEnglish(i+1);
+      doc.add(new Field("field2",  txt, Field.Store.YES, Field.Index.ANALYZED));
+      writer.addDocument(doc);
+    }
+    writer.optimize();
+    writer.close();
+
+    searcher = new IndexSearcher(directory, true);
+    searcher.setSimilarity(similarity);
+  }
 
-		query = newPhraseQuery("field", "twenty two", true);
-		QueryUtils.check(query);
+  public void test() throws IOException {
+    PayloadNearQuery query;
+    TopDocs hits;
+
+    query = newPhraseQuery("field", "twenty two", true);
+    QueryUtils.check(query);
 		
-		// all 10 hits should have score = 3 because adjacent terms have payloads of 2,4
-		// and all the similarity factors are set to 1
-		hits = searcher.search(query, null, 100);
-		assertTrue("hits is null and it shouldn't be", hits != null);
-		assertTrue("should be 10 hits", hits.totalHits == 10);
-		for (int j = 0; j < hits.scoreDocs.length; j++) {
-			ScoreDoc doc = hits.scoreDocs[j];
-			assertTrue(doc.score + " does not equal: " + 3, doc.score == 3);
-		}
-		for (int i=1;i<10;i++) {
-			query = newPhraseQuery("field", English.intToEnglish(i)+" hundred", true);
-			// all should have score = 3 because adjacent terms have payloads of 2,4
-			// and all the similarity factors are set to 1
-			hits = searcher.search(query, null, 100);
-			assertTrue("hits is null and it shouldn't be", hits != null);
-			assertTrue("should be 100 hits", hits.totalHits == 100);
-			for (int j = 0; j < hits.scoreDocs.length; j++) {
-				ScoreDoc doc = hits.scoreDocs[j];
-//				System.out.println("Doc: " + doc.toString());
-//				System.out.println("Explain: " + searcher.explain(query, doc.doc));
-				assertTrue(doc.score + " does not equal: " + 3, doc.score == 3);
-			}
-		}
-	}
-
-	public void testLongerSpan() throws IOException {
-		PayloadNearQuery query;
-		TopDocs hits;
-		query = newPhraseQuery("field", "nine hundred ninety nine", true);
-		hits = searcher.search(query, null, 100);
-		ScoreDoc doc = hits.scoreDocs[0];
-//		System.out.println("Doc: " + doc.toString());
-//		System.out.println("Explain: " + searcher.explain(query, doc.doc));
-		assertTrue("hits is null and it shouldn't be", hits != null);
-		assertTrue("there should only be one hit", hits.totalHits == 1);
-		// should have score = 3 because adjacent terms have payloads of 2,4
-		assertTrue(doc.score + " does not equal: " + 3, doc.score == 3); 
-	}
-
-	public void testComplexNested() throws IOException {
-		PayloadNearQuery query;
-		TopDocs hits;
-
-		// combine ordered and unordered spans with some nesting to make sure all payloads are counted
-
-		SpanQuery q1 = newPhraseQuery("field", "nine hundred", true);
-		SpanQuery q2 = newPhraseQuery("field", "ninety nine", true);
-		SpanQuery q3 = newPhraseQuery("field", "nine ninety", false);
-		SpanQuery q4 = newPhraseQuery("field", "hundred nine", false);
-		SpanQuery[]clauses = new SpanQuery[] {new PayloadNearQuery(new SpanQuery[] {q1,q2}, 0, true), new PayloadNearQuery(new SpanQuery[] {q3,q4}, 0, false)};
-		query = new PayloadNearQuery(clauses, 0, false);
-		hits = searcher.search(query, null, 100);
-		assertTrue("hits is null and it shouldn't be", hits != null);
-		// should be only 1 hit - doc 999
-		assertTrue("should only be one hit", hits.scoreDocs.length == 1);
-		// the score should be 3 - the average of all the underlying payloads
-		ScoreDoc doc = hits.scoreDocs[0];
-//		System.out.println("Doc: " + doc.toString());
-//		System.out.println("Explain: " + searcher.explain(query, doc.doc));
-		assertTrue(doc.score + " does not equal: " + 3, doc.score == 3);  
-	}
-	// must be static for weight serialization tests 
-	static class BoostingSimilarity extends DefaultSimilarity {
+    // all 10 hits should have score = 3 because adjacent terms have payloads of 2,4
+    // and all the similarity factors are set to 1
+    hits = searcher.search(query, null, 100);
+    assertTrue("hits is null and it shouldn't be", hits != null);
+    assertTrue("should be 10 hits", hits.totalHits == 10);
+    for (int j = 0; j < hits.scoreDocs.length; j++) {
+      ScoreDoc doc = hits.scoreDocs[j];
+      assertTrue(doc.score + " does not equal: " + 3, doc.score == 3);
+    }
+    for (int i=1;i<10;i++) {
+      query = newPhraseQuery("field", English.intToEnglish(i)+" hundred", true);
+      // all should have score = 3 because adjacent terms have payloads of 2,4
+      // and all the similarity factors are set to 1
+      hits = searcher.search(query, null, 100);
+      assertTrue("hits is null and it shouldn't be", hits != null);
+      assertTrue("should be 100 hits", hits.totalHits == 100);
+      for (int j = 0; j < hits.scoreDocs.length; j++) {
+        ScoreDoc doc = hits.scoreDocs[j];
+        //				System.out.println("Doc: " + doc.toString());
+        //				System.out.println("Explain: " + searcher.explain(query, doc.doc));
+        assertTrue(doc.score + " does not equal: " + 3, doc.score == 3);
+      }
+    }
+  }
 
-// TODO: Remove warning after API has been finalized
+
+  public void testPayloadNear() throws IOException {
+    SpanNearQuery q1, q2;
+    PayloadNearQuery query;
+    TopDocs hits;
+    //SpanNearQuery(clauses, 10000, false)
+    q1 = spanNearQuery("field2", "twenty two");
+    q2 = spanNearQuery("field2", "twenty three");
+    SpanQuery[] clauses = new SpanQuery[2];
+    clauses[0] = q1;
+    clauses[1] = q2;
+    query = new PayloadNearQuery(clauses, 10, false); 
+    //System.out.println(query.toString());
+    assertEquals(12, searcher.search(query, null, 100).totalHits);
+    /*
+    System.out.println(hits.totalHits);
+    for (int j = 0; j < hits.scoreDocs.length; j++) {
+      ScoreDoc doc = hits.scoreDocs[j];
+      System.out.println("doc: "+doc.doc+", score: "+doc.score);
+    }
+    */
+  }
+
+  private SpanNearQuery spanNearQuery(String fieldName, String words) {
+    String[] wordList = words.split("[\\s]+");
+    SpanQuery clauses[] = new SpanQuery[wordList.length];
+    for (int i=0;i<clauses.length;i++) {
+      clauses[i] = new PayloadTermQuery(new Term(fieldName, wordList[i]), new AveragePayloadFunction());  
+    } 
+    return new SpanNearQuery(clauses, 10000, false);
+  }
+
+  public void testLongerSpan() throws IOException {
+    PayloadNearQuery query;
+    TopDocs hits;
+    query = newPhraseQuery("field", "nine hundred ninety nine", true);
+    hits = searcher.search(query, null, 100);
+    ScoreDoc doc = hits.scoreDocs[0];
+    //		System.out.println("Doc: " + doc.toString());
+    //		System.out.println("Explain: " + searcher.explain(query, doc.doc));
+    assertTrue("hits is null and it shouldn't be", hits != null);
+    assertTrue("there should only be one hit", hits.totalHits == 1);
+    // should have score = 3 because adjacent terms have payloads of 2,4
+    assertTrue(doc.score + " does not equal: " + 3, doc.score == 3); 
+  }
+
+  public void testComplexNested() throws IOException {
+    PayloadNearQuery query;
+    TopDocs hits;
+
+    // combine ordered and unordered spans with some nesting to make sure all payloads are counted
+
+    SpanQuery q1 = newPhraseQuery("field", "nine hundred", true);
+    SpanQuery q2 = newPhraseQuery("field", "ninety nine", true);
+    SpanQuery q3 = newPhraseQuery("field", "nine ninety", false);
+    SpanQuery q4 = newPhraseQuery("field", "hundred nine", false);
+    SpanQuery[]clauses = new SpanQuery[] {new PayloadNearQuery(new SpanQuery[] {q1,q2}, 0, true), new PayloadNearQuery(new SpanQuery[] {q3,q4}, 0, false)};
+    query = new PayloadNearQuery(clauses, 0, false);
+    hits = searcher.search(query, null, 100);
+    assertTrue("hits is null and it shouldn't be", hits != null);
+    // should be only 1 hit - doc 999
+    assertTrue("should only be one hit", hits.scoreDocs.length == 1);
+    // the score should be 3 - the average of all the underlying payloads
+    ScoreDoc doc = hits.scoreDocs[0];
+    //		System.out.println("Doc: " + doc.toString());
+    //		System.out.println("Explain: " + searcher.explain(query, doc.doc));
+    assertTrue(doc.score + " does not equal: " + 3, doc.score == 3);  
+  }
+
+  // must be static for weight serialization tests 
+  static class BoostingSimilarity extends DefaultSimilarity {
+
+    // TODO: Remove warning after API has been finalized
     public float scorePayload(int docId, String fieldName, int start, int end, byte[] payload, int offset, int length) {
       //we know it is size 4 here, so ignore the offset/length
       return payload[0];
     }
-		//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-		//Make everything else 1 so we see the effect of the payload
-		//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-		public float lengthNorm(String fieldName, int numTerms) {
-			return 1;
-		}
-
-		public float queryNorm(float sumOfSquaredWeights) {
-			return 1;
-		}
-
-		public float sloppyFreq(int distance) {
-			return 1;
-		}
-
-		public float coord(int overlap, int maxOverlap) {
-			return 1;
-		}
-		public float tf(float freq) {
-			return 1;
-		}
-		// idf used for phrase queries
-		public float idf(Collection terms, Searcher searcher) {
-			return 1;
-		}
-	}
+    //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+    //Make everything else 1 so we see the effect of the payload
+    //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+    public float lengthNorm(String fieldName, int numTerms) {
+      return 1;
+    }
+
+    public float queryNorm(float sumOfSquaredWeights) {
+      return 1;
+    }
+
+    public float sloppyFreq(int distance) {
+      return 1;
+    }
+
+    public float coord(int overlap, int maxOverlap) {
+      return 1;
+    }
+    public float tf(float freq) {
+      return 1;
+    }
+    // idf used for phrase queries
+    public float idf(Collection terms, Searcher searcher) {
+      return 1;
+    }
+  }
 }