lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From romseyg...@apache.org
Subject svn commit: r1680208 [1/2] - in /lucene/dev/branches/branch_5x/lucene: ./ core/ core/src/java/org/apache/lucene/search/payloads/ core/src/java/org/apache/lucene/search/spans/ core/src/test/org/apache/lucene/search/ core/src/test/org/apache/lucene/searc...
Date Tue, 19 May 2015 09:21:25 GMT
Author: romseygeek
Date: Tue May 19 09:21:24 2015
New Revision: 1680208

URL: http://svn.apache.org/r1680208
Log:
LUCENE-6371: Add collection API to Spans, remove payload methods

Added:
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanCollector.java
      - copied unchanged from r1680205, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanCollector.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/SpanNearPayloadCheckQuery.java
      - copied unchanged from r1680205, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/payloads/SpanNearPayloadCheckQuery.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/SpanPayloadCheckQuery.java
      - copied unchanged from r1680205, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/payloads/SpanPayloadCheckQuery.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/BufferedSpanCollector.java
      - copied unchanged from r1680205, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/BufferedSpanCollector.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanCollector.java
      - copied unchanged from r1680205, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanCollector.java
Removed:
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansPayloadOrdered.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearPayloadCheckQuery.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanPayloadCheckQuery.java
Modified:
    lucene/dev/branches/branch_5x/lucene/   (props changed)
    lucene/dev/branches/branch_5x/lucene/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/branch_5x/lucene/core/   (props changed)
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/ContainSpans.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FilterSpans.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainQuery.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainingQuery.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanQuery.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanWithinQuery.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java
    lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java
    lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadBasics.java
    lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadSpans.java
    lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java
    lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java
    lucene/dev/branches/branch_5x/lucene/highlighter/   (props changed)
    lucene/dev/branches/branch_5x/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
    lucene/dev/branches/branch_5x/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
    lucene/dev/branches/branch_5x/lucene/test-framework/   (props changed)
    lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/search/spans/AssertingSpanQuery.java
    lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/search/spans/AssertingSpans.java

Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Tue May 19 09:21:24 2015
@@ -160,6 +160,13 @@ API Changes
 * LUCENE-6445: Two new methods in Highlighter's TokenSources; the existing
   methods are now marked deprecated. (David Smiley)
 
+* LUCENE-6371: Payload collection from Spans is moved to a more generic
+  SpanCollector framework.  Spans no longer implements .hasPayload() and
+  .getPayload() methods, and instead exposes a collect() method that allows
+  the collection of arbitrary postings information. SpanPayloadCheckQuery and
+  SpanPayloadNearCheckQuery have moved from the .spans package to the .payloads
+  package. (Alan Woodward, David Smiley, Paul Elschot)
+
 Other
 
 * LUCENE-6413: Test runner should report the number of suites completed/ 

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java Tue May 19 09:21:24 2015
@@ -17,11 +17,6 @@ package org.apache.lucene.search.payload
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.Objects;
-
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
@@ -29,8 +24,6 @@ import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.similarities.DefaultSimilarity;
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.search.similarities.Similarity.SimScorer;
-import org.apache.lucene.search.spans.NearSpansOrdered;
-import org.apache.lucene.search.spans.NearSpansUnordered;
 import org.apache.lucene.search.spans.SpanNearQuery;
 import org.apache.lucene.search.spans.SpanQuery;
 import org.apache.lucene.search.spans.SpanScorer;
@@ -40,6 +33,11 @@ import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.ToStringUtils;
 
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Objects;
+
 /**
  * This class is very similar to
  * {@link org.apache.lucene.search.spans.SpanNearQuery} except that it factors
@@ -55,8 +53,10 @@ import org.apache.lucene.util.ToStringUt
  * @see org.apache.lucene.search.similarities.Similarity.SimScorer#computePayloadFactor(int, int, int, BytesRef)
  */
 public class PayloadNearQuery extends SpanNearQuery {
+
   protected String fieldName;
   protected PayloadFunction function;
+  protected final PayloadSpanCollector payloadCollector = new PayloadSpanCollector();
 
   public PayloadNearQuery(SpanQuery[] clauses, int slop, boolean inOrder) {
     this(clauses, slop, inOrder, new AveragePayloadFunction());
@@ -129,17 +129,18 @@ public class PayloadNearQuery extends Sp
   }
 
   public class PayloadNearSpanWeight extends SpanWeight {
+
     public PayloadNearSpanWeight(SpanQuery query, IndexSearcher searcher)
         throws IOException {
-      super(query, searcher);
+      super(query, searcher, payloadCollector);
     }
 
     @Override
     public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException {
-      Spans spans = query.getSpans(context, acceptDocs, termContexts);
+      Spans spans = query.getSpans(context, acceptDocs, termContexts, payloadCollector);
       return (spans == null)
               ? null
-              : new PayloadNearSpanScorer(spans, this, similarity, similarity.simScorer(stats, context));
+              : new PayloadNearSpanScorer(spans, this, similarity.simScorer(stats, context));
     }
     
     @Override
@@ -176,31 +177,11 @@ public class PayloadNearQuery extends Sp
     protected float payloadScore;
     private int payloadsSeen;
 
-    protected PayloadNearSpanScorer(Spans spans, SpanWeight weight,
-        Similarity similarity, Similarity.SimScorer docScorer) throws IOException {
+    protected PayloadNearSpanScorer(Spans spans, SpanWeight weight, Similarity.SimScorer docScorer) throws IOException {
       super(spans, weight, docScorer);
       this.spans = spans;
     }
 
-    // Get the payloads associated with all underlying subspans
-    public void getPayloads(Spans[] subSpans) throws IOException {
-      for (int i = 0; i < subSpans.length; i++) {
-        if (subSpans[i] instanceof NearSpansOrdered) {
-          if (((NearSpansOrdered) subSpans[i]).isPayloadAvailable()) {
-            processPayloads(((NearSpansOrdered) subSpans[i]).getPayload(),
-                subSpans[i].startPosition(), subSpans[i].endPosition());
-          }
-          getPayloads(((NearSpansOrdered) subSpans[i]).getSubSpans());
-        } else if (subSpans[i] instanceof NearSpansUnordered) {
-          if (((NearSpansUnordered) subSpans[i]).isPayloadAvailable()) {
-            processPayloads(((NearSpansUnordered) subSpans[i]).getPayload(),
-                subSpans[i].startPosition(), subSpans[i].endPosition());
-          }
-          getPayloads(((NearSpansUnordered) subSpans[i]).getSubSpans());
-        }
-      }
-    }
-
     // TODO change the whole spans api to use bytesRef, or nuke spans
     BytesRef scratch = new BytesRef();
 
@@ -237,9 +218,9 @@ public class PayloadNearQuery extends Sp
       do {
         int matchLength = spans.endPosition() - startPos;
         freq += docScorer.computeSlopFactor(matchLength);
-        Spans[] spansArr = new Spans[1];
-        spansArr[0] = spans;
-        getPayloads(spansArr);            
+        payloadCollector.reset();
+        spans.collect(payloadCollector);
+        processPayloads(payloadCollector.getPayloads(), startPos, spans.endPosition());
         startPos = spans.nextStartPosition();
       } while (startPos != Spans.NO_MORE_POSITIONS);
     }

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java Tue May 19 09:21:24 2015
@@ -17,15 +17,6 @@ package org.apache.lucene.search.payload
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeSet;
-
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexReaderContext;
 import org.apache.lucene.index.LeafReaderContext;
@@ -46,6 +37,15 @@ import org.apache.lucene.search.spans.Sp
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.search.spans.Spans;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
+
 /**
  * Experimental class to get set of payloads for most standard Lucene queries.
  * Operates like Highlighter - IndexReader should only contain doc of interest,
@@ -187,17 +187,16 @@ public class PayloadSpanUtil {
     for (Term term : terms) {
       termContexts.put(term, TermContext.build(context, term));
     }
+
+    PayloadSpanCollector collector = new PayloadSpanCollector();
     for (LeafReaderContext leafReaderContext : context.leaves()) {
-      final Spans spans = query.getSpans(leafReaderContext, leafReaderContext.reader().getLiveDocs(), termContexts);
+      final Spans spans = query.getSpans(leafReaderContext, leafReaderContext.reader().getLiveDocs(), termContexts, collector);
       if (spans != null) {
         while (spans.nextDoc() != Spans.NO_MORE_DOCS) {
           while (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
-            if (spans.isPayloadAvailable()) {
-              Collection<byte[]> payload = spans.getPayload();
-              for (byte [] bytes : payload) {
-                payloads.add(bytes);
-              }
-            }
+            collector.reset();
+            spans.collect(collector);
+            payloads.addAll(collector.getPayloads());
           }
         }
       }

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java Tue May 19 09:21:24 2015
@@ -17,9 +17,6 @@ package org.apache.lucene.search.payload
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.util.Objects;
-
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.PostingsEnum;
 import org.apache.lucene.index.Term;
@@ -28,6 +25,8 @@ import org.apache.lucene.search.IndexSea
 import org.apache.lucene.search.similarities.DefaultSimilarity;
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.search.similarities.Similarity.SimScorer;
+import org.apache.lucene.search.spans.BufferedSpanCollector;
+import org.apache.lucene.search.spans.SpanCollector;
 import org.apache.lucene.search.spans.SpanQuery;
 import org.apache.lucene.search.spans.SpanScorer;
 import org.apache.lucene.search.spans.SpanTermQuery;
@@ -37,6 +36,9 @@ import org.apache.lucene.search.spans.Te
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 
+import java.io.IOException;
+import java.util.Objects;
+
 /**
  * This class is very similar to
  * {@link org.apache.lucene.search.spans.SpanTermQuery} except that it factors
@@ -67,19 +69,52 @@ public class PayloadTermQuery extends Sp
 
   @Override
   public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
-    return new PayloadTermWeight(this, searcher);
+    return new PayloadTermWeight(this, searcher, new PayloadTermCollector());
+  }
+
+  protected class PayloadTermCollector implements SpanCollector {
+
+    BytesRef payload;
+
+    @Override
+    public void reset() {
+      payload = null;
+    }
+
+    @Override
+    public int requiredPostings() {
+      return PostingsEnum.PAYLOADS;
+    }
+
+    @Override
+    public void collectLeaf(PostingsEnum postings, Term term) throws IOException {
+      payload = postings.getPayload();
+    }
+
+    @Override
+    public BufferedSpanCollector buffer() {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public SpanCollector bufferedCollector() {
+      throw new UnsupportedOperationException();
+    }
   }
 
   protected class PayloadTermWeight extends SpanWeight {
 
-    public PayloadTermWeight(PayloadTermQuery query, IndexSearcher searcher)
+    final PayloadTermCollector payloadCollector;
+
+    public PayloadTermWeight(PayloadTermQuery query, IndexSearcher searcher, PayloadTermCollector collector)
         throws IOException {
-      super(query, searcher);
+      super(query, searcher, collector);
+      this.payloadCollector = collector;
     }
 
     @Override
     public PayloadTermSpanScorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException {
-      TermSpans spans = (TermSpans) query.getSpans(context, acceptDocs, termContexts);
+      TermSpans spans = (TermSpans) query.getSpans(context, acceptDocs, termContexts, payloadCollector);
       return (spans == null)
               ? null
               : new PayloadTermSpanScorer(spans, this, similarity.simScorer(stats, context));
@@ -109,29 +144,22 @@ public class PayloadTermQuery extends Sp
 
           freq += docScorer.computeSlopFactor(matchLength);
           numMatches++;
-          processPayload(similarity);
+          payloadCollector.reset();
+          spans.collect(payloadCollector);
+          processPayload();
 
           startPos = spans.nextStartPosition();
         } while (startPos != Spans.NO_MORE_POSITIONS);
       }
 
-      protected void processPayload(Similarity similarity) throws IOException {
-        if (spans.isPayloadAvailable()) {
-          final PostingsEnum postings = termSpans.getPostings();
-          payload = postings.getPayload();
-          if (payload != null) {
-            payloadScore = function.currentScore(docID(), term.field(),
-                                                 spans.startPosition(), spans.endPosition(), payloadsSeen, payloadScore,
-                                                 docScorer.computePayloadFactor(docID(), spans.startPosition(), spans.endPosition(), payload));
-          } else {
-            payloadScore = function.currentScore(docID(), term.field(),
-                                                 spans.startPosition(), spans.endPosition(), payloadsSeen, payloadScore, 1F);
-          }
-          payloadsSeen++;
-
-        } else {
-          // zero out the payload?
-        }
+      protected void processPayload() throws IOException {
+
+        float payloadFactor = payloadCollector.payload == null ? 1F :
+            docScorer.computePayloadFactor(docID(), spans.startPosition(), spans.endPosition(), payloadCollector.payload);
+        payloadScore = function.currentScore(docID(), term.field(), spans.startPosition(), spans.endPosition(),
+                                             payloadsSeen, payloadScore, payloadFactor);
+        payloadsSeen++;
+
       }
 
       /**

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/ContainSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/ContainSpans.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/ContainSpans.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/ContainSpans.java Tue May 19 09:21:24 2015
@@ -19,7 +19,6 @@ package org.apache.lucene.search.spans;
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Objects;
 
 abstract class ContainSpans extends ConjunctionSpans {
@@ -49,12 +48,8 @@ abstract class ContainSpans extends Conj
   }
 
   @Override
-  public boolean isPayloadAvailable() throws IOException {
-    return sourceSpans.isPayloadAvailable();
+  public void collect(SpanCollector collector) throws IOException {
+    sourceSpans.collect(collector);
   }
 
-  @Override
-  public Collection<byte[]> getPayload() throws IOException {
-    return sourceSpans.getPayload();
-  }
 }

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java Tue May 19 09:21:24 2015
@@ -17,21 +17,20 @@ package org.apache.lucene.search.spans;
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.Set;
-import java.util.Objects;
-
-import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermContext;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Weight;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.ToStringUtils;
 
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
 /**
  * <p>Wrapper to allow {@link SpanQuery} objects participate in composite 
  * single-field SpanQueries by 'lying' about their search field. That is, 
@@ -97,8 +96,8 @@ public class FieldMaskingSpanQuery exten
   // ...this is done to be more consistent with things like SpanFirstQuery
   
   @Override
-  public Spans getSpans(LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) throws IOException {
-    return maskedQuery.getSpans(context, acceptDocs, termContexts);
+  public Spans getSpans(LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts, SpanCollector collector) throws IOException {
+    return maskedQuery.getSpans(context, acceptDocs, termContexts, collector);
   }
 
   @Override

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FilterSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FilterSpans.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FilterSpans.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FilterSpans.java Tue May 19 09:21:24 2015
@@ -17,12 +17,11 @@ package org.apache.lucene.search.spans;
  * limitations under the License.
  */
 
+import org.apache.lucene.search.TwoPhaseIterator;
+
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Objects;
 
-import org.apache.lucene.search.TwoPhaseIterator;
-
 /**
  * A {@link Spans} implementation wrapping another spans instance,
  * allowing to filter spans matches easily by implementing {@link #accept}
@@ -110,17 +109,12 @@ public abstract class FilterSpans extend
     return atFirstInCurrentDoc ? -1
           : (startPos != NO_MORE_POSITIONS) ? in.endPosition() : NO_MORE_POSITIONS;
   }
-  
-  @Override
-  public final Collection<byte[]> getPayload() throws IOException {
-    return in.getPayload();
-  }
 
   @Override
-  public final boolean isPayloadAvailable() throws IOException {
-    return in.isPayloadAvailable();
+  public void collect(SpanCollector collector) throws IOException {
+    in.collect(collector);
   }
-  
+
   @Override
   public final long cost() {
     return in.cost();

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java Tue May 19 09:21:24 2015
@@ -19,12 +19,9 @@ package org.apache.lucene.search.spans;
 
 import java.io.IOException;
 import java.util.List;
-import java.util.Collection;
 
 /** A Spans that is formed from the ordered subspans of a SpanNearQuery
- * where the subspans do not overlap and have a maximum slop between them,
- * and that does not need to collect payloads.
- * To also collect payloads, see {@link NearSpansPayloadOrdered}.
+ * where the subspans do not overlap and have a maximum slop between them.
  * <p>
  * The formed spans only contains minimum slop matches.<br>
  * The matching slop is computed from the distance(s) between
@@ -41,6 +38,9 @@ import java.util.Collection;
  * <pre>t1 t2 .. t3      </pre>
  * <pre>      t1 .. t2 t3</pre>
  *
+ * Because the algorithm used to minimize the size of a match consumes
+ * child Spans eagerly, this uses a BufferedSpanCollector to collect
+ * information from subspans.
  *
  * Expert:
  * Only public for subclassing.  Most implementations should not need this class
@@ -51,9 +51,13 @@ public class NearSpansOrdered extends Ne
   protected int matchStart = -1;
   protected int matchEnd = -1;
 
-  public NearSpansOrdered(SpanNearQuery query, List<Spans> subSpans) throws IOException {
+  protected final SpanCollector collector;
+  protected BufferedSpanCollector buffer;
+
+  public NearSpansOrdered(SpanNearQuery query, List<Spans> subSpans, SpanCollector collector) throws IOException {
     super(query, subSpans);
     this.atFirstInCurrentDoc = true; // -1 startPosition/endPosition also at doc -1
+    this.collector = collector;
   }
 
   @Override
@@ -140,10 +144,15 @@ public class NearSpansOrdered extends Ne
     matchStart = lastSubSpans.startPosition();
     matchEnd = lastSubSpans.endPosition();
 
+    buffer = collector.buffer();
+    buffer.collectCandidate(subSpans[subSpans.length - 1]);
+    buffer.accept();
+
     int matchSlop = 0;
     int lastStart = matchStart;
     for (int i = subSpans.length - 2; i >= 0; i--) {
       Spans prevSpans = subSpans[i];
+      buffer.collectCandidate(prevSpans);
 
       int prevStart = prevSpans.startPosition();
       int prevEnd = prevSpans.endPosition();
@@ -160,8 +169,11 @@ public class NearSpansOrdered extends Ne
         // prevSpans still before (lastStart, lastEnd)
         prevStart = ppStart;
         prevEnd = ppEnd;
+        buffer.collectCandidate(prevSpans);
       }
 
+      buffer.accept();
+
       assert prevStart <= matchStart;
       if (matchStart > prevEnd) { // Only non overlapping spans add to slop.
         matchSlop += (matchStart - prevEnd);
@@ -190,13 +202,10 @@ public class NearSpansOrdered extends Ne
   }
 
   @Override
-  public Collection<byte[]> getPayload() throws IOException {
-    return null;
-  }
-
-  @Override
-  public boolean isPayloadAvailable() {
-    return false;
+  public void collect(SpanCollector collector) {
+    assert collector == this.collector
+        : "You must collect using the same SpanCollector as was passed to the NearSpans constructor";
+    buffer.replay();
   }
 
   @Override

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java Tue May 19 09:21:24 2015
@@ -22,10 +22,7 @@ import org.apache.lucene.util.PriorityQu
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.List;
-import java.util.Set;
-import java.util.HashSet;
 
 /**
  * Similar to {@link NearSpansOrdered}, but for the unordered case.
@@ -118,13 +115,8 @@ public class NearSpansUnordered extends
     }
 
     @Override
-    public Collection<byte[]> getPayload() throws IOException {
-      return in.getPayload();
-    }
-
-    @Override
-    public boolean isPayloadAvailable() throws IOException {
-      return in.isPayloadAvailable();
+    public void collect(SpanCollector collector) throws IOException {
+      in.collect(collector);
     }
 
     @Override
@@ -249,31 +241,11 @@ public class NearSpansUnordered extends
           : maxEndPositionCell.endPosition();
   }
 
-
-  /**
-   * WARNING: The List is not necessarily in order of the positions.
-   * @return Collection of <code>byte[]</code> payloads
-   * @throws IOException if there is a low-level I/O error
-   */
   @Override
-  public Collection<byte[]> getPayload() throws IOException {
-    Set<byte[]> matchPayload = new HashSet<>();
+  public void collect(SpanCollector collector) throws IOException {
     for (SpansCell cell : subSpanCells) {
-      if (cell.isPayloadAvailable()) {
-        matchPayload.addAll(cell.getPayload());
-      }
-    }
-    return matchPayload;
-  }
-
-  @Override
-  public boolean isPayloadAvailable() throws IOException {
-    for (SpansCell cell : subSpanCells) {
-      if (cell.isPayloadAvailable()) {
-        return true;
-      }
+      cell.collect(collector);
     }
-    return false;
   }
 
   @Override

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainQuery.java Tue May 19 09:21:24 2015
@@ -17,19 +17,19 @@ package org.apache.lucene.search.spans;
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.Set;
-import java.util.Objects;
-
-import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermContext;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.Bits;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
 abstract class SpanContainQuery extends SpanQuery implements Cloneable {
   SpanQuery big;
   SpanQuery little;
@@ -55,12 +55,12 @@ abstract class SpanContainQuery extends
     little.extractTerms(terms);
   }
 
-  ArrayList<Spans> prepareConjunction(final LeafReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts) throws IOException {
-    Spans bigSpans = big.getSpans(context, acceptDocs, termContexts);
+  ArrayList<Spans> prepareConjunction(final LeafReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts, SpanCollector collector) throws IOException {
+    Spans bigSpans = big.getSpans(context, acceptDocs, termContexts, collector);
     if (bigSpans == null) {
       return null;
     }
-    Spans littleSpans = little.getSpans(context, acceptDocs, termContexts);
+    Spans littleSpans = little.getSpans(context, acceptDocs, termContexts, collector);
     if (littleSpans == null) {
       return null;
     }

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainingQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainingQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainingQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainingQuery.java Tue May 19 09:21:24 2015
@@ -17,15 +17,15 @@ package org.apache.lucene.search.spans;
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.ArrayList;
-
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.TermContext;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermContext;
 import org.apache.lucene.util.Bits;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Map;
+
 /** Keep matches that contain another Spans. */
 public class SpanContainingQuery extends SpanContainQuery {
   /** Construct a SpanContainingQuery matching spans from <code>big</code>
@@ -54,8 +54,8 @@ public class SpanContainingQuery extends
    * The payload is from the spans of <code>big</code>.
    */
   @Override
-  public Spans getSpans(final LeafReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts) throws IOException {
-    ArrayList<Spans> containerContained = prepareConjunction(context, acceptDocs, termContexts);
+  public Spans getSpans(final LeafReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts, SpanCollector collector) throws IOException {
+    ArrayList<Spans> containerContained = prepareConjunction(context, acceptDocs, termContexts, collector);
     if (containerContained == null) {
       return null;
     }

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java Tue May 19 09:21:24 2015
@@ -17,22 +17,22 @@ package org.apache.lucene.search.spans;
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.Set;
-import java.util.Objects;
-
-import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermContext;
+import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TopTermsRewrite;
 import org.apache.lucene.search.ScoringRewrite;
-import org.apache.lucene.search.BooleanClause.Occur; // javadocs only
+import org.apache.lucene.search.TopTermsRewrite;
 import org.apache.lucene.util.Bits;
 
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
 /**
  * Wraps any {@link MultiTermQuery} as a {@link SpanQuery}, 
  * so it can be nested within other SpanQuery classes.
@@ -99,7 +99,7 @@ public class SpanMultiTermQueryWrapper<Q
   }
   
   @Override
-  public Spans getSpans(LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) throws IOException {
+  public Spans getSpans(LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts, SpanCollector collector) throws IOException {
     throw new UnsupportedOperationException("Query should have been rewritten");
   }
 

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java Tue May 19 09:21:24 2015
@@ -17,15 +17,8 @@ package org.apache.lucene.search.spans;
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermContext;
 import org.apache.lucene.index.Terms;
@@ -33,6 +26,13 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.ToStringUtils;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 /** Matches spans which are near one another.  One can specify <i>slop</i>, the
  * maximum number of intervening unmatched positions, as well as whether
  * matches are required to be in-order.
@@ -118,11 +118,17 @@ public class SpanNearQuery extends SpanQ
   }
 
   @Override
-  public Spans getSpans(final LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) throws IOException {
-    ArrayList<Spans> subSpans = new ArrayList<>(clauses.size());
+  public Spans getSpans(final LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts, SpanCollector collector) throws IOException {
 
+    Terms terms = context.reader().terms(field);
+    if (terms == null) {
+      return null; // field does not exist
+    }
+
+    ArrayList<Spans> subSpans = new ArrayList<>(clauses.size());
+    SpanCollector subSpanCollector = inOrder ? collector.bufferedCollector() : collector;
     for (SpanQuery seq : clauses) {
-      Spans subSpan = seq.getSpans(context, acceptDocs, termContexts);
+      Spans subSpan = seq.getSpans(context, acceptDocs, termContexts, subSpanCollector);
       if (subSpan != null) {
         subSpans.add(subSpan);
       } else {
@@ -130,15 +136,9 @@ public class SpanNearQuery extends SpanQ
       }
     }
 
-    Terms terms = context.reader().terms(field);
-    if (terms == null) {
-      return null; // field does not exist
-    }
     
     // all NearSpans require at least two subSpans
-    return (! inOrder) ? new NearSpansUnordered(this, subSpans)
-          : collectPayloads && terms.hasPayloads() ? new NearSpansPayloadOrdered(this, subSpans)
-          : new NearSpansOrdered(this, subSpans);
+    return (! inOrder) ? new NearSpansUnordered(this, subSpans) : new NearSpansOrdered(this, subSpans, collector);
   }
 
   @Override

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java Tue May 19 09:21:24 2015
@@ -105,13 +105,13 @@ public class SpanNotQuery extends SpanQu
   }
 
   @Override
-  public Spans getSpans(final LeafReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts) throws IOException {
-    final Spans includeSpans = include.getSpans(context, acceptDocs, termContexts);
+  public Spans getSpans(final LeafReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts, SpanCollector collector) throws IOException {
+    final Spans includeSpans = include.getSpans(context, acceptDocs, termContexts, collector);
     if (includeSpans == null) {
       return null;
     }
 
-    final Spans excludeSpans = exclude.getSpans(context, acceptDocs, termContexts);
+    final Spans excludeSpans = exclude.getSpans(context, acceptDocs, termContexts, collector);
     if (excludeSpans == null) {
       return includeSpans;
     }

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java Tue May 19 09:21:24 2015
@@ -17,26 +17,24 @@ package org.apache.lucene.search.spans;
  * limitations under the License.
  */
 
-import java.io.IOException;
-
-import java.util.List;
-import java.util.Collection;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermContext;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.ToStringUtils;
-import org.apache.lucene.search.Query;
 import org.apache.lucene.search.DisiPriorityQueue;
 import org.apache.lucene.search.DisiWrapper;
-import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.DisjunctionDISIApproximation;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TwoPhaseIterator;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.ToStringUtils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 
 /** Matches the union of its clauses.
@@ -147,13 +145,13 @@ public class SpanOrQuery extends SpanQue
 
 
   @Override
-  public Spans getSpans(final LeafReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts)
+  public Spans getSpans(final LeafReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts, SpanCollector collector)
   throws IOException {
 
     final ArrayList<Spans> subSpans = new ArrayList<>(clauses.size());
 
     for (SpanQuery sq : clauses) {
-      Spans spans = sq.getSpans(context, acceptDocs, termContexts);
+      Spans spans = sq.getSpans(context, acceptDocs, termContexts, collector);
       if (spans != null) {
         subSpans.add(spans);
       }
@@ -306,17 +304,9 @@ public class SpanOrQuery extends SpanQue
       }
 
       @Override
-      public Collection<byte[]> getPayload() throws IOException {
-        return topPositionSpans == null
-                ? null
-                : topPositionSpans.isPayloadAvailable()
-                ? new ArrayList<>(topPositionSpans.getPayload())
-                : null;
-      }
-
-      @Override
-      public boolean isPayloadAvailable() throws IOException {
-        return (topPositionSpans != null) && topPositionSpans.isPayloadAvailable();
+      public void collect(SpanCollector collector) throws IOException {
+        if (topPositionSpans != null)
+          topPositionSpans.collect(collector);
       }
 
       @Override

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java Tue May 19 09:21:24 2015
@@ -77,8 +77,8 @@ public abstract class SpanPositionCheckQ
   protected abstract AcceptStatus acceptPosition(Spans spans) throws IOException;
 
   @Override
-  public Spans getSpans(final LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) throws IOException {
-    Spans matchSpans = match.getSpans(context, acceptDocs, termContexts);
+  public Spans getSpans(final LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts, SpanCollector collector) throws IOException {
+    Spans matchSpans = match.getSpans(context, acceptDocs, termContexts, collector);
     return (matchSpans == null) ? null : new FilterSpans(matchSpans) {
       @Override
       protected AcceptStatus accept(Spans candidate) throws IOException {

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanQuery.java Tue May 19 09:21:24 2015
@@ -17,10 +17,6 @@ package org.apache.lucene.search.spans;
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.Set;
-
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermContext;
@@ -29,13 +25,17 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.Bits;
 
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
 /** Base class for span-based queries. */
 public abstract class SpanQuery extends Query {
   /** Expert: Returns the matches for this query in an index.  
    *  Used internally to search for spans.
    *  This may return null to indicate that the SpanQuery has no results.
    */
-  public abstract Spans getSpans(LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) throws IOException;
+  public abstract Spans getSpans(LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts, SpanCollector collector) throws IOException;
 
   /**
    * Extract terms from these spans.
@@ -53,7 +53,7 @@ public abstract class SpanQuery extends
 
   @Override
   public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
-    return new SpanWeight(this, searcher);
+    return new SpanWeight(this, searcher, SpanCollector.NO_OP);
   }
 
 }

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java Tue May 19 09:21:24 2015
@@ -17,13 +17,8 @@ package org.apache.lucene.search.spans;
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.Set;
-import java.util.Objects;
-
-import org.apache.lucene.index.PostingsEnum;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.PostingsEnum;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermContext;
 import org.apache.lucene.index.TermState;
@@ -32,6 +27,11 @@ import org.apache.lucene.index.TermsEnum
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.ToStringUtils;
 
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
 /** Matches spans containing a term.
  * This should not be used for terms that are indexed at position Integer.MAX_VALUE.
  */
@@ -83,7 +83,7 @@ public class SpanTermQuery extends SpanQ
   }
 
   @Override
-  public Spans getSpans(final LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) throws IOException {
+  public Spans getSpans(final LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts, SpanCollector collector) throws IOException {
     TermContext termContext = termContexts.get(term);
     final TermState state;
     if (termContext == null) {
@@ -115,7 +115,7 @@ public class SpanTermQuery extends SpanQ
     final TermsEnum termsEnum = context.reader().terms(term.field()).iterator();
     termsEnum.seekExact(term.bytes(), state);
 
-    final PostingsEnum postings = termsEnum.postings(acceptDocs, null, PostingsEnum.PAYLOADS);
+    final PostingsEnum postings = termsEnum.postings(acceptDocs, null, collector.requiredPostings());
     return new TermSpans(postings, term);
   }
 }

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java Tue May 19 09:21:24 2015
@@ -17,12 +17,6 @@ package org.apache.lucene.search.spans;
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
 import org.apache.lucene.index.IndexReaderContext;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
@@ -37,6 +31,12 @@ import org.apache.lucene.search.similari
 import org.apache.lucene.search.similarities.Similarity.SimScorer;
 import org.apache.lucene.util.Bits;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
 /**
  * Expert-only.  Public for use by other weight implementations
  */
@@ -44,12 +44,14 @@ public class SpanWeight extends Weight {
   protected final Similarity similarity;
   protected final Map<Term,TermContext> termContexts;
   protected final SpanQuery query;
+  protected final SpanCollector collector;
   protected Similarity.SimWeight stats;
 
-  public SpanWeight(SpanQuery query, IndexSearcher searcher) throws IOException {
+  public SpanWeight(SpanQuery query, IndexSearcher searcher, SpanCollector collector) throws IOException {
     super(query);
     this.similarity = searcher.getSimilarity();
     this.query = query;
+    this.collector = collector;
 
     termContexts = new HashMap<>();
     TreeSet<Term> terms = new TreeSet<>();
@@ -97,7 +99,7 @@ public class SpanWeight extends Weight {
     if (terms != null && terms.hasPositions() == false) {
       throw new IllegalStateException("field \"" + query.getField() + "\" was indexed without position data; cannot run SpanQuery (query=" + query + ")");
     }
-    Spans spans = query.getSpans(context, acceptDocs, termContexts);
+    Spans spans = query.getSpans(context, acceptDocs, termContexts, collector);
     return (spans == null) ? null : new SpanScorer(spans, this, similarity.simScorer(stats, context));
   }
 

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanWithinQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanWithinQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanWithinQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanWithinQuery.java Tue May 19 09:21:24 2015
@@ -17,15 +17,15 @@ package org.apache.lucene.search.spans;
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.ArrayList;
-
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.TermContext;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermContext;
 import org.apache.lucene.util.Bits;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Map;
+
 /** Keep matches that are contained within another Spans. */
 public class SpanWithinQuery extends SpanContainQuery {
   /** Construct a SpanWithinQuery matching spans from <code>little</code>
@@ -54,8 +54,8 @@ public class SpanWithinQuery extends Spa
    * The payload is from the spans of <code>little</code>.
    */
   @Override
-  public Spans getSpans(final LeafReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts) throws IOException {
-    ArrayList<Spans> containerContained = prepareConjunction(context, acceptDocs, termContexts);
+  public Spans getSpans(final LeafReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts, SpanCollector collector) throws IOException {
+    ArrayList<Spans> containerContained = prepareConjunction(context, acceptDocs, termContexts, collector);
     if (containerContained == null) {
       return null;
     }

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java Tue May 19 09:21:24 2015
@@ -17,12 +17,11 @@ package org.apache.lucene.search.spans;
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.util.Collection;
-
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.TwoPhaseIterator;
 
+import java.io.IOException;
+
 /** Iterates through combinations of start/end positions per-doc.
  *  Each start/end position represents a range of term positions within the current document.
  *  These are enumerated in order, by increasing document number, within that by
@@ -51,33 +50,12 @@ public abstract class Spans extends DocI
   public abstract int endPosition();
 
   /**
-   * Returns the payload data for the current start/end position.
-   * This is only valid after {@link #nextStartPosition()}
-   * returned an available start position.
-   * This method must not be called more than once after each call
-   * of {@link #nextStartPosition()}. However, most payloads are loaded lazily,
-   * so if the payload data for the current position is not needed,
-   * this method may not be called at all for performance reasons.
-   * <br>
-   * Note that the return type is a collection, thus the ordering should not be relied upon.
-   * <br>
-   * @lucene.experimental
-   *
-   * @return a List of byte arrays containing the data of this payload, otherwise null if isPayloadAvailable is false
-   * @throws IOException if there is a low-level I/O error
-   */
-  public abstract Collection<byte[]> getPayload() throws IOException;
-
-  /**
-   * Checks if a payload can be loaded at the current start/end position.
-   * <p>
-   * Payloads can only be loaded once per call to
-   * {@link #nextStartPosition()}.
+   * Collect data from the current Spans
+   * @param collector a SpanCollector
    *
-   * @return true if there is a payload available at this start/end position
-   *              that can be loaded
+   * @lucene.experimental
    */
-  public abstract boolean isPayloadAvailable() throws IOException;
+  public abstract void collect(SpanCollector collector) throws IOException;
 
   /**
    * Optional method: Return a {@link TwoPhaseIterator} view of this

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java Tue May 19 09:21:24 2015
@@ -16,14 +16,11 @@ package org.apache.lucene.search.spans;
  */
 
 
-import org.apache.lucene.index.Term;
 import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.Term;
 import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.util.BytesRef;
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.Collection;
 import java.util.Objects;
 
 /**
@@ -109,6 +106,7 @@ public class TermSpans extends Spans {
     return postings.cost();
   }
 
+  /*
   @Override
   public Collection<byte[]> getPayload() throws IOException {
     final BytesRef payload = postings.getPayload();
@@ -127,6 +125,12 @@ public class TermSpans extends Spans {
   public boolean isPayloadAvailable() throws IOException {
     return readPayload == false && postings.getPayload() != null;
   }
+  */
+
+  @Override
+  public void collect(SpanCollector collector) throws IOException {
+    collector.collectLeaf(postings, term);
+  }
 
   @Override
   public String toString() {

Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java Tue May 19 09:21:24 2015
@@ -17,34 +17,37 @@ package org.apache.lucene.search;
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.io.StringReader;
-import java.nio.charset.StandardCharsets;
-import java.util.Collection;
-
-import org.apache.lucene.analysis.*;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockPayloadAnalyzer;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.SlowCompositeReaderWrapper;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.store.Directory;
+import org.apache.lucene.search.payloads.PayloadSpanCollector;
 import org.apache.lucene.search.payloads.PayloadSpanUtil;
 import org.apache.lucene.search.spans.MultiSpansWrapper;
 import org.apache.lucene.search.spans.SpanNearQuery;
 import org.apache.lucene.search.spans.SpanQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.search.spans.Spans;
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
 
 /**
  * Term position unit test.
@@ -53,7 +56,7 @@ import org.apache.lucene.util.BytesRef;
  */
 public class TestPositionIncrement extends LuceneTestCase {
 
-  final static boolean VERBOSE = false;
+  final static boolean VERBOSE = true;
 
   public void testSetPosition() throws Exception {
     Analyzer analyzer = new Analyzer() {
@@ -238,14 +241,17 @@ public class TestPositionIncrement exten
     if (VERBOSE) {
       System.out.println("\ngetPayloadSpans test");
     }
-    Spans pspans = MultiSpansWrapper.wrap(is.getIndexReader(), snq);
+    PayloadSpanCollector collector = new PayloadSpanCollector();
+    Spans pspans = MultiSpansWrapper.wrap(is.getIndexReader(), snq, collector);
     while (pspans.nextDoc() != Spans.NO_MORE_DOCS) {
       while (pspans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
         if (VERBOSE) {
           System.out.println("doc " + pspans.docID() + ": span " + pspans.startPosition()
               + " to " + pspans.endPosition());
         }
-        Collection<byte[]> payloads = pspans.getPayload();
+        collector.reset();
+        pspans.collect(collector);
+        Collection<byte[]> payloads = collector.getPayloads();
         sawZero |= pspans.startPosition() == 0;
         for (byte[] bytes : payloads) {
           count++;
@@ -256,7 +262,7 @@ public class TestPositionIncrement exten
       }
     }
     assertTrue(sawZero);
-    assertEquals(5, count);
+    assertEquals(8, count);
 
     // System.out.println("\ngetSpans test");
     Spans spans = MultiSpansWrapper.wrap(is.getIndexReader(), snq);
@@ -282,7 +288,7 @@ public class TestPositionIncrement exten
       //System.out.println(s);
       sawZero |= s.equals("pos: 0");
     }
-    assertEquals(5, count);
+    assertEquals(8, count);
     assertTrue(sawZero);
     writer.close();
     is.getIndexReader().close();

Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadBasics.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadBasics.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadBasics.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadBasics.java Tue May 19 09:21:24 2015
@@ -17,13 +17,6 @@ package org.apache.lucene.search.payload
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.analysis.SimplePayloadFilter;
@@ -36,9 +29,7 @@ import org.apache.lucene.index.Term;
 import org.apache.lucene.search.CheckHits;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.spans.SpanNearPayloadCheckQuery;
 import org.apache.lucene.search.spans.SpanNearQuery;
-import org.apache.lucene.search.spans.SpanPayloadCheckQuery;
 import org.apache.lucene.search.spans.SpanPositionRangeQuery;
 import org.apache.lucene.search.spans.SpanQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
@@ -50,6 +41,13 @@ import org.apache.lucene.util.TestUtil;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
 /** basic test of payload-spans */
 public class TestPayloadBasics extends LuceneTestCase {
   private static IndexSearcher searcher;

Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadSpans.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadSpans.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadSpans.java Tue May 19 09:21:24 2015
@@ -16,32 +16,23 @@ package org.apache.lucene.search.payload
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.io.StringReader;
-import java.nio.charset.StandardCharsets;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Set;
-
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.payloads.PayloadHelper;
-import org.apache.lucene.search.payloads.PayloadSpanUtil;
 import org.apache.lucene.search.similarities.DefaultSimilarity;
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.search.spans.MultiSpansWrapper;
@@ -55,6 +46,13 @@ import org.apache.lucene.store.Directory
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
 
+import java.io.IOException;
+import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
 public class TestPayloadSpans extends LuceneTestCase {
   private IndexSearcher searcher;
   private Similarity similarity = new DefaultSimilarity();
@@ -74,14 +72,15 @@ public class TestPayloadSpans extends Lu
     SpanTermQuery stq;
     Spans spans;
     stq = new SpanTermQuery(new Term(PayloadHelper.FIELD, "seventy"));
-    spans = MultiSpansWrapper.wrap(indexReader, stq);
+    PayloadSpanCollector collector = new PayloadSpanCollector();
+    spans = MultiSpansWrapper.wrap(indexReader, stq, collector);
     assertTrue("spans is null and it shouldn't be", spans != null);
-    checkSpans(spans, 100, 1, 1, 1);
+    checkSpans(spans, collector, 100, 1, 1, 1);
 
     stq = new SpanTermQuery(new Term(PayloadHelper.NO_PAYLOAD_FIELD, "seventy"));  
-    spans = MultiSpansWrapper.wrap(indexReader, stq);
+    spans = MultiSpansWrapper.wrap(indexReader, stq, collector);
     assertTrue("spans is null and it shouldn't be", spans != null);
-    checkSpans(spans, 100, 0, 0, 0);
+    checkSpans(spans, collector, 100, 0, 0, 0);
   }
 
   public void testSpanFirst() throws IOException {
@@ -90,19 +89,20 @@ public class TestPayloadSpans extends Lu
     SpanFirstQuery sfq;
     match = new SpanTermQuery(new Term(PayloadHelper.FIELD, "one"));
     sfq = new SpanFirstQuery(match, 2);
-    Spans spans = MultiSpansWrapper.wrap(indexReader, sfq);
-    checkSpans(spans, 109, 1, 1, 1);
+    PayloadSpanCollector collector = new PayloadSpanCollector();
+    Spans spans = MultiSpansWrapper.wrap(indexReader, sfq, collector);
+    checkSpans(spans, collector, 109, 1, 1, 1);
     //Test more complicated subclause
     SpanQuery[] clauses = new SpanQuery[2];
     clauses[0] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "one"));
     clauses[1] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "hundred"));
     match = new SpanNearQuery(clauses, 0, true);
     sfq = new SpanFirstQuery(match, 2);
-    checkSpans(MultiSpansWrapper.wrap(indexReader, sfq), 100, 2, 1, 1);
+    checkSpans(MultiSpansWrapper.wrap(indexReader, sfq, collector), collector, 100, 2, 1, 1);
 
     match = new SpanNearQuery(clauses, 0, false);
     sfq = new SpanFirstQuery(match, 2);
-    checkSpans(MultiSpansWrapper.wrap(indexReader, sfq), 100, 2, 1, 1);
+    checkSpans(MultiSpansWrapper.wrap(indexReader, sfq, collector), collector, 100, 2, 1, 1);
     
   }
   
@@ -124,9 +124,9 @@ public class TestPayloadSpans extends Lu
     writer.addDocument(doc);
     IndexReader reader = writer.getReader();
     writer.close();
-    
 
-    checkSpans(MultiSpansWrapper.wrap(reader, snq), 1,new int[]{2});
+    PayloadSpanCollector collector = new PayloadSpanCollector();
+    checkSpans(MultiSpansWrapper.wrap(reader, snq, collector), collector, 1, new int[]{2});
     reader.close();
     directory.close();
   }
@@ -135,8 +135,10 @@ public class TestPayloadSpans extends Lu
     SpanTermQuery stq;
     Spans spans;
     IndexSearcher searcher = getSearcher();
+    PayloadSpanCollector collector = new PayloadSpanCollector();
+
     stq = new SpanTermQuery(new Term(PayloadHelper.FIELD, "mark"));
-    spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), stq);
+    spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), stq, collector);
     assertNull(spans);
 
     SpanQuery[] clauses = new SpanQuery[3];
@@ -145,9 +147,9 @@ public class TestPayloadSpans extends Lu
     clauses[2] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "xx"));
     SpanNearQuery spanNearQuery = new SpanNearQuery(clauses, 12, false);
 
-    spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), spanNearQuery);
+    spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), spanNearQuery, collector);
     assertTrue("spans is null and it shouldn't be", spans != null);
-    checkSpans(spans, 2, new int[]{3,3});
+    checkSpans(spans, collector, 2, new int[]{3,3});
 
      
     clauses[0] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "xx"));
@@ -156,10 +158,10 @@ public class TestPayloadSpans extends Lu
 
     spanNearQuery = new SpanNearQuery(clauses, 6, true);
    
-    spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), spanNearQuery);
+    spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), spanNearQuery, collector);
 
     assertTrue("spans is null and it shouldn't be", spans != null);
-    checkSpans(spans, 1, new int[]{3});
+    checkSpans(spans, collector, 1, new int[]{3});
      
     clauses = new SpanQuery[2];
      
@@ -178,10 +180,9 @@ public class TestPayloadSpans extends Lu
     SpanNearQuery nestedSpanNearQuery = new SpanNearQuery(clauses2, 6, false);
     
     // yy within 6 of xx within 6 of rr
-
-    spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), nestedSpanNearQuery);
+    spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), nestedSpanNearQuery, collector);
     assertTrue("spans is null and it shouldn't be", spans != null);
-    checkSpans(spans, 2, new int[]{3,3});
+    checkSpans(spans, collector, 2, new int[]{3,3});
     closeIndexReader.close();
     directory.close();
   }
@@ -208,12 +209,13 @@ public class TestPayloadSpans extends Lu
      
     clauses3[0] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "np"));
     clauses3[1] = snq;
-     
+
+    PayloadSpanCollector collector = new PayloadSpanCollector();
     SpanNearQuery nestedSpanNearQuery = new SpanNearQuery(clauses3, 6, false);
-    spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), nestedSpanNearQuery);
+    spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), nestedSpanNearQuery, collector);
 
     assertTrue("spans is null and it shouldn't be", spans != null);
-    checkSpans(spans, 1, new int[]{3});
+    checkSpans(spans, collector, 1, new int[]{3});
     closeIndexReader.close();
     directory.close();
   }
@@ -248,9 +250,10 @@ public class TestPayloadSpans extends Lu
      
     SpanNearQuery nestedSpanNearQuery = new SpanNearQuery(clauses3, 6, false);
 
-    spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), nestedSpanNearQuery);
+    PayloadSpanCollector collector = new PayloadSpanCollector();
+    spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), nestedSpanNearQuery, collector);
     assertTrue("spans is null and it shouldn't be", spans != null);
-    checkSpans(spans, 2, new int[]{8, 8});
+    checkSpans(spans, collector, 2, new int[]{8, 8});
     closeIndexReader.close();
     directory.close();
   }
@@ -272,15 +275,17 @@ public class TestPayloadSpans extends Lu
     SpanTermQuery stq2 = new SpanTermQuery(new Term("content", "k"));
     SpanQuery[] sqs = { stq1, stq2 };
     SpanNearQuery snq = new SpanNearQuery(sqs, 1, true);
-    Spans spans = MultiSpansWrapper.wrap(is.getIndexReader(), snq);
+    PayloadSpanCollector collector = new PayloadSpanCollector();
+    Spans spans = MultiSpansWrapper.wrap(is.getIndexReader(), snq, collector);
 
     TopDocs topDocs = is.search(snq, 1);
     Set<String> payloadSet = new HashSet<>();
     for (int i = 0; i < topDocs.scoreDocs.length; i++) {
       while (spans.nextDoc() != Spans.NO_MORE_DOCS) {
         while (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
-          Collection<byte[]> payloads = spans.getPayload();
-  
+          collector.reset();
+          spans.collect(collector);
+          Collection<byte[]> payloads = collector.getPayloads();
           for (final byte [] payload : payloads) {
             payloadSet.add(new String(payload, StandardCharsets.UTF_8));
           }
@@ -310,14 +315,17 @@ public class TestPayloadSpans extends Lu
     SpanTermQuery stq2 = new SpanTermQuery(new Term("content", "k"));
     SpanQuery[] sqs = { stq1, stq2 };
     SpanNearQuery snq = new SpanNearQuery(sqs, 0, true);
-    Spans spans =  MultiSpansWrapper.wrap(is.getIndexReader(), snq);
+    PayloadSpanCollector collector = new PayloadSpanCollector();
+    Spans spans =  MultiSpansWrapper.wrap(is.getIndexReader(), snq, collector);
 
     TopDocs topDocs = is.search(snq, 1);
     Set<String> payloadSet = new HashSet<>();
     for (int i = 0; i < topDocs.scoreDocs.length; i++) {
       while (spans.nextDoc() != Spans.NO_MORE_DOCS) {
         while (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
-          Collection<byte[]> payloads = spans.getPayload();
+          collector.reset();
+          spans.collect(collector);
+          Collection<byte[]> payloads = collector.getPayloads();
   
           for (final byte [] payload : payloads) {
             payloadSet.add(new String(payload, StandardCharsets.UTF_8));
@@ -348,14 +356,17 @@ public class TestPayloadSpans extends Lu
     SpanTermQuery stq2 = new SpanTermQuery(new Term("content", "k"));
     SpanQuery[] sqs = { stq1, stq2 };
     SpanNearQuery snq = new SpanNearQuery(sqs, 0, true);
-    Spans spans =  MultiSpansWrapper.wrap(is.getIndexReader(), snq);
+    PayloadSpanCollector collector = new PayloadSpanCollector();
+    Spans spans =  MultiSpansWrapper.wrap(is.getIndexReader(), snq, collector);
 
     TopDocs topDocs = is.search(snq, 1);
     Set<String> payloadSet = new HashSet<>();
     for (int i = 0; i < topDocs.scoreDocs.length; i++) {
       while (spans.nextDoc() != Spans.NO_MORE_DOCS) {
         while (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
-          Collection<byte[]> payloads = spans.getPayload();
+          collector.reset();
+          spans.collect(collector);
+          Collection<byte[]> payloads = collector.getPayloads();
   
           for (final byte [] payload : payloads) {
             payloadSet.add(new String(payload, StandardCharsets.UTF_8));
@@ -401,7 +412,7 @@ public class TestPayloadSpans extends Lu
     directory.close();
   }
 
-  private void checkSpans(Spans spans, int expectedNumSpans, int expectedNumPayloads,
+  private void checkSpans(Spans spans, PayloadSpanCollector collector, int expectedNumSpans, int expectedNumPayloads,
                           int expectedPayloadLength, int expectedFirstByte) throws IOException {
     assertTrue("spans is null and it shouldn't be", spans != null);
     //each position match should have a span associated with it, since there is just one underlying term query, there should
@@ -409,16 +420,16 @@ public class TestPayloadSpans extends Lu
     int seen = 0;
     while (spans.nextDoc() != Spans.NO_MORE_DOCS) {
       while (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
-        assertEquals("isPayloadAvailable should return true/false as payloads are expected", expectedNumPayloads > 0, spans.isPayloadAvailable());
-        //See payload helper, for the PayloadHelper.FIELD field, there is a single byte payload at every token
-        if (spans.isPayloadAvailable()) {
-          Collection<byte[]> payload = spans.getPayload();
-          assertEquals("payload size", expectedNumPayloads, payload.size());
-          for (final byte [] thePayload : payload) {
-            assertEquals("payload length", expectedPayloadLength, thePayload.length);
-            assertEquals("payload first byte", expectedFirstByte, thePayload[0]);
-          }
+        collector.reset();
+        spans.collect(collector);
+
+        Collection<byte[]> payload = collector.getPayloads();
+        assertEquals("payload size", expectedNumPayloads, payload.size());
+        for (final byte [] thePayload : payload) {
+          assertEquals("payload length", expectedPayloadLength, thePayload.length);
+          assertEquals("payload first byte", expectedFirstByte, thePayload[0]);
         }
+
         seen++;
       }
     }
@@ -446,26 +457,26 @@ public class TestPayloadSpans extends Lu
     return searcher;
   }
   
-  private void checkSpans(Spans spans, int numSpans, int[] numPayloads) throws IOException {
+  private void checkSpans(Spans spans, PayloadSpanCollector collector, int numSpans, int[] numPayloads) throws IOException {
     int cnt = 0;
 
     while (spans.nextDoc() != Spans.NO_MORE_DOCS) {
       while (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
         if(VERBOSE)
           System.out.println("\nSpans Dump --");
-        if (spans.isPayloadAvailable()) {
-          Collection<byte[]> payload = spans.getPayload();
-          if(VERBOSE) {
-            System.out.println("payloads for span:" + payload.size());
-            for (final byte [] bytes : payload) {
-              System.out.println("doc:" + spans.docID() + " s:" + spans.startPosition() + " e:" + spans.endPosition() + " "
+        collector.reset();
+        spans.collect(collector);
+
+        Collection<byte[]> payload = collector.getPayloads();
+        if(VERBOSE) {
+          System.out.println("payloads for span:" + payload.size());
+          for (final byte [] bytes : payload) {
+            System.out.println("doc:" + spans.docID() + " s:" + spans.startPosition() + " e:" + spans.endPosition() + " "
                 + new String(bytes, StandardCharsets.UTF_8));
-            }
           }
-          assertEquals("payload size", numPayloads[cnt], payload.size());
-        } else { // no payload available
-          assertFalse("Expected spans:" + numPayloads[cnt] + " found: 0", numPayloads.length > 0 && numPayloads[cnt] > 0 );
         }
+        assertEquals("payload size", numPayloads[cnt], payload.size());
+
         cnt++;
       }
     }

Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java Tue May 19 09:21:24 2015
@@ -17,17 +17,16 @@ package org.apache.lucene.search.spans;
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-import java.util.Set;
-
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermContext;
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.util.Bits;
 
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
 /**
  * Holds all implementations of classes in the o.a.l.s.spans package as a
  * back-compatibility test. It does not run any tests per-se, however if
@@ -65,19 +64,14 @@ final class JustCompileSearchSpans {
     public int endPosition() {
       throw new UnsupportedOperationException(UNSUPPORTED_MSG);
     }
-    
-    @Override
-    public int nextStartPosition() throws IOException {
-      throw new UnsupportedOperationException(UNSUPPORTED_MSG);
-    }
 
     @Override
-    public Collection<byte[]> getPayload() {
-      throw new UnsupportedOperationException(UNSUPPORTED_MSG);
+    public void collect(SpanCollector collector) throws IOException {
+
     }
 
     @Override
-    public boolean isPayloadAvailable() {
+    public int nextStartPosition() throws IOException {
       throw new UnsupportedOperationException(UNSUPPORTED_MSG);
     }
 
@@ -100,7 +94,7 @@ final class JustCompileSearchSpans {
     }
 
     @Override
-    public Spans getSpans(LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) {
+    public Spans getSpans(LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts, SpanCollector collector) {
       throw new UnsupportedOperationException(UNSUPPORTED_MSG);
     }
 
@@ -137,19 +131,14 @@ final class JustCompileSearchSpans {
     public int endPosition() {
       throw new UnsupportedOperationException(UNSUPPORTED_MSG);
     }
-    
-    @Override
-    public int nextStartPosition() throws IOException {
-      throw new UnsupportedOperationException(UNSUPPORTED_MSG);
-    }
 
     @Override
-    public Collection<byte[]> getPayload() {
-      throw new UnsupportedOperationException(UNSUPPORTED_MSG);
+    public void collect(SpanCollector collector) throws IOException {
+
     }
 
     @Override
-    public boolean isPayloadAvailable() {
+    public int nextStartPosition() throws IOException {
       throw new UnsupportedOperationException(UNSUPPORTED_MSG);
     }
 

Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java Tue May 19 09:21:24 2015
@@ -17,11 +17,6 @@ package org.apache.lucene.search.spans;
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
@@ -30,6 +25,11 @@ import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermContext;
 import org.apache.lucene.util.Bits;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
 /**
  * 
  * A wrapper to perform span operations on a non-leaf reader context
@@ -40,6 +40,10 @@ import org.apache.lucene.util.Bits;
 public class MultiSpansWrapper {
 
   public static Spans wrap(IndexReader reader, SpanQuery spanQuery) throws IOException {
+    return wrap(reader, spanQuery, SpanCollector.NO_OP);
+  }
+
+  public static Spans wrap(IndexReader reader, SpanQuery spanQuery, SpanCollector collector) throws IOException {
     LeafReader lr = SlowCompositeReaderWrapper.wrap(reader); // slow, but ok for testing
     LeafReaderContext lrContext = lr.getContext();
     SpanQuery rewrittenQuery = (SpanQuery) spanQuery.rewrite(lr); // get the term contexts so getSpans can be called directly
@@ -50,7 +54,7 @@ public class MultiSpansWrapper {
       TermContext termContext = TermContext.build(lrContext, term);
       termContexts.put(term, termContext);
     }
-    Spans actSpans = spanQuery.getSpans(lrContext, new Bits.MatchAllBits(lr.numDocs()), termContexts);
+    Spans actSpans = spanQuery.getSpans(lrContext, new Bits.MatchAllBits(lr.numDocs()), termContexts, collector);
     return actSpans;
   }
 }



Mime
View raw message