lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1721134 [6/48] - in /lucene/dev/branches/lucene6835: ./ dev-tools/ dev-tools/eclipse/dot.settings/ dev-tools/idea/.idea/libraries/ dev-tools/maven/ dev-tools/maven/lucene/analysis/morfologik/ dev-tools/scripts/ lucene/ lucene/analysis/ luc...
Date Mon, 21 Dec 2015 11:40:14 GMT
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java Mon Dec 21 11:39:57 2015
@@ -21,6 +21,7 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -268,6 +269,9 @@ public final class SegmentInfo {
       if (!m.matches()) {
         throw new IllegalArgumentException("invalid codec filename '" + file + "', must match: " + IndexFileNames.CODEC_FILE_PATTERN.pattern());
       }
+      if (file.toLowerCase(Locale.ROOT).endsWith(".tmp")) {
+        throw new IllegalArgumentException("invalid codec filename '" + file + "', cannot end with .tmp extension");
+      }
     }
   }
   

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java Mon Dec 21 11:39:57 2015
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.util.List;
 
 import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.DimensionalWriter;
 import org.apache.lucene.codecs.DocValuesConsumer;
 import org.apache.lucene.codecs.FieldsConsumer;
 import org.apache.lucene.codecs.NormsConsumer;
@@ -108,6 +109,17 @@ final class SegmentMerger {
       long t1 = System.nanoTime();
       mergeState.infoStream.message("SM", ((t1-t0)/1000000) + " msec to merge doc values [" + numMerged + " docs]");
     }
+
+    if (mergeState.infoStream.isEnabled("SM")) {
+      t0 = System.nanoTime();
+    }
+    if (mergeState.mergeFieldInfos.hasDimensionalValues()) {
+      mergeDimensionalValues(segmentWriteState);
+    }
+    if (mergeState.infoStream.isEnabled("SM")) {
+      long t1 = System.nanoTime();
+      mergeState.infoStream.message("SM", ((t1-t0)/1000000) + " msec to merge dimensional values [" + numMerged + " docs]");
+    }
     
     if (mergeState.mergeFieldInfos.hasNorms()) {
       if (mergeState.infoStream.isEnabled("SM")) {
@@ -151,6 +163,12 @@ final class SegmentMerger {
     }
   }
 
+  private void mergeDimensionalValues(SegmentWriteState segmentWriteState) throws IOException {
+    try (DimensionalWriter writer = codec.dimensionalFormat().fieldsWriter(segmentWriteState)) {
+      writer.merge(mergeState);
+    }
+  }
+
   private void mergeNorms(SegmentWriteState segmentWriteState) throws IOException {
     try (NormsConsumer consumer = codec.normsFormat().normsConsumer(segmentWriteState)) {
       consumer.merge(mergeState);

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java Mon Dec 21 11:39:57 2015
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.util.Collections;
 
 import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.DimensionalReader;
 import org.apache.lucene.codecs.DocValuesProducer;
 import org.apache.lucene.codecs.FieldInfosFormat;
 import org.apache.lucene.codecs.FieldsProducer;
@@ -62,7 +63,7 @@ public final class SegmentReader extends
   // TODO: why is this public?
   public SegmentReader(SegmentCommitInfo si, IOContext context) throws IOException {
     this.si = si;
-    core = new SegmentCoreReaders(this, si.info.dir, si, context);
+    core = new SegmentCoreReaders(si.info.dir, si, context);
     segDocValues = new SegmentDocValues();
     
     boolean success = false;
@@ -217,6 +218,12 @@ public final class SegmentReader extends
   }
   
   @Override
+  public DimensionalValues getDimensionalValues() {
+    ensureOpen();
+    return core.dimensionalReader;
+  }
+
+  @Override
   public NormsProducer getNormsReader() {
     ensureOpen();
     return core.normsProducer;
@@ -235,6 +242,12 @@ public final class SegmentReader extends
   }
 
   @Override
+  public DimensionalReader getDimensionalReader() {
+    ensureOpen();
+    return core.dimensionalReader;
+  }
+
+  @Override
   public String toString() {
     // SegmentInfo.toString takes dir and number of
     // *pending* deletions; so we reverse compute that here:

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java Mon Dec 21 11:39:57 2015
@@ -20,6 +20,7 @@ package org.apache.lucene.index;
 import java.io.IOException;
 import java.util.Iterator;
 
+import org.apache.lucene.codecs.DimensionalReader;
 import org.apache.lucene.codecs.DocValuesProducer;
 import org.apache.lucene.codecs.FieldsProducer;
 import org.apache.lucene.codecs.NormsProducer;
@@ -45,7 +46,7 @@ public final class SlowCodecReaderWrappe
    */
   public static CodecReader wrap(final LeafReader reader) throws IOException {
     if (reader instanceof CodecReader) {
-      return (CodecReader)reader;
+      return (CodecReader) reader;
     } else {
       // simulate it slowly, over the leafReader api:
       reader.checkIntegrity();
@@ -91,6 +92,16 @@ public final class SlowCodecReaderWrappe
         }
 
         @Override
+        public DimensionalValues getDimensionalValues() {
+          return reader.getDimensionalValues();
+        }
+
+        @Override
+        public DimensionalReader getDimensionalReader() {
+          return dimensionalValuesToReader(reader.getDimensionalValues());
+        }
+
+        @Override
         public Bits getLiveDocs() {
           return reader.getLiveDocs();
         }
@@ -117,6 +128,32 @@ public final class SlowCodecReaderWrappe
       };
     }
   }
+
+  private static DimensionalReader dimensionalValuesToReader(DimensionalValues values) {
+    if (values == null) {
+      return null;
+    }
+    return new DimensionalReader() {
+      @Override
+      public void intersect(String fieldName, IntersectVisitor visitor) throws IOException {
+        values.intersect(fieldName, visitor);
+      }
+
+      @Override
+      public void checkIntegrity() throws IOException {
+        // We already checkIntegrity the entire reader up front
+      }
+
+      @Override
+      public void close() {
+      }
+
+      @Override
+      public long ramBytesUsed() {
+        return 0;
+      }
+    };
+  }
   
   private static NormsProducer readerToNormsProducer(final LeafReader reader) {
     return new NormsProducer() {

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java Mon Dec 21 11:39:57 2015
@@ -21,10 +21,10 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
-import org.apache.lucene.util.Bits;
 import org.apache.lucene.index.MultiDocValues.MultiSortedDocValues;
 import org.apache.lucene.index.MultiDocValues.MultiSortedSetDocValues;
 import org.apache.lucene.index.MultiDocValues.OrdinalMap;
+import org.apache.lucene.util.Bits;
 
 /**
  * This class forces a composite reader (eg a {@link
@@ -234,6 +234,12 @@ public final class SlowCompositeReaderWr
   }
 
   @Override
+  public DimensionalValues getDimensionalValues() {
+    ensureOpen();
+    return MultiDimensionalValues.get(in);
+  }
+
+  @Override
   public FieldInfos getFieldInfos() {
     ensureOpen();
     return MultiFields.getMergedFieldInfos(in);

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java Mon Dec 21 11:39:57 2015
@@ -80,7 +80,7 @@ final class StandardDirectoryReader exte
     // no need to process segments in reverse order
     final int numSegments = infos.size();
 
-    List<SegmentReader> readers = new ArrayList<>();
+    final List<SegmentReader> readers = new ArrayList<>(numSegments);
     final Directory dir = writer.getDirectory();
 
     final SegmentInfos segmentInfos = infos.clone();
@@ -136,7 +136,7 @@ final class StandardDirectoryReader exte
 
     // we put the old SegmentReaders in a map, that allows us
     // to lookup a reader using its segment name
-    final Map<String,Integer> segmentReaders = new HashMap<>();
+    final Map<String,Integer> segmentReaders = (oldReaders == null ? Collections.emptyMap() : new HashMap<>(oldReaders.size()));
 
     if (oldReaders != null) {
       // create a Map SegmentName->SegmentReader

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/StoredDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/StoredDocument.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/StoredDocument.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/StoredDocument.java Mon Dec 21 11:39:57 2015
@@ -21,10 +21,6 @@ import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
-import org.apache.lucene.document.DoubleField;
-import org.apache.lucene.document.FloatField;
-import org.apache.lucene.document.IntField;
-import org.apache.lucene.document.LongField;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.util.BytesRef;
 
@@ -151,8 +147,8 @@ public class StoredDocument implements I
     * Returns an array of values of the field specified as the method parameter.
     * This method returns an empty array when there are no
     * matching fields.  It never returns null.
-    * For {@link IntField}, {@link LongField}, {@link
-    * FloatField} and {@link DoubleField} it returns the string value of the number. If you want
+    * For {@link org.apache.lucene.document.LegacyIntField}, {@link org.apache.lucene.document.LegacyLongField}, {@link
+    * org.apache.lucene.document.LegacyFloatField} and {@link org.apache.lucene.document.LegacyDoubleField} it returns the string value of the number. If you want
     * the actual numeric field instances back, use {@link #getFields}.
     * @param name the name of the field
     * @return a <code>String[]</code> of field values
@@ -176,8 +172,8 @@ public class StoredDocument implements I
     * this document, or null.  If multiple fields exist with this name, this
     * method returns the first value added. If only binary fields with this name
     * exist, returns null.
-    * For {@link IntField}, {@link LongField}, {@link
-    * FloatField} and {@link DoubleField} it returns the string value of the number. If you want
+    * For {@link org.apache.lucene.document.LegacyIntField}, {@link org.apache.lucene.document.LegacyLongField}, {@link
+    * org.apache.lucene.document.LegacyFloatField} and {@link org.apache.lucene.document.LegacyDoubleField} it returns the string value of the number. If you want
     * the actual numeric field instance back, use {@link #getField}.
     */
    public final String get(String name) {

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/Term.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/Term.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/Term.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/Term.java Mon Dec 21 11:39:57 2015
@@ -24,6 +24,7 @@ import java.nio.charset.CodingErrorActio
 import java.nio.charset.StandardCharsets;
 
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
 
 /**
   A Term represents a word from text.  This is the unit of search.  It is
@@ -39,16 +40,22 @@ public final class Term implements Compa
 
   /** Constructs a Term with the given field and bytes.
    * <p>Note that a null field or null bytes value results in undefined
-   * behavior for most Lucene APIs that accept a Term parameter. 
+   * behavior for most Lucene APIs that accept a Term parameter.
    *
-   * <p>WARNING: the provided BytesRef is not copied, but used directly.
-   * Therefore the bytes should not be modified after construction, for
-   * example, you should clone a copy by {@link BytesRef#deepCopyOf}
-   * rather than pass reused bytes from a TermsEnum.
+   * <p>The provided BytesRef is copied when it is non null.
    */
   public Term(String fld, BytesRef bytes) {
     field = fld;
-    this.bytes = bytes;
+    this.bytes = bytes == null ? null : BytesRef.deepCopyOf(bytes);
+  }
+
+  /** Constructs a Term with the given field and the bytes from a builder.
+   * <p>Note that a null field value results in undefined
+   * behavior for most Lucene APIs that accept a Term parameter.
+   */
+  public Term(String fld, BytesRefBuilder bytesBuilder) {
+    field = fld;
+    this.bytes = bytesBuilder.toBytesRef();
   }
 
   /** Constructs a Term with the given field and text.
@@ -61,7 +68,7 @@ public final class Term implements Compa
   /** Constructs a Term with the given field and empty text.
    * This serves two purposes: 1) reuse of a Term with the same field.
    * 2) pattern for a query.
-   * 
+   *
    * @param fld field's name
    */
   public Term(String fld) {
@@ -75,10 +82,10 @@ public final class Term implements Compa
   /** Returns the text of this term.  In the case of words, this is simply the
     text of the word.  In the case of dates and other types, this is an
     encoding of the object as a string.  */
-  public final String text() { 
+  public final String text() {
     return toString(bytes);
   }
-  
+
   /** Returns human-readable form of the term text. If the term is not unicode,
    * the raw bytes will be printed instead. */
   public static final String toString(BytesRef termText) {
@@ -93,7 +100,7 @@ public final class Term implements Compa
     }
   }
 
-  /** Returns the bytes of this term. */
+  /** Returns the bytes of this term, these should not be modified. */
   public final BytesRef bytes() { return bytes; }
 
   @Override
@@ -141,8 +148,8 @@ public final class Term implements Compa
     }
   }
 
-  /** 
-   * Resets the field and text of a Term. 
+  /**
+   * Resets the field and text of a Term.
    * <p>WARNING: the provided BytesRef is not copied, but used directly.
    * Therefore the bytes should not be modified after construction, for
    * example, you should clone a copy rather than pass reused bytes from

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BlendedTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BlendedTermQuery.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BlendedTermQuery.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BlendedTermQuery.java Mon Dec 21 11:39:57 2015
@@ -92,7 +92,7 @@ public final class BlendedTermQuery exte
       terms = ArrayUtil.grow(terms, numTerms + 1);
       boosts = ArrayUtil.grow(boosts, numTerms + 1);
       contexts = ArrayUtil.grow(contexts, numTerms + 1);
-      terms[numTerms] = new Term(term.field(), BytesRef.deepCopyOf(term.bytes()));
+      terms[numTerms] = term;
       boosts[numTerms] = boost;
       contexts[numTerms] = context;
       numTerms += 1;

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java Mon Dec 21 11:39:57 2015
@@ -28,6 +28,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.BooleanClause.Occur;
@@ -222,36 +223,141 @@ public class BooleanQuery extends Query
 
   @Override
   public Query rewrite(IndexReader reader) throws IOException {
-    if (minimumNumberShouldMatch == 0 && clauses.size() == 1) {// optimize 1-clause queries
+    // optimize 1-clause queries
+    if (clauses.size() == 1) {
       BooleanClause c = clauses.get(0);
-      if (!c.isProhibited()) {  // just return clause
-
-        Query query = c.getQuery();
+      Query query = c.getQuery();
+      if (minimumNumberShouldMatch == 1 && c.getOccur() == Occur.SHOULD) {
+        return query;
+      } else if (minimumNumberShouldMatch == 0) {
+        switch (c.getOccur()) {
+          case SHOULD:
+          case MUST:
+            return query;
+          case FILTER:
+            // no scoring clauses, so return a score of 0
+            return new BoostQuery(new ConstantScoreQuery(query), 0);
+          case MUST_NOT:
+            // no positive clauses
+            return new MatchNoDocsQuery();
+          default:
+            throw new AssertionError();
+        }
+      }
+    }
 
-        if (c.isScoring() == false) {
-          // our single clause is a filter, so we need to disable scoring
-          query = new BoostQuery(new ConstantScoreQuery(query), 0);
+    // recursively rewrite
+    {
+      BooleanQuery.Builder builder = new BooleanQuery.Builder();
+      builder.setDisableCoord(isCoordDisabled());
+      builder.setMinimumNumberShouldMatch(getMinimumNumberShouldMatch());
+      boolean actuallyRewritten = false;
+      for (BooleanClause clause : this) {
+        Query query = clause.getQuery();
+        Query rewritten = query.rewrite(reader);
+        if (rewritten != query) {
+          actuallyRewritten = true;
         }
+        builder.add(rewritten, clause.getOccur());
+      }
+      if (actuallyRewritten) {
+        return builder.build();
+      }
+    }
 
-        return query;
+    // remove duplicate FILTER and MUST_NOT clauses
+    {
+      int clauseCount = 0;
+      for (Collection<Query> queries : clauseSets.values()) {
+        clauseCount += queries.size();
+      }
+      if (clauseCount != clauses.size()) {
+        // since clauseSets implicitly deduplicates FILTER and MUST_NOT
+        // clauses, this means there were duplicates
+        BooleanQuery.Builder rewritten = new BooleanQuery.Builder();
+        rewritten.setDisableCoord(disableCoord);
+        rewritten.setMinimumNumberShouldMatch(minimumNumberShouldMatch);
+        for (Map.Entry<Occur, Collection<Query>> entry : clauseSets.entrySet()) {
+          final Occur occur = entry.getKey();
+          for (Query query : entry.getValue()) {
+            rewritten.add(query, occur);
+          }
+        }
+        return rewritten.build();
       }
     }
 
-    BooleanQuery.Builder builder = new BooleanQuery.Builder();
-    builder.setDisableCoord(isCoordDisabled());
-    builder.setMinimumNumberShouldMatch(getMinimumNumberShouldMatch());
-    boolean actuallyRewritten = false;
-    for (BooleanClause clause : this) {
-      Query query = clause.getQuery();
-      Query rewritten = query.rewrite(reader);
-      if (rewritten != query) {
-        actuallyRewritten = true;
+    // remove FILTER clauses that are also MUST clauses
+    // or that match all documents
+    if (clauseSets.get(Occur.MUST).size() > 0 && clauseSets.get(Occur.FILTER).size() > 0) {
+      final Set<Query> filters = new HashSet<Query>(clauseSets.get(Occur.FILTER));
+      boolean modified = filters.remove(new MatchAllDocsQuery());
+      modified |= filters.removeAll(clauseSets.get(Occur.MUST));
+      if (modified) {
+        BooleanQuery.Builder builder = new BooleanQuery.Builder();
+        builder.setDisableCoord(isCoordDisabled());
+        builder.setMinimumNumberShouldMatch(getMinimumNumberShouldMatch());
+        for (BooleanClause clause : clauses) {
+          if (clause.getOccur() != Occur.FILTER) {
+            builder.add(clause);
+          }
+        }
+        for (Query filter : filters) {
+          builder.add(filter, Occur.FILTER);
+        }
+        return builder.build();
       }
-      builder.add(rewritten, clause.getOccur());
     }
-    if (actuallyRewritten) {
-      return builder.build();
+
+    // Rewrite queries whose single scoring clause is a MUST clause on a
+    // MatchAllDocsQuery to a ConstantScoreQuery
+    {
+      final Collection<Query> musts = clauseSets.get(Occur.MUST);
+      final Collection<Query> filters = clauseSets.get(Occur.FILTER);
+      if (musts.size() == 1
+          && filters.size() > 0) {
+        Query must = musts.iterator().next();
+        float boost = 1f;
+        if (must instanceof BoostQuery) {
+          BoostQuery boostQuery = (BoostQuery) must;
+          must = boostQuery.getQuery();
+          boost = boostQuery.getBoost();
+        }
+        if (must.getClass() == MatchAllDocsQuery.class) {
+          // our single scoring clause matches everything: rewrite to a CSQ on the filter
+          // ignore SHOULD clause for now
+          BooleanQuery.Builder builder = new BooleanQuery.Builder();
+          for (BooleanClause clause : clauses) {
+            switch (clause.getOccur()) {
+              case FILTER:
+              case MUST_NOT:
+                builder.add(clause);
+                break;
+              default:
+                // ignore
+                break;
+            }
+          }
+          Query rewritten = builder.build();
+          rewritten = new ConstantScoreQuery(rewritten);
+          if (boost != 1f) {
+            rewritten = new BoostQuery(rewritten, boost);
+          }
+
+          // now add back the SHOULD clauses
+          builder = new BooleanQuery.Builder()
+            .setDisableCoord(isCoordDisabled())
+            .setMinimumNumberShouldMatch(getMinimumNumberShouldMatch())
+            .add(rewritten, Occur.MUST);
+          for (Query query : clauseSets.get(Occur.SHOULD)) {
+            builder.add(query, Occur.SHOULD);
+          }
+          rewritten = builder.build();
+          return rewritten;
+        }
+      }
     }
+
     return super.rewrite(reader);
   }
 

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java Mon Dec 21 11:39:57 2015
@@ -83,11 +83,11 @@ final class BooleanScorer extends BulkSc
     }
 
     void advance(int min) throws IOException {
-      score(null, min, min);
+      score(orCollector, null, min, min);
     }
 
-    void score(Bits acceptDocs, int min, int max) throws IOException {
-      next = scorer.score(orCollector, acceptDocs, min, max);
+    void score(LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException {
+      next = scorer.score(collector, acceptDocs, min, max);
     }
   }
 
@@ -179,6 +179,9 @@ final class BooleanScorer extends BulkSc
     if (minShouldMatch < 1 || minShouldMatch > scorers.size()) {
       throw new IllegalArgumentException("minShouldMatch should be within 1..num_scorers. Got " + minShouldMatch);
     }
+    if (scorers.size() <= 1) {
+      throw new IllegalArgumentException("This scorer can only be used with two scorers or more, got " + scorers.size());
+    }
     for (int i = 0; i < buckets.length; i++) {
       buckets[i] = new Bucket();
     }
@@ -237,12 +240,12 @@ final class BooleanScorer extends BulkSc
     }
   }
 
-  private void scoreWindow(LeafCollector collector, Bits acceptDocs, int base, int min, int max,
-      BulkScorerAndDoc[] scorers, int numScorers) throws IOException {
+  private void scoreWindowIntoBitSetAndReplay(LeafCollector collector, Bits acceptDocs,
+      int base, int min, int max, BulkScorerAndDoc[] scorers, int numScorers) throws IOException {
     for (int i = 0; i < numScorers; ++i) {
       final BulkScorerAndDoc scorer = scorers[i];
       assert scorer.next < max;
-      scorer.score(acceptDocs, min, max);
+      scorer.score(orCollector, acceptDocs, min, max);
     }
 
     scoreMatches(collector, base);
@@ -270,14 +273,7 @@ final class BooleanScorer extends BulkSc
     return headTop;
   }
 
-  private void scoreWindow(LeafCollector collector, Bits acceptDocs, int windowBase, int windowMin, int windowMax) throws IOException {
-    // Fill 'leads' with all scorers from 'head' that are in the right window
-    leads[0] = head.pop();
-    int maxFreq = 1;
-    while (head.size() > 0 && head.top().next < windowMax) {
-      leads[maxFreq++] = head.pop();
-    }
-
+  private void scoreWindowMultipleScorers(LeafCollector collector, Bits acceptDocs, int windowBase, int windowMin, int windowMax, int maxFreq) throws IOException {
     while (maxFreq < minShouldMatch && maxFreq + tail.size() >= minShouldMatch) {
       // a match is still possible
       final BulkScorerAndDoc candidate = tail.pop();
@@ -296,7 +292,7 @@ final class BooleanScorer extends BulkSc
       }
       tail.clear();
 
-      scoreWindow(collector, acceptDocs, windowBase, windowMin, windowMax, leads, maxFreq);
+      scoreWindowIntoBitSetAndReplay(collector, acceptDocs, windowBase, windowMin, windowMax, leads, maxFreq);
     }
 
     // Push back scorers into head and tail
@@ -308,21 +304,64 @@ final class BooleanScorer extends BulkSc
     }
   }
 
+  private void scoreWindowSingleScorer(BulkScorerAndDoc bulkScorer, LeafCollector collector,
+      Bits acceptDocs, int windowMin, int windowMax, int max) throws IOException {
+    assert tail.size() == 0;
+    final int nextWindowBase = head.top().next & ~MASK;
+    final int end = Math.max(windowMax, Math.min(max, nextWindowBase));
+    
+    bulkScorer.score(collector, acceptDocs, windowMin, end);
+    
+    // reset the scorer that should be used for the general case
+    collector.setScorer(fakeScorer);
+  }
+
+  private BulkScorerAndDoc scoreWindow(BulkScorerAndDoc top, LeafCollector collector,
+      LeafCollector singleClauseCollector, Bits acceptDocs, int min, int max) throws IOException {
+    final int windowBase = top.next & ~MASK; // find the window that the next match belongs to
+    final int windowMin = Math.max(min, windowBase);
+    final int windowMax = Math.min(max, windowBase + SIZE);
+
+    // Fill 'leads' with all scorers from 'head' that are in the right window
+    leads[0] = head.pop();
+    int maxFreq = 1;
+    while (head.size() > 0 && head.top().next < windowMax) {
+      leads[maxFreq++] = head.pop();
+    }
+
+    if (minShouldMatch == 1 && maxFreq == 1) {
+      // special case: only one scorer can match in the current window,
+      // we can collect directly
+      final BulkScorerAndDoc bulkScorer = leads[0];
+      scoreWindowSingleScorer(bulkScorer, singleClauseCollector, acceptDocs, windowMin, windowMax, max);
+      return head.add(bulkScorer);
+    } else {
+      // general case, collect through a bit set first and then replay
+      scoreWindowMultipleScorers(collector, acceptDocs, windowBase, windowMin, windowMax, maxFreq);
+      return head.top();
+    }
+  }
+
   @Override
   public int score(LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException {
     fakeScorer.doc = -1;
     collector.setScorer(fakeScorer);
 
+    final LeafCollector singleClauseCollector;
+    if (coordFactors[1] == 1f) {
+      singleClauseCollector = collector;
+    } else {
+      singleClauseCollector = new FilterLeafCollector(collector) {
+        @Override
+        public void setScorer(Scorer scorer) throws IOException {
+          super.setScorer(new BooleanTopLevelScorers.BoostedScorer(scorer, coordFactors[1]));
+        }
+      };
+    }
+
     BulkScorerAndDoc top = advance(min);
     while (top.next < max) {
-
-      final int windowBase = top.next & ~MASK; // find the window that the next match belongs to
-      final int windowMin = Math.max(min, windowBase);
-      final int windowMax = Math.min(max, windowBase + SIZE);
-
-      // general case
-      scoreWindow(collector, acceptDocs, windowBase, windowMin, windowMax);
-      top = head.top();
+      top = scoreWindow(top, collector, singleClauseCollector, acceptDocs, min, max);
     }
 
     return top.next;

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BooleanTopLevelScorers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BooleanTopLevelScorers.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BooleanTopLevelScorers.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BooleanTopLevelScorers.java Mon Dec 21 11:39:57 2015
@@ -22,6 +22,8 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 
+import org.apache.lucene.util.Bits;
+
 /** Internal document-at-a-time scorers used to deal with stupid coord() computation */
 class BooleanTopLevelScorers {
   
@@ -48,7 +50,39 @@ class BooleanTopLevelScorers {
       return Collections.singleton(new ChildScorer(in, "BOOSTED"));
     }
   }
-  
+
+  /**
+   * Used when there is more than one scorer in a query, but a segment
+   * only had one non-null scorer.
+   */
+  static class BoostedBulkScorer extends BulkScorer {
+
+    final BulkScorer in;
+    final float boost;
+
+    BoostedBulkScorer(BulkScorer scorer, float boost) {
+      this.in = scorer;
+      this.boost = boost;
+    }
+
+    @Override
+    public int score(LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException {
+      final LeafCollector wrapped = new FilterLeafCollector(collector) {
+        @Override
+        public void setScorer(Scorer scorer) throws IOException {
+          super.setScorer(new BoostedScorer(scorer, boost));
+        }
+      };
+      return in.score(wrapped, acceptDocs, min, max);
+    }
+
+    @Override
+    public long cost() {
+      return in.cost();
+    }
+
+  }
+
   /** 
    * Used when there are both mandatory and optional clauses, but minShouldMatch
    * dictates that some of the optional clauses must match. The query is a conjunction,
@@ -89,22 +123,25 @@ class BooleanTopLevelScorers {
       this.coordReq = coordReq;
       this.coordBoth = coordBoth;
     }
-    
+
     @Override
     public float score() throws IOException {
+      // TODO: sum into a double and cast to float if we ever send required clauses to BS1
       int curDoc = reqScorer.docID();
-      float reqScore = reqScorer.score();
-      if (optScorer == null) {
-        return reqScore * coordReq;
+      float score = reqScorer.score();
+
+      int optScorerDoc = optIterator.docID();
+      if (optScorerDoc < curDoc) {
+        optScorerDoc = optIterator.advance(curDoc);
       }
       
-      int optScorerDoc = optScorer.docID();
-      if (optScorerDoc < curDoc && (optScorerDoc = optScorer.advance(curDoc)) == NO_MORE_DOCS) {
-        optScorer = null;
-        return reqScore * coordReq;
+      if (optScorerDoc == curDoc) {
+        score = (score + optScorer.score()) * coordBoth;
+      } else {
+        score = score * coordReq;
       }
       
-      return optScorerDoc == curDoc ? (reqScore + optScorer.score()) * coordBoth : reqScore * coordReq;
+      return score;
     }
   }
 
@@ -121,22 +158,25 @@ class BooleanTopLevelScorers {
       this.requiredCount = requiredCount;
       this.coords = coords;
     }
-    
+
     @Override
     public float score() throws IOException {
+      // TODO: sum into a double and cast to float if we ever send required clauses to BS1
       int curDoc = reqScorer.docID();
-      float reqScore = reqScorer.score();
-      if (optScorer == null) {
-        return reqScore * coords[requiredCount];
+      float score = reqScorer.score();
+
+      int optScorerDoc = optIterator.docID();
+      if (optScorerDoc < curDoc) {
+        optScorerDoc = optIterator.advance(curDoc);
       }
       
-      int optScorerDoc = optScorer.docID();
-      if (optScorerDoc < curDoc && (optScorerDoc = optScorer.advance(curDoc)) == NO_MORE_DOCS) {
-        optScorer = null;
-        return reqScore * coords[requiredCount];
+      if (optScorerDoc == curDoc) {
+        score = (score + optScorer.score()) * coords[requiredCount + optScorer.freq()];
+      } else {
+        score = score * coords[requiredCount];
       }
       
-      return optScorerDoc == curDoc ? (reqScore + optScorer.score()) * coords[requiredCount + optScorer.freq()] : reqScore * coords[requiredCount];
+      return score;
     }
   }
 }

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java Mon Dec 21 11:39:57 2015
@@ -190,7 +190,7 @@ final class BooleanWeight extends Weight
   /** Try to build a boolean scorer for this weight. Returns null if {@link BooleanScorer}
    *  cannot be used. */
   // pkg-private for forcing use of BooleanScorer in tests
-  BooleanScorer booleanScorer(LeafReaderContext context) throws IOException {
+  BulkScorer booleanScorer(LeafReaderContext context) throws IOException {
     List<BulkScorer> optional = new ArrayList<BulkScorer>();
     Iterator<BooleanClause> cIter = query.iterator();
     for (Weight w  : weights) {
@@ -222,12 +222,21 @@ final class BooleanWeight extends Weight
       return null;
     }
 
+    if (optional.size() == 1) {
+      BulkScorer opt = optional.get(0);
+      if (!disableCoord && maxCoord > 1) {
+        return new BooleanTopLevelScorers.BoostedBulkScorer(opt, coord(1, maxCoord));
+      } else {
+        return opt;
+      }
+    }
+
     return new BooleanScorer(this, disableCoord, maxCoord, optional, Math.max(1, query.getMinimumNumberShouldMatch()), needsScores);
   }
 
   @Override
   public BulkScorer bulkScorer(LeafReaderContext context) throws IOException {
-    final BooleanScorer bulkScorer = booleanScorer(context);
+    final BulkScorer bulkScorer = booleanScorer(context);
     if (bulkScorer != null) { // BooleanScorer is applicable
       // TODO: what is the right heuristic here?
       final long costThreshold;

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BulkScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BulkScorer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BulkScorer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/BulkScorer.java Mon Dec 21 11:39:57 2015
@@ -84,7 +84,7 @@ public abstract class BulkScorer {
   public abstract int score(LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException;
 
   /**
-   * Same as {@link Scorer#cost()} for bulk scorers.
+   * Same as {@link DocIdSetIterator#cost()} for bulk scorers.
    */
   public abstract long cost();
 }

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/CachingCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/CachingCollector.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/CachingCollector.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/CachingCollector.java Mon Dec 21 11:39:57 2015
@@ -62,23 +62,21 @@ public abstract class CachingCollector e
     private CachedScorer() { super(null); }
 
     @Override
-    public final float score() { return score; }
+    public DocIdSetIterator iterator() {
+      throw new UnsupportedOperationException();
+    }
 
     @Override
-    public final int advance(int target) { throw new UnsupportedOperationException(); }
+    public final float score() { return score; }
 
     @Override
-    public final int docID() { return doc; }
+    public int docID() {
+      return doc;
+    }
 
     @Override
     public final int freq() { throw new UnsupportedOperationException(); }
 
-    @Override
-    public final int nextDoc() { throw new UnsupportedOperationException(); }
-
-    @Override
-    public long cost() { return 1; }
-
   }
 
   private static class NoScoreCachingCollector extends CachingCollector {
@@ -99,6 +97,9 @@ public abstract class CachingCollector e
       return new NoScoreCachingLeafCollector(in, maxDocsToCache);
     }
 
+    // note: do *not* override needScore to say false. Just because we aren't caching the score doesn't mean the
+    //   wrapped collector doesn't need it to do its job.
+
     public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
       postCollection();
       final LeafCollector in = this.in.getLeafCollector(context);
@@ -177,6 +178,13 @@ public abstract class CachingCollector e
       scores.add(coll.cachedScores());
     }
 
+    /** Ensure the scores are collected so they can be replayed, even if the wrapped collector doesn't need them. */
+    @Override
+    public boolean needsScores() {
+      return true;
+    }
+
+    @Override
     protected void collect(LeafCollector collector, int i) throws IOException {
       final int[] docs = this.docs.get(i);
       final float[] scores = this.scores.get(i);
@@ -189,7 +197,6 @@ public abstract class CachingCollector e
         collector.collect(scorer.doc);
       }
     }
-
   }
 
   private class NoScoreCachingLeafCollector extends FilterLeafCollector {

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ConjunctionDISI.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ConjunctionDISI.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ConjunctionDISI.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ConjunctionDISI.java Mon Dec 21 11:39:57 2015
@@ -23,6 +23,7 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 
+import org.apache.lucene.search.spans.Spans;
 import org.apache.lucene.util.CollectionUtil;
 
 /** A conjunction of DocIdSetIterators.
@@ -32,16 +33,34 @@ import org.apache.lucene.util.Collection
  */
 public class ConjunctionDISI extends DocIdSetIterator {
 
-  /** Create a conjunction over the provided iterators, taking advantage of
-   *  {@link TwoPhaseIterator}. */
-  public static ConjunctionDISI intersect(List<? extends DocIdSetIterator> iterators) {
+  /** Create a conjunction over the provided {@link Scorer}s, taking advantage
+   *  of {@link TwoPhaseIterator}. */
+  public static ConjunctionDISI intersectScorers(List<Scorer> scorers) {
+    if (scorers.size() < 2) {
+      throw new IllegalArgumentException("Cannot make a ConjunctionDISI of less than 2 iterators");
+    }
+    final List<DocIdSetIterator> allIterators = new ArrayList<>();
+    final List<TwoPhaseIterator> twoPhaseIterators = new ArrayList<>();
+    for (Scorer scorer : scorers) {
+      addScorer(scorer, allIterators, twoPhaseIterators);
+    }
+
+    if (twoPhaseIterators.isEmpty()) {
+      return new ConjunctionDISI(allIterators);
+    } else {
+      return new TwoPhase(allIterators, twoPhaseIterators);
+    }
+  }
+
+  /** Create a conjunction over the provided DocIdSetIterators. */
+  public static ConjunctionDISI intersectIterators(List<DocIdSetIterator> iterators) {
     if (iterators.size() < 2) {
       throw new IllegalArgumentException("Cannot make a ConjunctionDISI of less than 2 iterators");
     }
     final List<DocIdSetIterator> allIterators = new ArrayList<>();
     final List<TwoPhaseIterator> twoPhaseIterators = new ArrayList<>();
-    for (DocIdSetIterator iter : iterators) {
-      addIterator(iter, allIterators, twoPhaseIterators);
+    for (DocIdSetIterator iterator : iterators) {
+      addIterator(iterator, allIterators, twoPhaseIterators);
     }
 
     if (twoPhaseIterators.isEmpty()) {
@@ -51,12 +70,48 @@ public class ConjunctionDISI extends Doc
     }
   }
 
-  /** Adds the iterator, possibly splitting up into two phases or collapsing if it is another conjunction */
+  /** Create a conjunction over the provided {@link Scorer}s, taking advantage
+   *  of {@link TwoPhaseIterator}. */
+  public static ConjunctionDISI intersectSpans(List<Spans> spanList) {
+    if (spanList.size() < 2) {
+      throw new IllegalArgumentException("Cannot make a ConjunctionDISI of less than 2 iterators");
+    }
+    final List<DocIdSetIterator> allIterators = new ArrayList<>();
+    final List<TwoPhaseIterator> twoPhaseIterators = new ArrayList<>();
+    for (Spans spans : spanList) {
+      addSpans(spans, allIterators, twoPhaseIterators);
+    }
+
+    if (twoPhaseIterators.isEmpty()) {
+      return new ConjunctionDISI(allIterators);
+    } else {
+      return new TwoPhase(allIterators, twoPhaseIterators);
+    }
+  }
+
+  /** Adds the scorer, possibly splitting up into two phases or collapsing if it is another conjunction */
+  private static void addScorer(Scorer scorer, List<DocIdSetIterator> allIterators, List<TwoPhaseIterator> twoPhaseIterators) {
+    TwoPhaseIterator twoPhaseIter = scorer.twoPhaseIterator();
+    if (twoPhaseIter != null) {
+      addTwoPhaseIterator(twoPhaseIter, allIterators, twoPhaseIterators);
+    } else { // no approximation support, use the iterator as-is
+      addIterator(scorer.iterator(), allIterators, twoPhaseIterators);
+    }
+  }
+
+  /** Adds the Spans. */
+  private static void addSpans(Spans spans, List<DocIdSetIterator> allIterators, List<TwoPhaseIterator> twoPhaseIterators) {
+    TwoPhaseIterator twoPhaseIter = spans.asTwoPhaseIterator();
+    if (twoPhaseIter != null) {
+      addTwoPhaseIterator(twoPhaseIter, allIterators, twoPhaseIterators);
+    } else { // no approximation support, use the iterator as-is
+      addIterator(spans, allIterators, twoPhaseIterators);
+    }
+  }
+
   private static void addIterator(DocIdSetIterator disi, List<DocIdSetIterator> allIterators, List<TwoPhaseIterator> twoPhaseIterators) {
     // Check for exactly this class for collapsing. Subclasses can do their own optimizations.
-    if (disi.getClass() == ConjunctionScorer.class) {
-      addIterator(((ConjunctionScorer) disi).disi, allIterators, twoPhaseIterators);
-    } else if (disi.getClass() == ConjunctionDISI.class || disi.getClass() == TwoPhase.class) {
+    if (disi.getClass() == ConjunctionDISI.class || disi.getClass() == TwoPhase.class) {
       ConjunctionDISI conjunction = (ConjunctionDISI) disi;
       // subconjuctions have already split themselves into two phase iterators and others, so we can take those
       // iterators as they are and move them up to this conjunction
@@ -67,16 +122,15 @@ public class ConjunctionDISI extends Doc
         Collections.addAll(twoPhaseIterators, twoPhase.twoPhaseView.twoPhaseIterators);
       }
     } else {
-      TwoPhaseIterator twoPhaseIter = TwoPhaseIterator.asTwoPhaseIterator(disi);
-      if (twoPhaseIter != null) {
-        allIterators.add(twoPhaseIter.approximation());
-        twoPhaseIterators.add(twoPhaseIter);
-      } else { // no approximation support, use the iterator as-is
-        allIterators.add(disi);
-      }
+      allIterators.add(disi);
     }
   }
 
+  private static void addTwoPhaseIterator(TwoPhaseIterator twoPhaseIter, List<DocIdSetIterator> allIterators, List<TwoPhaseIterator> twoPhaseIterators) {
+    addIterator(twoPhaseIter.approximation(), allIterators, twoPhaseIterators);
+    twoPhaseIterators.add(twoPhaseIter);
+  }
+
   final DocIdSetIterator lead;
   final DocIdSetIterator[] others;
 
@@ -155,7 +209,7 @@ public class ConjunctionDISI extends Doc
 
   @Override
   public long cost() {
-    return lead.cost();
+    return lead.cost(); // overestimate
   }
 
   /**
@@ -164,16 +218,33 @@ public class ConjunctionDISI extends Doc
   private static class TwoPhaseConjunctionDISI extends TwoPhaseIterator {
 
     private final TwoPhaseIterator[] twoPhaseIterators;
+    private final float matchCost;
 
     private TwoPhaseConjunctionDISI(List<? extends DocIdSetIterator> iterators, List<TwoPhaseIterator> twoPhaseIterators) {
       super(new ConjunctionDISI(iterators));
       assert twoPhaseIterators.size() > 0;
+
+      CollectionUtil.timSort(twoPhaseIterators, new Comparator<TwoPhaseIterator>() {
+        @Override
+        public int compare(TwoPhaseIterator o1, TwoPhaseIterator o2) {
+          return Float.compare(o1.matchCost(), o2.matchCost());
+        }
+      });
+
       this.twoPhaseIterators = twoPhaseIterators.toArray(new TwoPhaseIterator[twoPhaseIterators.size()]);
+
+      // Compute the matchCost as the total matchCost of the sub iterators.
+      // TODO: This could be too high because the matching is done cheapest first: give the lower matchCosts a higher weight.
+      float totalMatchCost = 0;
+      for (TwoPhaseIterator tpi : twoPhaseIterators) {
+        totalMatchCost += tpi.matchCost();
+      }
+      matchCost = totalMatchCost;
     }
 
     @Override
     public boolean matches() throws IOException {
-      for (TwoPhaseIterator twoPhaseIterator : twoPhaseIterators) {
+      for (TwoPhaseIterator twoPhaseIterator : twoPhaseIterators) { // match cheapest first
         if (twoPhaseIterator.matches() == false) {
           return false;
         }
@@ -181,6 +252,11 @@ public class ConjunctionDISI extends Doc
       return true;
     }
 
+    @Override
+    public float matchCost() {
+      return matchCost;
+    }
+
   }
 
   /**

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java Mon Dec 21 11:39:57 2015
@@ -29,27 +29,27 @@ class ConjunctionScorer extends Scorer {
   final Scorer[] scorers;
   final float coord;
 
-  ConjunctionScorer(Weight weight, List<? extends DocIdSetIterator> required, List<Scorer> scorers) {
+  ConjunctionScorer(Weight weight, List<Scorer> required, List<Scorer> scorers) {
     this(weight, required, scorers, 1f);
   }
 
   /** Create a new {@link ConjunctionScorer}, note that {@code scorers} must be a subset of {@code required}. */
-  ConjunctionScorer(Weight weight, List<? extends DocIdSetIterator> required, List<Scorer> scorers, float coord) {
+  ConjunctionScorer(Weight weight, List<Scorer> required, List<Scorer> scorers, float coord) {
     super(weight);
     assert required.containsAll(scorers);
     this.coord = coord;
-    this.disi = ConjunctionDISI.intersect(required);
+    this.disi = ConjunctionDISI.intersectScorers(required);
     this.scorers = scorers.toArray(new Scorer[scorers.size()]);
   }
 
   @Override
-  public TwoPhaseIterator asTwoPhaseIterator() {
+  public TwoPhaseIterator twoPhaseIterator() {
     return disi.asTwoPhaseIterator();
   }
 
   @Override
-  public int advance(int target) throws IOException {
-    return disi.advance(target);
+  public DocIdSetIterator iterator() {
+    return disi;
   }
 
   @Override
@@ -58,11 +58,6 @@ class ConjunctionScorer extends Scorer {
   }
 
   @Override
-  public int nextDoc() throws IOException {
-    return disi.nextDoc();
-  }
-
-  @Override
   public float score() throws IOException {
     double sum = 0.0d;
     for (Scorer scorer : scorers) {
@@ -77,11 +72,6 @@ class ConjunctionScorer extends Scorer {
   }
 
   @Override
-  public long cost() {
-    return disi.cost();
-  }
-
-  @Override
   public Collection<ChildScorer> getChildren() {
     ArrayList<ChildScorer> children = new ArrayList<>();
     for (Scorer scorer : scorers) {

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ConstantScoreScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ConstantScoreScorer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ConstantScoreScorer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ConstantScoreScorer.java Mon Dec 21 11:39:57 2015
@@ -54,18 +54,13 @@ public final class ConstantScoreScorer e
   }
 
   @Override
-  public TwoPhaseIterator asTwoPhaseIterator() {
-    return twoPhaseIterator;
-  }
-
-  @Override
-  public float score() throws IOException {
-    return score;
+  public DocIdSetIterator iterator() {
+    return disi;
   }
 
   @Override
-  public int freq() throws IOException {
-    return 1;
+  public TwoPhaseIterator twoPhaseIterator() {
+    return twoPhaseIterator;
   }
 
   @Override
@@ -74,18 +69,14 @@ public final class ConstantScoreScorer e
   }
 
   @Override
-  public int nextDoc() throws IOException {
-    return disi.nextDoc();
+  public float score() throws IOException {
+    return score;
   }
 
   @Override
-  public int advance(int target) throws IOException {
-    return disi.advance(target);
+  public int freq() throws IOException {
+    return 1;
   }
 
-  @Override
-  public long cost() {
-    return disi.cost();
-  }
 }
 

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ConstantScoreWeight.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ConstantScoreWeight.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ConstantScoreWeight.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ConstantScoreWeight.java Mon Dec 21 11:39:57 2015
@@ -82,9 +82,9 @@ public abstract class ConstantScoreWeigh
     if (s == null) {
       exists = false;
     } else {
-      final TwoPhaseIterator twoPhase = s.asTwoPhaseIterator();
+      final TwoPhaseIterator twoPhase = s.twoPhaseIterator();
       if (twoPhase == null) {
-        exists = s.advance(doc) == doc;
+        exists = s.iterator().advance(doc) == doc;
       } else {
         exists = twoPhase.approximation().advance(doc) == doc && twoPhase.matches();
       }

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ControlledRealTimeReopenThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ControlledRealTimeReopenThread.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ControlledRealTimeReopenThread.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ControlledRealTimeReopenThread.java Mon Dec 21 11:39:57 2015
@@ -181,7 +181,7 @@ public class ControlledRealTimeReopenThr
         if (maxMS < 0) {
           wait();
         } else {
-          long msLeft = (startMS + maxMS) - (System.nanoTime())/1000000;
+          long msLeft = (startMS + maxMS) - System.nanoTime()/1000000;
           if (msLeft <= 0) {
             return false;
           } else {
@@ -248,4 +248,9 @@ public class ControlledRealTimeReopenThr
       }
     }
   }
+
+  /** Returns which {@code generation} the current searcher is guaranteed to include. */
+  public long getSearchingGen() {
+    return searchingGen;
+  }
 }

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisiPriorityQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisiPriorityQueue.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisiPriorityQueue.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisiPriorityQueue.java Mon Dec 21 11:39:57 2015
@@ -28,8 +28,7 @@ import org.apache.lucene.util.PriorityQu
  * pluggable comparison function makes the rebalancing quite slow.
  * @lucene.internal
  */
-public final class DisiPriorityQueue<Iter extends DocIdSetIterator>
-implements Iterable<DisiWrapper<Iter>> {
+public final class DisiPriorityQueue implements Iterable<DisiWrapper> {
 
   static int leftNode(int node) {
     return ((node + 1) << 1) - 1;
@@ -43,10 +42,9 @@ implements Iterable<DisiWrapper<Iter>> {
     return ((node + 1) >>> 1) - 1;
   }
 
-  private final DisiWrapper<Iter>[] heap;
+  private final DisiWrapper[] heap;
   private int size;
 
-  @SuppressWarnings({"unchecked","rawtypes"})
   public DisiPriorityQueue(int maxSize) {
     heap = new DisiWrapper[maxSize];
     size = 0;
@@ -56,15 +54,15 @@ implements Iterable<DisiWrapper<Iter>> {
     return size;
   }
 
-  public DisiWrapper<Iter> top() {
+  public DisiWrapper top() {
     return heap[0];
   }
 
   /** Get the list of scorers which are on the current doc. */
-  public DisiWrapper<Iter> topList() {
-    final DisiWrapper<Iter>[] heap = this.heap;
+  public DisiWrapper topList() {
+    final DisiWrapper[] heap = this.heap;
     final int size = this.size;
-    DisiWrapper<Iter> list = heap[0];
+    DisiWrapper list = heap[0];
     list.next = null;
     if (size >= 3) {
       list = topList(list, heap, size, 1);
@@ -76,14 +74,14 @@ implements Iterable<DisiWrapper<Iter>> {
   }
 
   // prepend w1 (iterator) to w2 (list)
-  private DisiWrapper<Iter> prepend(DisiWrapper<Iter> w1, DisiWrapper<Iter> w2) {
+  private DisiWrapper prepend(DisiWrapper w1, DisiWrapper w2) {
     w1.next = w2;
     return w1;
   }
 
-  private DisiWrapper<Iter> topList(DisiWrapper<Iter> list, DisiWrapper<Iter>[] heap,
+  private DisiWrapper topList(DisiWrapper list, DisiWrapper[] heap,
                                     int size, int i) {
-    final DisiWrapper<Iter> w = heap[i];
+    final DisiWrapper w = heap[i];
     if (w.doc == list.doc) {
       list = prepend(w, list);
       final int left = leftNode(i);
@@ -98,8 +96,8 @@ implements Iterable<DisiWrapper<Iter>> {
     return list;
   }
 
-  public DisiWrapper<Iter> add(DisiWrapper<Iter> entry) {
-    final DisiWrapper<Iter>[] heap = this.heap;
+  public DisiWrapper add(DisiWrapper entry) {
+    final DisiWrapper[] heap = this.heap;
     final int size = this.size;
     heap[size] = entry;
     upHeap(size);
@@ -107,9 +105,9 @@ implements Iterable<DisiWrapper<Iter>> {
     return heap[0];
   }
 
-  public DisiWrapper<Iter> pop() {
-    final DisiWrapper<Iter>[] heap = this.heap;
-    final DisiWrapper<Iter> result = heap[0];
+  public DisiWrapper pop() {
+    final DisiWrapper[] heap = this.heap;
+    final DisiWrapper result = heap[0];
     final int i = --size;
     heap[0] = heap[i];
     heap[i] = null;
@@ -117,18 +115,18 @@ implements Iterable<DisiWrapper<Iter>> {
     return result;
   }
 
-  public DisiWrapper<Iter> updateTop() {
+  public DisiWrapper updateTop() {
     downHeap(size);
     return heap[0];
   }
 
-  DisiWrapper<Iter> updateTop(DisiWrapper<Iter> topReplacement) {
+  DisiWrapper updateTop(DisiWrapper topReplacement) {
     heap[0] = topReplacement;
     return updateTop();
   }
 
   void upHeap(int i) {
-    final DisiWrapper<Iter> node = heap[i];
+    final DisiWrapper node = heap[i];
     final int nodeDoc = node.doc;
     int j = parentNode(i);
     while (j >= 0 && nodeDoc < heap[j].doc) {
@@ -141,7 +139,7 @@ implements Iterable<DisiWrapper<Iter>> {
 
   void downHeap(int size) {
     int i = 0;
-    final DisiWrapper<Iter> node = heap[0];
+    final DisiWrapper node = heap[0];
     int j = leftNode(i);
     if (j < size) {
       int k = rightNode(j);
@@ -164,7 +162,7 @@ implements Iterable<DisiWrapper<Iter>> {
   }
 
   @Override
-  public Iterator<DisiWrapper<Iter>> iterator() {
+  public Iterator<DisiWrapper> iterator() {
     return Arrays.asList(heap).subList(0, size).iterator();
   }
 

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisiWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisiWrapper.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisiWrapper.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisiWrapper.java Mon Dec 21 11:39:57 2015
@@ -17,15 +17,19 @@ package org.apache.lucene.search;
  * limitations under the License.
  */
 
+import org.apache.lucene.search.spans.Spans;
+
 /**
  * Wrapper used in {@link DisiPriorityQueue}.
  * @lucene.internal
  */
-public class DisiWrapper<Iter extends DocIdSetIterator> {
-  public final Iter iterator;
+public class DisiWrapper {
+  public final DocIdSetIterator iterator;
+  public final Scorer scorer;
   public final long cost;
+  public final float matchCost; // the match cost for two-phase iterators, 0 otherwise
   public int doc; // the current doc, used for comparison
-  public DisiWrapper<Iter> next; // reference to a next element, see #topList
+  public DisiWrapper next; // reference to a next element, see #topList
 
   // An approximation of the iterator, or the iterator itself if it does not
   // support two-phase iteration
@@ -33,20 +37,43 @@ public class DisiWrapper<Iter extends Do
   // A two-phase view of the iterator, or null if the iterator does not support
   // two-phase iteration
   public final TwoPhaseIterator twoPhaseView;
-  
+
+  // FOR SPANS
+  public final Spans spans;
   public int lastApproxMatchDoc; // last doc of approximation that did match
   public int lastApproxNonMatchDoc; // last doc of approximation that did not match
 
-  public DisiWrapper(Iter iterator) {
-    this.iterator = iterator;
+  public DisiWrapper(Scorer scorer) {
+    this.scorer = scorer;
+    this.spans = null;
+    this.iterator = scorer.iterator();
+    this.cost = iterator.cost();
+    this.doc = -1;
+    this.twoPhaseView = scorer.twoPhaseIterator();
+      
+    if (twoPhaseView != null) {
+      approximation = twoPhaseView.approximation();
+      matchCost = twoPhaseView.matchCost();
+    } else {
+      approximation = iterator;
+      matchCost = 0f;
+    }
+  }
+
+  public DisiWrapper(Spans spans) {
+    this.scorer = null;
+    this.spans = spans;
+    this.iterator = spans;
     this.cost = iterator.cost();
     this.doc = -1;
-    this.twoPhaseView = TwoPhaseIterator.asTwoPhaseIterator(iterator);
+    this.twoPhaseView = spans.asTwoPhaseIterator();
       
     if (twoPhaseView != null) {
       approximation = twoPhaseView.approximation();
+      matchCost = twoPhaseView.matchCost();
     } else {
       approximation = iterator;
+      matchCost = 0f;
     }
     this.lastApproxNonMatchDoc = -2;
     this.lastApproxMatchDoc = -2;

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisjunctionDISIApproximation.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisjunctionDISIApproximation.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisjunctionDISIApproximation.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisjunctionDISIApproximation.java Mon Dec 21 11:39:57 2015
@@ -23,16 +23,15 @@ import java.io.IOException;
  * the provided iterators.
  * @lucene.internal
  */
-public class DisjunctionDISIApproximation<Iter extends DocIdSetIterator>
-extends DocIdSetIterator {
+public class DisjunctionDISIApproximation extends DocIdSetIterator {
 
-  final DisiPriorityQueue<Iter> subIterators;
+  final DisiPriorityQueue subIterators;
   final long cost;
 
-  public DisjunctionDISIApproximation(DisiPriorityQueue<Iter> subIterators) {
+  public DisjunctionDISIApproximation(DisiPriorityQueue subIterators) {
     this.subIterators = subIterators;
     long cost = 0;
-    for (DisiWrapper<Iter> w : subIterators) {
+    for (DisiWrapper w : subIterators) {
       cost += w.cost;
     }
     this.cost = cost;
@@ -50,7 +49,7 @@ extends DocIdSetIterator {
 
   @Override
   public int nextDoc() throws IOException {
-    DisiWrapper<Iter> top = subIterators.top();
+    DisiWrapper top = subIterators.top();
     final int doc = top.doc;
     do {
       top.doc = top.approximation.nextDoc();
@@ -62,7 +61,7 @@ extends DocIdSetIterator {
 
   @Override
   public int advance(int target) throws IOException {
-    DisiWrapper<Iter> top = subIterators.top();
+    DisiWrapper top = subIterators.top();
     do {
       top.doc = top.approximation.advance(target);
       top = subIterators.updateTop();

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java Mon Dec 21 11:39:57 2015
@@ -18,7 +18,9 @@ package org.apache.lucene.search;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
@@ -45,58 +47,36 @@ import org.apache.lucene.index.Term;
 public final class DisjunctionMaxQuery extends Query implements Iterable<Query> {
 
   /* The subqueries */
-  private ArrayList<Query> disjuncts = new ArrayList<>();
+  private final Query[] disjuncts;
 
   /* Multiple of the non-max disjunct scores added into our final score.  Non-zero values support tie-breaking. */
-  private float tieBreakerMultiplier = 0.0f;
-
-  /** Creates a new empty DisjunctionMaxQuery.  Use add() to add the subqueries.
-   * @param tieBreakerMultiplier the score of each non-maximum disjunct for a document is multiplied by this weight
-   *        and added into the final score.  If non-zero, the value should be small, on the order of 0.1, which says that
-   *        10 occurrences of word in a lower-scored field that is also in a higher scored field is just as good as a unique
-   *        word in the lower scored field (i.e., one that is not in any higher scored field.
-   */
-  public DisjunctionMaxQuery(float tieBreakerMultiplier) {
-    this.tieBreakerMultiplier = tieBreakerMultiplier;
-  }
+  private final float tieBreakerMultiplier;
 
   /**
    * Creates a new DisjunctionMaxQuery
    * @param disjuncts a {@code Collection<Query>} of all the disjuncts to add
-   * @param tieBreakerMultiplier   the weight to give to each matching non-maximum disjunct
+   * @param tieBreakerMultiplier  the score of each non-maximum disjunct for a document is multiplied by this weight
+   *        and added into the final score.  If non-zero, the value should be small, on the order of 0.1, which says that
+   *        10 occurrences of word in a lower-scored field that is also in a higher scored field is just as good as a unique
+   *        word in the lower scored field (i.e., one that is not in any higher scored field.
    */
   public DisjunctionMaxQuery(Collection<Query> disjuncts, float tieBreakerMultiplier) {
     Objects.requireNonNull(disjuncts, "Collection of Querys must not be null");
     this.tieBreakerMultiplier = tieBreakerMultiplier;
-    add(disjuncts);
-  }
-
-  /** Add a subquery to this disjunction
-   * @param query the disjunct added
-   */
-  public void add(Query query) {
-    disjuncts.add(Objects.requireNonNull(query, "Query must not be null"));
-  }
-
-  /** Add a collection of disjuncts to this disjunction
-   * via {@code Iterable<Query>}
-   * @param disjuncts a collection of queries to add as disjuncts.
-   */
-  public void add(Collection<Query> disjuncts) {
-    this.disjuncts.addAll(Objects.requireNonNull(disjuncts, "Query connection must not be null"));
+    this.disjuncts = disjuncts.toArray(new Query[disjuncts.size()]);
   }
 
   /** @return An {@code Iterator<Query>} over the disjuncts */
   @Override
   public Iterator<Query> iterator() {
-    return disjuncts.iterator();
+    return getDisjuncts().iterator();
   }
   
   /**
    * @return the disjuncts.
    */
-  public ArrayList<Query> getDisjuncts() {
-    return disjuncts;
+  public List<Query> getDisjuncts() {
+    return Collections.unmodifiableList(Arrays.asList(disjuncts));
   }
 
   /**
@@ -215,21 +195,22 @@ public final class DisjunctionMaxQuery e
    * @return an optimized copy of us (which may not be a copy if there is nothing to optimize) */
   @Override
   public Query rewrite(IndexReader reader) throws IOException {
-    int numDisjunctions = disjuncts.size();
-    if (numDisjunctions == 1) {
-      return disjuncts.get(0);
+    if (disjuncts.length == 1) {
+      return disjuncts[0];
     }
-    DisjunctionMaxQuery rewritten = new DisjunctionMaxQuery(tieBreakerMultiplier);
+
     boolean actuallyRewritten = false;
+    List<Query> rewrittenDisjuncts = new ArrayList<>();
     for (Query sub : disjuncts) {
       Query rewrittenSub = sub.rewrite(reader);
       actuallyRewritten |= rewrittenSub != sub;
-      rewritten.add(rewrittenSub);
+      rewrittenDisjuncts.add(rewrittenSub);
     }
 
     if (actuallyRewritten) {
-      return rewritten;
+      return new DisjunctionMaxQuery(rewrittenDisjuncts, tieBreakerMultiplier);
     }
+
     return super.rewrite(reader);
   }
 
@@ -241,16 +222,15 @@ public final class DisjunctionMaxQuery e
   public String toString(String field) {
     StringBuilder buffer = new StringBuilder();
     buffer.append("(");
-    int numDisjunctions = disjuncts.size();
-    for (int i = 0 ; i < numDisjunctions; i++) {
-      Query subquery = disjuncts.get(i);
+    for (int i = 0 ; i < disjuncts.length; i++) {
+      Query subquery = disjuncts[i];
       if (subquery instanceof BooleanQuery) {   // wrap sub-bools in parens
         buffer.append("(");
         buffer.append(subquery.toString(field));
         buffer.append(")");
       }
       else buffer.append(subquery.toString(field));
-      if (i != numDisjunctions-1) buffer.append(" | ");
+      if (i != disjuncts.length-1) buffer.append(" | ");
     }
     buffer.append(")");
     if (tieBreakerMultiplier != 0.0f) {
@@ -270,7 +250,7 @@ public final class DisjunctionMaxQuery e
     DisjunctionMaxQuery other = (DisjunctionMaxQuery)o;
     return super.equals(o)
             && this.tieBreakerMultiplier == other.tieBreakerMultiplier
-            && this.disjuncts.equals(other.disjuncts);
+            && Arrays.equals(disjuncts, other.disjuncts);
   }
 
   /** Compute a hash code for hashing us
@@ -280,7 +260,7 @@ public final class DisjunctionMaxQuery e
   public int hashCode() {
     int h = super.hashCode();
     h = 31 * h + Float.floatToIntBits(tieBreakerMultiplier);
-    h = 31 * h + disjuncts.hashCode();
+    h = 31 * h + Arrays.hashCode(disjuncts);
     return h;
   }
 

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java Mon Dec 21 11:39:57 2015
@@ -46,11 +46,11 @@ final class DisjunctionMaxScorer extends
   }
 
   @Override
-  protected float score(DisiWrapper<Scorer> topList) throws IOException {
+  protected float score(DisiWrapper topList) throws IOException {
     float scoreSum = 0;
     float scoreMax = 0;
-    for (DisiWrapper<Scorer> w = topList; w != null; w = w.next) {
-      final float subScore = w.iterator.score();
+    for (DisiWrapper w = topList; w != null; w = w.next) {
+      final float subScore = w.scorer.score();
       scoreSum += subScore;
       if (subScore > scoreMax) {
         scoreMax = subScore;

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java Mon Dec 21 11:39:57 2015
@@ -22,95 +22,144 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 
+import org.apache.lucene.util.PriorityQueue;
+
 /**
  * Base class for Scorers that score disjunctions.
  */
 abstract class DisjunctionScorer extends Scorer {
 
   private final boolean needsScores;
-  final DisiPriorityQueue<Scorer> subScorers;
-  private final long cost;
 
-  /** Linked list of scorers which are on the current doc */
-  private DisiWrapper<Scorer> topScorers;
+  private final DisiPriorityQueue subScorers;
+  private final DisjunctionDISIApproximation approximation;
+  private final TwoPhase twoPhase;
 
   protected DisjunctionScorer(Weight weight, List<Scorer> subScorers, boolean needsScores) {
     super(weight);
     if (subScorers.size() <= 1) {
       throw new IllegalArgumentException("There must be at least 2 subScorers");
     }
-    this.subScorers = new DisiPriorityQueue<Scorer>(subScorers.size());
-    long cost = 0;
+    this.subScorers = new DisiPriorityQueue(subScorers.size());
     for (Scorer scorer : subScorers) {
-      final DisiWrapper<Scorer> w = new DisiWrapper<>(scorer);
-      cost += w.cost;
+      final DisiWrapper w = new DisiWrapper(scorer);
       this.subScorers.add(w);
     }
-    this.cost = cost;
     this.needsScores = needsScores;
-  }
+    this.approximation = new DisjunctionDISIApproximation(this.subScorers);
 
-  @Override
-  public TwoPhaseIterator asTwoPhaseIterator() {
     boolean hasApproximation = false;
-    for (DisiWrapper<Scorer> w : subScorers) {
+    float sumMatchCost = 0;
+    long sumApproxCost = 0;
+    // Compute matchCost as the average over the matchCost of the subScorers.
+    // This is weighted by the cost, which is an expected number of matching documents.
+    for (DisiWrapper w : this.subScorers) {
+      long costWeight = (w.cost <= 1) ? 1 : w.cost;
+      sumApproxCost += costWeight;
       if (w.twoPhaseView != null) {
         hasApproximation = true;
-        break;
+        sumMatchCost += w.matchCost * costWeight;
       }
     }
 
-    if (! hasApproximation) {
-      // none of the sub scorers supports approximations
-      return null;
-    }
-
-    // note it is important to share the same pq as this scorer so that
-    // rebalancing the pq through the approximation will also rebalance
-    // the pq in this scorer.
-    return new TwoPhaseIterator(new DisjunctionDISIApproximation<Scorer>(subScorers)) {
-
-      @Override
-      public boolean matches() throws IOException {
-        DisiWrapper<Scorer> topScorers = subScorers.topList();
-        // remove the head of the list as long as it does not match
-        while (topScorers.twoPhaseView != null && ! topScorers.twoPhaseView.matches()) {
-          topScorers = topScorers.next;
-          if (topScorers == null) {
-            return false;
-          }
+    if (hasApproximation == false) { // no sub scorer supports approximations
+      twoPhase = null;
+    } else {
+      final float matchCost = sumMatchCost / sumApproxCost;
+      twoPhase = new TwoPhase(approximation, matchCost);
+    }
+  }
+
+  @Override
+  public DocIdSetIterator iterator() {
+    if (twoPhase != null) {
+      return TwoPhaseIterator.asDocIdSetIterator(twoPhase);
+    } else {
+      return approximation;
+    }
+  }
+
+  @Override
+  public TwoPhaseIterator twoPhaseIterator() {
+    return twoPhase;
+  }
+
+  private class TwoPhase extends TwoPhaseIterator {
+
+    private final float matchCost;
+    // list of verified matches on the current doc
+    DisiWrapper verifiedMatches;
+    // priority queue of approximations on the current doc that have not been verified yet
+    final PriorityQueue<DisiWrapper> unverifiedMatches;
+
+    private TwoPhase(DocIdSetIterator approximation, float matchCost) {
+      super(approximation);
+      this.matchCost = matchCost;
+      unverifiedMatches = new PriorityQueue<DisiWrapper>(DisjunctionScorer.this.subScorers.size()) {
+        @Override
+        protected boolean lessThan(DisiWrapper a, DisiWrapper b) {
+          return a.matchCost < b.matchCost;
+        }
+      };
+    }
+
+    DisiWrapper getSubMatches() throws IOException {
+      // iteration order does not matter
+      for (DisiWrapper w : unverifiedMatches) {
+        if (w.twoPhaseView.matches()) {
+          w.next = verifiedMatches;
+          verifiedMatches = w;
         }
-        // now we know we have at least one match since the first element of 'matchList' matches
-        if (needsScores) {
-          // if scores or freqs are needed, we also need to remove scorers
-          // from the top list that do not actually match
-          DisiWrapper<Scorer> previous = topScorers;
-          for (DisiWrapper<Scorer> w = topScorers.next; w != null; w = w.next) {
-            if (w.twoPhaseView != null && ! w.twoPhaseView.matches()) {
-              // w does not match, remove it
-              previous.next = w.next;
-            } else {
-              previous = w;
-            }
+      }
+      unverifiedMatches.clear();
+      return verifiedMatches;
+    }
+    
+    @Override
+    public boolean matches() throws IOException {
+      verifiedMatches = null;
+      unverifiedMatches.clear();
+      
+      for (DisiWrapper w = subScorers.topList(); w != null; ) {
+        DisiWrapper next = w.next;
+        
+        if (w.twoPhaseView == null) {
+          // implicitly verified, move it to verifiedMatches
+          w.next = verifiedMatches;
+          verifiedMatches = w;
+          
+          if (needsScores == false) {
+            // we can stop here
+            return true;
           }
         } else {
-          // since we don't need scores, let's pretend we have a single match
-          topScorers.next = null;
+          unverifiedMatches.add(w);
         }
-
-        // We need to explicitely set the list of top scorers to avoid the
-        // laziness of DisjunctionScorer.score() that would take all scorers
-        // positioned on the same doc as the top of the pq, including
-        // non-matching scorers
-        DisjunctionScorer.this.topScorers = topScorers;
+        w = next;
+      }
+      
+      if (verifiedMatches != null) {
         return true;
       }
-    };
-  }
-
-  @Override
-  public final long cost() {
-    return cost;
+      
+      // verify subs that have an two-phase iterator
+      // least-costly ones first
+      while (unverifiedMatches.size() > 0) {
+        DisiWrapper w = unverifiedMatches.pop();
+        if (w.twoPhaseView.matches()) {
+          w.next = null;
+          verifiedMatches = w;
+          return true;
+        }
+      }
+      
+      return false;
+    }
+    
+    @Override
+    public float matchCost() {
+      return matchCost;
+    }
   }
 
   @Override
@@ -118,38 +167,19 @@ abstract class DisjunctionScorer extends
    return subScorers.top().doc;
   }
 
-  @Override
-  public final int nextDoc() throws IOException {
-    topScorers = null;
-    DisiWrapper<Scorer> top = subScorers.top();
-    final int doc = top.doc;
-    do {
-      top.doc = top.iterator.nextDoc();
-      top = subScorers.updateTop();
-    } while (top.doc == doc);
-
-    return top.doc;
-  }
-
-  @Override
-  public final int advance(int target) throws IOException {
-    topScorers = null;
-    DisiWrapper<Scorer> top = subScorers.top();
-    do {
-      top.doc = top.iterator.advance(target);
-      top = subScorers.updateTop();
-    } while (top.doc < target);
-
-    return top.doc;
+  DisiWrapper getSubMatches() throws IOException {
+    if (twoPhase == null) {
+      return subScorers.topList();
+    } else {
+      return twoPhase.getSubMatches();
+    }
   }
 
   @Override
   public final int freq() throws IOException {
-    if (topScorers == null) {
-      topScorers = subScorers.topList();
-    }
+    DisiWrapper subMatches = getSubMatches();
     int freq = 1;
-    for (DisiWrapper<Scorer> w = topScorers.next; w != null; w = w.next) {
+    for (DisiWrapper w = subMatches.next; w != null; w = w.next) {
       freq += 1;
     }
     return freq;
@@ -157,20 +187,17 @@ abstract class DisjunctionScorer extends
 
   @Override
   public final float score() throws IOException {
-    if (topScorers == null) {
-      topScorers = subScorers.topList();
-    }
-    return score(topScorers);
+    return score(getSubMatches());
   }
 
   /** Compute the score for the given linked list of scorers. */
-  protected abstract float score(DisiWrapper<Scorer> topList) throws IOException;
+  protected abstract float score(DisiWrapper topList) throws IOException;
 
   @Override
   public final Collection<ChildScorer> getChildren() {
     ArrayList<ChildScorer> children = new ArrayList<>();
-    for (DisiWrapper<Scorer> scorer : subScorers) {
-      children.add(new ChildScorer(scorer.iterator, "SHOULD"));
+    for (DisiWrapper scorer : subScorers) {
+      children.add(new ChildScorer(scorer.scorer, "SHOULD"));
     }
     return children;
   }

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java Mon Dec 21 11:39:57 2015
@@ -21,7 +21,6 @@ import java.io.IOException;
 import java.util.List;
 
 /** A Scorer for OR like queries, counterpart of <code>ConjunctionScorer</code>.
- * This Scorer implements {@link Scorer#advance(int)} and uses advance() on the given Scorers. 
  */
 final class DisjunctionSumScorer extends DisjunctionScorer { 
   private final float[] coord;
@@ -37,11 +36,11 @@ final class DisjunctionSumScorer extends
   }
 
   @Override
-  protected float score(DisiWrapper<Scorer> topList) throws IOException {
+  protected float score(DisiWrapper topList) throws IOException {
     double score = 0;
     int freq = 0;
-    for (DisiWrapper<Scorer> w = topList; w != null; w = w.next) {
-      score += w.iterator.score();
+    for (DisiWrapper w = topList; w != null; w = w.next) {
+      score += w.scorer.score();
       freq += 1;
     }
     return (float)score * coord[freq];

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DocIdSetIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DocIdSetIterator.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DocIdSetIterator.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/DocIdSetIterator.java Mon Dec 21 11:39:57 2015
@@ -19,8 +19,6 @@ package org.apache.lucene.search;
 
 import java.io.IOException;
 
-import org.apache.lucene.search.spans.Spans;
-
 /**
  * This abstract class defines methods to iterate over a set of non-decreasing
  * doc ids. Note that this class assumes it iterates on doc Ids, and therefore

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java Mon Dec 21 11:39:57 2015
@@ -44,9 +44,11 @@ final class ExactPhraseScorer extends Sc
 
   private final Similarity.SimScorer docScorer;
   private final boolean needsScores;
+  private float matchCost;
 
   ExactPhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings,
-                    Similarity.SimScorer docScorer, boolean needsScores) throws IOException {
+                    Similarity.SimScorer docScorer, boolean needsScores,
+                    float matchCost) throws IOException {
     super(weight);
     this.docScorer = docScorer;
     this.needsScores = needsScores;
@@ -57,36 +59,29 @@ final class ExactPhraseScorer extends Sc
       iterators.add(posting.postings);
       postingsAndPositions.add(new PostingsAndPosition(posting.postings, posting.position));
     }
-    conjunction = ConjunctionDISI.intersect(iterators);
+    conjunction = ConjunctionDISI.intersectIterators(iterators);
     this.postings = postingsAndPositions.toArray(new PostingsAndPosition[postingsAndPositions.size()]);
+    this.matchCost = matchCost;
   }
 
   @Override
-  public TwoPhaseIterator asTwoPhaseIterator() {
+  public TwoPhaseIterator twoPhaseIterator() {
     return new TwoPhaseIterator(conjunction) {
       @Override
       public boolean matches() throws IOException {
         return phraseFreq() > 0;
       }
-    };
-  }
 
-  private int doNext(int doc) throws IOException {
-    for (;; doc = conjunction.nextDoc()) {
-      if (doc == NO_MORE_DOCS || phraseFreq() > 0) {
-        return doc;
+      @Override
+      public float matchCost() {
+        return matchCost;
       }
-    }
-  }
-
-  @Override
-  public int nextDoc() throws IOException {
-    return doNext(conjunction.nextDoc());
+    };
   }
 
   @Override
-  public int advance(int target) throws IOException {
-    return doNext(conjunction.advance(target));
+  public DocIdSetIterator iterator() {
+    return TwoPhaseIterator.asDocIdSetIterator(twoPhaseIterator());
   }
 
   @Override
@@ -172,8 +167,4 @@ final class ExactPhraseScorer extends Sc
     return this.freq = freq;
   }
 
-  @Override
-  public long cost() {
-    return conjunction.cost();
-  }
 }

Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/FakeScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/FakeScorer.java?rev=1721134&r1=1721133&r2=1721134&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/FakeScorer.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/search/FakeScorer.java Mon Dec 21 11:39:57 2015
@@ -29,11 +29,6 @@ final class FakeScorer extends Scorer {
   public FakeScorer() {
     super(null);
   }
-    
-  @Override
-  public int advance(int target) {
-    throw new UnsupportedOperationException("FakeScorer doesn't support advance(int)");
-  }
 
   @Override
   public int docID() {
@@ -44,11 +39,6 @@ final class FakeScorer extends Scorer {
   public int freq() {
     return freq;
   }
-
-  @Override
-  public int nextDoc() {
-    throw new UnsupportedOperationException("FakeScorer doesn't support nextDoc()");
-  }
     
   @Override
   public float score() {
@@ -56,8 +46,8 @@ final class FakeScorer extends Scorer {
   }
 
   @Override
-  public long cost() {
-    return 1;
+  public DocIdSetIterator iterator() {
+    throw new UnsupportedOperationException();
   }
 
   @Override




Mime
View raw message