lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1596817 [2/5] - in /lucene/dev/branches/lucene5675: ./ dev-tools/ dev-tools/idea/lucene/spatial/ dev-tools/idea/solr/contrib/analysis-extras/ dev-tools/scripts/ lucene/ lucene/codecs/ lucene/codecs/src/java/org/apache/lucene/codecs/memory/...
Date Thu, 22 May 2014 11:38:49 GMT
Modified: lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java Thu May 22 11:38:47 2014
@@ -33,6 +33,7 @@ import java.util.Set;
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.FieldInfosFormat;
 import org.apache.lucene.codecs.LiveDocsFormat;
 import org.apache.lucene.store.ChecksumIndexInput;
@@ -44,68 +45,79 @@ import org.apache.lucene.store.NoSuchDir
 import org.apache.lucene.util.IOUtils;
 
 /**
- * A collection of segmentInfo objects with methods for operating on
- * those segments in relation to the file system.
+ * A collection of segmentInfo objects with methods for operating on those
+ * segments in relation to the file system.
  * <p>
  * The active segments in the index are stored in the segment info file,
- * <tt>segments_N</tt>. There may be one or more <tt>segments_N</tt> files in the
- * index; however, the one with the largest generation is the active one (when
- * older segments_N files are present it's because they temporarily cannot be
- * deleted, or, a writer is in the process of committing, or a custom 
- * {@link org.apache.lucene.index.IndexDeletionPolicy IndexDeletionPolicy}
- * is in use). This file lists each segment by name and has details about the
- * codec and generation of deletes.
+ * <tt>segments_N</tt>. There may be one or more <tt>segments_N</tt> files in
+ * the index; however, the one with the largest generation is the active one
+ * (when older segments_N files are present it's because they temporarily cannot
+ * be deleted, or, a writer is in the process of committing, or a custom
+ * {@link org.apache.lucene.index.IndexDeletionPolicy IndexDeletionPolicy} is in
+ * use). This file lists each segment by name and has details about the codec
+ * and generation of deletes.
+ * </p>
+ * <p>
+ * There is also a file <tt>segments.gen</tt>. This file contains the current
+ * generation (the <tt>_N</tt> in <tt>segments_N</tt>) of the index. This is
+ * used only as a fallback in case the current generation cannot be accurately
+ * determined by directory listing alone (as is the case for some NFS clients
+ * with time-based directory cache expiration). This file simply contains an
+ * {@link DataOutput#writeInt Int32} version header (
+ * {@link #FORMAT_SEGMENTS_GEN_CURRENT}), followed by the generation recorded as
+ * {@link DataOutput#writeLong Int64}, written twice.
  * </p>
- * <p>There is also a file <tt>segments.gen</tt>. This file contains
- * the current generation (the <tt>_N</tt> in <tt>segments_N</tt>) of the index.
- * This is used only as a fallback in case the current generation cannot be
- * accurately determined by directory listing alone (as is the case for some NFS
- * clients with time-based directory cache expiration). This file simply contains
- * an {@link DataOutput#writeInt Int32} version header 
- * ({@link #FORMAT_SEGMENTS_GEN_CURRENT}), followed by the
- * generation recorded as {@link DataOutput#writeLong Int64}, written twice.</p>
  * <p>
  * Files:
  * <ul>
- *   <li><tt>segments.gen</tt>: GenHeader, Generation, Generation, Footer
- *   <li><tt>segments_N</tt>: Header, Version, NameCounter, SegCount,
- *    &lt;SegName, SegCodec, DelGen, DeletionCount, FieldInfosGen, UpdatesFiles&gt;<sup>SegCount</sup>, 
- *    CommitUserData, Footer
+ * <li><tt>segments.gen</tt>: GenHeader, Generation, Generation, Footer
+ * <li><tt>segments_N</tt>: Header, Version, NameCounter, SegCount, &lt;SegName,
+ * SegCodec, DelGen, DeletionCount, FieldInfosGen, DocValuesGen,
+ * UpdatesFiles&gt;<sup>SegCount</sup>, CommitUserData, Footer
  * </ul>
  * </p>
  * Data types:
  * <p>
  * <ul>
- *   <li>Header --&gt; {@link CodecUtil#writeHeader CodecHeader}</li>
- *   <li>GenHeader, NameCounter, SegCount, DeletionCount --&gt; {@link DataOutput#writeInt Int32}</li>
- *   <li>Generation, Version, DelGen, Checksum, FieldInfosGen --&gt; {@link DataOutput#writeLong Int64}</li>
- *   <li>SegName, SegCodec --&gt; {@link DataOutput#writeString String}</li>
- *   <li>CommitUserData --&gt; {@link DataOutput#writeStringStringMap Map&lt;String,String&gt;}</li>
- *   <li>UpdatesFiles --&gt; {@link DataOutput#writeStringSet(Set) Set&lt;String&gt;}</li>
- *   <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}</li>
+ * <li>Header --&gt; {@link CodecUtil#writeHeader CodecHeader}</li>
+ * <li>GenHeader, NameCounter, SegCount, DeletionCount --&gt;
+ * {@link DataOutput#writeInt Int32}</li>
+ * <li>Generation, Version, DelGen, Checksum, FieldInfosGen, DocValuesGen --&gt;
+ * {@link DataOutput#writeLong Int64}</li>
+ * <li>SegName, SegCodec --&gt; {@link DataOutput#writeString String}</li>
+ * <li>CommitUserData --&gt; {@link DataOutput#writeStringStringMap
+ * Map&lt;String,String&gt;}</li>
+ * <li>UpdatesFiles --&gt; Map&lt;{@link DataOutput#writeInt Int32},
+ * {@link DataOutput#writeStringSet(Set) Set&lt;String&gt;}&gt;</li>
+ * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}</li>
  * </ul>
  * </p>
  * Field Descriptions:
  * <p>
  * <ul>
- *   <li>Version counts how often the index has been changed by adding or deleting
- *       documents.</li>
- *   <li>NameCounter is used to generate names for new segment files.</li>
- *   <li>SegName is the name of the segment, and is used as the file name prefix for
- *       all of the files that compose the segment's index.</li>
- *   <li>DelGen is the generation count of the deletes file. If this is -1,
- *       there are no deletes. Anything above zero means there are deletes 
- *       stored by {@link LiveDocsFormat}.</li>
- *   <li>DeletionCount records the number of deleted documents in this segment.</li>
- *   <li>SegCodec is the {@link Codec#getName() name} of the Codec that encoded
- *       this segment.</li>
- *   <li>CommitUserData stores an optional user-supplied opaque
- *       Map&lt;String,String&gt; that was passed to 
- *       {@link IndexWriter#setCommitData(java.util.Map)}.</li>
- *   <li>FieldInfosGen is the generation count of the fieldInfos file. If this is -1,
- *       there are no updates to the fieldInfos in that segment. Anything above zero 
- *       means there are updates to fieldInfos stored by {@link FieldInfosFormat}.</li>
- *   <li>UpdatesFiles stores the list of files that were updated in that segment.</li>
+ * <li>Version counts how often the index has been changed by adding or deleting
+ * documents.</li>
+ * <li>NameCounter is used to generate names for new segment files.</li>
+ * <li>SegName is the name of the segment, and is used as the file name prefix
+ * for all of the files that compose the segment's index.</li>
+ * <li>DelGen is the generation count of the deletes file. If this is -1, there
+ * are no deletes. Anything above zero means there are deletes stored by
+ * {@link LiveDocsFormat}.</li>
+ * <li>DeletionCount records the number of deleted documents in this segment.</li>
+ * <li>SegCodec is the {@link Codec#getName() name} of the Codec that encoded
+ * this segment.</li>
+ * <li>CommitUserData stores an optional user-supplied opaque
+ * Map&lt;String,String&gt; that was passed to
+ * {@link IndexWriter#setCommitData(java.util.Map)}.</li>
+ * <li>FieldInfosGen is the generation count of the fieldInfos file. If this is
+ * -1, there are no updates to the fieldInfos in that segment. Anything above
+ * zero means there are updates to fieldInfos stored by {@link FieldInfosFormat}
+ * .</li>
+ * <li>DocValuesGen is the generation count of the updatable DocValues. If this
+ * is -1, there are no updates to DocValues in that segment. Anything above zero
+ * means there are updates to DocValues stored by {@link DocValuesFormat}.</li>
+ * <li>UpdatesFiles stores the set of files that were updated in that segment
+ * per field.</li>
  * </ul>
  * </p>
  * 
@@ -121,6 +133,9 @@ public final class SegmentInfos implemen
   
   /** The file format version for the segments_N codec header, since 4.8+ */
   public static final int VERSION_48 = 2;
+  
+  /** The file format version for the segments_N codec header, since 4.9+ */
+  public static final int VERSION_49 = 3;
 
   // Used for the segments.gen file only!
   // Whenever you add a new format, make it 1 smaller (negative version logic)!
@@ -330,7 +345,7 @@ public final class SegmentInfos implemen
         throw new IndexFormatTooOldException(input, magic, CodecUtil.CODEC_MAGIC, CodecUtil.CODEC_MAGIC);
       }
       // 4.0+
-      int format = CodecUtil.checkHeaderNoMagic(input, "segments", VERSION_40, VERSION_48);
+      int format = CodecUtil.checkHeaderNoMagic(input, "segments", VERSION_40, VERSION_49);
       version = input.readLong();
       counter = input.readInt();
       int numSegments = input.readInt();
@@ -352,19 +367,45 @@ public final class SegmentInfos implemen
         if (format >= VERSION_46) {
           fieldInfosGen = input.readLong();
         }
-        SegmentCommitInfo siPerCommit = new SegmentCommitInfo(info, delCount, delGen, fieldInfosGen);
+        long dvGen = -1;
+        if (format >= VERSION_49) {
+          dvGen = input.readLong();
+        } else {
+          dvGen = fieldInfosGen;
+        }
+        SegmentCommitInfo siPerCommit = new SegmentCommitInfo(info, delCount, delGen, fieldInfosGen, dvGen);
         if (format >= VERSION_46) {
-          int numGensUpdatesFiles = input.readInt();
-          final Map<Long,Set<String>> genUpdatesFiles;
-          if (numGensUpdatesFiles == 0) {
-            genUpdatesFiles = Collections.emptyMap();
+          if (format < VERSION_49) {
+            // Recorded per-generation files, which were buggy (see
+            // LUCENE-5636). We need to read and keep them so we continue to
+            // reference those files. Unfortunately it means that the files will
+            // be referenced even if the fields are updated again, until the
+            // segment is merged.
+            final int numGensUpdatesFiles = input.readInt();
+            final Map<Long,Set<String>> genUpdatesFiles;
+            if (numGensUpdatesFiles == 0) {
+              genUpdatesFiles = Collections.emptyMap();
+            } else {
+              genUpdatesFiles = new HashMap<>(numGensUpdatesFiles);
+              for (int i = 0; i < numGensUpdatesFiles; i++) {
+                genUpdatesFiles.put(input.readLong(), input.readStringSet());
+              }
+            }
+            siPerCommit.setGenUpdatesFiles(genUpdatesFiles);
           } else {
-            genUpdatesFiles = new HashMap<>(numGensUpdatesFiles);
-            for (int i = 0; i < numGensUpdatesFiles; i++) {
-              genUpdatesFiles.put(input.readLong(), input.readStringSet());
+            siPerCommit.setFieldInfosFiles(input.readStringSet());
+            final Map<Integer,Set<String>> dvUpdateFiles;
+            final int numDVFields = input.readInt();
+            if (numDVFields == 0) {
+              dvUpdateFiles = Collections.emptyMap();
+            } else {
+              dvUpdateFiles = new HashMap<>(numDVFields);
+              for (int i = 0; i < numDVFields; i++) {
+                dvUpdateFiles.put(input.readInt(), input.readStringSet());
+              }
             }
+            siPerCommit.setDocValuesUpdatesFiles(dvUpdateFiles);
           }
-          siPerCommit.setGenUpdatesFiles(genUpdatesFiles);
         }
         add(siPerCommit);
       }
@@ -429,7 +470,7 @@ public final class SegmentInfos implemen
 
     try {
       segnOutput = directory.createOutput(segmentFileName, IOContext.DEFAULT);
-      CodecUtil.writeHeader(segnOutput, "segments", VERSION_48);
+      CodecUtil.writeHeader(segnOutput, "segments", VERSION_49);
       segnOutput.writeLong(version); 
       segnOutput.writeInt(counter); // write counter
       segnOutput.writeInt(size()); // write infos
@@ -444,10 +485,12 @@ public final class SegmentInfos implemen
         }
         segnOutput.writeInt(delCount);
         segnOutput.writeLong(siPerCommit.getFieldInfosGen());
-        final Map<Long,Set<String>> genUpdatesFiles = siPerCommit.getUpdatesFiles();
-        segnOutput.writeInt(genUpdatesFiles.size());
-        for (Entry<Long,Set<String>> e : genUpdatesFiles.entrySet()) {
-          segnOutput.writeLong(e.getKey());
+        segnOutput.writeLong(siPerCommit.getDocValuesGen());
+        segnOutput.writeStringSet(siPerCommit.getFieldInfosFiles());
+        final Map<Integer,Set<String>> dvUpdatesFiles = siPerCommit.getDocValuesUpdatesFiles();
+        segnOutput.writeInt(dvUpdatesFiles.size());
+        for (Entry<Integer,Set<String>> e : dvUpdatesFiles.entrySet()) {
+          segnOutput.writeInt(e.getKey());
           segnOutput.writeStringSet(e.getValue());
         }
         assert si.dir == directory;

Modified: lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java Thu May 22 11:38:47 2014
@@ -23,7 +23,6 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.IdentityHashMap;
 import java.util.List;
-import java.util.Map.Entry;
 import java.util.Map;
 import java.util.Set;
 
@@ -41,6 +40,7 @@ import org.apache.lucene.store.IOContext
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.CloseableThreadLocal;
 import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.Version;
 
 /**
  * IndexReader implementation over a single segment. 
@@ -112,7 +112,7 @@ public final class SegmentReader extends
         liveDocs = null;
       }
       numDocs = si.info.getDocCount() - si.getDelCount();
-      
+
       if (fieldInfos.hasDocValues()) {
         initDocValuesProducers(codec);
       }
@@ -175,24 +175,88 @@ public final class SegmentReader extends
   }
 
   // initialize the per-field DocValuesProducer
+  @SuppressWarnings("deprecation")
   private void initDocValuesProducers(Codec codec) throws IOException {
     final Directory dir = core.cfsReader != null ? core.cfsReader : si.info.dir;
     final DocValuesFormat dvFormat = codec.docValuesFormat();
-    final Map<Long,List<FieldInfo>> genInfos = getGenInfos();
-    
-//      System.out.println("[" + Thread.currentThread().getName() + "] SR.initDocValuesProducers: segInfo=" + si + "; gens=" + genInfos.keySet());
-    
-    // TODO: can we avoid iterating over fieldinfos several times and creating maps of all this stuff if dv updates do not exist?
-    
-    for (Entry<Long,List<FieldInfo>> e : genInfos.entrySet()) {
-      Long gen = e.getKey();
-      List<FieldInfo> infos = e.getValue();
-      DocValuesProducer dvp = segDocValues.getDocValuesProducer(gen, si, IOContext.READ, dir, dvFormat, infos);
-      dvGens.add(gen);
-      for (FieldInfo fi : infos) {
+
+    if (!si.hasFieldUpdates()) {
+      // simple case, no DocValues updates
+      final DocValuesProducer dvp = segDocValues.getDocValuesProducer(-1L, si, IOContext.READ, dir, dvFormat, fieldInfos);
+      dvGens.add(-1L);
+      dvProducers.add(dvp);
+      for (FieldInfo fi : fieldInfos) {
+        if (!fi.hasDocValues()) continue;
+        assert fi.getDocValuesGen() == -1;
         dvProducersByField.put(fi.name, dvp);
       }
-      dvProducers.add(dvp);
+      return;
+    }
+
+    Version ver;
+    try {
+      ver = Version.parseLeniently(si.info.getVersion());
+    } catch (IllegalArgumentException e) {
+      // happened in TestBackwardsCompatibility on a 4.0.0.2 index (no matching
+      // Version constant), anyway it's a pre-4.9 index.
+      ver = null;
+    }
+    if (ver != null && ver.onOrAfter(Version.LUCENE_4_9)) {
+      DocValuesProducer baseProducer = null;
+      for (FieldInfo fi : fieldInfos) {
+        if (!fi.hasDocValues()) continue;
+        long docValuesGen = fi.getDocValuesGen();
+        if (docValuesGen == -1) {
+          if (baseProducer == null) {
+//        System.out.println("[" + Thread.currentThread().getName() + "] SR.initDocValuesProducers: segInfo=" + si + "; gen=" + docValuesGen + "; field=" + fi.name);
+            // the base producer gets all the fields, so the Codec can validate properly
+            baseProducer = segDocValues.getDocValuesProducer(docValuesGen, si, IOContext.READ, dir, dvFormat, fieldInfos);
+            dvGens.add(docValuesGen);
+            dvProducers.add(baseProducer);
+          }
+//        System.out.println("[" + Thread.currentThread().getName() + "] SR.initDocValuesProducers: segInfo=" + si + "; gen=" + docValuesGen + "; field=" + fi.name);
+          dvProducersByField.put(fi.name, baseProducer);
+        } else {
+          assert !dvGens.contains(docValuesGen);
+//        System.out.println("[" + Thread.currentThread().getName() + "] SR.initDocValuesProducers: segInfo=" + si + "; gen=" + docValuesGen + "; field=" + fi.name);
+          final DocValuesProducer dvp = segDocValues.getDocValuesProducer(docValuesGen, si, IOContext.READ, dir, dvFormat, new FieldInfos(new FieldInfo[] { fi }));
+          dvGens.add(docValuesGen);
+          dvProducers.add(dvp);
+          dvProducersByField.put(fi.name, dvp);
+        }
+      }
+    } else {
+      // For pre-4.9 indexes, especially with doc-values updates, multiple
+      // FieldInfos could belong to the same dvGen. Therefore need to make sure
+      // we initialize each DocValuesProducer once per gen.
+      Map<Long,List<FieldInfo>> genInfos = new HashMap<>();
+      for (FieldInfo fi : fieldInfos) {
+        if (!fi.hasDocValues()) continue;
+        List<FieldInfo> genFieldInfos = genInfos.get(fi.getDocValuesGen());
+        if (genFieldInfos == null) {
+          genFieldInfos = new ArrayList<>();
+          genInfos.put(fi.getDocValuesGen(), genFieldInfos);
+        }
+        genFieldInfos.add(fi);
+      }
+      
+      for (Map.Entry<Long,List<FieldInfo>> e : genInfos.entrySet()) {
+        long docValuesGen = e.getKey();
+        List<FieldInfo> infos = e.getValue();
+        final DocValuesProducer dvp;
+        if (docValuesGen == -1) {
+          // we need to send all FieldInfos to gen=-1, but later we need to
+          // record the DVP only for the "true" gen=-1 fields (not updated)
+          dvp = segDocValues.getDocValuesProducer(docValuesGen, si, IOContext.READ, dir, dvFormat, fieldInfos);
+        } else {
+          dvp = segDocValues.getDocValuesProducer(docValuesGen, si, IOContext.READ, dir, dvFormat, new FieldInfos(infos.toArray(new FieldInfo[infos.size()])));
+        }
+        dvGens.add(docValuesGen);
+        dvProducers.add(dvp);
+        for (FieldInfo fi : infos) {
+          dvProducersByField.put(fi.name, dvp);
+        }
+      }
     }
   }
   
@@ -229,24 +293,6 @@ public final class SegmentReader extends
     }
   }
   
-  // returns a gen->List<FieldInfo> mapping. Fields without DV updates have gen=-1
-  private Map<Long,List<FieldInfo>> getGenInfos() {
-    final Map<Long,List<FieldInfo>> genInfos = new HashMap<>();
-    for (FieldInfo fi : fieldInfos) {
-      if (fi.getDocValuesType() == null) {
-        continue;
-      }
-      long gen = fi.getDocValuesGen();
-      List<FieldInfo> infos = genInfos.get(gen);
-      if (infos == null) {
-        infos = new ArrayList<>();
-        genInfos.put(gen, infos);
-      }
-      infos.add(fi);
-    }
-    return genInfos;
-  }
-
   @Override
   public Bits getLiveDocs() {
     ensureOpen();

Modified: lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/index/TrackingIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/index/TrackingIndexWriter.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/index/TrackingIndexWriter.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/index/TrackingIndexWriter.java Thu May 22 11:38:47 2014
@@ -83,7 +83,7 @@ public class TrackingIndexWriter {
     return indexingGen.get();
   }
 
-  /** Calls {@link IndexWriter#deleteDocuments(Term)} and
+  /** Calls {@link IndexWriter#deleteDocuments(Term...)} and
    *  returns the generation that reflects this change. */
   public long deleteDocuments(Term t) throws IOException {
     writer.deleteDocuments(t);
@@ -99,7 +99,7 @@ public class TrackingIndexWriter {
     return indexingGen.get();
   }
 
-  /** Calls {@link IndexWriter#deleteDocuments(Query)} and
+  /** Calls {@link IndexWriter#deleteDocuments(Query...)} and
    *  returns the generation that reflects this change. */
   public long deleteDocuments(Query q) throws IOException {
     writer.deleteDocuments(q);

Modified: lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java Thu May 22 11:38:47 2014
@@ -19,6 +19,7 @@ package org.apache.lucene.search;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
@@ -341,6 +342,11 @@ public class BooleanQuery extends Query 
     @Override
     public Scorer scorer(AtomicReaderContext context, Bits acceptDocs)
         throws IOException {
+      // initially the user provided value,
+      // but if minNrShouldMatch == optional.size(),
+      // we will optimize and move these to required, making this 0
+      int minShouldMatch = minNrShouldMatch;
+
       List<Scorer> required = new ArrayList<>();
       List<Scorer> prohibited = new ArrayList<>();
       List<Scorer> optional = new ArrayList<>();
@@ -360,34 +366,69 @@ public class BooleanQuery extends Query 
           optional.add(subScorer);
         }
       }
-
-      if (required.size() == 0 && optional.size() == 0) {
+      
+      // scorer simplifications:
+      
+      if (optional.size() == minShouldMatch) {
+        // any optional clauses are in fact required
+        required.addAll(optional);
+        optional.clear();
+        minShouldMatch = 0;
+      }
+      
+      if (required.isEmpty() && optional.isEmpty()) {
         // no required and optional clauses.
         return null;
-      } else if (optional.size() < minNrShouldMatch) {
+      } else if (optional.size() < minShouldMatch) {
         // either >1 req scorer, or there are 0 req scorers and at least 1
         // optional scorer. Therefore if there are not enough optional scorers
         // no documents will be matched by the query
         return null;
       }
       
-      // simple conjunction
-      if (optional.size() == 0 && prohibited.size() == 0) {
-        float coord = disableCoord ? 1.0f : coord(required.size(), maxCoord);
-        return new ConjunctionScorer(this, required.toArray(new Scorer[required.size()]), coord);
+      // three cases: conjunction, disjunction, or mix
+      
+      // pure conjunction
+      if (optional.isEmpty()) {
+        return excl(req(required, disableCoord), prohibited);
       }
       
-      // simple disjunction
-      if (required.size() == 0 && prohibited.size() == 0 && minNrShouldMatch <= 1 && optional.size() > 1) {
-        float coord[] = new float[optional.size()+1];
-        for (int i = 0; i < coord.length; i++) {
-          coord[i] = disableCoord ? 1.0f : coord(i, maxCoord);
-        }
-        return new DisjunctionSumScorer(this, optional.toArray(new Scorer[optional.size()]), coord);
+      // pure disjunction
+      if (required.isEmpty()) {
+        return excl(opt(optional, minShouldMatch, disableCoord), prohibited);
       }
       
-      // Return a BooleanScorer2
-      return new BooleanScorer2(this, disableCoord, minNrShouldMatch, required, prohibited, optional, maxCoord);
+      // conjunction-disjunction mix:
+      // we create the required and optional pieces with coord disabled, and then
+      // combine the two: if minNrShouldMatch > 0, then its a conjunction: because the
+      // optional side must match. otherwise its required + optional, factoring the
+      // number of optional terms into the coord calculation
+      
+      Scorer req = excl(req(required, true), prohibited);
+      Scorer opt = opt(optional, minShouldMatch, true);
+
+      // TODO: clean this up: its horrible
+      if (disableCoord) {
+        if (minShouldMatch > 0) {
+          return new ConjunctionScorer(this, new Scorer[] { req, opt }, 1F);
+        } else {
+          return new ReqOptSumScorer(req, opt);          
+        }
+      } else if (optional.size() == 1) {
+        if (minShouldMatch > 0) {
+          return new ConjunctionScorer(this, new Scorer[] { req, opt }, coord(required.size()+1, maxCoord));
+        } else {
+          float coordReq = coord(required.size(), maxCoord);
+          float coordBoth = coord(required.size() + 1, maxCoord);
+          return new BooleanTopLevelScorers.ReqSingleOptScorer(req, opt, coordReq, coordBoth);
+        }
+      } else {
+        if (minShouldMatch > 0) {
+          return new BooleanTopLevelScorers.CoordinatingConjunctionScorer(this, coords(), req, required.size(), opt);
+        } else {
+          return new BooleanTopLevelScorers.ReqMultiOptScorer(req, opt, required.size(), coords()); 
+        }
+      }
     }
     
     @Override
@@ -396,17 +437,89 @@ public class BooleanQuery extends Query 
         // BS2 (in-order) will be used by scorer()
         return false;
       }
+      int optionalCount = 0;
       for (BooleanClause c : clauses) {
         if (c.isRequired()) {
           // BS2 (in-order) will be used by scorer()
           return false;
+        } else if (!c.isProhibited()) {
+          optionalCount++;
         }
       }
       
+      if (optionalCount == minNrShouldMatch) {
+        return false; // BS2 (in-order) will be used, as this means conjunction
+      }
+      
       // scorer() will return an out-of-order scorer if requested.
       return true;
     }
     
+    private Scorer req(List<Scorer> required, boolean disableCoord) {
+      if (required.size() == 1) {
+        Scorer req = required.get(0);
+        if (!disableCoord && maxCoord > 1) {
+          return new BooleanTopLevelScorers.BoostedScorer(req, coord(1, maxCoord));
+        } else {
+          return req;
+        }
+      } else {
+        return new ConjunctionScorer(this, 
+                                     required.toArray(new Scorer[required.size()]),
+                                     disableCoord ? 1.0F : coord(required.size(), maxCoord));
+      }
+    }
+    
+    private Scorer excl(Scorer main, List<Scorer> prohibited) throws IOException {
+      if (prohibited.isEmpty()) {
+        return main;
+      } else if (prohibited.size() == 1) {
+        return new ReqExclScorer(main, prohibited.get(0));
+      } else {
+        float coords[] = new float[prohibited.size()+1];
+        Arrays.fill(coords, 1F);
+        // TODO: don't score here.
+        return new ReqExclScorer(main, 
+                                 new DisjunctionSumScorer(this, 
+                                                          prohibited.toArray(new Scorer[prohibited.size()]), 
+                                                          coords));
+      }
+    }
+    
+    private Scorer opt(List<Scorer> optional, int minShouldMatch, boolean disableCoord) throws IOException {
+      if (optional.size() == 1) {
+        Scorer opt = optional.get(0);
+        if (!disableCoord && maxCoord > 1) {
+          return new BooleanTopLevelScorers.BoostedScorer(opt, coord(1, maxCoord));
+        } else {
+          return opt;
+        }
+      } else {
+        float coords[];
+        if (disableCoord) {
+          coords = new float[optional.size()+1];
+          Arrays.fill(coords, 1F);
+        } else {
+          coords = coords();
+        }
+        if (minShouldMatch > 1) {
+          return new MinShouldMatchSumScorer(this, optional, minShouldMatch, coords);
+        } else {
+          return new DisjunctionSumScorer(this, 
+                                          optional.toArray(new Scorer[optional.size()]), 
+                                          coords);
+        }
+      }
+    }
+    
+    private float[] coords() {
+      float[] coords = new float[maxCoord+1];
+      coords[0] = 0F;
+      for (int i = 1; i < coords.length; i++) {
+        coords[i] = coord(i, maxCoord);
+      }
+      return coords;
+    }
   }
 
   @Override

Modified: lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java Thu May 22 11:38:47 2014
@@ -42,7 +42,7 @@ abstract class DisjunctionScorer extends
    * Organize subScorers into a min heap with scorers generating the earliest document on top.
    */
   protected final void heapify() {
-    for (int i = (numScorers >> 1) - 1; i >= 0; i--) {
+    for (int i = (numScorers >>> 1) - 1; i >= 0; i--) {
       heapAdjust(i);
     }
   }
@@ -55,7 +55,7 @@ abstract class DisjunctionScorer extends
     Scorer scorer = subScorers[root];
     int doc = scorer.docID();
     int i = root;
-    while (i <= (numScorers >> 1) - 1) {
+    while (i <= (numScorers >>> 1) - 1) {
       int lchild = (i << 1) + 1;
       Scorer lscorer = subScorers[lchild];
       int ldoc = lscorer.docID();

Modified: lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/MinShouldMatchSumScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/MinShouldMatchSumScorer.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/MinShouldMatchSumScorer.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/MinShouldMatchSumScorer.java Thu May 22 11:38:47 2014
@@ -59,6 +59,8 @@ class MinShouldMatchSumScorer extends Sc
   /** The number of subscorers that provide the current match. */
   protected int nrMatchers = -1;
   private double score = Float.NaN;
+  
+  private final float coord[];
 
   /**
    * Construct a <code>MinShouldMatchSumScorer</code>.
@@ -72,7 +74,7 @@ class MinShouldMatchSumScorer extends Sc
    * <br>When minimumNrMatchers equals the number of subScorers,
    * it is more efficient to use <code>ConjunctionScorer</code>.
    */
-  public MinShouldMatchSumScorer(Weight weight, List<Scorer> subScorers, int minimumNrMatchers) throws IOException {
+  public MinShouldMatchSumScorer(Weight weight, List<Scorer> subScorers, int minimumNrMatchers, float coord[]) throws IOException {
     super(weight);
     this.nrInHeap = this.numScorers = subScorers.size();
 
@@ -105,17 +107,10 @@ class MinShouldMatchSumScorer extends Sc
     for (int i = 0; i < nrInHeap; i++) {
       this.subScorers[i] = this.sortedSubScorers[mm-1+i];
     }
+    this.coord = coord;
     minheapHeapify();
     assert minheapCheck();
   }
-  
-  /**
-   * Construct a <code>DisjunctionScorer</code>, using one as the minimum number
-   * of matching subscorers.
-   */
-  public MinShouldMatchSumScorer(Weight weight, List<Scorer> subScorers) throws IOException {
-    this(weight, subScorers, 1);
-  }
 
   @Override
   public final Collection<ChildScorer> getChildren() {
@@ -223,7 +218,7 @@ class MinShouldMatchSumScorer extends Sc
    */
   @Override
   public float score() throws IOException {
-    return (float) score;
+    return coord[nrMatchers] * (float) score;
   }
 
   @Override

Modified: lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/QueryRescorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/QueryRescorer.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/QueryRescorer.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/QueryRescorer.java Thu May 22 11:38:47 2014
@@ -85,18 +85,23 @@ public abstract class QueryRescorer exte
         scorer = weight.scorer(readerContext, null);
       }
 
-      int targetDoc = docID - docBase;
-      int actualDoc = scorer.docID();
-      if (actualDoc < targetDoc) {
-        actualDoc = scorer.advance(targetDoc);
-      }
+      if(scorer != null) {
+        int targetDoc = docID - docBase;
+        int actualDoc = scorer.docID();
+        if (actualDoc < targetDoc) {
+          actualDoc = scorer.advance(targetDoc);
+        }
 
-      if (actualDoc == targetDoc) {
-        // Query did match this doc:
-        hit.score = combine(hit.score, true, scorer.score());
+        if (actualDoc == targetDoc) {
+          // Query did match this doc:
+          hit.score = combine(hit.score, true, scorer.score());
+        } else {
+          // Query did not match this doc:
+          assert actualDoc > targetDoc;
+          hit.score = combine(hit.score, false, 0.0f);
+        }
       } else {
         // Query did not match this doc:
-        assert actualDoc > targetDoc;
         hit.score = combine(hit.score, false, 0.0f);
       }
 

Modified: lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java Thu May 22 11:38:47 2014
@@ -111,7 +111,7 @@ class ReqExclScorer extends Scorer {
 
   @Override
   public Collection<ChildScorer> getChildren() {
-    return Collections.singleton(new ChildScorer(reqScorer, "FILTERED"));
+    return Collections.singleton(new ChildScorer(reqScorer, "MUST"));
   }
 
   @Override

Modified: lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java Thu May 22 11:38:47 2014
@@ -29,8 +29,8 @@ class ReqOptSumScorer extends Scorer {
   /** The scorers passed from the constructor.
    * These are set to null as soon as their next() or skipTo() returns false.
    */
-  private Scorer reqScorer;
-  private Scorer optScorer;
+  protected Scorer reqScorer;
+  protected Scorer optScorer;
 
   /** Construct a <code>ReqOptScorer</code>.
    * @param reqScorer The required scorer. This must match.

Modified: lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java Thu May 22 11:38:47 2014
@@ -32,23 +32,21 @@ import java.util.Collections;
  * several places, however all they have in hand is a {@link Scorer} object, and
  * might end up computing the score of a document more than once.
  */
-public class ScoreCachingWrappingScorer extends Scorer {
+public class ScoreCachingWrappingScorer extends FilterScorer {
 
-  private final Scorer scorer;
   private int curDoc = -1;
   private float curScore;
 
   /** Creates a new instance by wrapping the given scorer. */
   public ScoreCachingWrappingScorer(Scorer scorer) {
-    super(scorer.weight);
-    this.scorer = scorer;
+    super(scorer);
   }
 
   @Override
   public float score() throws IOException {
-    int doc = scorer.docID();
+    int doc = in.docID();
     if (doc != curDoc) {
-      curScore = scorer.score();
+      curScore = in.score();
       curDoc = doc;
     }
 
@@ -56,32 +54,7 @@ public class ScoreCachingWrappingScorer 
   }
 
   @Override
-  public int freq() throws IOException {
-    return scorer.freq();
-  }
-
-  @Override
-  public int docID() {
-    return scorer.docID();
-  }
-
-  @Override
-  public int nextDoc() throws IOException {
-    return scorer.nextDoc();
-  }
-  
-  @Override
-  public int advance(int target) throws IOException {
-    return scorer.advance(target);
-  }
-
-  @Override
   public Collection<ChildScorer> getChildren() {
-    return Collections.singleton(new ChildScorer(scorer, "CACHED"));
-  }
-
-  @Override
-  public long cost() {
-    return scorer.cost();
+    return Collections.singleton(new ChildScorer(in, "CACHED"));
   }
 }

Modified: lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java Thu May 22 11:38:47 2014
@@ -22,9 +22,10 @@ import org.apache.lucene.util.IOUtils;
 
 import java.io.File;
 import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.io.FilenameFilter;
+import java.io.FilterOutputStream;
 import java.io.IOException;
-import java.io.RandomAccessFile;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.Set;
@@ -262,7 +263,7 @@ public abstract class FSDirectory extend
     ensureOpen();
 
     ensureCanWrite(name);
-    return new FSIndexOutput(this, name);
+    return new FSIndexOutput(name);
   }
 
   protected void ensureCanWrite(String name) throws IOException {
@@ -338,58 +339,37 @@ public abstract class FSDirectory extend
     return this.getClass().getSimpleName() + "@" + directory + " lockFactory=" + getLockFactory();
   }
 
-  /**
-   * Writes output with {@link RandomAccessFile#write(byte[], int, int)}
-   */
-  protected static class FSIndexOutput extends BufferedIndexOutput {
+  final class FSIndexOutput extends OutputStreamIndexOutput {
     /**
-     * The maximum chunk size is 8192 bytes, because {@link RandomAccessFile} mallocs
+     * The maximum chunk size is 8192 bytes, because {@link FileOutputStream} mallocs
      * a native buffer outside of stack if the write buffer size is larger.
      */
-    private static final int CHUNK_SIZE = 8192;
+    static final int CHUNK_SIZE = 8192;
     
-    private final FSDirectory parent;
     private final String name;
-    private final RandomAccessFile file;
-    private volatile boolean isOpen; // remember if the file is open, so that we don't try to close it more than once
-    
-    public FSIndexOutput(FSDirectory parent, String name) throws IOException {
-      super(CHUNK_SIZE);
-      this.parent = parent;
-      this.name = name;
-      file = new RandomAccessFile(new File(parent.directory, name), "rw");
-      isOpen = true;
-    }
 
-    @Override
-    protected void flushBuffer(byte[] b, int offset, int size) throws IOException {
-      assert isOpen;
-      while (size > 0) {
-        final int toWrite = Math.min(CHUNK_SIZE, size);
-        file.write(b, offset, toWrite);
-        offset += toWrite;
-        size -= toWrite;
-      }
-      assert size == 0;
+    public FSIndexOutput(String name) throws IOException {
+      super(new FilterOutputStream(new FileOutputStream(new File(directory, name))) {
+        // This implementation ensures, that we never write more than CHUNK_SIZE bytes:
+        @Override
+        public void write(byte[] b, int offset, int length) throws IOException {
+          while (length > 0) {
+            final int chunk = Math.min(length, CHUNK_SIZE);
+            out.write(b, offset, chunk);
+            length -= chunk;
+            offset += chunk;
+          }
+        }
+      }, CHUNK_SIZE);
+      this.name = name;
     }
     
     @Override
     public void close() throws IOException {
-      parent.onIndexOutputClosed(name);
-      // only close the file if it has not been closed yet
-      if (isOpen) {
-        boolean success = false;
-        try {
-          super.close();
-          success = true;
-        } finally {
-          isOpen = false;
-          if (success) {
-            IOUtils.close(file);
-          } else {
-            IOUtils.closeWhileHandlingException(file);
-          }
-        }
+      try {
+        onIndexOutputClosed(name);
+      } finally {
+        super.close();
       }
     }
   }

Modified: lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java Thu May 22 11:38:47 2014
@@ -1,4 +1,5 @@
 package org.apache.lucene.store;
+
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -15,6 +16,7 @@ package org.apache.lucene.store;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 import java.io.IOException;
 
 /**
@@ -22,48 +24,58 @@ import java.io.IOException;
  * 
  * @lucene.internal
  */
-final class RateLimitedIndexOutput extends BufferedIndexOutput {
+final class RateLimitedIndexOutput extends IndexOutput {
   
   private final IndexOutput delegate;
-  private final BufferedIndexOutput bufferedDelegate;
   private final RateLimiter rateLimiter;
 
   /** How many bytes we've written since we last called rateLimiter.pause. */
   private long bytesSinceLastPause;
+  
+  /** Cached here not not always have to call RateLimiter#getMinPauseCheckBytes()
+   * which does volatile read. */
+  private long currentMinPauseCheckBytes;
 
   RateLimitedIndexOutput(final RateLimiter rateLimiter, final IndexOutput delegate) {
-    // TODO should we make buffer size configurable
-    if (delegate instanceof BufferedIndexOutput) {
-      bufferedDelegate = (BufferedIndexOutput) delegate;
-      this.delegate = delegate;
-    } else {
-      this.delegate = delegate;
-      bufferedDelegate = null;
-    }
+    this.delegate = delegate;
     this.rateLimiter = rateLimiter;
+    this.currentMinPauseCheckBytes = rateLimiter.getMinPauseCheckBytes();
   }
   
   @Override
-  protected void flushBuffer(byte[] b, int offset, int len) throws IOException {
-    bytesSinceLastPause += len;
-    if (bytesSinceLastPause > rateLimiter.getMinPauseCheckBytes()) {
-      rateLimiter.pause(bytesSinceLastPause);
-      bytesSinceLastPause = 0;
-    }
-    if (bufferedDelegate != null) {
-      bufferedDelegate.flushBuffer(b, offset, len);
-    } else {
-      delegate.writeBytes(b, offset, len);
-    }
-    
+  public void close() throws IOException {
+    delegate.close();
   }
 
   @Override
-  public void close() throws IOException {
-    try {
-      super.close();
-    } finally {
-      delegate.close();
-    }
+  public long getFilePointer() {
+    return delegate.getFilePointer();
+  }
+
+  @Override
+  public long getChecksum() throws IOException {
+    return delegate.getChecksum();
+  }
+
+  @Override
+  public void writeByte(byte b) throws IOException {
+    bytesSinceLastPause++;
+    checkRate();
+    delegate.writeByte(b);
+  }
+
+  @Override
+  public void writeBytes(byte[] b, int offset, int length) throws IOException {
+    bytesSinceLastPause += length;
+    checkRate();
+    delegate.writeBytes(b, offset, length);
+  }
+  
+  private void checkRate() {
+    if (bytesSinceLastPause > currentMinPauseCheckBytes) {
+      rateLimiter.pause(bytesSinceLastPause);
+      bytesSinceLastPause = 0;
+      currentMinPauseCheckBytes = rateLimiter.getMinPauseCheckBytes();
+    }    
   }
 }

Modified: lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java Thu May 22 11:38:47 2014
@@ -129,6 +129,14 @@ public final class ByteSequenceOutputs e
   }
 
   @Override
+  public void skipOutput(DataInput in) throws IOException {
+    final int len = in.readVInt();
+    if (len != 0) {
+      in.skipBytes(len);
+    }
+  }
+
+  @Override
   public BytesRef getNoOutput() {
     return NO_OUTPUT;
   }

Modified: lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java Thu May 22 11:38:47 2014
@@ -362,7 +362,7 @@ class BytesStore extends DataOutput {
       }
 
       @Override
-      public void skipBytes(int count) {
+      public void skipBytes(long count) {
         setPosition(getPosition() + count);
       }
 
@@ -430,7 +430,7 @@ class BytesStore extends DataOutput {
       }
 
       @Override
-      public void skipBytes(int count) {
+      public void skipBytes(long count) {
         setPosition(getPosition() - count);
       }
 

Modified: lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java Thu May 22 11:38:47 2014
@@ -132,6 +132,14 @@ public final class CharSequenceOutputs e
       return output;
     }
   }
+  
+  @Override
+  public void skipOutput(DataInput in) throws IOException {
+    final int len = in.readVInt();
+    for(int idx=0;idx<len;idx++) {
+      in.readVInt();
+    }
+  }
 
   @Override
   public CharsRef getNoOutput() {

Modified: lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/FST.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/FST.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/FST.java Thu May 22 11:38:47 2014
@@ -893,10 +893,10 @@ public final class FST<T> {
           // skip this arc:
           readLabel(in);
           if (arc.flag(BIT_ARC_HAS_OUTPUT)) {
-            outputs.read(in);
+            outputs.skipOutput(in);
           }
           if (arc.flag(BIT_ARC_HAS_FINAL_OUTPUT)) {
-            outputs.readFinalOutput(in);
+            outputs.skipFinalOutput(in);
           }
           if (arc.flag(BIT_STOP_NODE)) {
           } else if (arc.flag(BIT_TARGET_NEXT)) {
@@ -1252,11 +1252,11 @@ public final class FST<T> {
       readLabel(in);
 
       if (flag(flags, BIT_ARC_HAS_OUTPUT)) {
-        outputs.read(in);
+        outputs.skipOutput(in);
       }
 
       if (flag(flags, BIT_ARC_HAS_FINAL_OUTPUT)) {
-        outputs.readFinalOutput(in);
+        outputs.skipFinalOutput(in);
       }
 
       if (!flag(flags, BIT_STOP_NODE) && !flag(flags, BIT_TARGET_NEXT)) {
@@ -1330,9 +1330,6 @@ public final class FST<T> {
     /** Returns true if this reader uses reversed bytes
      *  under-the-hood. */
     public abstract boolean reversed();
-
-    /** Skips bytes. */
-    public abstract void skipBytes(int count);
   }
 
   private static class ArcAndState<T> {

Modified: lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/ForwardBytesReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/ForwardBytesReader.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/ForwardBytesReader.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/ForwardBytesReader.java Thu May 22 11:38:47 2014
@@ -41,7 +41,7 @@ final class ForwardBytesReader extends F
   }
 
   @Override
-  public void skipBytes(int count) {
+  public void skipBytes(long count) {
     pos += count;
   }
 

Modified: lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java Thu May 22 11:38:47 2014
@@ -131,6 +131,17 @@ public final class IntSequenceOutputs ex
       return output;
     }
   }
+  
+  @Override
+  public void skipOutput(DataInput in) throws IOException {
+    final int len = in.readVInt();
+    if (len == 0) {
+      return;
+    }
+    for(int idx=0;idx<len;idx++) {
+      in.readVInt();
+    }
+  }
 
   @Override
   public IntsRef getNoOutput() {

Modified: lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java Thu May 22 11:38:47 2014
@@ -63,12 +63,25 @@ public abstract class Outputs<T> {
    *  #write(Object, DataOutput)}. */
   public abstract T read(DataInput in) throws IOException;
 
+  /** Skip the output; defaults to just calling {@link #read}
+   *  and discarding the result. */
+  public void skipOutput(DataInput in) throws IOException {
+    read(in);
+  }
+
   /** Decode an output value previously written with {@link
    *  #writeFinalOutput(Object, DataOutput)}.  By default this
    *  just calls {@link #read(DataInput)}. */
   public T readFinalOutput(DataInput in) throws IOException {
     return read(in);
   }
+  
+  /** Skip the output previously written with {@link #writeFinalOutput};
+   *  defaults to just calling {@link #readFinalOutput} and discarding
+   *  the result. */
+  public void skipFinalOutput(DataInput in) throws IOException {
+    skipOutput(in);
+  }
 
   /** NOTE: this output is compared with == so you must
    *  ensure that all methods return the single object if

Modified: lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java Thu May 22 11:38:47 2014
@@ -153,6 +153,12 @@ public class PairOutputs<A,B> extends Ou
     B output2 = outputs2.read(in);
     return newPair(output1, output2);
   }
+  
+  @Override
+  public void skipOutput(DataInput in) throws IOException {
+    outputs1.skipOutput(in);
+    outputs2.skipOutput(in);
+  }
 
   @Override
   public Pair<A,B> getNoOutput() {

Modified: lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/ReverseBytesReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/ReverseBytesReader.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/ReverseBytesReader.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/java/org/apache/lucene/util/fst/ReverseBytesReader.java Thu May 22 11:38:47 2014
@@ -39,7 +39,7 @@ final class ReverseBytesReader extends F
   }
 
   @Override
-  public void skipBytes(int count) {
+  public void skipBytes(long count) {
     pos -= count;
   }
 

Modified: lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java Thu May 22 11:38:47 2014
@@ -1285,27 +1285,23 @@ public class TestBinaryDocValuesUpdates 
     
     Document doc = new Document();
     doc.add(new StringField("id", "d0", Store.NO));
-    doc.add(new BinaryDocValuesField("f", toBytes(1L)));
+    doc.add(new BinaryDocValuesField("f1", toBytes(1L)));
+    doc.add(new BinaryDocValuesField("f2", toBytes(1L)));
     writer.addDocument(doc);
 
-    // create first gen of update files
-    writer.updateBinaryDocValue(new Term("id", "d0"), "f", toBytes(2L));
-    writer.commit();
-    int numFiles = dir.listAll().length;
-
-    DirectoryReader r = DirectoryReader.open(dir);
-    BytesRef scratch = new BytesRef();
-    assertEquals(2L, getValue(r.leaves().get(0).reader().getBinaryDocValues("f"), 0, scratch));
-    r.close();
-    
-    // create second gen of update files, first gen should be deleted
-    writer.updateBinaryDocValue(new Term("id", "d0"), "f", toBytes(5L));
-    writer.commit();
-    assertEquals(numFiles, dir.listAll().length);
-
-    r = DirectoryReader.open(dir);
-    assertEquals(5L, getValue(r.leaves().get(0).reader().getBinaryDocValues("f"), 0, scratch));
-    r.close();
+    // update each field twice to make sure all unneeded files are deleted
+    for (String f : new String[] { "f1", "f2" }) {
+      writer.updateBinaryDocValue(new Term("id", "d0"), f, toBytes(2L));
+      writer.commit();
+      int numFiles = dir.listAll().length;
+      
+      // update again, number of files shouldn't change (old field's gen is
+      // removed) 
+      writer.updateBinaryDocValue(new Term("id", "d0"), f, toBytes(3L));
+      writer.commit();
+      
+      assertEquals(numFiles, dir.listAll().length);
+    }
 
     writer.shutdown();
     dir.close();

Modified: lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/index/TestDoc.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/index/TestDoc.java Thu May 22 11:38:47 2014
@@ -44,7 +44,6 @@ import org.apache.lucene.store.TrackingD
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.InfoStream;
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TestUtil;
 
 
 /** JUnit adaptation of an older test case DocTest. */
@@ -240,7 +239,7 @@ public class TestDoc extends LuceneTestC
         }
       }
 
-      return new SegmentCommitInfo(info, 0, -1L, -1L);
+      return new SegmentCommitInfo(info, 0, -1L, -1L, -1L);
    }
 
 

Modified: lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java Thu May 22 11:38:47 2014
@@ -325,7 +325,7 @@ public class TestIndexWriterThreadsToSeg
             segSeen.add(segName);
             SegmentInfo si = new Lucene46SegmentInfoFormat().getSegmentInfoReader().read(dir, segName, IOContext.DEFAULT);
             si.setCodec(codec);
-            SegmentCommitInfo sci = new SegmentCommitInfo(si, 0, -1, -1);
+            SegmentCommitInfo sci = new SegmentCommitInfo(si, 0, -1, -1, -1);
             SegmentReader sr = new SegmentReader(sci, IOContext.DEFAULT);
             try {
               thread0Count += sr.docFreq(new Term("field", "threadID0"));

Modified: lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java Thu May 22 11:38:47 2014
@@ -1264,27 +1264,24 @@ public class TestNumericDocValuesUpdates
     
     Document doc = new Document();
     doc.add(new StringField("id", "d0", Store.NO));
-    doc.add(new NumericDocValuesField("f", 1L));
+    doc.add(new NumericDocValuesField("f1", 1L));
+    doc.add(new NumericDocValuesField("f2", 1L));
     writer.addDocument(doc);
 
-    // create first gen of update files
-    writer.updateNumericDocValue(new Term("id", "d0"), "f", 2L);
-    writer.commit();
-    int numFiles = dir.listAll().length;
-
-    DirectoryReader r = DirectoryReader.open(dir);
-    assertEquals(2L, r.leaves().get(0).reader().getNumericDocValues("f").get(0));
-    r.close();
+    // update each field twice to make sure all unneeded files are deleted
+    for (String f : new String[] { "f1", "f2" }) {
+      writer.updateNumericDocValue(new Term("id", "d0"), f, 2L);
+      writer.commit();
+      int numFiles = dir.listAll().length;
+      
+      // update again, number of files shouldn't change (old field's gen is
+      // removed) 
+      writer.updateNumericDocValue(new Term("id", "d0"), f, 3L);
+      writer.commit();
+      
+      assertEquals(numFiles, dir.listAll().length);
+    }
     
-    // create second gen of update files, first gen should be deleted
-    writer.updateNumericDocValue(new Term("id", "d0"), "f", 5L);
-    writer.commit();
-    assertEquals(numFiles, dir.listAll().length);
-
-    r = DirectoryReader.open(dir);
-    assertEquals(5L, r.leaves().get(0).reader().getNumericDocValues("f").get(0));
-    r.close();
-
     writer.shutdown();
     dir.close();
   }
@@ -1455,7 +1452,6 @@ public class TestNumericDocValuesUpdates
     writer.updateNumericDocValue(new Term("id", "doc-0"), "val", 100L);
     DirectoryReader reader = DirectoryReader.open(writer, true); // flush
     assertEquals(0, cachingDir.listCachedFiles().length);
-    for (String f : cachingDir.listAll()) System.out.println(f + " " + cachingDir.fileLength(f));
     
     IOUtils.close(reader, writer, cachingDir);
   }

Modified: lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java Thu May 22 11:38:47 2014
@@ -30,7 +30,6 @@ import org.apache.lucene.util.FixedBitSe
 import org.apache.lucene.util.InfoStream;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;
-import org.apache.lucene.util.TestUtil;
 
 public class TestSegmentMerger extends LuceneTestCase {
   //The variables for the new merged segment
@@ -91,7 +90,7 @@ public class TestSegmentMerger extends L
     SegmentReader mergedReader = new SegmentReader(new SegmentCommitInfo(
                                                          new SegmentInfo(mergedDir, Constants.LUCENE_MAIN_VERSION, mergedSegment, docsMerged,
                                                                          false, codec, null),
-                                                         0, -1L, -1L),
+                                                         0, -1L, -1L, -1L),
                                                    newIOContext(random()));
     assertTrue(mergedReader != null);
     assertTrue(mergedReader.numDocs() == 2);

Modified: lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java Thu May 22 11:38:47 2014
@@ -107,6 +107,46 @@ public class TestQueryRescorer extends L
     dir.close();
   }
 
+  // Test LUCENE-5682
+  public void testNullScorerTermQuery() throws Exception {
+    Directory dir = newDirectory();
+    RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+
+    Document doc = new Document();
+    doc.add(newStringField("id", "0", Field.Store.YES));
+    doc.add(newTextField("field", "wizard the the the the the oz", Field.Store.NO));
+    w.addDocument(doc);
+    doc = new Document();
+    doc.add(newStringField("id", "1", Field.Store.YES));
+    // 1 extra token, but wizard and oz are close;
+    doc.add(newTextField("field", "wizard oz the the the the the the", Field.Store.NO));
+    w.addDocument(doc);
+    IndexReader r = w.getReader();
+    w.shutdown();
+
+    // Do ordinary BooleanQuery:
+    BooleanQuery bq = new BooleanQuery();
+    bq.add(new TermQuery(new Term("field", "wizard")), Occur.SHOULD);
+    bq.add(new TermQuery(new Term("field", "oz")), Occur.SHOULD);
+    IndexSearcher searcher = getSearcher(r);
+    searcher.setSimilarity(new DefaultSimilarity());
+
+    TopDocs hits = searcher.search(bq, 10);
+    assertEquals(2, hits.totalHits);
+    assertEquals("0", searcher.doc(hits.scoreDocs[0].doc).get("id"));
+    assertEquals("1", searcher.doc(hits.scoreDocs[1].doc).get("id"));
+
+    // Now, resort using TermQuery on term that does not exist.
+    TermQuery tq = new TermQuery(new Term("field", "gold"));
+    TopDocs hits2 = QueryRescorer.rescore(searcher, hits, tq, 2.0, 10);
+
+    // Just testing that null scorer is handled.
+    assertEquals(2, hits2.totalHits);
+
+    r.close();
+    dir.close();
+  }
+
   public void testCustomCombine() throws Exception {
     Directory dir = newDirectory();
     RandomIndexWriter w = new RandomIndexWriter(random(), dir);

Modified: lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java (original)
+++ lucene/dev/branches/lucene5675/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java Thu May 22 11:38:47 2014
@@ -171,7 +171,8 @@ public class TestSubScorerFreqs extends 
       boolean includeOptional = occur.contains("SHOULD");
       for (int i = 0; i < maxDocs; i++) {
         Map<Query, Float> doc0 = c.docCounts.get(i);
-        assertEquals(includeOptional ? 5 : 4, doc0.size());
+        // Y doesnt exist in the index, so its not in the scorer tree
+        assertEquals(4, doc0.size());
         assertEquals(1.0F, doc0.get(aQuery), FLOAT_TOLERANCE);
         assertEquals(4.0F, doc0.get(dQuery), FLOAT_TOLERANCE);
         if (includeOptional) {
@@ -179,7 +180,8 @@ public class TestSubScorerFreqs extends 
         }
 
         Map<Query, Float> doc1 = c.docCounts.get(++i);
-        assertEquals(includeOptional ? 5 : 4, doc1.size());
+        // Y doesnt exist in the index, so its not in the scorer tree
+        assertEquals(4, doc1.size());
         assertEquals(1.0F, doc1.get(aQuery), FLOAT_TOLERANCE);
         assertEquals(1.0F, doc1.get(dQuery), FLOAT_TOLERANCE);
         if (includeOptional) {

Modified: lucene/dev/branches/lucene5675/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/lucene5675/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Thu May 22 11:38:47 2014
@@ -448,7 +448,7 @@ public class MemoryIndex {
 
       if (!fieldInfos.containsKey(fieldName)) {
         fieldInfos.put(fieldName, 
-            new FieldInfo(fieldName, true, fieldInfos.size(), false, false, false, this.storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS , null, null, null));
+            new FieldInfo(fieldName, true, fieldInfos.size(), false, false, false, this.storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS , null, null, -1, null));
       }
       TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class);
       PositionIncrementAttribute posIncrAttribute = stream.addAttribute(PositionIncrementAttribute.class);

Modified: lucene/dev/branches/lucene5675/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java (original)
+++ lucene/dev/branches/lucene5675/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java Thu May 22 11:38:47 2014
@@ -140,7 +140,9 @@ public class IndexSplitter {
       // Same info just changing the dir:
       SegmentInfo newInfo = new SegmentInfo(destFSDir, info.getVersion(), info.name, info.getDocCount(), 
                                             info.getUseCompoundFile(), info.getCodec(), info.getDiagnostics());
-      destInfos.add(new SegmentCommitInfo(newInfo, infoPerCommit.getDelCount(), infoPerCommit.getDelGen(), infoPerCommit.getFieldInfosGen()));
+      destInfos.add(new SegmentCommitInfo(newInfo, infoPerCommit.getDelCount(),
+          infoPerCommit.getDelGen(), infoPerCommit.getFieldInfosGen(),
+          infoPerCommit.getDocValuesGen()));
       // now copy files over
       Collection<String> files = infoPerCommit.files();
       for (final String srcName : files) {

Modified: lucene/dev/branches/lucene5675/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java (original)
+++ lucene/dev/branches/lucene5675/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java Thu May 22 11:38:47 2014
@@ -894,10 +894,15 @@ public class DocTermOrds {
     @Override
     public long lookupTerm(BytesRef key) {
       try {
-        if (te.seekCeil(key) == SeekStatus.FOUND) {
-          return te.ord();
-        } else {
-          return -te.ord()-1;
+        switch (te.seekCeil(key)) {
+          case FOUND:           
+            assert te.ord() >= 0;
+            return te.ord();
+          case NOT_FOUND:
+            assert te.ord() >= 0;
+            return -te.ord()-1;
+          default: /* END */
+            return -numTerms()-1;
         }
       } catch (IOException e) {
         throw new RuntimeException(e);

Modified: lucene/dev/branches/lucene5675/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java (original)
+++ lucene/dev/branches/lucene5675/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java Thu May 22 11:38:47 2014
@@ -215,7 +215,7 @@ public class UninvertingReader extends F
         }
       }
       filteredInfos.add(new FieldInfo(fi.name, fi.isIndexed(), fi.number, fi.hasVectors(), fi.omitsNorms(),
-                                      fi.hasPayloads(), fi.getIndexOptions(), type, fi.getNormType(), null));
+                                      fi.hasPayloads(), fi.getIndexOptions(), type, fi.getNormType(), -1, null));
     }
     fieldInfos = new FieldInfos(filteredInfos.toArray(new FieldInfo[filteredInfos.size()]));
   }

Modified: lucene/dev/branches/lucene5675/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java (original)
+++ lucene/dev/branches/lucene5675/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java Thu May 22 11:38:47 2014
@@ -122,6 +122,11 @@ public final class ListOfOutputs<T> exte
   public Object read(DataInput in) throws IOException {
     return outputs.read(in);
   }
+  
+  @Override
+  public void skipOutput(DataInput in) throws IOException {
+    outputs.skipOutput(in);
+  }
 
   @Override
   public Object readFinalOutput(DataInput in) throws IOException {
@@ -136,6 +141,14 @@ public final class ListOfOutputs<T> exte
       return outputList;
     }
   }
+  
+  @Override
+  public void skipFinalOutput(DataInput in) throws IOException {
+    int count = in.readVInt();
+    for(int i=0;i<count;i++) {
+      outputs.skipOutput(in);
+    }
+  }
 
   @Override
   public Object getNoOutput() {

Modified: lucene/dev/branches/lucene5675/lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java (original)
+++ lucene/dev/branches/lucene5675/lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java Thu May 22 11:38:47 2014
@@ -579,6 +579,16 @@ public class TestDocTermOrds extends Luc
     termsEnum.seekExact(2);
     assertEquals("world", termsEnum.term().utf8ToString());
     assertEquals(2, termsEnum.ord());
+    
+    // lookupTerm(BytesRef) 
+    assertEquals(-1, dv.lookupTerm(new BytesRef("apple")));
+    assertEquals(0, dv.lookupTerm(new BytesRef("beer")));
+    assertEquals(-2, dv.lookupTerm(new BytesRef("car")));
+    assertEquals(1, dv.lookupTerm(new BytesRef("hello")));
+    assertEquals(-3, dv.lookupTerm(new BytesRef("matter")));
+    assertEquals(2, dv.lookupTerm(new BytesRef("world")));
+    assertEquals(-4, dv.lookupTerm(new BytesRef("zany")));
+
     ireader.close();
     directory.close();
   }

Modified: lucene/dev/branches/lucene5675/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java (original)
+++ lucene/dev/branches/lucene5675/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java Thu May 22 11:38:47 2014
@@ -240,7 +240,7 @@ public class AnalyzingInfixSuggester ext
       searcherMgr = new SearcherManager(writer, true, null);
       success = true;
     } finally {
-      if (success == false) {
+      if (success == false && writer != null) {
         writer.rollback();
         writer = null;
       }

Modified: lucene/dev/branches/lucene5675/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java (original)
+++ lucene/dev/branches/lucene5675/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java Thu May 22 11:38:47 2014
@@ -357,7 +357,7 @@ public abstract class BasePostingsFormat
 
       fieldInfoArray[fieldUpto] = new FieldInfo(field, true, fieldUpto, false, false, true,
                                                 IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
-                                                null, DocValuesType.NUMERIC, null);
+                                                null, DocValuesType.NUMERIC, -1, null);
       fieldUpto++;
 
       SortedMap<BytesRef,Long> postings = new TreeMap<>();
@@ -680,6 +680,7 @@ public abstract class BasePostingsFormat
                                                    indexOptions,
                                                    null,
                                                    DocValuesType.NUMERIC,
+                                                   -1,
                                                    null);
     }
 

Modified: lucene/dev/branches/lucene5675/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/lucene5675/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java Thu May 22 11:38:47 2014
@@ -524,10 +524,6 @@ public class MockDirectoryWrapper extend
     }
     //System.out.println(Thread.currentThread().getName() + ": MDW: create " + name);
     IndexOutput delegateOutput = in.createOutput(name, LuceneTestCase.newIOContext(randomState, context));
-    if (randomState.nextInt(10) == 0){
-      // once in a while wrap the IO in a Buffered IO with random buffer sizes
-      delegateOutput = new BufferedIndexOutputWrapper(1+randomState.nextInt(BufferedIndexOutput.DEFAULT_BUFFER_SIZE), delegateOutput);
-    } 
     final IndexOutput io = new MockIndexOutputWrapper(this, delegateOutput, name);
     addFileHandle(io, name, Handle.Output);
     openFilesForWrite.add(name);
@@ -947,29 +943,6 @@ public class MockDirectoryWrapper extend
     in.copy(to, src, dest, context);
   }
   
-  final class BufferedIndexOutputWrapper extends BufferedIndexOutput {
-    private final IndexOutput io;
-    
-    public BufferedIndexOutputWrapper(int bufferSize, IndexOutput io) {
-      super(bufferSize);
-      this.io = io;
-    }
-    
-    @Override
-    protected void flushBuffer(byte[] b, int offset, int len) throws IOException {
-      io.writeBytes(b, offset, len);
-    }
-    
-    @Override
-    public void close() throws IOException {
-      try {
-        super.close();
-      } finally {
-        io.close();
-      }
-    }
-  }
-
   /** Use this when throwing fake {@code IOException},
    *  e.g. from {@link MockDirectoryWrapper.Failure}. */
   public static class FakeIOException extends IOException {



Mime
View raw message