lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1634823 [4/9] - in /lucene/dev/branches/lucene6005: ./ dev-tools/ lucene/ lucene/analysis/ lucene/analysis/common/ lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/ lucene/analysis/common/src/java/org/apache/lucene/analy...
Date Tue, 28 Oct 2014 09:19:29 GMT
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java Tue Oct 28 09:19:21 2014
@@ -34,7 +34,6 @@ import org.apache.lucene.store.Directory
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.StringHelper;
 import org.apache.lucene.util.Version;
 
 /**
@@ -42,20 +41,19 @@ import org.apache.lucene.util.Version;
  * <p>
  * Files:
  * <ul>
- *   <li><tt>.si</tt>: Header, SegVersion, SegSize, IsCompoundFile, Diagnostics, Files, Id, Footer
+ *   <li><tt>.si</tt>: Header, SegVersion, SegSize, IsCompoundFile, Diagnostics, Files, Footer
  * </ul>
  * </p>
  * Data types:
  * <p>
  * <ul>
- *   <li>Header --&gt; {@link CodecUtil#writeHeader CodecHeader}</li>
+ *   <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}</li>
  *   <li>SegSize --&gt; {@link DataOutput#writeInt Int32}</li>
  *   <li>SegVersion --&gt; {@link DataOutput#writeString String}</li>
  *   <li>Files --&gt; {@link DataOutput#writeStringSet Set&lt;String&gt;}</li>
  *   <li>Diagnostics --&gt; {@link DataOutput#writeStringStringMap Map&lt;String,String&gt;}</li>
  *   <li>IsCompoundFile --&gt; {@link DataOutput#writeByte Int8}</li>
  *   <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}</li>
- *   <li>Id --&gt; {@link DataOutput#writeString String}</li>
  * </ul>
  * </p>
  * Field Descriptions:
@@ -84,15 +82,16 @@ public class Lucene50SegmentInfoFormat e
   }
   
   @Override
-  public SegmentInfo read(Directory dir, String segment, IOContext context) throws IOException {
+  public SegmentInfo read(Directory dir, String segment, byte[] segmentID, IOContext context) throws IOException {
     final String fileName = IndexFileNames.segmentFileName(segment, "", Lucene50SegmentInfoFormat.SI_EXTENSION);
     try (ChecksumIndexInput input = dir.openChecksumInput(fileName, context)) {
       Throwable priorE = null;
       SegmentInfo si = null;
       try {
-        CodecUtil.checkHeader(input, Lucene50SegmentInfoFormat.CODEC_NAME,
-                                     Lucene50SegmentInfoFormat.VERSION_START,
-                                     Lucene50SegmentInfoFormat.VERSION_CURRENT);
+        CodecUtil.checkIndexHeader(input, Lucene50SegmentInfoFormat.CODEC_NAME,
+                                          Lucene50SegmentInfoFormat.VERSION_START,
+                                          Lucene50SegmentInfoFormat.VERSION_CURRENT,
+                                          segmentID, "");
         final Version version = Version.fromBits(input.readInt(), input.readInt(), input.readInt());
         
         final int docCount = input.readInt();
@@ -103,10 +102,7 @@ public class Lucene50SegmentInfoFormat e
         final Map<String,String> diagnostics = input.readStringStringMap();
         final Set<String> files = input.readStringSet();
         
-        byte[] id = new byte[StringHelper.ID_LENGTH];
-        input.readBytes(id, 0, id.length);
-        
-        si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, id);
+        si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, segmentID);
         si.setFiles(files);
       } catch (Throwable exception) {
         priorE = exception;
@@ -124,7 +120,11 @@ public class Lucene50SegmentInfoFormat e
 
     boolean success = false;
     try (IndexOutput output = dir.createOutput(fileName, ioContext)) {
-      CodecUtil.writeHeader(output, Lucene50SegmentInfoFormat.CODEC_NAME, Lucene50SegmentInfoFormat.VERSION_CURRENT);
+      CodecUtil.writeIndexHeader(output, 
+                                   Lucene50SegmentInfoFormat.CODEC_NAME, 
+                                   Lucene50SegmentInfoFormat.VERSION_CURRENT,
+                                   si.getId(),
+                                   "");
       Version version = si.getVersion();
       if (version.major < 5) {
         throw new IllegalArgumentException("invalid major version: should be >= 5 but got: " + version.major + " segment=" + si);
@@ -145,11 +145,6 @@ public class Lucene50SegmentInfoFormat e
         }
       }
       output.writeStringSet(files);
-      byte[] id = si.getId();
-      if (id.length != StringHelper.ID_LENGTH) {
-        throw new IllegalArgumentException("invalid id, got=" + StringHelper.idToString(id));
-      }
-      output.writeBytes(id, 0, id.length);
       CodecUtil.writeFooter(output);
       success = true;
     } finally {

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java Tue Oct 28 09:19:21 2014
@@ -52,7 +52,7 @@ import org.apache.lucene.util.packed.Pac
  * <p>Here is a more detailed description of the field data file format:</p>
  * <ul>
  * <li>FieldData (.fdt) --&gt; &lt;Header&gt;, PackedIntsVersion, &lt;Chunk&gt;<sup>ChunkCount</sup></li>
- * <li>Header --&gt; {@link CodecUtil#writeSegmentHeader SegmentHeader}</li>
+ * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}</li>
  * <li>PackedIntsVersion --&gt; {@link PackedInts#VERSION_CURRENT} as a {@link DataOutput#writeVInt VInt}</li>
  * <li>ChunkCount is not known in advance and is the number of chunks necessary to store all document of the segment</li>
  * <li>Chunk --&gt; DocBase, ChunkDocs, DocFieldCounts, DocLengths, &lt;CompressedDocs&gt;</li>
@@ -104,7 +104,7 @@ import org.apache.lucene.util.packed.Pac
  * <p>A fields index file (extension <tt>.fdx</tt>).</p>
  * <ul>
  * <li>FieldsIndex (.fdx) --&gt; &lt;Header&gt;, &lt;ChunkIndex&gt;</li>
- * <li>Header --&gt; {@link CodecUtil#writeSegmentHeader SegmentHeader}</li>
+ * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}</li>
  * <li>ChunkIndex: See {@link CompressingStoredFieldsIndexWriter}</li>
  * </ul>
  * </li>

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50TermVectorsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50TermVectorsFormat.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50TermVectorsFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50TermVectorsFormat.java Tue Oct 28 09:19:21 2014
@@ -59,7 +59,7 @@ import org.apache.lucene.util.packed.Pac
  * <p>Here is a more detailed description of the field data file format:</p>
  * <ul>
  * <li>VectorData (.tvd) --&gt; &lt;Header&gt;, PackedIntsVersion, ChunkSize, &lt;Chunk&gt;<sup>ChunkCount</sup>, Footer</li>
- * <li>Header --&gt; {@link CodecUtil#writeSegmentHeader SegmentHeader}</li>
+ * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}</li>
  * <li>PackedIntsVersion --&gt; {@link PackedInts#VERSION_CURRENT} as a {@link DataOutput#writeVInt VInt}</li>
  * <li>ChunkSize is the number of bytes of terms to accumulate before flushing, as a {@link DataOutput#writeVInt VInt}</li>
  * <li>ChunkCount is not known in advance and is the number of chunks necessary to store all document of the segment</li>
@@ -113,7 +113,7 @@ import org.apache.lucene.util.packed.Pac
  * <p>An index file (extension <tt>.tvx</tt>).</p>
  * <ul>
  * <li>VectorIndex (.tvx) --&gt; &lt;Header&gt;, &lt;ChunkIndex&gt;, Footer</li>
- * <li>Header --&gt; {@link CodecUtil#writeSegmentHeader SegmentHeader}</li>
+ * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}</li>
  * <li>ChunkIndex: See {@link CompressingStoredFieldsIndexWriter}</li>
  * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}</li>
  * </ul>

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/package.html?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/package.html (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/codecs/lucene50/package.html Tue Oct 28 09:19:21 2014
@@ -154,20 +154,20 @@ its title, url, or an identifier to acce
 returned for each hit when searching. This is keyed by document number.
 </li>
 <li>
-{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term dictionary}. 
+{@link org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat Term dictionary}. 
 A dictionary containing all of the terms used in all of the
 indexed fields of all of the documents. The dictionary also contains the number
 of documents which contain the term, and pointers to the term's frequency and
 proximity data.
 </li>
 <li>
-{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term Frequency data}. 
+{@link org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat Term Frequency data}. 
 For each term in the dictionary, the numbers of all the
 documents that contain that term, and the frequency of the term in that
 document, unless frequencies are omitted (IndexOptions.DOCS_ONLY)
 </li>
 <li>
-{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term Proximity data}. 
+{@link org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat Term Proximity data}. 
 For each term in the dictionary, the positions that the
 term occurs in each document. Note that this will not exist if all fields in
 all documents omit position data.
@@ -185,7 +185,7 @@ term frequency. To add Term Vectors to y
 {@link org.apache.lucene.document.Field Field} constructors
 </li>
 <li>
-{@link org.apache.lucene.codecs.lucene410.Lucene410DocValuesFormat Per-document values}. 
+{@link org.apache.lucene.codecs.lucene50.Lucene50DocValuesFormat Per-document values}. 
 Like stored values, these are also keyed by document
 number, but are generally intended to be loaded into main memory for fast
 access. Whereas stored values are generally intended for summary results from
@@ -264,27 +264,27 @@ systems that frequently run out of file 
 <td>The stored fields for documents</td>
 </tr>
 <tr>
-<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term Dictionary}</td>
+<td>{@link org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat Term Dictionary}</td>
 <td>.tim</td>
 <td>The term dictionary, stores term info</td>
 </tr>
 <tr>
-<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Term Index}</td>
+<td>{@link org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat Term Index}</td>
 <td>.tip</td>
 <td>The index into the Term Dictionary</td>
 </tr>
 <tr>
-<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Frequencies}</td>
+<td>{@link org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat Frequencies}</td>
 <td>.doc</td>
 <td>Contains the list of docs which contain each term along with frequency</td>
 </tr>
 <tr>
-<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Positions}</td>
+<td>{@link org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat Positions}</td>
 <td>.pos</td>
 <td>Stores position information about where a term occurs in the index</td>
 </tr>
 <tr>
-<td>{@link org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat Payloads}</td>
+<td>{@link org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat Payloads}</td>
 <td>.pay</td>
 <td>Stores additional per-position metadata information such as character offsets and user payloads</td>
 </tr>
@@ -294,7 +294,7 @@ systems that frequently run out of file 
 <td>Encodes length and boost factors for docs and fields</td>
 </tr>
 <tr>
-<td>{@link org.apache.lucene.codecs.lucene410.Lucene410DocValuesFormat Per-Document Values}</td>
+<td>{@link org.apache.lucene.codecs.lucene50.Lucene50DocValuesFormat Per-Document Values}</td>
 <td>.dvd, .dvm</td>
 <td>Encodes additional scoring factors or other per-document information.</td>
 </tr>

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java Tue Oct 28 09:19:21 2014
@@ -31,8 +31,8 @@ import org.apache.lucene.codecs.CodecUti
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter;
-import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
 import org.apache.lucene.codecs.lucene50.Lucene50Codec;
+import org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.FieldInfo.DocValuesType;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
@@ -112,6 +112,22 @@ import org.apache.lucene.util.Version;
 
 // nocommit move to oal.index?
 
+// nocommit maxTokenLength?
+
+// nocommit per-field norms format?  then we can commit these "tradeoffs"
+
+// nocommit sortMissingFirst/Last
+
+// nocommit default value?
+
+// nocommit getTermFilter?
+
+// nocommit default qp operator
+
+// nocommit copy field?
+
+// nocommit controlling compression of stored fields, norms
+
 // nocommit back compat: how to handle pre-schema indices
 
 // nocommit should we try to have "garbage collection" here?
@@ -1027,7 +1043,7 @@ public class FieldTypes {
         } else if (field.blockTreeMinItemsInBlock != null) {
           assert field.blockTreeMaxItemsInBlock != null;
           // nocommit do we now have cleaner API for this?  Ie "get me default PF, changing these settings"...
-          return new Lucene41PostingsFormat(field.blockTreeMinItemsInBlock.intValue(),
+          return new Lucene50PostingsFormat(field.blockTreeMinItemsInBlock.intValue(),
                                             field.blockTreeMaxItemsInBlock.intValue());
         }
         return super.getPostingsFormatForField(fieldName); 
@@ -2419,8 +2435,7 @@ public class FieldTypes {
 
   // nocommit somehow move this to IndexReader?
   public static FieldTypes getFieldTypes(Directory dir, Analyzer defaultQueryAnalyzer) throws IOException {
-    SegmentInfos infos = new SegmentInfos();
-    infos.read(dir);
+    SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
     return getFieldTypes(infos.getUserData(), defaultQueryAnalyzer, IndexSearcher.getDefaultSimilarity());
   }
 

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Tue Oct 28 09:19:21 2014
@@ -32,8 +32,6 @@ import java.util.Map;
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.blocktree.FieldReader;
-import org.apache.lucene.codecs.blocktree.Stats;
 import org.apache.lucene.index.CheckIndex.Status.DocValuesStatus;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.search.DocIdSetIterator;
@@ -278,7 +276,7 @@ public class CheckIndex implements Close
        *  tree terms dictionary (this is only set if the
        *  {@link PostingsFormat} for this segment uses block
        *  tree. */
-      public Map<String,Stats> blockTreeStats = null;
+      public Map<String,Object> blockTreeStats = null;
     }
 
     /**
@@ -453,7 +451,7 @@ public class CheckIndex implements Close
   public Status checkIndex(List<String> onlySegments) throws IOException {
     ensureOpen();
     NumberFormat nf = NumberFormat.getInstance(Locale.ROOT);
-    SegmentInfos sis = new SegmentInfos();
+    SegmentInfos sis = null;
     Status result = new Status();
     result.dir = dir;
     String[] files = dir.listAll();
@@ -464,7 +462,7 @@ public class CheckIndex implements Close
     try {
       // Do not use SegmentInfos.read(Directory) since the spooky
       // retrying it does is not necessary here (we hold the write lock):
-      sis.read(dir, lastSegmentsFile);
+      sis = SegmentInfos.readCommit(dir, lastSegmentsFile);
     } catch (Throwable t) {
       if (failFast) {
         IOUtils.reThrow(t);
@@ -594,7 +592,7 @@ public class CheckIndex implements Close
       segInfoStat.docCount = info.info.getDocCount();
       
       final Version version = info.info.getVersion();
-      if (info.info.getDocCount() <= 0 && version != null && version.onOrAfter(Version.LUCENE_4_5_0)) {
+      if (info.info.getDocCount() <= 0) {
         throw new RuntimeException("illegal number of documents: maxDoc=" + info.info.getDocCount());
       }
 
@@ -1289,14 +1287,12 @@ public class CheckIndex implements Close
         // docs got deleted and then merged away):
         
       } else {
-        if (fieldTerms instanceof FieldReader) {
-          final Stats stats = ((FieldReader) fieldTerms).computeStats();
-          assert stats != null;
-          if (status.blockTreeStats == null) {
-            status.blockTreeStats = new HashMap<>();
-          }
-          status.blockTreeStats.put(field, stats);
+        final Object stats = fieldTerms.getStats();
+        assert stats != null;
+        if (status.blockTreeStats == null) {
+          status.blockTreeStats = new HashMap<>();
         }
+        status.blockTreeStats.put(field, stats);
         
         if (sumTotalTermFreq != 0) {
           final long v = fields.terms(field).getSumTotalTermFreq();
@@ -1423,7 +1419,7 @@ public class CheckIndex implements Close
     }
     
     if (verbose && status.blockTreeStats != null && infoStream != null && status.termCount > 0) {
-      for(Map.Entry<String,Stats> ent : status.blockTreeStats.entrySet()) {
+      for(Map.Entry<String, Object> ent : status.blockTreeStats.entrySet()) {
         infoStream.println("      field \"" + ent.getKey() + "\":");
         infoStream.println("      " + ent.getValue().toString().replace("\n", "\n      "));
       }

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java Tue Oct 28 09:19:21 2014
@@ -130,6 +130,10 @@ final class DefaultIndexingChain extends
         PerField perField = fieldHash[i];
         while (perField != null) {
           if (perField.docValuesWriter != null) {
+            if (perField.fieldInfo.hasDocValues() == false) {
+              // BUG
+              throw new AssertionError("segment=" + state.segmentInfo + ": field=\"" + perField.fieldInfo.name + "\" has no docValues but wrote them");
+            }
             if (dvConsumer == null) {
               // lazy init
               DocValuesFormat fmt = state.segmentInfo.getCodec().docValuesFormat();
@@ -139,6 +143,9 @@ final class DefaultIndexingChain extends
             perField.docValuesWriter.finish(docCount);
             perField.docValuesWriter.flush(state, dvConsumer);
             perField.docValuesWriter = null;
+          } else if (perField.fieldInfo.hasDocValues()) {
+            // BUG
+            throw new AssertionError("segment=" + state.segmentInfo + ": field=\"" + perField.fieldInfo.name + "\" has docValues but did not write them");
           }
           perField = perField.next;
         }
@@ -156,6 +163,16 @@ final class DefaultIndexingChain extends
         IOUtils.closeWhileHandlingException(dvConsumer);
       }
     }
+
+    if (state.fieldInfos.hasDocValues() == false) {
+      if (dvConsumer != null) {
+        // BUG
+        throw new AssertionError("segment=" + state.segmentInfo + ": fieldInfos has no docValues but wrote them");
+      }
+    } else if (dvConsumer == null) {
+      // BUG
+      throw new AssertionError("segment=" + state.segmentInfo + ": fieldInfos has docValues but did not wrote them");
+    }
   }
 
   /** Catch up for all docs before us that had no stored
@@ -387,12 +404,12 @@ final class DefaultIndexingChain extends
 
     boolean hasDocValues = fp.fieldInfo.hasDocValues();
 
-    // This will throw an exc if the caller tried to
-    // change the DV type for the field:
-    fp.fieldInfo.setDocValuesType(dvType);
     if (hasDocValues == false) {
+      // This will throw an exc if the caller tried to
+      // change the DV type for the field:
       fieldInfos.globalFieldNumbers.setDocValuesType(fp.fieldInfo.number, fp.fieldInfo.name, dvType);
     }
+    fp.fieldInfo.setDocValuesType(dvType);
 
     int docID = docState.docID;
 

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java Tue Oct 28 09:19:21 2014
@@ -225,8 +225,7 @@ public abstract class DirectoryReader ex
 
     List<IndexCommit> commits = new ArrayList<>();
 
-    SegmentInfos latest = new SegmentInfos();
-    latest.read(dir);
+    SegmentInfos latest = SegmentInfos.readLatestCommit(dir);
     final long currentGen = latest.getGeneration();
 
     commits.add(new StandardDirectoryReader.ReaderCommit(latest, dir));
@@ -239,11 +238,11 @@ public abstract class DirectoryReader ex
           !fileName.equals(IndexFileNames.OLD_SEGMENTS_GEN) &&
           SegmentInfos.generationFromSegmentsFileName(fileName) < currentGen) {
 
-        SegmentInfos sis = new SegmentInfos();
+        SegmentInfos sis = null;
         try {
           // IOException allowed to throw there, in case
           // segments_N is corrupt
-          sis.read(dir, fileName);
+          sis = SegmentInfos.readCommit(dir, fileName);
         } catch (FileNotFoundException | NoSuchFileException fnfe) {
           // LUCENE-948: on NFS (and maybe others), if
           // you have writers switching back and forth
@@ -252,7 +251,6 @@ public abstract class DirectoryReader ex
           // file segments_X exists when in fact it
           // doesn't.  So, we catch this and handle it
           // as if the file does not exist
-          sis = null;
         }
 
         if (sis != null)

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Tue Oct 28 09:19:21 2014
@@ -78,9 +78,8 @@ class DocumentsWriterPerThread {
       this.infoStream = infoStream;
     }
 
-    // Only called by asserts
-    public boolean testPoint(String name) {
-      return docWriter.testPoint(name);
+    public void testPoint(String name) {
+      docWriter.testPoint(name);
     }
 
     public void clear() {
@@ -204,11 +203,10 @@ class DocumentsWriterPerThread {
     return retval;
   }
   
-  final boolean testPoint(String message) {
+  final void testPoint(String message) {
     if (infoStream.isEnabled("TP")) {
       infoStream.message("TP", message);
     }
-    return true;
   }
 
   /** Anything that will add N docs to the index should reserve first to
@@ -222,7 +220,7 @@ class DocumentsWriterPerThread {
   }
 
   public void updateDocument(IndexDocument doc, Analyzer analyzer, Term delTerm) throws IOException {
-    assert testPoint("DocumentsWriterPerThread addDocument start");
+    testPoint("DocumentsWriterPerThread addDocument start");
     assert deleteQueue != null;
     docState.doc = doc;
     docState.analyzer = analyzer;
@@ -260,7 +258,7 @@ class DocumentsWriterPerThread {
   }
 
   public int updateDocuments(Iterable<? extends IndexDocument> docs, Analyzer analyzer, Term delTerm) throws IOException {
-    assert testPoint("DocumentsWriterPerThread addDocuments start");
+    testPoint("DocumentsWriterPerThread addDocuments start");
     assert deleteQueue != null;
     docState.analyzer = analyzer;
     if (INFO_VERBOSE && infoStream.isEnabled("DWPT")) {

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterStallControl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterStallControl.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterStallControl.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterStallControl.java Tue Oct 28 09:19:21 2014
@@ -70,9 +70,9 @@ final class DocumentsWriterStallControl 
         if (stalled) { // react on the first wakeup call!
           // don't loop here, higher level logic will re-stall!
           try {
-            assert incWaiters();
+            incWaiters();
             wait();
-            assert  decrWaiters();
+            decrWaiters();
           } catch (InterruptedException e) {
             throw new ThreadInterruptedException(e);
           }
@@ -86,17 +86,16 @@ final class DocumentsWriterStallControl 
   }
   
   
-  private boolean incWaiters() {
+  private void incWaiters() {
     numWaiting++;
     assert waiting.put(Thread.currentThread(), Boolean.TRUE) == null;
-    
-    return numWaiting > 0;
+    assert numWaiting > 0;
   }
   
-  private boolean decrWaiters() {
+  private void decrWaiters() {
     numWaiting--;
     assert waiting.remove(Thread.currentThread()) != null;
-    return numWaiting >= 0;
+    assert numWaiting >= 0;
   }
   
   synchronized boolean hasBlocked() { // for tests

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java Tue Oct 28 09:19:21 2014
@@ -165,9 +165,6 @@ public final class FieldInfo {
       if (omitNorms) {
         throw new IllegalStateException("non-indexed field '" + name + "' cannot omit norms");
       }
-      if (indexOptions != null) {
-        throw new IllegalStateException("non-indexed field '" + name + "' cannot have index options");
-      }
     }
     
     if (dvGen != -1 && docValueType == null) {

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java Tue Oct 28 09:19:21 2014
@@ -220,11 +220,17 @@ public class FieldInfos implements Itera
       return fieldNumber.intValue();
     }
 
-    // used by assert
-    synchronized boolean containsConsistent(Integer number, String name, DocValuesType dvType) {
-      return name.equals(numberToName.get(number))
-          && number.equals(nameToNumber.get(name)) &&
-        (dvType == null || docValuesType.get(name) == null || dvType == docValuesType.get(name));
+    synchronized void verifyConsistent(Integer number, String name, DocValuesType dvType) {
+      if (name.equals(numberToName.get(number)) == false) {
+        throw new IllegalArgumentException("field number " + number + " is already mapped to field name \"" + numberToName.get(number) + "\", not \"" + name + "\"");
+      }
+      if (number.equals(nameToNumber.get(name)) == false) {
+        throw new IllegalArgumentException("field name \"" + name + "\" is already mapped to field number \"" + nameToNumber.get(name) + "\", not \"" + number + "\"");
+      }
+      DocValuesType currentDVType = docValuesType.get(name);
+      if (dvType != null && currentDVType != null && dvType != currentDVType) {
+        throw new IllegalArgumentException("cannot change DocValues type from " + currentDVType + " to " + dvType + " for field \"" + name + "\"");
+      }
     }
 
     /**
@@ -248,7 +254,7 @@ public class FieldInfos implements Itera
     }
 
     synchronized void setDocValuesType(int number, String name, DocValuesType dvType) {
-      assert containsConsistent(number, name, dvType);
+      verifyConsistent(number, name, dvType);
       docValuesType.put(name, dvType);
     }
   }
@@ -302,20 +308,22 @@ public class FieldInfos implements Itera
         final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber, docValues);
         fi = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, -1, null);
         assert !byName.containsKey(fi.name);
-        assert globalFieldNumbers.containsConsistent(Integer.valueOf(fi.number), fi.name, fi.getDocValuesType());
+        globalFieldNumbers.verifyConsistent(Integer.valueOf(fi.number), fi.name, fi.getDocValuesType());
         byName.put(fi.name, fi);
       } else {
         fi.update(storeTermVector, omitNorms, storePayloads, indexOptions);
 
         if (docValues != null) {
-          // only pay the synchronization cost if fi does not already have a DVType
+          // Only pay the synchronization cost if fi does not already have a DVType
           boolean updateGlobal = !fi.hasDocValues();
-          fi.setDocValuesType(docValues); // this will also perform the consistency check.
           if (updateGlobal) {
-            // must also update docValuesType map so it's
-            // aware of this field's DocValueType 
+            // Must also update docValuesType map so it's
+            // aware of this field's DocValueType.  This will throw IllegalArgumentException if
+            // an illegal type change was attempted.
             globalFieldNumbers.setDocValuesType(fi.number, name, docValues);
           }
+
+          fi.setDocValuesType(docValues); // this will also perform the consistency check.
         }
       }
       return fi;

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java Tue Oct 28 09:19:21 2014
@@ -151,6 +151,11 @@ public class FilterLeafReader extends Le
     public boolean hasPayloads() {
       return in.hasPayloads();
     }
+
+    @Override
+    public Object getStats() throws IOException {
+      return in.getStats();
+    }
   }
 
   /** Base class for filtering {@link TermsEnum} implementations. */

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java Tue Oct 28 09:19:21 2014
@@ -110,8 +110,6 @@ final class FreqProxTermsWriterPerField 
   void newTerm(final int termID) {
     // First time we're seeing this term since the last
     // flush
-    assert docState.testPoint("FreqProxTermsWriterPerField.newTerm start");
-
     final FreqProxPostingsArray postings = freqProxPostingsArray;
 
     postings.lastDocIDs[termID] = docState.docID;
@@ -136,9 +134,6 @@ final class FreqProxTermsWriterPerField 
 
   @Override
   void addTerm(final int termID) {
-
-    assert docState.testPoint("FreqProxTermsWriterPerField.addTerm start");
-
     final FreqProxPostingsArray postings = freqProxPostingsArray;
 
     assert !hasFreq || postings.termFreqs[termID] > 0;

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java Tue Oct 28 09:19:21 2014
@@ -165,9 +165,9 @@ final class IndexFileDeleter implements 
             if (infoStream.isEnabled("IFD")) {
               infoStream.message("IFD", "init: load commit \"" + fileName + "\"");
             }
-            SegmentInfos sis = new SegmentInfos();
+            SegmentInfos sis = null;
             try {
-              sis.read(directory, fileName);
+              sis = SegmentInfos.readCommit(directory, fileName);
             } catch (FileNotFoundException | NoSuchFileException e) {
               // LUCENE-948: on NFS (and maybe others), if
               // you have writers switching back and forth
@@ -179,7 +179,6 @@ final class IndexFileDeleter implements 
               if (infoStream.isEnabled("IFD")) {
                 infoStream.message("IFD", "init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
               }
-              sis = null;
             } catch (IOException e) {
               if (SegmentInfos.generationFromSegmentsFileName(fileName) <= currentGen && directory.fileLength(fileName) > 0) {
                 throw e;
@@ -187,7 +186,6 @@ final class IndexFileDeleter implements 
                 // Most likely we are opening an index that
                 // has an aborted "future" commit, so suppress
                 // exc in this case
-                sis = null;
               }
             }
             if (sis != null) {
@@ -215,9 +213,9 @@ final class IndexFileDeleter implements 
       // listing was stale (eg when index accessed via NFS
       // client with stale directory listing cache).  So we
       // try now to explicitly open this commit point:
-      SegmentInfos sis = new SegmentInfos();
+      SegmentInfos sis = null;
       try {
-        sis.read(directory, currentSegmentsFile);
+        sis = SegmentInfos.readCommit(directory, currentSegmentsFile);
       } catch (IOException e) {
         throw new CorruptIndexException("unable to read current segments_N file", currentSegmentsFile, e);
       }

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Tue Oct 28 09:19:21 2014
@@ -775,7 +775,6 @@ public class IndexWriter implements Clos
 
       // If index is too old, reading the segments will throw
       // IndexFormatTooOldException.
-      segmentInfos = new SegmentInfos();
 
       boolean initialIndexExists = true;
 
@@ -784,13 +783,17 @@ public class IndexWriter implements Clos
         // against an index that's currently open for
         // searching.  In this case we write the next
         // segments_N file with no segments:
+        SegmentInfos sis = null;
         try {
-          segmentInfos.read(directory);
-          segmentInfos.clear();
+          sis = SegmentInfos.readLatestCommit(directory);
+          sis.clear();
         } catch (IOException e) {
           // Likely this means it's a fresh directory
           initialIndexExists = false;
+          sis = new SegmentInfos();
         }
+        
+        segmentInfos = sis;
 
         // Record that we have a change (zero out all
         // segments) pending:
@@ -804,7 +807,7 @@ public class IndexWriter implements Clos
 
         // Do not use SegmentInfos.read(Directory) since the spooky
         // retrying it does is not necessary here (we hold the write lock):
-        segmentInfos.read(directory, lastSegmentsFile);
+        segmentInfos = SegmentInfos.readCommit(directory, lastSegmentsFile);
 
         IndexCommit commit = config.getIndexCommit();
         if (commit != null) {
@@ -815,8 +818,7 @@ public class IndexWriter implements Clos
           // points.
           if (commit.getDirectory() != directory)
             throw new IllegalArgumentException("IndexCommit's directory doesn't match my directory");
-          SegmentInfos oldInfos = new SegmentInfos();
-          oldInfos.read(directory, commit.getSegmentsFileName());
+          SegmentInfos oldInfos = SegmentInfos.readCommit(directory, commit.getSegmentsFileName());
           segmentInfos.replace(oldInfos);
           changed();
           if (infoStream.isEnabled("IW")) {
@@ -2034,7 +2036,7 @@ public class IndexWriter implements Clos
           infoStream.message("IW", "rollback: infos=" + segString(segmentInfos));
         }
 
-        assert testPoint("rollback before checkpoint");
+        testPoint("rollback before checkpoint");
 
         // Ask deleter to locate unreferenced files & remove
         // them:
@@ -2422,8 +2424,7 @@ public class IndexWriter implements Clos
           if (infoStream.isEnabled("IW")) {
             infoStream.message("IW", "addIndexes: process directory " + dir);
           }
-          SegmentInfos sis = new SegmentInfos(); // read infos from dir
-          sis.read(dir);
+          SegmentInfos sis = SegmentInfos.readLatestCommit(dir); // read infos from dir
           totalDocCount += sis.totalDocCount();
 
           fieldTypes.addAll(FieldTypes.getFieldTypes(sis.getUserData(), analyzer, fieldTypes.getSimilarity()));
@@ -2736,7 +2737,7 @@ public class IndexWriter implements Clos
       }
 
       doBeforeFlush();
-      assert testPoint("startDoFlush");
+      testPoint("startDoFlush");
       SegmentInfos toCommit = null;
       boolean anySegmentsFlushed = false;
 
@@ -3024,7 +3025,7 @@ public class IndexWriter implements Clos
     }
 
     doBeforeFlush();
-    assert testPoint("startDoFlush");
+    testPoint("startDoFlush");
     boolean success = false;
     try {
 
@@ -3107,9 +3108,7 @@ public class IndexWriter implements Clos
 
   // for testing only
   DocumentsWriter getDocsWriter() {
-    boolean test = false;
-    assert test = true;
-    return test ? docWriter : null;
+    return docWriter;
   }
 
   /** Expert:  Return the number of documents currently
@@ -3194,7 +3193,7 @@ public class IndexWriter implements Clos
    */
   synchronized private ReadersAndUpdates commitMergedDeletesAndUpdates(MergePolicy.OneMerge merge, MergeState mergeState) throws IOException {
 
-    assert testPoint("startCommitMergeDeletes");
+    testPoint("startCommitMergeDeletes");
 
     final List<SegmentCommitInfo> sourceSegments = merge.segments;
 
@@ -3379,7 +3378,7 @@ public class IndexWriter implements Clos
 
   synchronized private boolean commitMerge(MergePolicy.OneMerge merge, MergeState mergeState) throws IOException {
 
-    assert testPoint("startCommitMerge");
+    testPoint("startCommitMerge");
 
     if (tragedy != null) {
       throw new IllegalStateException("this writer hit an unrecoverable error; cannot complete merge", tragedy);
@@ -3723,7 +3722,7 @@ public class IndexWriter implements Clos
 
   synchronized private void _mergeInit(MergePolicy.OneMerge merge) throws IOException {
 
-    assert testPoint("startMergeInit");
+    testPoint("startMergeInit");
 
     assert merge.registerDone;
     assert merge.maxNumSegments == -1 || merge.maxNumSegments > 0;
@@ -3960,8 +3959,8 @@ public class IndexWriter implements Clos
       // we pass merge.getMergeReaders() instead of merge.readers to allow the
       // OneMerge to return a view over the actual segments to merge
       final SegmentMerger merger = new SegmentMerger(merge.getMergeReaders(),
-          merge.info.info, infoStream, dirWrapper,
-          checkAbort, globalFieldNumberMap, 
+                                                     merge.info.info, infoStream, dirWrapper,
+                                                     checkAbort, globalFieldNumberMap, 
                                                      context);
 
       merge.checkAborted(directory);
@@ -4261,7 +4260,7 @@ public class IndexWriter implements Clos
    *  it. */
   private void startCommit(final SegmentInfos toSync) throws IOException {
 
-    assert testPoint("startStartCommit");
+    testPoint("startStartCommit");
     assert pendingCommit == null;
 
     if (tragedy != null) {
@@ -4297,13 +4296,13 @@ public class IndexWriter implements Clos
         assert filesExist(toSync);
       }
 
-      assert testPoint("midStartCommit");
+      testPoint("midStartCommit");
 
       boolean pendingCommitSet = false;
 
       try {
 
-        assert testPoint("midStartCommit2");
+        testPoint("midStartCommit2");
 
         synchronized(this) {
 
@@ -4341,7 +4340,7 @@ public class IndexWriter implements Clos
           infoStream.message("IW", "done all syncs: " + filesToSync);
         }
 
-        assert testPoint("midStartCommitSuccess");
+        testPoint("midStartCommitSuccess");
 
       } finally {
         synchronized(this) {
@@ -4365,7 +4364,7 @@ public class IndexWriter implements Clos
     } catch (OutOfMemoryError oom) {
       tragicEvent(oom, "startCommit");
     }
-    assert testPoint("finishStartCommit");
+    testPoint("finishStartCommit");
   }
 
   /**
@@ -4442,7 +4441,7 @@ public class IndexWriter implements Clos
     IOUtils.reThrowUnchecked(tragedy);
   }
 
-  // Used only by assert for testing.  Current points:
+  // Used for testing.  Current points:
   //   startDoFlush
   //   startCommitMerge
   //   startStartCommit
@@ -4453,11 +4452,10 @@ public class IndexWriter implements Clos
   //   startCommitMergeDeletes
   //   startMergeInit
   //   DocumentsWriter.ThreadState.init start
-  private final boolean testPoint(String message) {
+  private final void testPoint(String message) {
     if (infoStream.isEnabled("TP")) {
       infoStream.message("TP", message);
     }
-    return true;
   }
 
   synchronized boolean nrtIsCurrent(SegmentInfos infos) {

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java Tue Oct 28 09:19:21 2014
@@ -1,17 +1,5 @@
 package org.apache.lucene.index;
 
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.lucene.codecs.DocValuesFormat;
-import org.apache.lucene.codecs.DocValuesProducer;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.RefCount;
-
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -29,6 +17,18 @@ import org.apache.lucene.util.RefCount;
  * limitations under the License.
  */
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RefCount;
+
 /**
  * Manages the {@link DocValuesProducer} held by {@link SegmentReader} and
  * keeps track of their reference counting.

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java Tue Oct 28 09:19:21 2014
@@ -34,7 +34,6 @@ import org.apache.lucene.util.Accountabl
 import org.apache.lucene.util.Accountables;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.RamUsageEstimator;
-import org.apache.lucene.util.Version;
 
 /** Encapsulates multiple producers when there are docvalues updates as one producer */
 // TODO: try to clean up close? no-op?
@@ -52,63 +51,26 @@ class SegmentDocValuesProducer extends D
   SegmentDocValuesProducer(SegmentCommitInfo si, Directory dir, FieldInfos fieldInfos, SegmentDocValues segDocValues, DocValuesFormat dvFormat) throws IOException {
     boolean success = false;
     try {
-      Version ver = si.info.getVersion();
-      if (ver != null && ver.onOrAfter(Version.LUCENE_4_9_0)) {
-        DocValuesProducer baseProducer = null;
-        for (FieldInfo fi : fieldInfos) {
-          if (!fi.hasDocValues()) {
-            continue;
-          }
-          long docValuesGen = fi.getDocValuesGen();
-          if (docValuesGen == -1) {
-            if (baseProducer == null) {
-              // the base producer gets all the fields, so the Codec can validate properly
-              baseProducer = segDocValues.getDocValuesProducer(docValuesGen, si, IOContext.READ, dir, dvFormat, fieldInfos);
-              dvGens.add(docValuesGen);
-              dvProducers.add(baseProducer);
-            }
-            dvProducersByField.put(fi.name, baseProducer);
-          } else {
-            assert !dvGens.contains(docValuesGen);
-            final DocValuesProducer dvp = segDocValues.getDocValuesProducer(docValuesGen, si, IOContext.READ, dir, dvFormat, new FieldInfos(new FieldInfo[] { fi }));
-            dvGens.add(docValuesGen);
-            dvProducers.add(dvp);
-            dvProducersByField.put(fi.name, dvp);
-          }
-        }
-      } else {
-        // For pre-4.9 indexes, especially with doc-values updates, multiple
-        // FieldInfos could belong to the same dvGen. Therefore need to make sure
-        // we initialize each DocValuesProducer once per gen.
-        Map<Long,List<FieldInfo>> genInfos = new HashMap<>();
-        for (FieldInfo fi : fieldInfos) {
-          if (!fi.hasDocValues()) {
-            continue;
-          }
-          List<FieldInfo> genFieldInfos = genInfos.get(fi.getDocValuesGen());
-          if (genFieldInfos == null) {
-            genFieldInfos = new ArrayList<>();
-            genInfos.put(fi.getDocValuesGen(), genFieldInfos);
-          }
-          genFieldInfos.add(fi);
+      DocValuesProducer baseProducer = null;
+      for (FieldInfo fi : fieldInfos) {
+        if (!fi.hasDocValues()) {
+          continue;
         }
-      
-        for (Map.Entry<Long,List<FieldInfo>> e : genInfos.entrySet()) {
-          long docValuesGen = e.getKey();
-          List<FieldInfo> infos = e.getValue();
-          final DocValuesProducer dvp;
-          if (docValuesGen == -1) {
-            // we need to send all FieldInfos to gen=-1, but later we need to
-            // record the DVP only for the "true" gen=-1 fields (not updated)
-            dvp = segDocValues.getDocValuesProducer(docValuesGen, si, IOContext.READ, dir, dvFormat, fieldInfos);
-          } else {
-            dvp = segDocValues.getDocValuesProducer(docValuesGen, si, IOContext.READ, dir, dvFormat, new FieldInfos(infos.toArray(new FieldInfo[infos.size()])));
+        long docValuesGen = fi.getDocValuesGen();
+        if (docValuesGen == -1) {
+          if (baseProducer == null) {
+            // the base producer gets all the fields, so the Codec can validate properly
+            baseProducer = segDocValues.getDocValuesProducer(docValuesGen, si, IOContext.READ, dir, dvFormat, fieldInfos);
+            dvGens.add(docValuesGen);
+            dvProducers.add(baseProducer);
           }
+          dvProducersByField.put(fi.name, baseProducer);
+        } else {
+          assert !dvGens.contains(docValuesGen);
+          final DocValuesProducer dvp = segDocValues.getDocValuesProducer(docValuesGen, si, IOContext.READ, dir, dvFormat, new FieldInfos(new FieldInfo[] { fi }));
           dvGens.add(docValuesGen);
           dvProducers.add(dvp);
-          for (FieldInfo fi : infos) {
-            dvProducersByField.put(fi.name, dvp);
-          }
+          dvProducersByField.put(fi.name, dvp);
         }
       }
       success = true;

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java Tue Oct 28 09:19:21 2014
@@ -100,7 +100,7 @@ public final class SegmentInfo {
     this.codec = codec;
     this.diagnostics = diagnostics;
     this.id = id;
-    if (id != null && id.length != StringHelper.ID_LENGTH) {
+    if (id.length != StringHelper.ID_LENGTH) {
       throw new IllegalArgumentException("invalid id: " + Arrays.toString(id));
     }
   }
@@ -223,7 +223,7 @@ public final class SegmentInfo {
 
   /** Return the id that uniquely identifies this segment. */
   public byte[] getId() {
-    return id == null ? null : id.clone();
+    return id.clone();
   }
 
   private Set<String> setFiles;

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java Tue Oct 28 09:19:21 2014
@@ -60,18 +60,20 @@ import org.apache.lucene.util.StringHelp
  * Files:
  * <ul>
  * <li><tt>segments_N</tt>: Header, Version, NameCounter, SegCount, &lt;SegName,
- * SegCodec, DelGen, DeletionCount, FieldInfosGen, DocValuesGen,
+ * HasSegID, SegID, SegCodec, DelGen, DeletionCount, FieldInfosGen, DocValuesGen,
  * UpdatesFiles&gt;<sup>SegCount</sup>, CommitUserData, Footer
  * </ul>
  * </p>
  * Data types:
  * <p>
  * <ul>
- * <li>Header --&gt; {@link CodecUtil#writeHeader CodecHeader}</li>
- * <li>GenHeader, NameCounter, SegCount, DeletionCount --&gt;
+ * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}</li>
+ * <li>NameCounter, SegCount, DeletionCount --&gt;
  * {@link DataOutput#writeInt Int32}</li>
  * <li>Generation, Version, DelGen, Checksum, FieldInfosGen, DocValuesGen --&gt;
  * {@link DataOutput#writeLong Int64}</li>
+ * <li>HasSegID --&gt; {@link DataOutput#writeByte Int8}</li>
+ * <li>SegID --&gt; {@link DataOutput#writeByte Int8<sup>ID_LENGTH</sup>}</li>
  * <li>SegName, SegCodec --&gt; {@link DataOutput#writeString String}</li>
  * <li>CommitUserData --&gt; {@link DataOutput#writeStringStringMap
  * Map&lt;String,String&gt;}</li>
@@ -94,6 +96,10 @@ import org.apache.lucene.util.StringHelp
  * <li>DeletionCount records the number of deleted documents in this segment.</li>
  * <li>SegCodec is the {@link Codec#getName() name} of the Codec that encoded
  * this segment.</li>
+ * <li>HasSegID is nonzero if the segment has an identifier. Otherwise, when it is 0
+ * the identifier is {@code null} and no SegID is written. Null only happens for Lucene
+ * 4.x segments referenced in commits.</li>
+ * <li>SegID is the identifier of the Codec that encoded this segment. </li>
  * <li>CommitUserData stores an optional user-supplied opaque
  * Map&lt;String,String&gt; that was passed to
  * {@link IndexWriter#setCommitData(java.util.Map)}.</li>
@@ -113,18 +119,6 @@ import org.apache.lucene.util.StringHelp
  */
 public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo> {
 
-  /** The file format version for the segments_N codec header, up to 4.5. */
-  public static final int VERSION_40 = 0;
-
-  /** The file format version for the segments_N codec header, since 4.6+. */
-  public static final int VERSION_46 = 1;
-  
-  /** The file format version for the segments_N codec header, since 4.8+ */
-  public static final int VERSION_48 = 2;
-  
-  /** The file format version for the segments_N codec header, since 4.9+ */
-  public static final int VERSION_49 = 3;
-
   /** The file format version for the segments_N codec header, since 5.0+ */
   public static final int VERSION_50 = 4;
 
@@ -155,8 +149,8 @@ public final class SegmentInfos implemen
   private byte[] id;
 
   /** Sole constructor. Typically you call this and then
-   *  use {@link #read(Directory) or
-   *  #read(Directory,String)} to populate each {@link
+   *  use {@link #readLatestCommit(Directory) or
+   *  #readCommit(Directory,String)} to populate each {@link
    *  SegmentCommitInfo}.  Alternatively, you can add/remove your
    *  own {@link SegmentCommitInfo}s. */
   public SegmentInfos() {
@@ -245,27 +239,20 @@ public final class SegmentInfos implemen
       throw new IllegalArgumentException("fileName \"" + fileName + "\" is not a segments file");
     }
   }
-
-  /**
-   * Get the next pending_segments_N filename that will be written.
-   */
-  public String getNextPendingSegmentFileName() {
-    long nextGeneration;
-
+  
+  /** return generation of the next pending_segments_N that will be written */
+  private long getNextPendingGeneration() {
     if (generation == -1) {
-      nextGeneration = 1;
+      return 1;
     } else {
-      nextGeneration = generation+1;
+      return generation+1;
     }
-    return IndexFileNames.fileNameFromGeneration(IndexFileNames.PENDING_SEGMENTS,
-                                                 "",
-                                                 nextGeneration);
   }
 
   /** Since Lucene 5.0, every commit (segments_N) writes a unique id.  This will
-   *  return that id, or null if this commit was prior to 5.0. */
+   *  return that id */
   public byte[] getId() {
-    return id == null ? null : id.clone();
+    return id.clone();
   }
 
   /**
@@ -277,131 +264,81 @@ public final class SegmentInfos implemen
    * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
    */
-  public final void read(Directory directory, String segmentFileName) throws IOException {
-    boolean success = false;
+  public static final SegmentInfos readCommit(Directory directory, String segmentFileName) throws IOException {
 
-    // Clear any previous segments:
-    this.clear();
-
-    generation = generationFromSegmentsFileName(segmentFileName);
-
-    lastGeneration = generation;
-
-    ChecksumIndexInput input = directory.openChecksumInput(segmentFileName, IOContext.READ);
-    try {
+    long generation = generationFromSegmentsFileName(segmentFileName);
+    try (ChecksumIndexInput input = directory.openChecksumInput(segmentFileName, IOContext.READ)) {
       // NOTE: as long as we want to throw indexformattooold (vs corruptindexexception), we need
       // to read the magic ourselves.
       int magic = input.readInt();
       if (magic != CodecUtil.CODEC_MAGIC) {
         throw new IndexFormatTooOldException(input, magic, CodecUtil.CODEC_MAGIC, CodecUtil.CODEC_MAGIC);
       }
-      // 4.0+
-      int format = CodecUtil.checkHeaderNoMagic(input, "segments", VERSION_40, VERSION_50);
-      version = input.readLong();
-      counter = input.readInt();
+      CodecUtil.checkHeaderNoMagic(input, "segments", VERSION_50, VERSION_50);
+      byte id[] = new byte[StringHelper.ID_LENGTH];
+      input.readBytes(id, 0, id.length);
+      CodecUtil.checkIndexHeaderSuffix(input, Long.toString(generation, Character.MAX_RADIX));
+      
+      SegmentInfos infos = new SegmentInfos();
+      infos.id = id;
+      infos.generation = generation;
+      infos.lastGeneration = generation;
+      infos.version = input.readLong();
+      infos.counter = input.readInt();
       int numSegments = input.readInt();
       if (numSegments < 0) {
         throw new CorruptIndexException("invalid segment count: " + numSegments, input);
       }
       for (int seg = 0; seg < numSegments; seg++) {
         String segName = input.readString();
+        final byte segmentID[];
+        byte hasID = input.readByte();
+        if (hasID == 1) {
+          segmentID = new byte[StringHelper.ID_LENGTH];
+          input.readBytes(segmentID, 0, segmentID.length);
+        } else {
+          throw new CorruptIndexException("invalid hasID byte, got: " + hasID, input);
+        }
         Codec codec = Codec.forName(input.readString());
-        //System.out.println("SIS.read seg=" + seg + " codec=" + codec);
-        SegmentInfo info = codec.segmentInfoFormat().read(directory, segName, IOContext.READ);
+        SegmentInfo info = codec.segmentInfoFormat().read(directory, segName, segmentID, IOContext.READ);
         info.setCodec(codec);
         long delGen = input.readLong();
         int delCount = input.readInt();
         if (delCount < 0 || delCount > info.getDocCount()) {
           throw new CorruptIndexException("invalid deletion count: " + delCount + " vs docCount=" + info.getDocCount(), input);
         }
-        long fieldInfosGen = -1;
-        if (format >= VERSION_46) {
-          fieldInfosGen = input.readLong();
-        }
-        long dvGen = -1;
-        if (format >= VERSION_49) {
-          dvGen = input.readLong();
-        } else {
-          dvGen = fieldInfosGen;
-        }
+        long fieldInfosGen = input.readLong();
+        long dvGen = input.readLong();
         SegmentCommitInfo siPerCommit = new SegmentCommitInfo(info, delCount, delGen, fieldInfosGen, dvGen);
-        if (format >= VERSION_46) {
-          if (format < VERSION_49) {
-            // Recorded per-generation files, which were buggy (see
-            // LUCENE-5636). We need to read and keep them so we continue to
-            // reference those files. Unfortunately it means that the files will
-            // be referenced even if the fields are updated again, until the
-            // segment is merged.
-            final int numGensUpdatesFiles = input.readInt();
-            final Map<Long,Set<String>> genUpdatesFiles;
-            if (numGensUpdatesFiles == 0) {
-              genUpdatesFiles = Collections.emptyMap();
-            } else {
-              genUpdatesFiles = new HashMap<>(numGensUpdatesFiles);
-              for (int i = 0; i < numGensUpdatesFiles; i++) {
-                genUpdatesFiles.put(input.readLong(), input.readStringSet());
-              }
-            }
-            siPerCommit.setGenUpdatesFiles(genUpdatesFiles);
-          } else {
-            siPerCommit.setFieldInfosFiles(input.readStringSet());
-            final Map<Integer,Set<String>> dvUpdateFiles;
-            final int numDVFields = input.readInt();
-            if (numDVFields == 0) {
-              dvUpdateFiles = Collections.emptyMap();
-            } else {
-              dvUpdateFiles = new HashMap<>(numDVFields);
-              for (int i = 0; i < numDVFields; i++) {
-                dvUpdateFiles.put(input.readInt(), input.readStringSet());
-              }
-            }
-            siPerCommit.setDocValuesUpdatesFiles(dvUpdateFiles);
+        siPerCommit.setFieldInfosFiles(input.readStringSet());
+        final Map<Integer,Set<String>> dvUpdateFiles;
+        final int numDVFields = input.readInt();
+        if (numDVFields == 0) {
+          dvUpdateFiles = Collections.emptyMap();
+        } else {
+          dvUpdateFiles = new HashMap<>(numDVFields);
+          for (int i = 0; i < numDVFields; i++) {
+            dvUpdateFiles.put(input.readInt(), input.readStringSet());
           }
         }
-        add(siPerCommit);
-      }
-      userData = input.readStringStringMap();
-      if (format >= VERSION_50) {
-        id = new byte[StringHelper.ID_LENGTH];
-        input.readBytes(id, 0, id.length);
+        siPerCommit.setDocValuesUpdatesFiles(dvUpdateFiles);
+        infos.add(siPerCommit);
       }
+      infos.userData = input.readStringStringMap();
 
-      if (format >= VERSION_48) {
-        CodecUtil.checkFooter(input);
-      } else {
-        final long checksumNow = input.getChecksum();
-        final long checksumThen = input.readLong();
-        if (checksumNow != checksumThen) {
-          throw new CorruptIndexException("checksum failed (hardware problem?) : expected=" + Long.toHexString(checksumThen) +  
-                                          " actual=" + Long.toHexString(checksumNow), input);
-        }
-        CodecUtil.checkEOF(input);
-      }
+      CodecUtil.checkFooter(input);
 
-      success = true;
-    } finally {
-      if (!success) {
-        // Clear any segment infos we had loaded so we
-        // have a clean slate on retry:
-        this.clear();
-        IOUtils.closeWhileHandlingException(input);
-      } else {
-        input.close();
-      }
+      return infos;
     }
   }
 
   /** Find the latest commit ({@code segments_N file}) and
    *  load all {@link SegmentCommitInfo}s. */
-  public final void read(Directory directory) throws IOException {
-    generation = lastGeneration = -1;
-
-    new FindSegmentsFile(directory) {
-
+  public static final SegmentInfos readLatestCommit(Directory directory) throws IOException {
+    return new FindSegmentsFile<SegmentInfos>(directory) {
       @Override
-      protected Object doBody(String segmentFileName) throws IOException {
-        read(directory, segmentFileName);
-        return null;
+      protected SegmentInfos doBody(String segmentFileName) throws IOException {
+        return readCommit(directory, segmentFileName);
       }
     }.run();
   }
@@ -412,27 +349,38 @@ public final class SegmentInfos implemen
 
   private void write(Directory directory) throws IOException {
 
-    String segmentFileName = getNextPendingSegmentFileName();
+    long nextGeneration = getNextPendingGeneration();
+    String segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.PENDING_SEGMENTS,
+                                                                   "",
+                                                                   nextGeneration);
     
     // Always advance the generation on write:
-    if (generation == -1) {
-      generation = 1;
-    } else {
-      generation++;
-    }
+    generation = nextGeneration;
     
     IndexOutput segnOutput = null;
     boolean success = false;
 
     try {
       segnOutput = directory.createOutput(segmentFileName, IOContext.DEFAULT);
-      CodecUtil.writeHeader(segnOutput, "segments", VERSION_50);
+      CodecUtil.writeIndexHeader(segnOutput, "segments", VERSION_50, 
+                                   StringHelper.randomId(), Long.toString(nextGeneration, Character.MAX_RADIX));
       segnOutput.writeLong(version); 
       segnOutput.writeInt(counter); // write counter
       segnOutput.writeInt(size()); // write infos
       for (SegmentCommitInfo siPerCommit : this) {
         SegmentInfo si = siPerCommit.info;
         segnOutput.writeString(si.name);
+        byte segmentID[] = si.getId();
+        // TODO: remove this in lucene 6, we don't need to include 4.x segments in commits anymore
+        if (segmentID == null) {
+          segnOutput.writeByte((byte)0);
+        } else {
+          if (segmentID.length != StringHelper.ID_LENGTH) {
+            throw new IllegalStateException("cannot write segment: invalid id segment=" + si.name + "id=" + StringHelper.idToString(segmentID));
+          }
+          segnOutput.writeByte((byte)1);
+          segnOutput.writeBytes(segmentID, segmentID.length);
+        }
         segnOutput.writeString(si.getCodec().getName());
         segnOutput.writeLong(siPerCommit.getDelGen());
         int delCount = siPerCommit.getDelCount();
@@ -452,8 +400,6 @@ public final class SegmentInfos implemen
         assert si.dir == directory;
       }
       segnOutput.writeStringStringMap(userData);
-      byte[] id = StringHelper.randomId();
-      segnOutput.writeBytes(id, 0, id.length);
       CodecUtil.writeFooter(segnOutput);
       segnOutput.close();
       directory.sync(Collections.singleton(segmentFileName));
@@ -547,7 +493,7 @@ public final class SegmentInfos implemen
    * time, etc., it could have been deleted due to a writer
    * commit finishing.
    */
-  public abstract static class FindSegmentsFile {
+  public abstract static class FindSegmentsFile<T> {
 
     final Directory directory;
 
@@ -558,12 +504,12 @@ public final class SegmentInfos implemen
 
     /** Locate the most recent {@code segments} file and
      *  run {@link #doBody} on it. */
-    public Object run() throws IOException {
+    public T run() throws IOException {
       return run(null);
     }
     
     /** Run {@link #doBody} on the provided commit. */
-    public Object run(IndexCommit commit) throws IOException {
+    public T run(IndexCommit commit) throws IOException {
       if (commit != null) {
         if (directory != commit.getDirectory())
           throw new IOException("the specified commit does not match the specified Directory");
@@ -607,11 +553,11 @@ public final class SegmentInfos implemen
           String segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
         
           try {
-            Object v = doBody(segmentFileName);
+            T t = doBody(segmentFileName);
             if (infoStream != null) {
               message("success on " + segmentFileName);
             }
-            return v;
+            return t;
           } catch (IOException err) {
             // Save the original root cause:
             if (exc == null) {
@@ -634,7 +580,7 @@ public final class SegmentInfos implemen
      * during the processing that could have been caused by
      * a writer committing.
      */
-    protected abstract Object doBody(String segmentFileName) throws IOException;
+    protected abstract T doBody(String segmentFileName) throws IOException;
   }
 
   // Carry over generation numbers from another SegmentInfos

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java Tue Oct 28 09:19:21 2014
@@ -30,8 +30,8 @@ import org.apache.lucene.index.MultiDocV
 
 /**
  * This class forces a composite reader (eg a {@link
- * MultiReader} or {@link DirectoryReader}) to emulate an
- * atomic reader.  This requires implementing the postings
+ * MultiReader} or {@link DirectoryReader}) to emulate a
+ * {@link LeafReader}.  This requires implementing the postings
  * APIs on-the-fly, using the static methods in {@link
  * MultiFields}, {@link MultiDocValues}, by stepping through
  * the sub-readers to merge fields/terms, appending docs, etc.
@@ -40,7 +40,7 @@ import org.apache.lucene.index.MultiDocV
  * performance hit.  If this is important to your use case,
  * you'll get better performance by gathering the sub readers using
  * {@link IndexReader#getContext()} to get the
- * atomic leaves and then operate per-LeafReader,
+ * leaves and then operate per-LeafReader,
  * instead of using this class.
  */
 public final class SlowCompositeReaderWrapper extends LeafReader {

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java Tue Oct 28 09:19:21 2014
@@ -47,11 +47,10 @@ final class StandardDirectoryReader exte
 
   /** called from DirectoryReader.open(...) methods */
   static DirectoryReader open(final Directory directory, final IndexCommit commit) throws IOException {
-    return (DirectoryReader) new SegmentInfos.FindSegmentsFile(directory) {
+    return new SegmentInfos.FindSegmentsFile<DirectoryReader>(directory) {
       @Override
-      protected Object doBody(String segmentFileName) throws IOException {
-        SegmentInfos sis = new SegmentInfos();
-        sis.read(directory, segmentFileName);
+      protected DirectoryReader doBody(String segmentFileName) throws IOException {
+        SegmentInfos sis = SegmentInfos.readCommit(directory, segmentFileName);
         final SegmentReader[] readers = new SegmentReader[sis.size()];
         for (int i = sis.size()-1; i >= 0; i--) {
           boolean success = false;
@@ -309,11 +308,10 @@ final class StandardDirectoryReader exte
   }
 
   private DirectoryReader doOpenFromCommit(IndexCommit commit) throws IOException {
-    return (DirectoryReader) new SegmentInfos.FindSegmentsFile(directory) {
+    return new SegmentInfos.FindSegmentsFile<DirectoryReader>(directory) {
       @Override
-      protected Object doBody(String segmentFileName) throws IOException {
-        final SegmentInfos infos = new SegmentInfos();
-        infos.read(directory, segmentFileName);
+      protected DirectoryReader doBody(String segmentFileName) throws IOException {
+        final SegmentInfos infos = SegmentInfos.readCommit(directory, segmentFileName);
         return doOpenIfChanged(infos);
       }
     }.run(commit);
@@ -338,8 +336,7 @@ final class StandardDirectoryReader exte
       // IndexWriter.prepareCommit has been called (but not
       // yet commit), then the reader will still see itself as
       // current:
-      SegmentInfos sis = new SegmentInfos();
-      sis.read(directory);
+      SegmentInfos sis = SegmentInfos.readLatestCommit(directory);
 
       // we loaded SegmentInfos from the directory
       return sis.getVersion() == segmentInfos.getVersion();

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java Tue Oct 28 09:19:21 2014
@@ -93,8 +93,6 @@ final class TermVectorsConsumer extends 
   @Override
   void finishDocument() throws IOException {
 
-    assert docWriter.testPoint("TermVectorsTermsWriter.finishDocument start");
-
     if (!hasVectors) {
       return;
     }
@@ -119,7 +117,6 @@ final class TermVectorsConsumer extends 
 
     super.reset();
     resetFields();
-    assert docWriter.testPoint("TermVectorsTermsWriter.finishDocument end");
   }
 
   @Override

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java Tue Oct 28 09:19:21 2014
@@ -63,8 +63,6 @@ final class TermVectorsConsumerPerField 
 
     doVectors = false;
 
-    assert docState.testPoint("TermVectorsTermsWriterPerField.finish start");
-
     final int numPostings = bytesHash.size();
 
     final BytesRef flushTerm = termsWriter.flushTerm;
@@ -226,7 +224,6 @@ final class TermVectorsConsumerPerField 
 
   @Override
   void newTerm(final int termID) {
-    assert docState.testPoint("TermVectorsTermsWriterPerField.newTerm start");
     TermVectorsPostingsArray postings = termVectorsPostingsArray;
 
     postings.freqs[termID] = 1;
@@ -238,7 +235,6 @@ final class TermVectorsConsumerPerField 
 
   @Override
   void addTerm(final int termID) {
-    assert docState.testPoint("TermVectorsTermsWriterPerField.addTerm start");
     TermVectorsPostingsArray postings = termVectorsPostingsArray;
 
     postings.freqs[termID]++;

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/Terms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/Terms.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/Terms.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/Terms.java Tue Oct 28 09:19:21 2014
@@ -193,4 +193,18 @@ public abstract class Terms {
       scratch.grow(scratch.length());
     }
   }
+  
+  /** 
+   * Expert: returns additional information about this Terms instance
+   * for debugging purposes.
+   */
+  public Object getStats() throws IOException {
+    StringBuilder sb = new StringBuilder();
+    sb.append("impl=" + getClass().getSimpleName());
+    sb.append(",size=" + size());
+    sb.append(",docCount=" + getDocCount());
+    sb.append(",sumTotalTermFreq=" + getSumTotalTermFreq());
+    sb.append(",sumDocFreq=" + getSumDocFreq());
+    return sb.toString();
+  }
 }

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/DocValuesDocIdSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/DocValuesDocIdSet.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/DocValuesDocIdSet.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/DocValuesDocIdSet.java Tue Oct 28 09:19:21 2014
@@ -19,6 +19,7 @@ package org.apache.lucene.search;
 import java.io.IOException;
 
 import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BitDocIdSet;
 import org.apache.lucene.util.FixedBitSet;
 
 /**
@@ -26,8 +27,7 @@ import org.apache.lucene.util.FixedBitSe
  * of its iterator is very stupid and slow if the implementation of the
  * {@link #matchDoc} method is not optimized, as iterators simply increment
  * the document id until {@code matchDoc(int)} returns true. Because of this
- * {@code matchDoc(int)} must be as fast as possible and in no case do any
- * I/O.
+ * {@code matchDoc(int)} must be as fast as possible.
  * @lucene.internal
  */
 public abstract class DocValuesDocIdSet extends DocIdSet {
@@ -65,7 +65,7 @@ public abstract class DocValuesDocIdSet 
     } : new Bits() {
       @Override
       public boolean get(int docid) {
-        return matchDoc(docid) && acceptDocs.get(docid);
+        return acceptDocs.get(docid) && matchDoc(docid);
       }
 
       @Override
@@ -116,7 +116,7 @@ public abstract class DocValuesDocIdSet 
     } else if (acceptDocs instanceof FixedBitSet) {
       // special case for FixedBitSet: use the iterator and filter it
       // (used e.g. when Filters are chained by FilteredQuery)
-      return new FilteredDocIdSetIterator(((DocIdSet) acceptDocs).iterator()) {
+      return new FilteredDocIdSetIterator(new BitDocIdSet((FixedBitSet) acceptDocs).iterator()) {
         @Override
         protected boolean match(int doc) {
           return DocValuesDocIdSet.this.matchDoc(doc);
@@ -134,19 +134,13 @@ public abstract class DocValuesDocIdSet 
       
         @Override
         public int nextDoc() {
-          do {
-            doc++;
-            if (doc >= maxDoc) {
-              return doc = NO_MORE_DOCS;
-            }
-          } while (!(matchDoc(doc) && acceptDocs.get(doc)));
-          return doc;
+          return advance(doc + 1);
         }
       
         @Override
         public int advance(int target) {
           for(doc=target; doc<maxDoc; doc++) {
-            if (matchDoc(doc) && acceptDocs.get(doc)) {
+            if (acceptDocs.get(doc) && matchDoc(doc)) {
               return doc;
             }
           }

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java Tue Oct 28 09:19:21 2014
@@ -261,7 +261,11 @@ public abstract class FieldComparator<T>
     private double bottom;
     private double topValue;
 
-    DoubleComparator(int numHits, String field, Double missingValue) {
+    /** 
+     * Creates a new comparator based on {@link Double#compare} for {@code numHits}.
+     * When a document has no value for the field, {@code missingValue} is substituted. 
+     */
+    public DoubleComparator(int numHits, String field, Double missingValue) {
       super(field, missingValue);
       values = new double[numHits];
     }
@@ -329,7 +333,11 @@ public abstract class FieldComparator<T>
     private float bottom;
     private float topValue;
 
-    FloatComparator(int numHits, String field, Float missingValue) {
+    /** 
+     * Creates a new comparator based on {@link Float#compare} for {@code numHits}.
+     * When a document has no value for the field, {@code missingValue} is substituted. 
+     */
+    public FloatComparator(int numHits, String field, Float missingValue) {
       super(field, missingValue);
       values = new float[numHits];
     }
@@ -398,7 +406,11 @@ public abstract class FieldComparator<T>
     private int bottom;                           // Value of bottom of queue
     private int topValue;
 
-    IntComparator(int numHits, String field, Integer missingValue) {
+    /** 
+     * Creates a new comparator based on {@link Integer#compare} for {@code numHits}.
+     * When a document has no value for the field, {@code missingValue} is substituted. 
+     */
+    public IntComparator(int numHits, String field, Integer missingValue) {
       super(field, missingValue);
       values = new int[numHits];
     }
@@ -466,7 +478,11 @@ public abstract class FieldComparator<T>
     private long bottom;
     private long topValue;
 
-    LongComparator(int numHits, String field, Long missingValue) {
+    /** 
+     * Creates a new comparator based on {@link Long#compare} for {@code numHits}.
+     * When a document has no value for the field, {@code missingValue} is substituted. 
+     */
+    public LongComparator(int numHits, String field, Long missingValue) {
       super(field, missingValue);
       values = new long[numHits];
     }
@@ -541,7 +557,8 @@ public abstract class FieldComparator<T>
     private Scorer scorer;
     private float topValue;
 
-    RelevanceComparator(int numHits) {
+    /** Creates a new comparator based on relevance for {@code numHits}. */
+    public RelevanceComparator(int numHits) {
       scores = new float[numHits];
     }
 
@@ -618,7 +635,8 @@ public abstract class FieldComparator<T>
     private int bottom;
     private int topValue;
 
-    DocComparator(int numHits) {
+    /** Creates a new comparator based on document ids for {@code numHits} */
+    public DocComparator(int numHits) {
       docIDs = new int[numHits];
     }
 

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java Tue Oct 28 09:19:21 2014
@@ -510,9 +510,8 @@ final class SloppyPhraseScorer extends S
     HashMap<Term,Integer> tg = new HashMap<>();
     Term[] t = tord.keySet().toArray(new Term[0]);
     for (int i=0; i<bb.size(); i++) { // i is the group no.
-      DocIdSetIterator bits = bb.get(i).iterator();
-      int ord;
-      while ((ord=bits.nextDoc())!=NO_MORE_DOCS) {
+      FixedBitSet bits = bb.get(i);
+      for (int ord = bits.nextSetBit(0); ord != DocIdSetIterator.NO_MORE_DOCS; ord = ord + 1 >= bits.length() ? DocIdSetIterator.NO_MORE_DOCS : bits.nextSetBit(ord + 1)) {
         tg.put(t[ord],i);
       }
     }

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java Tue Oct 28 09:19:21 2014
@@ -37,10 +37,19 @@ public final class DocIdSetBuilder {
   // to re-compute approximateCardinality on the sparse set every time 
   private long costUpperBound;
 
-  /** Sole constructor. */
-  public DocIdSetBuilder(int maxDoc) {
+  /** Create a new instance that can hold <code>maxDoc</code> documents and is optionally <code>full</code>. */
+  public DocIdSetBuilder(int maxDoc, boolean full) {
     this.maxDoc = maxDoc;
     threshold = maxDoc >>> 10;
+    if (full) {
+      denseSet = new FixedBitSet(maxDoc);
+      denseSet.set(0, maxDoc);
+    }
+  }
+
+  /** Create a new empty instance. */
+  public DocIdSetBuilder(int maxDoc) {
+    this(maxDoc, false);
   }
 
   /**
@@ -63,7 +72,7 @@ public final class DocIdSetBuilder {
         denseSet = new FixedBitSet(maxDoc);
         denseSet.or(it);
         if (sparseSet != null) {
-          denseSet.or(sparseSet.iterator());
+          denseSet.or(new BitSetIterator(sparseSet, 0L));
         }
         return;
       }
@@ -77,6 +86,28 @@ public final class DocIdSetBuilder {
   }
 
   /**
+   * Removes from this builder documents that are not contained in <code>it</code>.
+   */
+  public void and(DocIdSetIterator it) throws IOException {
+    if (denseSet != null) {
+      denseSet.and(it);
+    } else if (sparseSet != null) {
+      sparseSet.and(it);
+    }
+  }
+
+  /**
+   * Removes from this builder documents that are contained in <code>it</code>.
+   */
+  public void andNot(DocIdSetIterator it) throws IOException {
+    if (denseSet != null) {
+      denseSet.andNot(it);
+    } else if (denseSet != null) {
+      denseSet.andNot(it);
+    }
+  }
+
+  /**
    * Build a {@link DocIdSet} that contains all doc ids that have been added.
    * This method may return <tt>null</tt> if no documents were addded to this
    * builder.
@@ -84,7 +115,14 @@ public final class DocIdSetBuilder {
    * anymore after this method has been called.
    */
   public DocIdSet build() {
-    final DocIdSet result = denseSet != null ? denseSet : sparseSet;
+    final DocIdSet result;
+    if (denseSet != null) {
+      result = new BitDocIdSet(denseSet);
+    } else if (sparseSet != null) {
+      result = new BitDocIdSet(sparseSet);
+    } else {
+      result = null;
+    }
     denseSet = null;
     sparseSet = null;
     costUpperBound = 0;



Mime
View raw message