lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From romseyg...@apache.org
Subject svn commit: r1435287 [7/41] - in /lucene/dev/branches/LUCENE-2878: ./ dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/analysis/icu/ dev-tools/maven/ dev-tools/maven/lucene/benchmark/ dev-tools/maven/solr/ dev-tools/m...
Date Fri, 18 Jan 2013 18:31:23 GMT
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java Fri Jan 18 18:30:54 2013
@@ -71,18 +71,27 @@ public abstract class TermVectorsWriter 
    *  has no vector fields, in this case <code>numVectorFields</code> 
    *  will be zero. */
   public abstract void startDocument(int numVectorFields) throws IOException;
-  
+
+  /** Called after a doc and all its fields have been added. */
+  public void finishDocument() throws IOException {};
+
   /** Called before writing the terms of the field.
    *  {@link #startTerm(BytesRef, int)} will be called <code>numTerms</code> times. */
   public abstract void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets, boolean payloads) throws IOException;
-  
+
+  /** Called after a field and all its terms have been added. */
+  public void finishField() throws IOException {};
+
   /** Adds a term and its term frequency <code>freq</code>.
    * If this field has positions and/or offsets enabled, then
    * {@link #addPosition(int, int, int, BytesRef)} will be called 
    * <code>freq</code> times respectively.
    */
   public abstract void startTerm(BytesRef term, int freq) throws IOException;
-  
+
+  /** Called after a term and all its positions have been added. */
+  public void finishTerm() throws IOException {}
+
   /** Adds a term position and offsets */
   public abstract void addPosition(int position, int startOffset, int endOffset, BytesRef payload) throws IOException;
   
@@ -97,7 +106,7 @@ public abstract class TermVectorsWriter 
    *  check that this is the case to detect the JRE bug described 
    *  in LUCENE-1282. */
   public abstract void finish(FieldInfos fis, int numDocs) throws IOException;
-  
+
   /** 
    * Called by IndexWriter when writing new segments.
    * <p>
@@ -197,6 +206,7 @@ public abstract class TermVectorsWriter 
   protected final void addAllDocVectors(Fields vectors, MergeState mergeState) throws IOException {
     if (vectors == null) {
       startDocument(0);
+      finishDocument();
       return;
     }
 
@@ -275,15 +285,19 @@ public abstract class TermVectorsWriter 
             addPosition(pos, startOffset, endOffset, payload);
           }
         }
+        finishTerm();
       }
       assert termCount == numTerms;
+      finishField();
     }
     assert fieldCount == numFields;
+    finishDocument();
   }
   
   /** Return the BytesRef Comparator used to sort terms
    *  before feeding to this API. */
   public abstract Comparator<BytesRef> getComparator() throws IOException;
 
+  @Override
   public abstract void close() throws IOException;
 }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/TermsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/TermsConsumer.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/TermsConsumer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/TermsConsumer.java Fri Jan 18 18:30:54 2013
@@ -23,6 +23,7 @@ import java.util.Comparator;
 import org.apache.lucene.index.FieldInfo; // javadocs
 import org.apache.lucene.index.DocsAndPositionsEnum;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.MergeState;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.MultiDocsEnum;
@@ -103,7 +104,7 @@ public abstract class TermsConsumer {
       while((term = termsEnum.next()) != null) {
         // We can pass null for liveDocs, because the
         // mapping enum will skip the non-live docs:
-        docsEnumIn = (MultiDocsEnum) termsEnum.docs(null, docsEnumIn, 0);
+        docsEnumIn = (MultiDocsEnum) termsEnum.docs(null, docsEnumIn, DocsEnum.FLAG_NONE);
         if (docsEnumIn != null) {
           docsEnum.reset(docsEnumIn);
           final PostingsConsumer postingsConsumer = startTerm(term);

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java Fri Jan 18 18:30:54 2013
@@ -48,16 +48,30 @@ import org.apache.lucene.store.IOContext
 public class CompressingStoredFieldsFormat extends StoredFieldsFormat {
 
   private final String formatName;
+  private final String segmentSuffix;
   private final CompressionMode compressionMode;
   private final int chunkSize;
 
   /**
+   * Create a new {@link CompressingStoredFieldsFormat} with an empty segment 
+   * suffix.
+   * 
+   * @see CompressingStoredFieldsFormat#CompressingStoredFieldsFormat(String, String, CompressionMode, int)
+   */
+  public CompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode, int chunkSize) {
+    this(formatName, "", compressionMode, chunkSize);
+  }
+  
+  /**
    * Create a new {@link CompressingStoredFieldsFormat}.
    * <p>
    * <code>formatName</code> is the name of the format. This name will be used
    * in the file formats to perform
    * {@link CodecUtil#checkHeader(org.apache.lucene.store.DataInput, String, int, int) codec header checks}.
    * <p>
+   * <code>segmentSuffix</code> is the segment suffix. This suffix is added to 
+   * the result file name only if it's not the empty string.
+   * <p>
    * The <code>compressionMode</code> parameter allows you to choose between
    * compression algorithms that have various compression and decompression
    * speeds so that you can pick the one that best fits your indexing and
@@ -81,25 +95,29 @@ public class CompressingStoredFieldsForm
    * @param chunkSize the minimum number of bytes of a single chunk of stored documents
    * @see CompressionMode
    */
-  public CompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode, int chunkSize) {
+  public CompressingStoredFieldsFormat(String formatName, String segmentSuffix, 
+                                       CompressionMode compressionMode, int chunkSize) {
     this.formatName = formatName;
+    this.segmentSuffix = segmentSuffix;
     this.compressionMode = compressionMode;
     if (chunkSize < 1) {
       throw new IllegalArgumentException("chunkSize must be >= 1");
     }
     this.chunkSize = chunkSize;
+    
   }
 
   @Override
   public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si,
       FieldInfos fn, IOContext context) throws IOException {
-    return new CompressingStoredFieldsReader(directory, si, fn, context, formatName, compressionMode);
+    return new CompressingStoredFieldsReader(directory, si, segmentSuffix, fn, 
+        context, formatName, compressionMode);
   }
 
   @Override
   public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si,
       IOContext context) throws IOException {
-    return new CompressingStoredFieldsWriter(directory, si, context,
+    return new CompressingStoredFieldsWriter(directory, si, segmentSuffix, context,
         formatName, compressionMode, chunkSize);
   }
 

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java Fri Jan 18 18:30:54 2013
@@ -54,7 +54,11 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.packed.PackedInts;
 
-final class CompressingStoredFieldsReader extends StoredFieldsReader {
+/**
+ * {@link StoredFieldsReader} impl for {@link CompressingStoredFieldsFormat}.
+ * @lucene.experimental
+ */
+public final class CompressingStoredFieldsReader extends StoredFieldsReader {
 
   private final FieldInfos fieldInfos;
   private final CompressingStoredFieldsIndexReader indexReader;
@@ -79,7 +83,8 @@ final class CompressingStoredFieldsReade
     this.closed = false;
   }
 
-  public CompressingStoredFieldsReader( Directory d, SegmentInfo si, FieldInfos fn,
+  /** Sole constructor. */
+  public CompressingStoredFieldsReader(Directory d, SegmentInfo si, String segmentSuffix, FieldInfos fn,
       IOContext context, String formatName, CompressionMode compressionMode) throws IOException {
     this.compressionMode = compressionMode;
     final String segment = si.name;
@@ -88,8 +93,8 @@ final class CompressingStoredFieldsReade
     numDocs = si.getDocCount();
     IndexInput indexStream = null;
     try {
-      fieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION), context);
-      final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION);
+      fieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION), context);
+      final String indexStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_INDEX_EXTENSION);
       indexStream = d.openInput(indexStreamFN, context);
 
       final String codecNameIdx = formatName + CODEC_SFX_IDX;
@@ -123,6 +128,9 @@ final class CompressingStoredFieldsReade
     }
   }
 
+  /** 
+   * Close the underlying {@link IndexInput}s.
+   */
   @Override
   public void close() throws IOException {
     if (!closed) {
@@ -387,8 +395,10 @@ final class CompressingStoredFieldsReade
      * Copy compressed data.
      */
     void copyCompressedData(DataOutput out) throws IOException {
-      final int chunkSize = chunkSize();
-      decompressor.copyCompressedData(fieldsStream, chunkSize, out);
+      final long chunkEnd = docBase + chunkDocs == numDocs
+          ? fieldsStream.length()
+          : indexReader.getStartPointer(docBase + chunkDocs);
+      out.copyBytes(fieldsStream, chunkEnd - fieldsStream.getFilePointer());
     }
 
   }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java Fri Jan 18 18:30:54 2013
@@ -47,7 +47,11 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.packed.PackedInts;
 
-final class CompressingStoredFieldsWriter extends StoredFieldsWriter {
+/**
+ * {@link StoredFieldsWriter} impl for {@link CompressingStoredFieldsFormat}.
+ * @lucene.experimental
+ */
+public final class CompressingStoredFieldsWriter extends StoredFieldsWriter {
 
   static final int         STRING = 0x00;
   static final int       BYTE_ARR = 0x01;
@@ -66,6 +70,7 @@ final class CompressingStoredFieldsWrite
 
   private final Directory directory;
   private final String segment;
+  private final String segmentSuffix;
   private CompressingStoredFieldsIndexWriter indexWriter;
   private IndexOutput fieldsStream;
 
@@ -79,11 +84,13 @@ final class CompressingStoredFieldsWrite
   private int docBase; // doc ID at the beginning of the chunk
   private int numBufferedDocs; // docBase + numBufferedDocs == current doc ID
 
-  public CompressingStoredFieldsWriter(Directory directory, SegmentInfo si, IOContext context,
+  /** Sole constructor. */
+  public CompressingStoredFieldsWriter(Directory directory, SegmentInfo si, String segmentSuffix, IOContext context,
       String formatName, CompressionMode compressionMode, int chunkSize) throws IOException {
     assert directory != null;
     this.directory = directory;
     this.segment = si.name;
+    this.segmentSuffix = segmentSuffix;
     this.compressionMode = compressionMode;
     this.compressor = compressionMode.newCompressor();
     this.chunkSize = chunkSize;
@@ -94,9 +101,9 @@ final class CompressingStoredFieldsWrite
     this.numBufferedDocs = 0;
 
     boolean success = false;
-    IndexOutput indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION), context);
+    IndexOutput indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_INDEX_EXTENSION), context);
     try {
-      fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION), context);
+      fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION), context);
 
       final String codecNameIdx = formatName + CODEC_SFX_IDX;
       final String codecNameDat = formatName + CODEC_SFX_DAT;
@@ -129,19 +136,8 @@ final class CompressingStoredFieldsWrite
     }
   }
 
-  private void endWithPreviousDocument() throws IOException {
-    if (numBufferedDocs > 0) {
-      endOffsets[numBufferedDocs - 1] = bufferedDocs.length;
-    }
-  }
-
   @Override
   public void startDocument(int numStoredFields) throws IOException {
-    endWithPreviousDocument();
-    if (triggerFlush()) {
-      flush();
-    }
-
     if (numBufferedDocs == this.numStoredFields.length) {
       final int newLength = ArrayUtil.oversize(numBufferedDocs + 1, 4);
       this.numStoredFields = Arrays.copyOf(this.numStoredFields, newLength);
@@ -151,6 +147,14 @@ final class CompressingStoredFieldsWrite
     ++numBufferedDocs;
   }
 
+  @Override
+  public void finishDocument() throws IOException {
+    endOffsets[numBufferedDocs - 1] = bufferedDocs.length;
+    if (triggerFlush()) {
+      flush();
+    }
+  }
+
   private static void saveInts(int[] values, int length, DataOutput out) throws IOException {
     assert length > 0;
     if (length == 1) {
@@ -282,15 +286,16 @@ final class CompressingStoredFieldsWrite
   public void abort() {
     IOUtils.closeWhileHandlingException(this);
     IOUtils.deleteFilesIgnoringExceptions(directory,
-        IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION),
-        IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION));
+        IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION),
+        IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_INDEX_EXTENSION));
   }
 
   @Override
   public void finish(FieldInfos fis, int numDocs) throws IOException {
-    endWithPreviousDocument();
     if (numBufferedDocs > 0) {
       flush();
+    } else {
+      assert bufferedDocs.length == 0;
     }
     if (docBase != numDocs) {
       throw new RuntimeException("Wrote " + docBase + " docs, finish called with numDocs=" + numDocs);
@@ -344,17 +349,13 @@ final class CompressingStoredFieldsWrite
             }
 
             if (compressionMode == matchingFieldsReader.getCompressionMode() // same compression mode
-                && (numBufferedDocs == 0 || triggerFlush()) // starting a new chunk
+                && numBufferedDocs == 0 // starting a new chunk
                 && startOffsets[it.chunkDocs - 1] < chunkSize // chunk is small enough
                 && startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] >= chunkSize // chunk is large enough
                 && nextDeletedDoc(it.docBase, liveDocs, it.docBase + it.chunkDocs) == it.docBase + it.chunkDocs) { // no deletion in the chunk
               assert docID == it.docBase;
 
               // no need to decompress, just copy data
-              endWithPreviousDocument();
-              if (triggerFlush()) {
-                flush();
-              }
               indexWriter.writeIndex(it.chunkDocs, fieldsStream.getFilePointer());
               writeHeader(this.docBase, it.chunkDocs, it.numStoredFields, it.lengths);
               it.copyCompressedData(fieldsStream);
@@ -373,6 +374,7 @@ final class CompressingStoredFieldsWrite
                 final int diff = docID - it.docBase;
                 startDocument(it.numStoredFields[diff]);
                 bufferedDocs.writeBytes(it.bytes.bytes, it.bytes.offset + startOffsets[diff], it.lengths[diff]);
+                finishDocument();
                 ++docCount;
                 mergeState.checkAbort.work(300);
               }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java Fri Jan 18 18:30:54 2013
@@ -141,14 +141,6 @@ public abstract class CompressionMode {
     }
 
     @Override
-    public void copyCompressedData(DataInput in, int originalLength, DataOutput out) throws IOException {
-      final int copied = LZ4.copyCompressedData(in, originalLength, out);
-      if (copied != originalLength) {
-        throw new CorruptIndexException("Currupted compressed stream: expected " + originalLength + " bytes, but got at least" + copied);
-      }
-    }
-
-    @Override
     public Decompressor clone() {
       return this;
     }
@@ -225,13 +217,6 @@ public abstract class CompressionMode {
     }
 
     @Override
-    public void copyCompressedData(DataInput in, int originalLength, DataOutput out) throws IOException {
-      final int compressedLength = in.readVInt();
-      out.writeVInt(compressedLength);
-      out.copyBytes(in, compressedLength);
-    }
-
-    @Override
     public Decompressor clone() {
       return new DeflateDecompressor();
     }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java Fri Jan 18 18:30:54 2013
@@ -24,7 +24,10 @@ import org.apache.lucene.store.DataOutpu
 /**
  * A data compressor.
  */
-abstract class Compressor {
+public abstract class Compressor {
+
+  /** Sole constructor, typically called from sub-classes. */
+  protected Compressor() {}
 
   /**
    * Compress bytes into <code>out</code>. It it the responsibility of the

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/Decompressor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/Decompressor.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/Decompressor.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/Decompressor.java Fri Jan 18 18:30:54 2013
@@ -20,13 +20,15 @@ package org.apache.lucene.codecs.compres
 import java.io.IOException;
 
 import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.DataOutput;
 import org.apache.lucene.util.BytesRef;
 
 /**
- * An decompressor.
+ * A decompressor.
  */
-abstract class Decompressor implements Cloneable {
+public abstract class Decompressor implements Cloneable {
+
+  /** Sole constructor, typically called from sub-classes. */
+  protected Decompressor() {}
 
   /**
    * Decompress bytes that were stored between offsets <code>offset</code> and
@@ -44,10 +46,6 @@ abstract class Decompressor implements C
    */
   public abstract void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException;
 
-  /** Copy a compressed stream whose original length is
-   * <code>originalLength</code> from <code>in</code> to <code>out</code>. */
-  public abstract void copyCompressedData(DataInput in, int originalLength, DataOutput out) throws IOException;
-
   @Override
   public abstract Decompressor clone();
 

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/LZ4.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/LZ4.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/LZ4.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/compressing/LZ4.java Fri Jan 18 18:30:54 2013
@@ -121,7 +121,7 @@ class LZ4 {
       matchLen += MIN_MATCH;
 
       // copying a multiple of 8 bytes can make decompression from 5% to 10% faster
-      final int fastLen = ((matchLen - 1) & 0xFFFFFFF8) + 8;
+      final int fastLen = (matchLen + 7) & 0xFFFFFFF8;
       if (matchDec < matchLen || dOff + fastLen > destEnd) {
         // overlap -> naive incremental copy
         for (int ref = dOff - matchDec, end = dOff + matchLen; dOff < end; ++ref, ++dOff) {
@@ -222,7 +222,7 @@ class LZ4 {
         }
 
         // compute match length
-        final int matchLen = MIN_MATCH + commonBytes(bytes, ref + 4, off + 4, limit);
+        final int matchLen = MIN_MATCH + commonBytes(bytes, ref + MIN_MATCH, off + MIN_MATCH, limit);
 
         encodeSequence(bytes, anchor, ref, off, matchLen, out);
         off += matchLen;
@@ -506,51 +506,4 @@ class LZ4 {
     encodeLastLiterals(src, anchor, srcEnd - anchor, out);
   }
 
-  /** Copy bytes from <code>in</code> to <code>out</code> where
-   *  <code>in</code> is a LZ4-encoded stream. This method copies enough bytes
-   *  so that <code>out</code> can be used later on to restore the first
-   *  <code>length</code> bytes of the stream. This method always reads at
-   *  least one byte from <code>in</code> so make sure not to call this method
-   *  if <code>in</code> reached the end of the stream, even if
-   *  <code>length=0</code>. */
-  public static int copyCompressedData(DataInput in, int length, DataOutput out) throws IOException {
-    int n = 0;
-    do {
-      // literals
-      final byte token = in.readByte();
-      out.writeByte(token);
-      int literalLen = (token & 0xFF) >>> 4;
-      if (literalLen == 0x0F) {
-        byte len;
-        while ((len = in.readByte()) == (byte) 0xFF) {
-          literalLen += 0xFF;
-          out.writeByte(len);
-        }
-        literalLen += len & 0xFF;
-        out.writeByte(len);
-      }
-      out.copyBytes(in, literalLen);
-      n += literalLen;
-      if (n >= length) {
-        break;
-      }
-
-      // matchs
-      out.copyBytes(in, 2); // match dec
-      int matchLen = token & 0x0F;
-      if (matchLen == 0x0F) {
-        byte len;
-        while ((len = in.readByte()) == (byte) 0xFF) {
-          matchLen += 0xFF;
-          out.writeByte(len);
-        }
-        matchLen += len & 0xFF;
-        out.writeByte(len);
-      }
-      matchLen += MIN_MATCH;
-      n += matchLen;
-    } while (n < length);
-    return n;
-  }
-
 }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/BitVector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/BitVector.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/BitVector.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/BitVector.java Fri Jan 18 18:30:54 2013
@@ -26,6 +26,7 @@ import org.apache.lucene.store.Directory
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.MutableBits;
 
 /** Optimized implementation of a vector of bits.  This is more-or-less like
@@ -109,6 +110,7 @@ final class BitVector implements Cloneab
   }
 
   /** Sets the value of <code>bit</code> to zero. */
+  @Override
   public final void clear(int bit) {
     if (bit >= size) {
       throw new ArrayIndexOutOfBoundsException(bit);
@@ -138,6 +140,7 @@ final class BitVector implements Cloneab
 
   /** Returns <code>true</code> if <code>bit</code> is one and
     <code>false</code> if it is zero. */
+  @Override
   public final boolean get(int bit) {
     assert bit >= 0 && bit < size: "bit " + bit + " is out of bounds 0.." + (size-1);
     return (bits[bit >> 3] & (1 << (bit & 7))) != 0;
@@ -236,7 +239,7 @@ final class BitVector implements Cloneab
       }
       assert verifyCount();
     } finally {
-      output.close();
+      IOUtils.close(output);
     }
   }
 

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java Fri Jan 18 18:30:54 2013
@@ -124,6 +124,7 @@ public final class Lucene40StoredFieldsR
    *
    * @throws IOException If an I/O error occurs
    */
+  @Override
   public final void close() throws IOException {
     if (!closed) {
       IOUtils.close(fieldsStream, indexStream);

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java Fri Jan 18 18:30:54 2013
@@ -113,11 +113,13 @@ public final class Lucene40StoredFieldsW
   // and adds a new entry for this document into the index
   // stream.  This assumes the buffer was already written
   // in the correct fields format.
+  @Override
   public void startDocument(int numStoredFields) throws IOException {
     indexStream.writeLong(fieldsStream.getFilePointer());
     fieldsStream.writeVInt(numStoredFields);
   }
 
+  @Override
   public void close() throws IOException {
     try {
       IOUtils.close(fieldsStream, indexStream);
@@ -126,6 +128,7 @@ public final class Lucene40StoredFieldsW
     }
   }
 
+  @Override
   public void abort() {
     try {
       close();
@@ -135,6 +138,7 @@ public final class Lucene40StoredFieldsW
         IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION));
   }
 
+  @Override
   public void writeField(FieldInfo info, StorableField field) throws IOException {
     fieldsStream.writeVInt(info.number);
     int bits = 0;

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java Fri Jan 18 18:30:54 2013
@@ -124,17 +124,16 @@ public final class Lucene40TermVectorsWr
     if (payloads)
       bits |= Lucene40TermVectorsReader.STORE_PAYLOAD_WITH_TERMVECTOR;
     tvf.writeByte(bits);
-    
-    assert fieldCount <= numVectorFields;
-    if (fieldCount == numVectorFields) {
-      // last field of the document
-      // this is crazy because the file format is crazy!
-      for (int i = 1; i < fieldCount; i++) {
-        tvd.writeVLong(fps[i] - fps[i-1]);
-      }
-    }
   }
   
+  @Override
+  public void finishDocument() throws IOException {
+    assert fieldCount == numVectorFields;
+    for (int i = 1; i < fieldCount; i++) {
+      tvd.writeVLong(fps[i] - fps[i-1]);
+    }
+  }
+
   private final BytesRef lastTerm = new BytesRef(10);
 
   // NOTE: we override addProx, so we don't need to buffer when indexing.
@@ -222,20 +221,6 @@ public final class Lucene40TermVectorsWr
       }
       
       bufferedIndex++;
-      
-      // dump buffer if we are done
-      if (bufferedIndex == bufferedFreq) {
-        if (payloads) {
-          tvf.writeBytes(payloadData.bytes, payloadData.offset, payloadData.length);
-        }
-        for (int i = 0; i < bufferedIndex; i++) {
-          if (offsets) {
-            tvf.writeVInt(offsetStartBuffer[i] - lastOffset);
-            tvf.writeVInt(offsetEndBuffer[i] - offsetStartBuffer[i]);
-            lastOffset = offsetEndBuffer[i];
-          }
-        }
-      }
     } else if (positions) {
       // write position delta
       writePosition(position - lastPosition, payload);
@@ -248,6 +233,25 @@ public final class Lucene40TermVectorsWr
     }
   }
   
+  @Override
+  public void finishTerm() throws IOException {
+    if (bufferedIndex > 0) {
+      // dump buffer
+      assert positions && (offsets || payloads);
+      assert bufferedIndex == bufferedFreq;
+      if (payloads) {
+        tvf.writeBytes(payloadData.bytes, payloadData.offset, payloadData.length);
+      }
+      for (int i = 0; i < bufferedIndex; i++) {
+        if (offsets) {
+          tvf.writeVInt(offsetStartBuffer[i] - lastOffset);
+          tvf.writeVInt(offsetEndBuffer[i] - offsetStartBuffer[i]);
+          lastOffset = offsetEndBuffer[i];
+        }
+      }
+    }
+  }
+
   private void writePosition(int delta, BytesRef payload) throws IOException {
     if (payloads) {
       int payloadLength = payload == null ? 0 : payload.length;

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java Fri Jan 18 18:30:54 2013
@@ -469,6 +469,7 @@ public final class Bytes {
       }
     }
     
+    @Override
     public int getValueSize() {
       return size;
     }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DirectSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DirectSource.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DirectSource.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DirectSource.java Fri Jan 18 18:30:54 2013
@@ -130,6 +130,7 @@ abstract class DirectSource extends Sour
       throw new UnsupportedOperationException("ints are not supported");
     }
 
+    @Override
     double toDouble(IndexInput input) throws IOException {
       return Float.intBitsToFloat(input.readInt());
     }
@@ -141,6 +142,7 @@ abstract class DirectSource extends Sour
       throw new UnsupportedOperationException("ints are not supported");
     }
 
+    @Override
     double toDouble(IndexInput input) throws IOException {
       return Double.longBitsToDouble(input.readLong());
     }
@@ -153,6 +155,7 @@ abstract class DirectSource extends Sour
       return input.readLong();
     }
 
+    @Override
     double toDouble(IndexInput input) {
       throw new UnsupportedOperationException("doubles are not supported");
     }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java Fri Jan 18 18:30:54 2013
@@ -122,7 +122,7 @@ public final class Lucene41PostingsReade
   /**
    * Read values that have been written using variable-length encoding instead of bit-packing.
    */
-  private static void readVIntBlock(IndexInput docIn, int[] docBuffer,
+  static void readVIntBlock(IndexInput docIn, int[] docBuffer,
       int[] freqBuffer, int num, boolean indexHasFreq) throws IOException {
     if (indexHasFreq) {
       for(int i=0;i<num;i++) {

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene41/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene41/package.html?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene41/package.html (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/codecs/lucene41/package.html Fri Jan 18 18:30:54 2013
@@ -147,7 +147,7 @@ its numbering.</p>
    This contains the set of field names used in the index.
 </li>
 <li>
-{@link org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat Stored Field values}. 
+{@link org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat Stored Field values}. 
 This contains, for each document, a list of attribute-value pairs, where the attributes 
 are field names. These are used to store auxiliary information about the document, such as 
 its title, url, or an identifier to access a database. The set of stored fields are what is 
@@ -254,12 +254,12 @@ systems that frequently run out of file 
 <td>Stores information about the fields</td>
 </tr>
 <tr>
-<td>{@link org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat Field Index}</td>
+<td>{@link org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat Field Index}</td>
 <td>.fdx</td>
 <td>Contains pointers to field data</td>
 </tr>
 <tr>
-<td>{@link org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat Field Data}</td>
+<td>{@link org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat Field Data}</td>
 <td>.fdt</td>
 <td>The stored fields for documents</td>
 </tr>

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/DocumentStoredFieldVisitor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/DocumentStoredFieldVisitor.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/DocumentStoredFieldVisitor.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/DocumentStoredFieldVisitor.java Fri Jan 18 18:30:54 2013
@@ -39,12 +39,15 @@ public class DocumentStoredFieldVisitor 
   private final StoredDocument doc = new StoredDocument();
   private final Set<String> fieldsToAdd;
 
-  /** Load only fields named in the provided <code>Set&lt;String&gt;</code>. */
+  /** 
+   * Load only fields named in the provided <code>Set&lt;String&gt;</code>. 
+   * @param fieldsToAdd Set of fields to load, or <code>null</code> (all fields).
+   */
   public DocumentStoredFieldVisitor(Set<String> fieldsToAdd) {
     this.fieldsToAdd = fieldsToAdd;
   }
 
-  /** Load only fields named in the provided <code>Set&lt;String&gt;</code>. */
+  /** Load only fields named in the provided fields. */
   public DocumentStoredFieldVisitor(String... fields) {
     fieldsToAdd = new HashSet<String>(fields.length);
     for(String field : fields) {

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/Field.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/Field.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/Field.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/Field.java Fri Jan 18 18:30:54 2013
@@ -266,6 +266,7 @@ public class Field implements IndexableF
    * binary value is used. Exactly one of stringValue(), readerValue(), and
    * getBinaryValue() must be set.
    */
+  @Override
   public String stringValue() {
     if (fieldsData instanceof String || fieldsData instanceof Number) {
       return fieldsData.toString();
@@ -279,6 +280,7 @@ public class Field implements IndexableF
    * binary value is used. Exactly one of stringValue(), readerValue(), and
    * getBinaryValue() must be set.
    */
+  @Override
   public Reader readerValue() {
     return fieldsData instanceof Reader ? (Reader) fieldsData : null;
   }
@@ -345,7 +347,7 @@ public class Field implements IndexableF
       throw new IllegalArgumentException("cannot change value type from " + fieldsData.getClass().getSimpleName() + " to BytesRef");
     }
     if (type.indexed()) {
-      throw new IllegalArgumentException("cannot set a Reader value on an indexed field");
+      throw new IllegalArgumentException("cannot set a BytesRef value on an indexed field");
     }
     fieldsData = value;
   }
@@ -431,6 +433,7 @@ public class Field implements IndexableF
     this.tokenStream = tokenStream;
   }
   
+  @Override
   public String name() {
     return name;
   }
@@ -441,6 +444,7 @@ public class Field implements IndexableF
    * The default value is <code>1.0f</code> (no boost).
    * @see #setBoost(float)
    */
+  @Override
   public float boost() {
     return boost;
   }
@@ -460,6 +464,7 @@ public class Field implements IndexableF
     this.boost = boost;
   }
 
+  @Override
   public Number numericValue() {
     if (fieldsData instanceof Number) {
       return (Number) fieldsData;
@@ -468,6 +473,7 @@ public class Field implements IndexableF
     }
   }
 
+  @Override
   public BytesRef binaryValue() {
     if (fieldsData instanceof BytesRef) {
       return (BytesRef) fieldsData;
@@ -494,10 +500,12 @@ public class Field implements IndexableF
   }
   
   /** Returns the {@link FieldType} for this field. */
+  @Override
   public FieldType fieldType() {
     return type;
   }
 
+  @Override
   public TokenStream tokenStream(Analyzer analyzer) throws IOException {
     if (!fieldType().indexed()) {
       return null;

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/FieldType.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/FieldType.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/document/FieldType.java Fri Jan 18 18:30:54 2013
@@ -102,6 +102,7 @@ public class FieldType implements Indexa
    * The default is <code>false</code>.
    * @see #setIndexed(boolean)
    */
+  @Override
   public boolean indexed() {
     return this.indexed;
   }
@@ -124,6 +125,7 @@ public class FieldType implements Indexa
    * The default is <code>false</code>.
    * @see #setStored(boolean)
    */
+  @Override
   public boolean stored() {
     return this.stored;
   }
@@ -146,6 +148,7 @@ public class FieldType implements Indexa
    * The default is <code>true</code>.
    * @see #setTokenized(boolean)
    */
+  @Override
   public boolean tokenized() {
     return this.tokenized;
   }
@@ -169,6 +172,7 @@ public class FieldType implements Indexa
    * The default is <code>false</code>. 
    * @see #setStoreTermVectors(boolean)
    */
+  @Override
   public boolean storeTermVectors() {
     return this.storeTermVectors;
   }
@@ -192,6 +196,7 @@ public class FieldType implements Indexa
    * The default is <code>false</code>.
    * @see #setStoreTermVectorOffsets(boolean)
    */
+  @Override
   public boolean storeTermVectorOffsets() {
     return this.storeTermVectorOffsets;
   }
@@ -215,6 +220,7 @@ public class FieldType implements Indexa
    * The default is <code>false</code>.
    * @see #setStoreTermVectorPositions(boolean)
    */
+  @Override
   public boolean storeTermVectorPositions() {
     return this.storeTermVectorPositions;
   }
@@ -238,6 +244,7 @@ public class FieldType implements Indexa
    * The default is <code>false</code>.
    * @see #setStoreTermVectorPayloads(boolean) 
    */
+  @Override
   public boolean storeTermVectorPayloads() {
     return this.storeTermVectorPayloads;
   }
@@ -261,6 +268,7 @@ public class FieldType implements Indexa
    * The default is <code>false</code>.
    * @see #setOmitNorms(boolean)
    */
+  @Override
   public boolean omitNorms() {
     return this.omitNorms;
   }
@@ -283,6 +291,7 @@ public class FieldType implements Indexa
    * The default is {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS}.
    * @see #setIndexOptions(org.apache.lucene.index.FieldInfo.IndexOptions)
    */
+  @Override
   public IndexOptions indexOptions() {
     return this.indexOptions;
   }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java Fri Jan 18 18:30:54 2013
@@ -90,6 +90,7 @@ public abstract class AtomicReader exten
    * field does not exists.  This method does not take into
    * account deleted documents that have not yet been merged
    * away. */
+  @Override
   public final long totalTermFreq(Term term) throws IOException {
     final Fields fields = fields();
     if (fields == null) {

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/BitsSlice.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/BitsSlice.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/BitsSlice.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/BitsSlice.java Fri Jan 18 18:30:54 2013
@@ -37,6 +37,7 @@ final class BitsSlice implements Bits {
     assert length >= 0: "length=" + length;
   }
     
+  @Override
   public boolean get(int doc) {
     if (doc >= length) {
       throw new RuntimeException("doc " + doc + " is out of bounds 0 .. " + (length-1));
@@ -45,6 +46,7 @@ final class BitsSlice implements Bits {
     return parent.get(doc+start);
   }
 
+  @Override
   public int length() {
     return length;
   }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java Fri Jan 18 18:30:54 2013
@@ -168,6 +168,8 @@ class BufferedDeletesStream {
       infoStream.message("BD", "applyDeletes: infos=" + infos + " packetCount=" + deletes.size());
     }
 
+    final long gen = nextGen++;
+
     List<SegmentInfoPerCommit> infos2 = new ArrayList<SegmentInfoPerCommit>();
     infos2.addAll(infos);
     Collections.sort(infos2, sortSegInfoByDelGen);
@@ -255,7 +257,7 @@ class BufferedDeletesStream {
          */
         delIDX--;
         infosIDX--;
-        info.setBufferedDeletesGen(nextGen);
+        info.setBufferedDeletesGen(gen);
 
       } else {
         //System.out.println("  gt");
@@ -290,7 +292,7 @@ class BufferedDeletesStream {
             infoStream.message("BD", "seg=" + info + " segGen=" + segGen + " coalesced deletes=[" + (coalescedDeletes == null ? "null" : coalescedDeletes) + "] newDelCount=" + delCount + (segAllDeletes ? " 100% deleted" : ""));
           }
         }
-        info.setBufferedDeletesGen(nextGen);
+        info.setBufferedDeletesGen(gen);
 
         infosIDX--;
       }
@@ -302,7 +304,7 @@ class BufferedDeletesStream {
     }
     // assert infos != segmentInfos || !any() : "infos=" + infos + " segmentInfos=" + segmentInfos + " any=" + any;
 
-    return new ApplyDeletesResult(anyNewDeletes, nextGen++, allDeleted);
+    return new ApplyDeletesResult(anyNewDeletes, gen, allDeleted);
   }
 
   synchronized long getNextGen() {
@@ -397,7 +399,7 @@ class BufferedDeletesStream {
 
       if (termsEnum.seekExact(term.bytes(), false)) {
         // we don't need term frequencies for this
-        DocsEnum docsEnum = termsEnum.docs(rld.getLiveDocs(), docs, 0);
+        DocsEnum docsEnum = termsEnum.docs(rld.getLiveDocs(), docs, DocsEnum.FLAG_NONE);
         //System.out.println("BDS: got docsEnum=" + docsEnum);
 
         if (docsEnum != null) {

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Fri Jan 18 18:30:54 2013
@@ -905,7 +905,7 @@ public class CheckIndex {
               totalTermFreq += docsNoDel.freq();
             }
           } else {
-            final DocsEnum docsNoDel = termsEnum.docs(null, docs, 0);
+            final DocsEnum docsNoDel = termsEnum.docs(null, docs, DocsEnum.FLAG_NONE);
             docCount = 0;
             totalTermFreq = -1;
             while(docsNoDel.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
@@ -991,7 +991,7 @@ public class CheckIndex {
         } else {
           for(int idx=0;idx<7;idx++) {
             final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8);
-            docs = termsEnum.docs(liveDocs, docs, 0);
+            docs = termsEnum.docs(liveDocs, docs, DocsEnum.FLAG_NONE);
             final int docID = docs.advance(skipDocID);
             if (docID == DocIdSetIterator.NO_MORE_DOCS) {
               break;
@@ -1057,7 +1057,7 @@ public class CheckIndex {
           }
           
           int expectedDocFreq = termsEnum.docFreq();
-          DocsEnum d = termsEnum.docs(null, null, 0);
+          DocsEnum d = termsEnum.docs(null, null, DocsEnum.FLAG_NONE);
           int docFreq = 0;
           while (d.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
             docFreq++;
@@ -1098,7 +1098,7 @@ public class CheckIndex {
                 throw new RuntimeException("seek to existing term " + seekTerms[i] + " failed");
               }
               
-              docs = termsEnum.docs(liveDocs, docs, 0);
+              docs = termsEnum.docs(liveDocs, docs, DocsEnum.FLAG_NONE);
               if (docs == null) {
                 throw new RuntimeException("null DocsEnum from to existing term " + seekTerms[i]);
               }
@@ -1116,7 +1116,7 @@ public class CheckIndex {
               }
               
               totDocFreq += termsEnum.docFreq();
-              docs = termsEnum.docs(null, docs, 0);
+              docs = termsEnum.docs(null, docs, DocsEnum.FLAG_NONE);
               if (docs == null) {
                 throw new RuntimeException("null DocsEnum from to existing term " + seekTerms[i]);
               }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java Fri Jan 18 18:30:54 2013
@@ -55,14 +55,15 @@ public class ConcurrentMergeScheduler ex
   // forcefully pause the larger ones, letting the smaller
   // ones run, up until maxMergeCount merges at which point
   // we forcefully pause incoming threads (that presumably
-  // are the ones causing so much merging).  We dynamically
-  // default this from 1 to 3, depending on how many cores
-  // you have:
-  private int maxThreadCount = Math.max(1, Math.min(3, Runtime.getRuntime().availableProcessors()/2));
+  // are the ones causing so much merging).  We default to 1
+  // here: tests on spinning-magnet drives showed slower
+  // indexing perf if more than one merge thread runs at
+  // once (though on an SSD it was faster):
+  private int maxThreadCount = 1;
 
   // Max number of merges we accept before forcefully
   // throttling the incoming threads
-  private int maxMergeCount = maxThreadCount+2;
+  private int maxMergeCount = 2;
 
   /** {@link Directory} that holds the index. */
   protected Directory dir;
@@ -144,6 +145,7 @@ public class ConcurrentMergeScheduler ex
 
   /** Sorts {@link MergeThread}s; larger merges come first. */
   protected static final Comparator<MergeThread> compareByMergeDocCount = new Comparator<MergeThread>() {
+    @Override
     public int compare(MergeThread t1, MergeThread t2) {
       final MergePolicy.OneMerge m1 = t1.getCurrentMerge();
       final MergePolicy.OneMerge m2 = t2.getCurrentMerge();
@@ -478,6 +480,14 @@ public class ConcurrentMergeScheduler ex
           // Subsequent times through the loop we do any new
           // merge that writer says is necessary:
           merge = tWriter.getNextMerge();
+
+          // Notify here in case any threads were stalled;
+          // they will notice that the pending merge has
+          // been pulled and possibly resume:
+          synchronized(ConcurrentMergeScheduler.this) {
+            ConcurrentMergeScheduler.this.notifyAll();
+          }
+
           if (merge != null) {
             updateMergeThreads();
             if (verbose()) {

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java Fri Jan 18 18:30:54 2013
@@ -327,6 +327,7 @@ final class DocFieldProcessor extends Do
   }
 
   private static final Comparator<DocFieldProcessorPerField> fieldsComp = new Comparator<DocFieldProcessorPerField>() {
+    @Override
     public int compare(DocFieldProcessorPerField o1, DocFieldProcessorPerField o2) {
       return o1.fieldInfo.name.compareTo(o2.fieldInfo.name);
     }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java Fri Jan 18 18:30:54 2013
@@ -106,75 +106,72 @@ final class DocInverterPerField extends 
           OffsetAttribute offsetAttribute = fieldState.attributeSource.addAttribute(OffsetAttribute.class);
           PositionIncrementAttribute posIncrAttribute = fieldState.attributeSource.addAttribute(PositionIncrementAttribute.class);
 
-          consumer.start(field);
+          if (hasMoreTokens) {
+            consumer.start(field);
 
-          for (;;) {
-
-            // If we hit an exception in stream.next below
-            // (which is fairly common, eg if analyzer
-            // chokes on a given document), then it's
-            // non-aborting and (above) this one document
-            // will be marked as deleted, but still
-            // consume a docID
-
-            if (!hasMoreTokens) break;
-
-            final int posIncr = posIncrAttribute.getPositionIncrement();
-            if (posIncr < 0) {
-              throw new IllegalArgumentException("position increment must be >=0 (got " + posIncr + ")");
-            }
-            if (fieldState.position == 0 && posIncr == 0) {
-              throw new IllegalArgumentException("first position increment must be > 0 (got 0)");
-            }
-            int position = fieldState.position + posIncr;
-            if (position > 0) {
-              // NOTE: confusing: this "mirrors" the
-              // position++ we do below
-              position--;
-            } else if (position < 0) {
-              throw new IllegalArgumentException("position overflow for field '" + field.name() + "'");
-            }
-            
-            // position is legal, we can safely place it in fieldState now.
-            // not sure if anything will use fieldState after non-aborting exc...
-            fieldState.position = position;
-
-            if (posIncr == 0)
-              fieldState.numOverlap++;
-            
-            if (checkOffsets) {
-              int startOffset = fieldState.offset + offsetAttribute.startOffset();
-              int endOffset = fieldState.offset + offsetAttribute.endOffset();
-              if (startOffset < 0 || endOffset < startOffset) {
-                throw new IllegalArgumentException("startOffset must be non-negative, and endOffset must be >= startOffset, "
-                    + "startOffset=" + startOffset + ",endOffset=" + endOffset);
+            do {
+              // If we hit an exception in stream.next below
+              // (which is fairly common, eg if analyzer
+              // chokes on a given document), then it's
+              // non-aborting and (above) this one document
+              // will be marked as deleted, but still
+              // consume a docID
+
+              final int posIncr = posIncrAttribute.getPositionIncrement();
+              if (posIncr < 0) {
+                throw new IllegalArgumentException("position increment must be >=0 (got " + posIncr + ")");
               }
-              if (startOffset < lastStartOffset) {
-                throw new IllegalArgumentException("offsets must not go backwards startOffset=" 
-                     + startOffset + " is < lastStartOffset=" + lastStartOffset);
+              if (fieldState.position == 0 && posIncr == 0) {
+                throw new IllegalArgumentException("first position increment must be > 0 (got 0)");
               }
-              lastStartOffset = startOffset;
-            }
-
-            boolean success = false;
-            try {
-              // If we hit an exception in here, we abort
-              // all buffered documents since the last
-              // flush, on the likelihood that the
-              // internal state of the consumer is now
-              // corrupt and should not be flushed to a
-              // new segment:
-              consumer.add();
-              success = true;
-            } finally {
-              if (!success) {
-                docState.docWriter.setAborting();
+              int position = fieldState.position + posIncr;
+              if (position > 0) {
+                // NOTE: confusing: this "mirrors" the
+                // position++ we do below
+                position--;
+              } else if (position < 0) {
+                throw new IllegalArgumentException("position overflow for field '" + field.name() + "'");
+              }
+              
+              // position is legal, we can safely place it in fieldState now.
+              // not sure if anything will use fieldState after non-aborting exc...
+              fieldState.position = position;
+
+              if (posIncr == 0)
+                fieldState.numOverlap++;
+              
+              if (checkOffsets) {
+                int startOffset = fieldState.offset + offsetAttribute.startOffset();
+                int endOffset = fieldState.offset + offsetAttribute.endOffset();
+                if (startOffset < 0 || endOffset < startOffset) {
+                  throw new IllegalArgumentException("startOffset must be non-negative, and endOffset must be >= startOffset, "
+                      + "startOffset=" + startOffset + ",endOffset=" + endOffset);
+                }
+                if (startOffset < lastStartOffset) {
+                  throw new IllegalArgumentException("offsets must not go backwards startOffset=" 
+                       + startOffset + " is < lastStartOffset=" + lastStartOffset);
+                }
+                lastStartOffset = startOffset;
               }
-            }
-            fieldState.length++;
-            fieldState.position++;
 
-            hasMoreTokens = stream.incrementToken();
+              boolean success = false;
+              try {
+                // If we hit an exception in here, we abort
+                // all buffered documents since the last
+                // flush, on the likelihood that the
+                // internal state of the consumer is now
+                // corrupt and should not be flushed to a
+                // new segment:
+                consumer.add();
+                success = true;
+              } finally {
+                if (!success) {
+                  docState.docWriter.setAborting();
+                }
+              }
+              fieldState.length++;
+              fieldState.position++;
+            } while (stream.incrementToken());
           }
           // trigger streams to perform end-of-stream operations
           stream.end();

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java Fri Jan 18 18:30:54 2013
@@ -363,7 +363,7 @@ public class DocTermOrds {
       final int df = te.docFreq();
       if (df <= maxTermDocFreq) {
 
-        docsEnum = te.docs(liveDocs, docsEnum, 0);
+        docsEnum = te.docs(liveDocs, docsEnum, DocsEnum.FLAG_NONE);
 
         // dF, but takes deletions into account
         int actualDF = 0;

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocValues.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocValues.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocValues.java Fri Jan 18 18:30:54 2013
@@ -163,6 +163,7 @@ public abstract class DocValues implemen
    * by the creator of this {@link DocValues} instance. API users should not
    * close {@link DocValues} instances.
    */
+  @Override
   public void close() throws IOException {
     cache.close(this);
   }
@@ -784,6 +785,7 @@ public abstract class DocValues implemen
       public DirectSourceCache() {
       }
 
+      @Override
       public synchronized Source load(DocValues values) throws IOException {
         if (ref == null) {
           ref = values.loadSource();
@@ -791,6 +793,7 @@ public abstract class DocValues implemen
         return ref;
       }
 
+      @Override
       public synchronized void invalidate(DocValues values) {
         ref = null;
         directSourceCache.close();

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java Fri Jan 18 18:30:54 2013
@@ -24,6 +24,7 @@ import org.apache.lucene.util.BytesRef;
 
 /** Also iterates through positions. */
 public abstract class DocsAndPositionsEnum extends DocsEnum {
+  
   /** Flag to pass to {@link TermsEnum#docsAndPositions(Bits,DocsAndPositionsEnum,int)}
    *  if you require offsets in the returned enum. */
   public static final int FLAG_OFFSETS = 0x1;

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java Fri Jan 18 18:30:54 2013
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
 
 import java.io.IOException;
 
+import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.Bits; // javadocs
@@ -27,6 +28,14 @@ import org.apache.lucene.util.Bits; // j
  *  NOTE: you must first call {@link #nextDoc} before using
  *  any of the per-doc methods. */
 public abstract class DocsEnum extends DocIdSetIterator {
+  
+  /**
+   * Flag to pass to {@link TermsEnum#docs(Bits,DocsEnum,int)} if you don't
+   * require term frequencies in the returned enum. When passed to
+   * {@link TermsEnum#docsAndPositions(Bits,DocsAndPositionsEnum,int)} means
+   * that no offsets and payloads will be returned.
+   */
+  public static final int FLAG_NONE = 0x0;
 
   /** Flag to pass to {@link TermsEnum#docs(Bits,DocsEnum,int)}
    *  if you require term frequencies in the returned enum. */
@@ -39,10 +48,16 @@ public abstract class DocsEnum extends D
   protected DocsEnum() {
   }
 
-  /** Returns term frequency in the current document.  Do
-   *  not call this before {@link #nextDoc} is first called,
-   *  nor after {@link #nextDoc} returns NO_MORE_DOCS. 
-   **/
+  /**
+   * Returns term frequency in the current document, or 1 if the field was
+   * indexed with {@link IndexOptions#DOCS_ONLY}. Do not call this before
+   * {@link #nextDoc} is first called, nor after {@link #nextDoc} returns
+   * {@link DocIdSetIterator#NO_MORE_DOCS}.
+   * 
+   * <p>
+   * <b>NOTE:</b> if the {@link DocsEnum} was obtain with {@link #FLAG_NONE},
+   * the result of this method is undefined.
+   */
   public abstract int freq() throws IOException;
   
   /** Returns the related attributes. */

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java Fri Jan 18 18:30:54 2013
@@ -384,14 +384,17 @@ final class DocumentsWriterFlushControl 
     return new Iterator<ThreadState>() {
       int i = 0;
 
+      @Override
       public boolean hasNext() {
         return i < upto;
       }
 
+      @Override
       public ThreadState next() {
         return perThreadPool.getThreadState(i++);
       }
 
+      @Override
       public void remove() {
         throw new UnsupportedOperationException("remove() not supported.");
       }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java Fri Jan 18 18:30:54 2013
@@ -178,6 +178,7 @@ class DocumentsWriterFlushQueue {
     protected GlobalDeletesTicket(FrozenBufferedDeletes frozenDeletes) {
       super(frozenDeletes);
     }
+    @Override
     protected void publish(DocumentsWriter writer) throws IOException {
       assert !published : "ticket was already publised - can not publish twice";
       published = true;
@@ -185,6 +186,7 @@ class DocumentsWriterFlushQueue {
       writer.finishFlush(null, frozenDeletes);
     }
 
+    @Override
     protected boolean canPublish() {
       return true;
     }
@@ -198,6 +200,7 @@ class DocumentsWriterFlushQueue {
       super(frozenDeletes);
     }
     
+    @Override
     protected void publish(DocumentsWriter writer) throws IOException {
       assert !published : "ticket was already publised - can not publish twice";
       published = true;
@@ -214,6 +217,7 @@ class DocumentsWriterFlushQueue {
       failed = true;
     }
 
+    @Override
     protected boolean canPublish() {
       return segment != null || failed;
     }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Fri Jan 18 18:30:54 2013
@@ -296,7 +296,9 @@ class DocumentsWriterPerThread {
       infoStream.message("DWPT", Thread.currentThread().getName() + " update delTerm=" + delTerm + " docID=" + docState.docID + " seg=" + segmentInfo.name);
     }
     int docCount = 0;
+    boolean allDocsIndexed = false;
     try {
+      
       for(IndexDocument doc : docs) {
         docState.doc = doc;
         docState.docID = numDocsInRAM;
@@ -309,20 +311,7 @@ class DocumentsWriterPerThread {
         } finally {
           if (!success) {
             // An exc is being thrown...
-
             if (!aborting) {
-              // One of the documents hit a non-aborting
-              // exception (eg something happened during
-              // analysis).  We now go and mark any docs
-              // from this batch that we had already indexed
-              // as deleted:
-              int docID = docState.docID;
-              final int endDocID = docID - docCount;
-              while (docID > endDocID) {
-                deleteDocID(docID);
-                docID--;
-              }
-
               // Incr here because finishDocument will not
               // be called (because an exc is being thrown):
               numDocsInRAM++;
@@ -343,6 +332,7 @@ class DocumentsWriterPerThread {
 
         finishDocument(null);
       }
+      allDocsIndexed = true;
 
       // Apply delTerm only after all indexing has
       // succeeded, but apply it only to docs prior to when
@@ -354,6 +344,16 @@ class DocumentsWriterPerThread {
       }
 
     } finally {
+      if (!allDocsIndexed && !aborting) {
+        // the iterator threw an exception that is not aborting 
+        // go and mark all docs from this block as deleted
+        int docID = numDocsInRAM-1;
+        final int endDocID = docID - docCount;
+        while (docID > endDocID) {
+          deleteDocID(docID);
+          docID--;
+        }
+      }
       docState.clear();
     }
 
@@ -637,6 +637,7 @@ class DocumentsWriterPerThread {
     }
     
     /* Allocate another int[] from the shared pool */
+    @Override
     public int[] getIntBlock() {
       int[] b = new int[IntBlockPool.INT_BLOCK_SIZE];
       bytesUsed.addAndGet(IntBlockPool.INT_BLOCK_SIZE
@@ -644,6 +645,7 @@ class DocumentsWriterPerThread {
       return b;
     }
     
+    @Override
     public void recycleIntBlocks(int[][] blocks, int offset, int length) {
       bytesUsed.addAndGet(-(length * (IntBlockPool.INT_BLOCK_SIZE * RamUsageEstimator.NUM_BYTES_INT)));
     }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java Fri Jan 18 18:30:54 2013
@@ -131,6 +131,7 @@ public class FieldInfos implements Itera
    * ordered by ascending field number
    */
   // TODO: what happens if in fact a different order is used?
+  @Override
   public Iterator<FieldInfo> iterator() {
     return values.iterator();
   }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/Fields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/Fields.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/Fields.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/Fields.java Fri Jan 18 18:30:54 2013
@@ -32,6 +32,7 @@ public abstract class Fields implements 
 
   /** Returns an iterator that will step through all fields
    *  names.  This will not return null.  */
+  @Override
   public abstract Iterator<String> iterator();
 
   /** Get the {@link Terms} for this field.  This will return

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java Fri Jan 18 18:30:54 2013
@@ -87,7 +87,7 @@ public class FilterAtomicReader extends 
     }
 
     @Override
-    public Comparator<BytesRef> getComparator() throws IOException {
+    public Comparator<BytesRef> getComparator() {
       return in.getComparator();
     }
 

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java Fri Jan 18 18:30:54 2013
@@ -78,6 +78,7 @@ final class FreqProxTermsWriterPerField 
   @Override
   void skippingLongTerm() {}
 
+  @Override
   public int compareTo(FreqProxTermsWriterPerField other) {
     return fieldInfo.name.compareTo(other.fieldInfo.name);
   }

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexCommit.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexCommit.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexCommit.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexCommit.java Fri Jan 18 18:30:54 2013
@@ -107,10 +107,11 @@ public abstract class IndexCommit implem
   public abstract long getGeneration();
 
   /** Returns userData, previously passed to {@link
-   *  IndexWriter#commit(Map)} for this commit.  Map is
+   *  IndexWriter#setCommitData(Map)} for this commit.  Map is
    *  String -> String. */
   public abstract Map<String,String> getUserData() throws IOException;
   
+  @Override
   public int compareTo(IndexCommit commit) {
     if (getDirectory() != commit.getDirectory()) {
       throw new UnsupportedOperationException("cannot compare IndexCommits from different Directory instances");

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java Fri Jan 18 18:30:54 2013
@@ -17,6 +17,7 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
+import java.io.Closeable;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -69,7 +70,7 @@ import org.apache.lucene.util.InfoStream
  * directly with no retry logic.
  */
 
-final class IndexFileDeleter {
+final class IndexFileDeleter implements Closeable {
 
   /* Files that we tried to delete but failed (likely
    * because they are open and we are running on Windows),
@@ -435,8 +436,9 @@ final class IndexFileDeleter {
     assert locked();
 
     assert Thread.holdsLock(writer);
-
+    long t0 = 0;
     if (infoStream.isEnabled("IFD")) {
+      t0 = System.nanoTime();
       infoStream.message("IFD", "now checkpoint \"" + writer.segString(writer.toLiveInfos(segmentInfos)) + "\" [" + segmentInfos.size() + " segments " + "; isCommit = " + isCommit + "]");
     }
 
@@ -466,6 +468,10 @@ final class IndexFileDeleter {
       // Save files so we can decr on next checkpoint/commit:
       lastFiles.add(segmentInfos.files(directory, false));
     }
+    if (infoStream.isEnabled("IFD")) {
+      long t1 = System.nanoTime();
+      infoStream.message("IFD", ((t1-t0)/1000000) + " msec to checkpoint");
+    }
   }
 
   void incRef(SegmentInfos segmentInfos, boolean isCommit) throws IOException {

Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexReader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/IndexReader.java Fri Jan 18 18:30:54 2013
@@ -370,6 +370,7 @@ public abstract class IndexReader implem
    * No other methods should be called after this has been called.
    * @throws IOException if there is a low-level IO error
    */
+  @Override
   public final synchronized void close() throws IOException {
     if (!closed) {
       decRef();



Mime
View raw message