lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sh...@apache.org
Subject svn commit: r1537832 [1/5] - in /lucene/dev/branches/branch_4x: ./ lucene/ lucene/codecs/ lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/ lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/ lucene/core/src/java/org/apache/lucene/cod...
Date Fri, 01 Nov 2013 06:03:54 GMT
Author: shaie
Date: Fri Nov  1 06:03:52 2013
New Revision: 1537832

URL: http://svn.apache.org/r1537832
Log:
LUCENE-5189: add NumericDocValues field updates

Added:
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene46/
      - copied from r1527154, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoFormat.java
      - copied unchanged from r1527391, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoFormat.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoReader.java
      - copied unchanged from r1527391, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoReader.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java
      - copied unchanged from r1527391, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/NumericFieldUpdates.java
      - copied unchanged from r1532670, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/NumericFieldUpdates.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/NumericUpdate.java   (props changed)
      - copied unchanged from r1523461, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/NumericUpdate.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java
      - copied, changed from r1535526, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/RefCount.java   (contents, props changed)
      - copied, changed from r1523461, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/RefCount.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java   (contents, props changed)
      - copied, changed from r1523461, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
    lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/codecs/facet46/
      - copied from r1527154, lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/codecs/facet46/
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java
      - copied unchanged from r1527391, lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosWriter.java
      - copied unchanged from r1527154, lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosWriter.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene45/
      - copied from r1527154, lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene45/
Removed:
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosWriter.java
    lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/codecs/facet45/
Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/lucene/   (props changed)
    lucene/dev/branches/branch_4x/lucene/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/branch_4x/lucene/codecs/   (props changed)
    lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
    lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java
    lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java
    lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java
    lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoReader.java
    lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java
    lucene/dev/branches/branch_4x/lucene/core/   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/Codec.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosReader.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosWriter.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosReader.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfoReader.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42Codec.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosReader.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45Codec.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46Codec.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene46/package.html
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/package.html
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BufferedDeletes.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CoalescedDeletes.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedDeletes.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MergeState.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/ReadersAndLiveDocs.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
    lucene/dev/branches/branch_4x/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/lucene3x/TestTermInfosReaderIndex.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java   (contents, props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/TestNamedSPILoader.java
    lucene/dev/branches/branch_4x/lucene/facet/   (props changed)
    lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/FacetTestCase.java
    lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestDemoFacets.java
    lucene/dev/branches/branch_4x/lucene/misc/   (props changed)
    lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java
    lucene/dev/branches/branch_4x/lucene/misc/src/test/org/apache/lucene/index/sorter/TestSortingMergePolicy.java
    lucene/dev/branches/branch_4x/lucene/suggest/   (props changed)
    lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
    lucene/dev/branches/branch_4x/lucene/test-framework/   (props changed)
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardCodec.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldInfosReader.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWFieldInfosWriter.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40RWCodec.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41/Lucene41RWCodec.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42RWCodec.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene45/Lucene45RWCodec.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java
    lucene/dev/branches/branch_4x/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
    lucene/dev/branches/branch_4x/solr/   (props changed)
    lucene/dev/branches/branch_4x/solr/core/   (props changed)
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java

Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Fri Nov  1 06:03:52 2013
@@ -73,6 +73,10 @@ New Features
 * LUCENE-5296: Add DirectDocValuesFormat, which holds all doc values
   in heap as uncompressed java native arrays.  (Mike McCandless)
 
+* LUCENE-5189: Add IndexWriter.updateNumericDocValues, to update
+  numeric DocValues fields of documents, without re-indexing them.
+  (Shai Erera, Mike McCandless, Robert Muir)
+  
 Bug Fixes
 
 * LUCENE-4998: Fixed a few places to pass IOContext.READONCE instead

Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java Fri Nov  1 06:03:52 2013
@@ -17,6 +17,16 @@ package org.apache.lucene.codecs.simplet
  * limitations under the License.
  */
 
+import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.END;
+import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.FIELD;
+import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.LENGTH;
+import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.MAXLENGTH;
+import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.MINVALUE;
+import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.NUMVALUES;
+import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.ORDPATTERN;
+import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.PATTERN;
+import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.TYPE;
+
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.math.BigInteger;
@@ -31,27 +41,17 @@ import org.apache.lucene.codecs.DocValue
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.StringHelper;
 
-import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.END;
-import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.FIELD;
-import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.LENGTH;
-import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.MAXLENGTH;
-import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.MINVALUE;
-import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.NUMVALUES;
-import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.ORDPATTERN;
-import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.PATTERN;
-import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.TYPE;
-
 class SimpleTextDocValuesReader extends DocValuesProducer {
 
   static class OneField {
@@ -62,8 +62,7 @@ class SimpleTextDocValuesReader extends 
     boolean fixedLength;
     long minValue;
     long numValues;
-    
-  };
+  }
 
   final int maxDoc;
   final IndexInput data;
@@ -71,7 +70,7 @@ class SimpleTextDocValuesReader extends 
   final Map<String,OneField> fields = new HashMap<String,OneField>();
   
   public SimpleTextDocValuesReader(SegmentReadState state, String ext) throws IOException {
-    //System.out.println("dir=" + state.directory + " seg=" + state.segmentInfo.name + " ext=" + ext);
+    // System.out.println("dir=" + state.directory + " seg=" + state.segmentInfo.name + " file=" + IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, ext));
     data = state.directory.openInput(IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, ext), state.context);
     maxDoc = state.segmentInfo.getDocCount();
     while(true) {
@@ -83,8 +82,6 @@ class SimpleTextDocValuesReader extends 
       assert startsWith(FIELD) : scratch.utf8ToString();
       String fieldName = stripPrefix(FIELD);
       //System.out.println("  field=" + fieldName);
-      FieldInfo fieldInfo = state.fieldInfos.fieldInfo(fieldName);
-      assert fieldInfo != null;
 
       OneField field = new OneField();
       fields.put(fieldName, field);

Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java Fri Nov  1 06:03:52 2013
@@ -55,7 +55,7 @@ class SimpleTextDocValuesWriter extends 
   private final Set<String> fieldsSeen = new HashSet<String>(); // for asserting
   
   public SimpleTextDocValuesWriter(SegmentWriteState state, String ext) throws IOException {
-    //System.out.println("WRITE: " + IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, ext) + " " + state.segmentInfo.getDocCount() + " docs");
+    // System.out.println("WRITE: " + IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, ext) + " " + state.segmentInfo.getDocCount() + " docs");
     data = state.directory.createOutput(IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, ext), state.context);
     numDocs = state.segmentInfo.getDocCount();
   }

Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java Fri Nov  1 06:03:52 2013
@@ -47,8 +47,8 @@ import static org.apache.lucene.codecs.s
 public class SimpleTextFieldInfosReader extends FieldInfosReader {
 
   @Override
-  public FieldInfos read(Directory directory, String segmentName, IOContext iocontext) throws IOException {
-    final String fileName = IndexFileNames.segmentFileName(segmentName, "", FIELD_INFOS_EXTENSION);
+  public FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext iocontext) throws IOException {
+    final String fileName = IndexFileNames.segmentFileName(segmentName, segmentSuffix, FIELD_INFOS_EXTENSION);
     IndexInput input = directory.openInput(fileName, iocontext);
     BytesRef scratch = new BytesRef();
     
@@ -105,6 +105,10 @@ public class SimpleTextFieldInfosReader 
         final DocValuesType docValuesType = docValuesType(dvType);
         
         SimpleTextUtil.readLine(input, scratch);
+        assert StringHelper.startsWith(scratch, DOCVALUES_GEN);
+        final long dvGen = Long.parseLong(readString(DOCVALUES_GEN.length, scratch));
+        
+        SimpleTextUtil.readLine(input, scratch);
         assert StringHelper.startsWith(scratch, NUM_ATTS);
         int numAtts = Integer.parseInt(readString(NUM_ATTS.length, scratch));
         Map<String,String> atts = new HashMap<String,String>();
@@ -122,6 +126,7 @@ public class SimpleTextFieldInfosReader 
 
         infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, 
           omitNorms, storePayloads, indexOptions, docValuesType, normsType, Collections.unmodifiableMap(atts));
+        infos[i].setDocValuesGen(dvGen);
       }
 
       if (input.getFilePointer() != input.length()) {

Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java Fri Nov  1 06:03:52 2013
@@ -53,14 +53,15 @@ public class SimpleTextFieldInfosWriter 
   static final BytesRef NORMS           =  new BytesRef("  norms ");
   static final BytesRef NORMS_TYPE      =  new BytesRef("  norms type ");
   static final BytesRef DOCVALUES       =  new BytesRef("  doc values ");
+  static final BytesRef DOCVALUES_GEN   =  new BytesRef("  doc values gen ");
   static final BytesRef INDEXOPTIONS    =  new BytesRef("  index options ");
   static final BytesRef NUM_ATTS        =  new BytesRef("  attributes ");
   final static BytesRef ATT_KEY         =  new BytesRef("    key ");
   final static BytesRef ATT_VALUE       =  new BytesRef("    value ");
   
   @Override
-  public void write(Directory directory, String segmentName, FieldInfos infos, IOContext context) throws IOException {
-    final String fileName = IndexFileNames.segmentFileName(segmentName, "", FIELD_INFOS_EXTENSION);
+  public void write(Directory directory, String segmentName, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
+    final String fileName = IndexFileNames.segmentFileName(segmentName, segmentSuffix, FIELD_INFOS_EXTENSION);
     IndexOutput out = directory.createOutput(fileName, context);
     BytesRef scratch = new BytesRef();
     boolean success = false;
@@ -108,6 +109,10 @@ public class SimpleTextFieldInfosWriter 
         SimpleTextUtil.write(out, DOCVALUES);
         SimpleTextUtil.write(out, getDocValuesType(fi.getDocValuesType()), scratch);
         SimpleTextUtil.writeNewline(out);
+        
+        SimpleTextUtil.write(out, DOCVALUES_GEN);
+        SimpleTextUtil.write(out, Long.toString(fi.getDocValuesGen()), scratch);
+        SimpleTextUtil.writeNewline(out);
                
         Map<String,String> atts = fi.attributes();
         int numAtts = atts == null ? 0 : atts.size();

Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoReader.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoReader.java Fri Nov  1 06:03:52 2013
@@ -17,8 +17,16 @@ package org.apache.lucene.codecs.simplet
  * limitations under the License.
  */
 
+import static org.apache.lucene.codecs.simpletext.SimpleTextSegmentInfoWriter.SI_DIAG_KEY;
+import static org.apache.lucene.codecs.simpletext.SimpleTextSegmentInfoWriter.SI_DIAG_VALUE;
+import static org.apache.lucene.codecs.simpletext.SimpleTextSegmentInfoWriter.SI_DOCCOUNT;
+import static org.apache.lucene.codecs.simpletext.SimpleTextSegmentInfoWriter.SI_FILE;
+import static org.apache.lucene.codecs.simpletext.SimpleTextSegmentInfoWriter.SI_NUM_DIAG;
+import static org.apache.lucene.codecs.simpletext.SimpleTextSegmentInfoWriter.SI_NUM_FILES;
+import static org.apache.lucene.codecs.simpletext.SimpleTextSegmentInfoWriter.SI_USECOMPOUND;
+import static org.apache.lucene.codecs.simpletext.SimpleTextSegmentInfoWriter.SI_VERSION;
+
 import java.io.IOException;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -34,8 +42,6 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.StringHelper;
 
-import static org.apache.lucene.codecs.simpletext.SimpleTextSegmentInfoWriter.*;
-
 /**
  * reads plaintext segments files
  * <p>
@@ -80,22 +86,6 @@ public class SimpleTextSegmentInfoReader
       }
       
       SimpleTextUtil.readLine(input, scratch);
-      assert StringHelper.startsWith(scratch, SI_NUM_ATTS);
-      int numAtts = Integer.parseInt(readString(SI_NUM_ATTS.length, scratch));
-      Map<String,String> attributes = new HashMap<String,String>();
-
-      for (int i = 0; i < numAtts; i++) {
-        SimpleTextUtil.readLine(input, scratch);
-        assert StringHelper.startsWith(scratch, SI_ATT_KEY);
-        String key = readString(SI_ATT_KEY.length, scratch);
-      
-        SimpleTextUtil.readLine(input, scratch);
-        assert StringHelper.startsWith(scratch, SI_ATT_VALUE);
-        String value = readString(SI_ATT_VALUE.length, scratch);
-        attributes.put(key, value);
-      }
-
-      SimpleTextUtil.readLine(input, scratch);
       assert StringHelper.startsWith(scratch, SI_NUM_FILES);
       int numFiles = Integer.parseInt(readString(SI_NUM_FILES.length, scratch));
       Set<String> files = new HashSet<String>();
@@ -108,7 +98,7 @@ public class SimpleTextSegmentInfoReader
       }
 
       SegmentInfo info = new SegmentInfo(directory, version, segmentName, docCount, 
-                                         isCompoundFile, null, diagnostics, Collections.unmodifiableMap(attributes));
+                                         isCompoundFile, null, diagnostics);
       info.setFiles(files);
       success = true;
       return info;

Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java Fri Nov  1 06:03:52 2013
@@ -45,9 +45,6 @@ public class SimpleTextSegmentInfoWriter
   final static BytesRef SI_NUM_DIAG         = new BytesRef("    diagnostics ");
   final static BytesRef SI_DIAG_KEY         = new BytesRef("      key ");
   final static BytesRef SI_DIAG_VALUE       = new BytesRef("      value ");
-  final static BytesRef SI_NUM_ATTS         = new BytesRef("    attributes ");
-  final static BytesRef SI_ATT_KEY          = new BytesRef("      key ");
-  final static BytesRef SI_ATT_VALUE        = new BytesRef("      value ");
   final static BytesRef SI_NUM_FILES        = new BytesRef("    files ");
   final static BytesRef SI_FILE             = new BytesRef("      file ");
   
@@ -93,24 +90,6 @@ public class SimpleTextSegmentInfoWriter
         }
       }
       
-      Map<String,String> atts = si.attributes();
-      int numAtts = atts == null ? 0 : atts.size();
-      SimpleTextUtil.write(output, SI_NUM_ATTS);
-      SimpleTextUtil.write(output, Integer.toString(numAtts), scratch);
-      SimpleTextUtil.writeNewline(output);
-    
-      if (numAtts > 0) {
-        for (Map.Entry<String,String> entry : atts.entrySet()) {
-          SimpleTextUtil.write(output, SI_ATT_KEY);
-          SimpleTextUtil.write(output, entry.getKey(), scratch);
-          SimpleTextUtil.writeNewline(output);
-        
-          SimpleTextUtil.write(output, SI_ATT_VALUE);
-          SimpleTextUtil.write(output, entry.getValue(), scratch);
-          SimpleTextUtil.writeNewline(output);
-        }
-      }
-
       Set<String> files = si.files();
       int numFiles = files == null ? 0 : files.size();
       SimpleTextUtil.write(output, SI_NUM_FILES);

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/Codec.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/Codec.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/Codec.java Fri Nov  1 06:03:52 2013
@@ -119,7 +119,7 @@ public abstract class Codec implements N
     loader.reload(classloader);
   }
   
-  private static Codec defaultCodec = Codec.forName("Lucene45");
+  private static Codec defaultCodec = Codec.forName("Lucene46");
   
   /** expert: returns the default codec used for newly created
    *  {@link IndexWriterConfig}s.

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosReader.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosReader.java Fri Nov  1 06:03:52 2013
@@ -35,5 +35,5 @@ public abstract class FieldInfosReader {
 
   /** Read the {@link FieldInfos} previously written with {@link
    *  FieldInfosWriter}. */
-  public abstract FieldInfos read(Directory directory, String segmentName, IOContext iocontext) throws IOException;
+  public abstract FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext iocontext) throws IOException;
 }

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosWriter.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FieldInfosWriter.java Fri Nov  1 06:03:52 2013
@@ -35,5 +35,5 @@ public abstract class FieldInfosWriter {
 
   /** Writes the provided {@link FieldInfos} to the
    *  directory. */
-  public abstract void write(Directory directory, String segmentName, FieldInfos infos, IOContext context) throws IOException;
+  public abstract void write(Directory directory, String segmentName, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException;
 }

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java Fri Nov  1 06:03:52 2013
@@ -21,13 +21,13 @@ package org.apache.lucene.codecs;
  * A codec that forwards all its method calls to another codec.
  * <p>
  * Extend this class when you need to reuse the functionality of an existing
- * codec. For example, if you want to build a codec that redefines Lucene45's
+ * codec. For example, if you want to build a codec that redefines Lucene46's
  * {@link LiveDocsFormat}:
  * <pre class="prettyprint">
  *   public final class CustomCodec extends FilterCodec {
  *
  *     public CustomCodec() {
- *       super("CustomCodec", new Lucene45Codec());
+ *       super("CustomCodec", new Lucene46Codec());
  *     }
  *
  *     public LiveDocsFormat liveDocsFormat() {

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosReader.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFieldInfosReader.java Fri Nov  1 06:03:52 2013
@@ -57,7 +57,7 @@ class Lucene3xFieldInfosReader extends F
   static final byte OMIT_POSITIONS = -128;
 
   @Override
-  public FieldInfos read(Directory directory, String segmentName, IOContext iocontext) throws IOException {
+  public FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext iocontext) throws IOException {
     final String fileName = IndexFileNames.segmentFileName(segmentName, "", FIELD_INFOS_EXTENSION);
     IndexInput input = directory.openInput(fileName, iocontext);
     

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfoReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfoReader.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfoReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfoReader.java Fri Nov  1 06:03:52 2013
@@ -244,7 +244,7 @@ public class Lucene3xSegmentInfoReader e
                                        null, diagnostics, Collections.unmodifiableMap(attributes));
     info.setFiles(files);
 
-    SegmentInfoPerCommit infoPerCommit = new SegmentInfoPerCommit(info, delCount, delGen);
+    SegmentInfoPerCommit infoPerCommit = new SegmentInfoPerCommit(info, delCount, delGen, -1);
     return infoPerCommit;
   }
 

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java Fri Nov  1 06:03:52 2013
@@ -82,7 +82,7 @@ public class Lucene40Codec extends Codec
   }
   
   @Override
-  public final SegmentInfoFormat segmentInfoFormat() {
+  public SegmentInfoFormat segmentInfoFormat() {
     return infosFormat;
   }
   

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java Fri Nov  1 06:03:52 2013
@@ -49,7 +49,7 @@ class Lucene40FieldInfosReader extends F
   }
 
   @Override
-  public FieldInfos read(Directory directory, String segmentName, IOContext iocontext) throws IOException {
+  public FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext iocontext) throws IOException {
     final String fileName = IndexFileNames.segmentFileName(segmentName, "", Lucene40FieldInfosFormat.FIELD_INFOS_EXTENSION);
     IndexInput input = directory.openInput(fileName, iocontext);
     

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java Fri Nov  1 06:03:52 2013
@@ -67,10 +67,11 @@ import org.apache.lucene.store.DataOutpu
  * 
  * @see SegmentInfos
  * @lucene.experimental
+ * @deprecated Only for reading old 4.0-4.5 segments
  */
+@Deprecated
 public class Lucene40SegmentInfoFormat extends SegmentInfoFormat {
   private final SegmentInfoReader reader = new Lucene40SegmentInfoReader();
-  private final SegmentInfoWriter writer = new Lucene40SegmentInfoWriter();
 
   /** Sole constructor. */
   public Lucene40SegmentInfoFormat() {
@@ -83,7 +84,7 @@ public class Lucene40SegmentInfoFormat e
 
   @Override
   public SegmentInfoWriter getSegmentInfoWriter() {
-    return writer;
+    throw new UnsupportedOperationException("this codec can only be used for reading");
   }
 
   /** File extension used to store {@link SegmentInfo}. */

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java Fri Nov  1 06:03:52 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.codecs.lucene4
  */
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.Map;
 import java.util.Set;
 
@@ -37,7 +36,9 @@ import org.apache.lucene.util.IOUtils;
  * 
  * @see Lucene40SegmentInfoFormat
  * @lucene.experimental
+ * @deprecated Only for reading old 4.0-4.5 segments
  */
+@Deprecated
 public class Lucene40SegmentInfoReader extends SegmentInfoReader {
 
   /** Sole constructor. */
@@ -60,15 +61,14 @@ public class Lucene40SegmentInfoReader e
       }
       final boolean isCompoundFile = input.readByte() == SegmentInfo.YES;
       final Map<String,String> diagnostics = input.readStringStringMap();
-      final Map<String,String> attributes = input.readStringStringMap();
+      input.readStringStringMap(); // read deprecated attributes
       final Set<String> files = input.readStringSet();
       
       if (input.getFilePointer() != input.length()) {
         throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")");
       }
 
-      final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile,
-                                             null, diagnostics, Collections.unmodifiableMap(attributes));
+      final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics);
       si.setFiles(files);
 
       success = true;

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java Fri Nov  1 06:03:52 2013
@@ -101,7 +101,7 @@ public class Lucene41Codec extends Codec
   }
   
   @Override
-  public final SegmentInfoFormat segmentInfoFormat() {
+  public SegmentInfoFormat segmentInfoFormat() {
     return infosFormat;
   }
   

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42Codec.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42Codec.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42Codec.java Fri Nov  1 06:03:52 2013
@@ -95,12 +95,12 @@ public class Lucene42Codec extends Codec
   }
   
   @Override
-  public final FieldInfosFormat fieldInfosFormat() {
+  public FieldInfosFormat fieldInfosFormat() {
     return fieldInfosFormat;
   }
   
   @Override
-  public final SegmentInfoFormat segmentInfoFormat() {
+  public SegmentInfoFormat segmentInfoFormat() {
     return infosFormat;
   }
   

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java Fri Nov  1 06:03:52 2013
@@ -83,10 +83,11 @@ import org.apache.lucene.store.DataOutpu
  * </ul>
  *
  * @lucene.experimental
+ * @deprecated Only for reading old 4.2-4.5 segments
  */
-public final class Lucene42FieldInfosFormat extends FieldInfosFormat {
+@Deprecated
+public class Lucene42FieldInfosFormat extends FieldInfosFormat {
   private final FieldInfosReader reader = new Lucene42FieldInfosReader();
-  private final FieldInfosWriter writer = new Lucene42FieldInfosWriter();
   
   /** Sole constructor. */
   public Lucene42FieldInfosFormat() {
@@ -99,7 +100,7 @@ public final class Lucene42FieldInfosFor
 
   @Override
   public FieldInfosWriter getFieldInfosWriter() throws IOException {
-    return writer;
+    throw new UnsupportedOperationException("this codec can only be used for reading");
   }
   
   /** Extension of field infos */

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosReader.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosReader.java Fri Nov  1 06:03:52 2013
@@ -38,8 +38,10 @@ import org.apache.lucene.util.IOUtils;
  * Lucene 4.2 FieldInfos reader.
  * 
  * @lucene.experimental
+ * @deprecated Only for reading old 4.2-4.5 segments
  * @see Lucene42FieldInfosFormat
  */
+@Deprecated
 final class Lucene42FieldInfosReader extends FieldInfosReader {
 
   /** Sole constructor. */
@@ -47,7 +49,7 @@ final class Lucene42FieldInfosReader ext
   }
 
   @Override
-  public FieldInfos read(Directory directory, String segmentName, IOContext iocontext) throws IOException {
+  public FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext iocontext) throws IOException {
     final String fileName = IndexFileNames.segmentFileName(segmentName, "", Lucene42FieldInfosFormat.EXTENSION);
     IndexInput input = directory.openInput(fileName, iocontext);
     

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45Codec.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45Codec.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45Codec.java Fri Nov  1 06:03:52 2013
@@ -45,10 +45,12 @@ import org.apache.lucene.codecs.perfield
  *
  * @see org.apache.lucene.codecs.lucene45 package documentation for file format details.
  * @lucene.experimental
+ * @deprecated Only for reading old 4.3-4.5 segments
  */
 // NOTE: if we make largish changes in a minor release, easier to just make Lucene46Codec or whatever
 // if they are backwards compatible or smallish we can probably do the backwards in the postingsreader
 // (it writes a minor version, etc).
+@Deprecated
 public class Lucene45Codec extends Codec {
   private final StoredFieldsFormat fieldsFormat = new Lucene41StoredFieldsFormat();
   private final TermVectorsFormat vectorsFormat = new Lucene42TermVectorsFormat();
@@ -92,12 +94,12 @@ public class Lucene45Codec extends Codec
   }
   
   @Override
-  public final FieldInfosFormat fieldInfosFormat() {
+  public FieldInfosFormat fieldInfosFormat() {
     return fieldInfosFormat;
   }
   
   @Override
-  public final SegmentInfoFormat segmentInfoFormat() {
+  public SegmentInfoFormat segmentInfoFormat() {
     return infosFormat;
   }
   

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46Codec.java?rev=1537832&r1=1527154&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46Codec.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46Codec.java Fri Nov  1 06:03:52 2013
@@ -28,7 +28,6 @@ import org.apache.lucene.codecs.SegmentI
 import org.apache.lucene.codecs.StoredFieldsFormat;
 import org.apache.lucene.codecs.TermVectorsFormat;
 import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat;
-import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoFormat;
 import org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat;
 import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
 import org.apache.lucene.codecs.lucene42.Lucene42TermVectorsFormat;
@@ -52,7 +51,7 @@ public class Lucene46Codec extends Codec
   private final StoredFieldsFormat fieldsFormat = new Lucene41StoredFieldsFormat();
   private final TermVectorsFormat vectorsFormat = new Lucene42TermVectorsFormat();
   private final FieldInfosFormat fieldInfosFormat = new Lucene46FieldInfosFormat();
-  private final SegmentInfoFormat infosFormat = new Lucene40SegmentInfoFormat();
+  private final SegmentInfoFormat segmentInfosFormat = new Lucene46SegmentInfoFormat();
   private final LiveDocsFormat liveDocsFormat = new Lucene40LiveDocsFormat();
   
   private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
@@ -96,7 +95,7 @@ public class Lucene46Codec extends Codec
   
   @Override
   public final SegmentInfoFormat segmentInfoFormat() {
-    return infosFormat;
+    return segmentInfosFormat;
   }
   
   @Override

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene46/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene46/package.html?rev=1537832&r1=1527154&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene46/package.html (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene46/package.html Fri Nov  1 06:03:52 2013
@@ -138,7 +138,7 @@ its numbering.</p>
 <p>Each segment index maintains the following:</p>
 <ul>
 <li>
-{@link org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoFormat Segment info}.
+{@link org.apache.lucene.codecs.lucene46.Lucene46SegmentInfoFormat Segment info}.
    This contains metadata about a segment, such as the number of documents,
    what files it uses, 
 </li>

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/package.html?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/package.html (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/package.html Fri Nov  1 06:03:52 2013
@@ -61,13 +61,13 @@ name of your codec.
   If you just want to customise the {@link org.apache.lucene.codecs.PostingsFormat}, or use different postings
   formats for different fields, then you can register your custom postings format in the same way (in
   META-INF/services/org.apache.lucene.codecs.PostingsFormat), and then extend the default
-  {@link org.apache.lucene.codecs.lucene45.Lucene45Codec} and override
-  {@link org.apache.lucene.codecs.lucene45.Lucene45Codec#getPostingsFormatForField(String)} to return your custom
+  {@link org.apache.lucene.codecs.lucene46.Lucene46Codec} and override
+  {@link org.apache.lucene.codecs.lucene46.Lucene46Codec#getPostingsFormatForField(String)} to return your custom
   postings format.
 </p>
 <p>
   Similarly, if you just want to customise the {@link org.apache.lucene.codecs.DocValuesFormat} per-field, have 
-  a look at {@link org.apache.lucene.codecs.lucene45.Lucene45Codec#getDocValuesFormatForField(String)}.
+  a look at {@link org.apache.lucene.codecs.lucene46.Lucene46Codec#getDocValuesFormatForField(String)}.
 </p>
 </body>
 </html>

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java Fri Nov  1 06:03:52 2013
@@ -22,13 +22,13 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.IdentityHashMap;
 import java.util.Map;
-import java.util.ServiceLoader; // javadocs
+import java.util.ServiceLoader;
 import java.util.TreeMap;
 
-import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.DocValuesConsumer;
-import org.apache.lucene.codecs.DocValuesProducer;
 import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.NumericDocValues;
@@ -76,11 +76,10 @@ public abstract class PerFieldDocValuesF
   }
 
   @Override
-  public final DocValuesConsumer fieldsConsumer(SegmentWriteState state)
-      throws IOException {
+  public final DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
     return new FieldsWriter(state);
   }
-  
+
   static class ConsumerAndSuffix implements Closeable {
     DocValuesConsumer consumer;
     int suffix;
@@ -97,7 +96,7 @@ public abstract class PerFieldDocValuesF
     private final Map<String,Integer> suffixes = new HashMap<String,Integer>();
     
     private final SegmentWriteState segmentWriteState;
-
+    
     public FieldsWriter(SegmentWriteState state) {
       segmentWriteState = state;
     }
@@ -123,32 +122,53 @@ public abstract class PerFieldDocValuesF
     }
 
     private DocValuesConsumer getInstance(FieldInfo field) throws IOException {
-      final DocValuesFormat format = getDocValuesFormatForField(field.name);
+      DocValuesFormat format = null;
+      if (field.getDocValuesGen() != -1) {
+        final String formatName = field.getAttribute(PER_FIELD_FORMAT_KEY);
+        // this means the field never existed in that segment, yet is applied updates
+        if (formatName != null) {
+          format = DocValuesFormat.forName(formatName);
+        }
+      }
+      if (format == null) {
+        format = getDocValuesFormatForField(field.name);
+      }
       if (format == null) {
         throw new IllegalStateException("invalid null DocValuesFormat for field=\"" + field.name + "\"");
       }
       final String formatName = format.getName();
       
       String previousValue = field.putAttribute(PER_FIELD_FORMAT_KEY, formatName);
-      assert previousValue == null: "formatName=" + formatName + " prevValue=" + previousValue;
+      assert field.getDocValuesGen() != -1 || previousValue == null: "formatName=" + formatName + " prevValue=" + previousValue;
       
-      Integer suffix;
+      Integer suffix = null;
       
       ConsumerAndSuffix consumer = formats.get(format);
       if (consumer == null) {
         // First time we are seeing this format; create a new instance
+
+        if (field.getDocValuesGen() != -1) {
+          final String suffixAtt = field.getAttribute(PER_FIELD_SUFFIX_KEY);
+          // even when dvGen is != -1, it can still be a new field, that never
+          // existed in the segment, and therefore doesn't have the recorded
+          // attributes yet.
+          if (suffixAtt != null) {
+            suffix = Integer.valueOf(suffixAtt);
+          }
+        }
         
-        // bump the suffix
-        suffix = suffixes.get(formatName);
         if (suffix == null) {
-          suffix = 0;
-        } else {
-          suffix = suffix + 1;
+          // bump the suffix
+          suffix = suffixes.get(formatName);
+          if (suffix == null) {
+            suffix = 0;
+          } else {
+            suffix = suffix + 1;
+          }
         }
         suffixes.put(formatName, suffix);
         
-        final String segmentSuffix = getFullSegmentSuffix(field.name,
-                                                          segmentWriteState.segmentSuffix,
+        final String segmentSuffix = getFullSegmentSuffix(segmentWriteState.segmentSuffix,
                                                           getSuffix(formatName, Integer.toString(suffix)));
         consumer = new ConsumerAndSuffix();
         consumer.consumer = format.fieldsConsumer(new SegmentWriteState(segmentWriteState, segmentSuffix));
@@ -161,10 +181,10 @@ public abstract class PerFieldDocValuesF
       }
       
       previousValue = field.putAttribute(PER_FIELD_SUFFIX_KEY, Integer.toString(suffix));
-      assert previousValue == null;
+      assert field.getDocValuesGen() != -1 || previousValue == null : "suffix=" + Integer.toString(suffix) + " prevValue=" + previousValue;
 
       // TODO: we should only provide the "slice" of FIS
-      // that this PF actually sees ...
+      // that this DVF actually sees ...
       return consumer.consumer;
     }
 
@@ -179,14 +199,11 @@ public abstract class PerFieldDocValuesF
     return formatName + "_" + suffix;
   }
 
-  static String getFullSegmentSuffix(String fieldName, String outerSegmentSuffix, String segmentSuffix) {
+  static String getFullSegmentSuffix(String outerSegmentSuffix, String segmentSuffix) {
     if (outerSegmentSuffix.length() == 0) {
       return segmentSuffix;
     } else {
-      // TODO: support embedding; I think it should work but
-      // we need a test confirm to confirm
-      // return outerSegmentSuffix + "_" + segmentSuffix;
-      throw new IllegalStateException("cannot embed PerFieldPostingsFormat inside itself (field \"" + fieldName + "\" returned PerFieldPostingsFormat)");
+      return outerSegmentSuffix + "_" + segmentSuffix;
     }
   }
 
@@ -210,7 +227,7 @@ public abstract class PerFieldDocValuesF
               final String suffix = fi.getAttribute(PER_FIELD_SUFFIX_KEY);
               assert suffix != null;
               DocValuesFormat format = DocValuesFormat.forName(formatName);
-              String segmentSuffix = getSuffix(formatName, suffix);
+              String segmentSuffix = getFullSegmentSuffix(readState.segmentSuffix, getSuffix(formatName, suffix));
               if (!formats.containsKey(segmentSuffix)) {
                 formats.put(segmentSuffix, format.fieldsProducer(new SegmentReadState(readState, segmentSuffix)));
               }

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BufferedDeletes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BufferedDeletes.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BufferedDeletes.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BufferedDeletes.java Fri Nov  1 06:03:52 2013
@@ -19,25 +19,26 @@ package org.apache.lucene.index;
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.RamUsageEstimator;
 
-/* Holds buffered deletes, by docID, term or query for a
+/* Holds buffered deletes and updates, by docID, term or query for a
  * single segment. This is used to hold buffered pending
- * deletes against the to-be-flushed segment.  Once the
- * deletes are pushed (on flush in DocumentsWriter), these
- * deletes are converted to a FrozenDeletes instance. */
+ * deletes and updates against the to-be-flushed segment.  Once the
+ * deletes and updates are pushed (on flush in DocumentsWriter), they
+ * are converted to a FrozenDeletes instance. */
 
 // NOTE: instances of this class are accessed either via a private
 // instance on DocumentWriterPerThread, or via sync'd code by
 // DocumentsWriterDeleteQueue
 
-class BufferedDeletes {
+class BufferedDeletes { // TODO (DVU_RENAME) BufferedUpdates?
 
   /* Rough logic: HashMap has an array[Entry] w/ varying
      load factor (say 2 * POINTER).  Entry is object w/ Term
@@ -63,11 +64,50 @@ class BufferedDeletes {
      undercount (say 24 bytes).  Integer is OBJ_HEADER + INT. */
   final static int BYTES_PER_DEL_QUERY = 5*RamUsageEstimator.NUM_BYTES_OBJECT_REF + 2*RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 2*RamUsageEstimator.NUM_BYTES_INT + 24;
 
+  /* Rough logic: NumericUpdate calculates its actual size,
+   * including the update Term and DV field (String). The 
+   * per-field map holds a reference to the updated field, and
+   * therefore we only account for the object reference and 
+   * map space itself. This is incremented when we first see
+   * an updated field.
+   * 
+   * HashMap has an array[Entry] w/ varying load
+   * factor (say 2*POINTER). Entry is an object w/ String key, 
+   * LinkedHashMap val, int hash, Entry next (OBJ_HEADER + 3*POINTER + INT).
+   * 
+   * LinkedHashMap (val) is counted as OBJ_HEADER, array[Entry] ref + header, 4*INT, 1*FLOAT,
+   * Set (entrySet) (2*OBJ_HEADER + ARRAY_HEADER + 2*POINTER + 4*INT + FLOAT)
+   */
+  final static int BYTES_PER_NUMERIC_FIELD_ENTRY =
+      7*RamUsageEstimator.NUM_BYTES_OBJECT_REF + 3*RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 
+      RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + 5*RamUsageEstimator.NUM_BYTES_INT + RamUsageEstimator.NUM_BYTES_FLOAT;
+      
+  /* Rough logic: Incremented when we see another Term for an already updated
+   * field.
+   * LinkedHashMap has an array[Entry] w/ varying load factor 
+   * (say 2*POINTER). Entry is an object w/ Term key, NumericUpdate val, 
+   * int hash, Entry next, Entry before, Entry after (OBJ_HEADER + 5*POINTER + INT).
+   * 
+   * Term (key) is counted only as POINTER.
+   * NumericUpdate (val) counts its own size and isn't accounted for here.
+   */
+  final static int BYTES_PER_NUMERIC_UPDATE_ENTRY = 7*RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.NUM_BYTES_INT;
+  
   final AtomicInteger numTermDeletes = new AtomicInteger();
+  final AtomicInteger numNumericUpdates = new AtomicInteger();
   final Map<Term,Integer> terms = new HashMap<Term,Integer>();
   final Map<Query,Integer> queries = new HashMap<Query,Integer>();
   final List<Integer> docIDs = new ArrayList<Integer>();
 
+  // Map<dvField,Map<updateTerm,NumericUpdate>>
+  // For each field we keep an ordered list of NumericUpdates, key'd by the
+  // update Term. LinkedHashMap guarantees we will later traverse the map in
+  // insertion order (so that if two terms affect the same document, the last
+  // one that came in wins), and helps us detect faster if the same Term is
+  // used to update the same field multiple times (so we later traverse it
+  // only once).
+  final Map<String,LinkedHashMap<Term,NumericUpdate>> numericUpdates = new HashMap<String,LinkedHashMap<Term,NumericUpdate>>();
+
   public static final Integer MAX_INT = Integer.valueOf(Integer.MAX_VALUE);
 
   final AtomicLong bytesUsed;
@@ -75,21 +115,17 @@ class BufferedDeletes {
   private final static boolean VERBOSE_DELETES = false;
 
   long gen;
+  
   public BufferedDeletes() {
-    this(new AtomicLong());
-  }
-
-  BufferedDeletes(AtomicLong bytesUsed) {
-    assert bytesUsed != null;
-    this.bytesUsed = bytesUsed;
+    this.bytesUsed = new AtomicLong();
   }
 
   @Override
   public String toString() {
     if (VERBOSE_DELETES) {
       return "gen=" + gen + " numTerms=" + numTermDeletes + ", terms=" + terms
-        + ", queries=" + queries + ", docIDs=" + docIDs + ", bytesUsed="
-        + bytesUsed;
+        + ", queries=" + queries + ", docIDs=" + docIDs + ", numericUpdates=" + numericUpdates
+        + ", bytesUsed=" + bytesUsed;
     } else {
       String s = "gen=" + gen;
       if (numTermDeletes.get() != 0) {
@@ -101,6 +137,9 @@ class BufferedDeletes {
       if (docIDs.size() != 0) {
         s += " " + docIDs.size() + " deleted docIDs";
       }
+      if (numNumericUpdates.get() != 0) {
+        s += " " + numNumericUpdates.get() + " numeric updates (unique count=" + numericUpdates.size() + ")";
+      }
       if (bytesUsed.get() != 0) {
         s += " bytesUsed=" + bytesUsed.get();
       }
@@ -145,20 +184,46 @@ class BufferedDeletes {
     }
   }
  
+  public void addNumericUpdate(NumericUpdate update, int docIDUpto) {
+    LinkedHashMap<Term,NumericUpdate> fieldUpdates = numericUpdates.get(update.field);
+    if (fieldUpdates == null) {
+      fieldUpdates = new LinkedHashMap<Term,NumericUpdate>();
+      numericUpdates.put(update.field, fieldUpdates);
+      bytesUsed.addAndGet(BYTES_PER_NUMERIC_FIELD_ENTRY);
+    }
+    final NumericUpdate current = fieldUpdates.get(update.term);
+    if (current != null && docIDUpto < current.docIDUpto) {
+      // Only record the new number if it's greater than or equal to the current
+      // one. This is important because if multiple threads are replacing the
+      // same doc at nearly the same time, it's possible that one thread that
+      // got a higher docID is scheduled before the other threads.
+      return;
+    }
+
+    update.docIDUpto = docIDUpto;
+    // since it's a LinkedHashMap, we must first remove the Term entry so that
+    // it's added last (we're interested in insertion-order).
+    if (current != null) {
+      fieldUpdates.remove(update.term);
+    }
+    fieldUpdates.put(update.term, update);
+    numNumericUpdates.incrementAndGet();
+    if (current == null) {
+      bytesUsed.addAndGet(BYTES_PER_NUMERIC_UPDATE_ENTRY + update.sizeInBytes());
+    }
+  }
+  
   void clear() {
     terms.clear();
     queries.clear();
     docIDs.clear();
+    numericUpdates.clear();
     numTermDeletes.set(0);
+    numNumericUpdates.set(0);
     bytesUsed.set(0);
   }
   
-  void clearDocIDs() {
-    bytesUsed.addAndGet(-docIDs.size()*BYTES_PER_DEL_DOCID);
-    docIDs.clear();
-  }
-  
   boolean any() {
-    return terms.size() > 0 || docIDs.size() > 0 || queries.size() > 0;
+    return terms.size() > 0 || docIDs.size() > 0 || queries.size() > 0 || numericUpdates.size() > 0;
   }
 }

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java Fri Nov  1 06:03:52 2013
@@ -18,10 +18,13 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
-import java.util.List;
 import java.util.ArrayList;
-import java.util.Comparator;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -35,12 +38,12 @@ import org.apache.lucene.util.InfoStream
 
 /* Tracks the stream of {@link BufferedDeletes}.
  * When DocumentsWriterPerThread flushes, its buffered
- * deletes are appended to this stream.  We later
- * apply these deletes (resolve them to the actual
+ * deletes and updates are appended to this stream.  We later
+ * apply them (resolve them to the actual
  * docIDs, per segment) when a merge is started
  * (only to the to-be-merged segments).  We
  * also apply to all segments when NRT reader is pulled,
- * commit/close is called, or when too many deletes are
+ * commit/close is called, or when too many deletes or  updates are
  * buffered and must be flushed (by RAM usage or by count).
  *
  * Each packet is assigned a generation, and each flushed or
@@ -48,7 +51,7 @@ import org.apache.lucene.util.InfoStream
  * track which BufferedDeletes packets to apply to any given
  * segment. */
 
-class BufferedDeletesStream {
+class BufferedDeletesStream { // TODO (DVU_RENAME) BufferedUpdatesStream
 
   // TODO: maybe linked list?
   private final List<FrozenBufferedDeletes> deletes = new ArrayList<FrozenBufferedDeletes>();
@@ -114,6 +117,7 @@ class BufferedDeletesStream {
   }
 
   public static class ApplyDeletesResult {
+    
     // True if any actual deletes took place:
     public final boolean anyDeletes;
 
@@ -190,7 +194,7 @@ class BufferedDeletesStream {
       final long segGen = info.getBufferedDeletesGen();
 
       if (packet != null && segGen < packet.delGen()) {
-        //System.out.println("  coalesce");
+//        System.out.println("  coalesce");
         if (coalescedDeletes == null) {
           coalescedDeletes = new CoalescedDeletes();
         }
@@ -217,15 +221,21 @@ class BufferedDeletesStream {
         int delCount = 0;
         final boolean segAllDeletes;
         try {
+          Map<String,NumericFieldUpdates> fieldUpdates = null;
           if (coalescedDeletes != null) {
             //System.out.println("    del coalesced");
             delCount += applyTermDeletes(coalescedDeletes.termsIterable(), rld, reader);
             delCount += applyQueryDeletes(coalescedDeletes.queriesIterable(), rld, reader);
+            fieldUpdates = applyNumericDocValuesUpdates(coalescedDeletes.numericDVUpdates, rld, reader, fieldUpdates);
           }
           //System.out.println("    del exact");
           // Don't delete by Term here; DocumentsWriterPerThread
           // already did that on flush:
           delCount += applyQueryDeletes(packet.queriesIterable(), rld, reader);
+          fieldUpdates = applyNumericDocValuesUpdates(Arrays.asList(packet.updates), rld, reader, fieldUpdates);
+          if (!fieldUpdates.isEmpty()) {
+            rld.writeFieldUpdates(info.info.dir, fieldUpdates);
+          }
           final int fullDelCount = rld.info.getDelCount() + rld.getPendingDeleteCount();
           assert fullDelCount <= rld.info.info.getDocCount();
           segAllDeletes = fullDelCount == rld.info.info.getDocCount();
@@ -272,10 +282,14 @@ class BufferedDeletesStream {
           try {
             delCount += applyTermDeletes(coalescedDeletes.termsIterable(), rld, reader);
             delCount += applyQueryDeletes(coalescedDeletes.queriesIterable(), rld, reader);
+            Map<String,NumericFieldUpdates> fieldUpdates = applyNumericDocValuesUpdates(coalescedDeletes.numericDVUpdates, rld, reader, null);
+            if (!fieldUpdates.isEmpty()) {
+              rld.writeFieldUpdates(info.info.dir, fieldUpdates);
+            }
             final int fullDelCount = rld.info.getDelCount() + rld.getPendingDeleteCount();
             assert fullDelCount <= rld.info.info.getDocCount();
             segAllDeletes = fullDelCount == rld.info.info.getDocCount();
-          } finally {   
+          } finally {
             rld.release(reader);
             readerPool.release(rld);
           }
@@ -289,7 +303,7 @@ class BufferedDeletesStream {
           }
 
           if (infoStream.isEnabled("BD")) {
-            infoStream.message("BD", "seg=" + info + " segGen=" + segGen + " coalesced deletes=[" + (coalescedDeletes == null ? "null" : coalescedDeletes) + "] newDelCount=" + delCount + (segAllDeletes ? " 100% deleted" : ""));
+            infoStream.message("BD", "seg=" + info + " segGen=" + segGen + " coalesced deletes=[" + coalescedDeletes + "] newDelCount=" + delCount + (segAllDeletes ? " 100% deleted" : ""));
           }
         }
         info.setBufferedDeletesGen(gen);
@@ -409,15 +423,15 @@ class BufferedDeletesStream {
             if (docID == DocIdSetIterator.NO_MORE_DOCS) {
               break;
             }   
+            if (!any) {
+              rld.initWritableLiveDocs();
+              any = true;
+            }
             // NOTE: there is no limit check on the docID
             // when deleting by Term (unlike by Query)
             // because on flush we apply all Term deletes to
             // each segment.  So all Term deleting here is
             // against prior segments:
-            if (!any) {
-              rld.initWritableLiveDocs();
-              any = true;
-            }
             if (rld.delete(docID)) {
               delCount++;
             }
@@ -429,6 +443,87 @@ class BufferedDeletesStream {
     return delCount;
   }
 
+  // NumericDocValues Updates
+  // If otherFieldUpdates != null, we need to merge the updates into them
+  private synchronized Map<String,NumericFieldUpdates> applyNumericDocValuesUpdates(Iterable<NumericUpdate> updates, 
+      ReadersAndLiveDocs rld, SegmentReader reader, Map<String,NumericFieldUpdates> otherFieldUpdates) throws IOException {
+    Fields fields = reader.fields();
+    if (fields == null) {
+      // This reader has no postings
+      return Collections.emptyMap();
+    }
+
+    // TODO: we can process the updates per DV field, from last to first so that
+    // if multiple terms affect same document for the same field, we add an update
+    // only once (that of the last term). To do that, we can keep a bitset which
+    // marks which documents have already been updated. So e.g. if term T1
+    // updates doc 7, and then we process term T2 and it updates doc 7 as well,
+    // we don't apply the update since we know T1 came last and therefore wins
+    // the update.
+    // We can also use that bitset as 'liveDocs' to pass to TermEnum.docs(), so
+    // that these documents aren't even returned.
+    
+    String currentField = null;
+    TermsEnum termsEnum = null;
+    DocsEnum docs = null;
+    final Map<String,NumericFieldUpdates> result = otherFieldUpdates == null ? new HashMap<String,NumericFieldUpdates>() : otherFieldUpdates;
+    //System.out.println(Thread.currentThread().getName() + " numericDVUpdate reader=" + reader);
+    for (NumericUpdate update : updates) {
+      Term term = update.term;
+      int limit = update.docIDUpto;
+      
+      // TODO: we traverse the terms in update order (not term order) so that we
+      // apply the updates in the correct order, i.e. if two terms udpate the
+      // same document, the last one that came in wins, irrespective of the
+      // terms lexical order.
+      // we can apply the updates in terms order if we keep an updatesGen (and
+      // increment it with every update) and attach it to each NumericUpdate. Note
+      // that we cannot rely only on docIDUpto because an app may send two updates
+      // which will get same docIDUpto, yet will still need to respect the order
+      // those updates arrived.
+      
+      if (!term.field().equals(currentField)) {
+        // if we change the code to process updates in terms order, enable this assert
+//        assert currentField == null || currentField.compareTo(term.field()) < 0;
+        currentField = term.field();
+        Terms terms = fields.terms(currentField);
+        if (terms != null) {
+          termsEnum = terms.iterator(termsEnum);
+        } else {
+          termsEnum = null;
+          continue; // no terms in that field
+        }
+      }
+
+      if (termsEnum == null) {
+        continue;
+      }
+      // System.out.println("  term=" + term);
+
+      if (termsEnum.seekExact(term.bytes())) {
+        // we don't need term frequencies for this
+        DocsEnum docsEnum = termsEnum.docs(rld.getLiveDocs(), docs, DocsEnum.FLAG_NONE);
+      
+        //System.out.println("BDS: got docsEnum=" + docsEnum);
+
+        NumericFieldUpdates fieldUpdates = result.get(update.field);
+        if (fieldUpdates == null) {
+          fieldUpdates = new NumericFieldUpdates.PackedNumericFieldUpdates(reader.maxDoc());
+          result.put(update.field, fieldUpdates);
+        }
+        int doc;
+        while ((doc = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
+          //System.out.println(Thread.currentThread().getName() + " numericDVUpdate term=" + term + " doc=" + docID);
+          if (doc >= limit) {
+            break; // no more docs that can be updated for this term
+          }
+          fieldUpdates.add(doc, update.value);
+        }
+      }
+    }
+    return result;
+  }
+
   public static class QueryAndLimit {
     public final Query query;
     public final int limit;

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Fri Nov  1 06:03:52 2013
@@ -540,11 +540,6 @@ public class CheckIndex {
           msg(infoStream, "    diagnostics = " + diagnostics);
         }
 
-        Map<String,String> atts = info.info.attributes();
-        if (atts != null && !atts.isEmpty()) {
-          msg(infoStream, "    attributes = " + atts);
-        }
-
         if (!info.hasDeletions()) {
           msg(infoStream, "    no deletions");
           segInfoStat.hasDeletions = false;

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CoalescedDeletes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CoalescedDeletes.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CoalescedDeletes.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CoalescedDeletes.java Fri Nov  1 06:03:52 2013
@@ -30,20 +30,27 @@ import org.apache.lucene.util.MergedIter
 class CoalescedDeletes {
   final Map<Query,Integer> queries = new HashMap<Query,Integer>();
   final List<Iterable<Term>> iterables = new ArrayList<Iterable<Term>>();
-
+  final List<NumericUpdate> numericDVUpdates = new ArrayList<NumericUpdate>();
+  
   @Override
   public String toString() {
     // note: we could add/collect more debugging information
-    return "CoalescedDeletes(termSets=" + iterables.size() + ",queries=" + queries.size() + ")";
+    return "CoalescedDeletes(termSets=" + iterables.size() + ",queries=" + queries.size() + ",numericUpdates=" + numericDVUpdates.size() + ")";
   }
 
   void update(FrozenBufferedDeletes in) {
     iterables.add(in.termsIterable());
 
-    for(int queryIdx=0;queryIdx<in.queries.length;queryIdx++) {
+    for (int queryIdx = 0; queryIdx < in.queries.length; queryIdx++) {
       final Query query = in.queries[queryIdx];
       queries.put(query, BufferedDeletes.MAX_INT);
     }
+    
+    for (NumericUpdate nu : in.updates) {
+      NumericUpdate clone = new NumericUpdate(nu.term, nu.field, nu.value);
+      clone.docIDUpto = Integer.MAX_VALUE;
+      numericDVUpdates.add(clone);
+    }
   }
 
  public Iterable<Term> termsIterable() {

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java Fri Nov  1 06:03:52 2013
@@ -85,7 +85,7 @@ final class DocFieldProcessor extends Do
     // FreqProxTermsWriter does this with
     // FieldInfo.storePayload.
     FieldInfosWriter infosWriter = codec.fieldInfosFormat().getFieldInfosWriter();
-    infosWriter.write(state.directory, state.segmentInfo.name, state.fieldInfos, IOContext.DEFAULT);
+    infosWriter.write(state.directory, state.segmentInfo.name, "", state.fieldInfos, IOContext.DEFAULT);
   }
 
   @Override

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java Fri Nov  1 06:03:52 2013
@@ -158,6 +158,13 @@ final class DocumentsWriter {
     return applyAllDeletes( deleteQueue);
   }
 
+  synchronized boolean updateNumericDocValue(Term term, String field, Long value) throws IOException {
+    final DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue;
+    deleteQueue.addNumericUpdate(new NumericUpdate(term, field, value));
+    flushControl.doOnDelete();
+    return applyAllDeletes(deleteQueue);
+  }
+
   DocumentsWriterDeleteQueue currentDeleteSession() {
     return deleteQueue;
   }

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java Fri Nov  1 06:03:52 2013
@@ -107,6 +107,11 @@ final class DocumentsWriterDeleteQueue {
     tryApplyGlobalSlice();
   }
 
+  void addNumericUpdate(NumericUpdate update) {
+    add(new NumericUpdateNode(update));
+    tryApplyGlobalSlice();
+  }
+
   /**
    * invariant for document update
    */
@@ -380,6 +385,22 @@ final class DocumentsWriterDeleteQueue {
     }
   }
 
+  private static final class NumericUpdateNode extends Node<NumericUpdate> {
+
+    NumericUpdateNode(NumericUpdate update) {
+      super(update);
+    }
+
+    @Override
+    void apply(BufferedDeletes bufferedDeletes, int docIDUpto) {
+      bufferedDeletes.addNumericUpdate(item, docIDUpto);
+    }
+
+    @Override
+    public String toString() {
+      return "update=" + item;
+    }
+  }
 
   private boolean forceApplyGlobalSlice() {
     globalBufferLock.lock();



Mime
View raw message