lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject svn commit: r1204297 [4/10] - in /lucene/dev/branches/solrcloud: ./ dev-tools/eclipse/ dev-tools/idea/lucene/contrib/ dev-tools/idea/lucene/contrib/instantiated/ dev-tools/maven/lucene/contrib/ dev-tools/maven/lucene/contrib/instantiated/ lucene/ lucen...
Date Sun, 20 Nov 2011 23:45:35 GMT
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java Sun Nov 20 23:45:25 2011
@@ -18,12 +18,8 @@ package org.apache.lucene.index.codecs;
  */
 
 import java.io.ByteArrayOutputStream;
-import java.io.Closeable;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.OutputStreamWriter;
 import java.io.PrintStream;
-import java.io.Writer;
 import java.util.Collection;
 import java.util.Comparator;
 import java.util.Iterator;
@@ -197,9 +193,6 @@ public class BlockTreeTermsReader extend
     try {
       IOUtils.close(in, postingsReader);
     } finally { 
-      for(FieldReader field : fields.values()) {
-        field.close();
-      }
       // Clear so refs to terms index is GCable even if
       // app hangs onto us:
       fields.clear();
@@ -221,6 +214,11 @@ public class BlockTreeTermsReader extend
     return fields.get(field);
   }
 
+  @Override
+  public int getUniqueFieldCount() {
+    return fields.size();
+  }
+
   // Iterates through all fields
   private class TermFieldsEnum extends FieldsEnum {
     final Iterator<FieldReader> it;
@@ -242,8 +240,8 @@ public class BlockTreeTermsReader extend
     }
     
     @Override
-    public TermsEnum terms() throws IOException {
-      return current.iterator();
+    public Terms terms() throws IOException {
+      return current;
     }
   }
 
@@ -390,7 +388,7 @@ public class BlockTreeTermsReader extend
   final Outputs<BytesRef> fstOutputs = ByteSequenceOutputs.getSingleton();
   final BytesRef NO_OUTPUT = fstOutputs.getNoOutput();
 
-  public final class FieldReader extends Terms implements Closeable {
+  public final class FieldReader extends Terms {
     final long numTerms;
     final FieldInfo fieldInfo;
     final long sumTotalTermFreq;
@@ -449,12 +447,7 @@ public class BlockTreeTermsReader extend
     }
 
     @Override
-    public void close() {
-      super.close();
-    }
-    
-    @Override
-    public TermsEnum iterator() throws IOException {
+    public TermsEnum iterator(TermsEnum reuse) throws IOException {
       return new SegmentTermsEnum();
     }
 
@@ -742,7 +735,7 @@ public class BlockTreeTermsReader extend
         }
       }
 
-      private final BytesRef savedStartTerm;
+      private BytesRef savedStartTerm;
       
       // TODO: in some cases we can filter by length?  eg
       // regexp foo*bar must be at least length 6 bytes
@@ -782,7 +775,7 @@ public class BlockTreeTermsReader extend
         f.load(rootCode);
 
         // for assert:
-        savedStartTerm = startTerm == null ? null : new BytesRef(startTerm);
+        assert setSavedStartTerm(startTerm);
 
         currentFrame = f;
         if (startTerm != null) {
@@ -790,6 +783,12 @@ public class BlockTreeTermsReader extend
         }
       }
 
+      // only for assert:
+      private boolean setSavedStartTerm(BytesRef startTerm) {
+        savedStartTerm = startTerm == null ? null : new BytesRef(startTerm);
+        return true;
+      }
+
       @Override
       public TermState termState() throws IOException {
         currentFrame.decodeMetaData();
@@ -914,8 +913,6 @@ public class BlockTreeTermsReader extend
 
         for(int idx=0;idx<=target.length;idx++) {
 
-          boolean lastIsSubBlock = false;
-
           while (true) {
             final int savePos = currentFrame.suffixesReader.getPosition();
             final int saveStartBytePos = currentFrame.startBytePos;
@@ -950,7 +947,6 @@ public class BlockTreeTermsReader extend
                     return;
                   }
                 }
-                lastIsSubBlock = isSubBlock;
                 continue;
               } else if (cmp == 0) {
                 //if (DEBUG) System.out.println("  return term=" + brToString(term));
@@ -1164,7 +1160,7 @@ public class BlockTreeTermsReader extend
 
     // Iterates through terms in this field
     private final class SegmentTermsEnum extends TermsEnum {
-      private final IndexInput in;
+      private IndexInput in;
 
       private Frame[] stack;
       private final Frame staticFrame;
@@ -1183,29 +1179,21 @@ public class BlockTreeTermsReader extend
 
       final BytesRef term = new BytesRef();
 
-      @SuppressWarnings("unchecked") private FST.Arc<BytesRef>[] arcs = new FST.Arc[5];      
+      @SuppressWarnings("unchecked") private FST.Arc<BytesRef>[] arcs = new FST.Arc[1];
 
       public SegmentTermsEnum() throws IOException {
         //if (DEBUG) System.out.println("BTTR.init seg=" + segment);
-        in = (IndexInput) BlockTreeTermsReader.this.in.clone();
-        stack = new Frame[5];
-        for(int stackOrd=0;stackOrd<stack.length;stackOrd++) {
-          stack[stackOrd] = new Frame(stackOrd);
-        }
+        stack = new Frame[0];
+        
         // Used to hold seek by TermState, or cached seek
         staticFrame = new Frame(-1);
 
+        // Init w/ root block; don't use index since it may
+        // not (and need not) have been loaded
         for(int arcIdx=0;arcIdx<arcs.length;arcIdx++) {
           arcs[arcIdx] = new FST.Arc<BytesRef>();
         }
 
-        // Init w/ root block; don't use index since it may
-        // not (and need not) have been loaded
-        //final FST.Arc<BytesRef> arc = index.getFirstArc(arcs[0]);
-
-        // Empty string prefix must have an output in the index!
-        //assert arc.isFinal();
-
         currentFrame = staticFrame;
         final FST.Arc<BytesRef> arc;
         if (index != null) {
@@ -1215,8 +1203,9 @@ public class BlockTreeTermsReader extend
         } else {
           arc = null;
         }
-        currentFrame = pushFrame(arc, rootCode, 0);
-        currentFrame.loadBlock();
+        currentFrame = staticFrame;
+        //currentFrame = pushFrame(arc, rootCode, 0);
+        //currentFrame.loadBlock();
         validIndexPrefix = 0;
         // if (DEBUG) {
         //   System.out.println("init frame state " + currentFrame.ord);
@@ -1227,6 +1216,12 @@ public class BlockTreeTermsReader extend
         // computeBlockStats().print(System.out);
       }
 
+      private void initIndexInput() {
+        if (this.in == null) {
+          this.in = (IndexInput) BlockTreeTermsReader.this.in.clone();
+        }
+      }
+
       /** Runs next() through the entire terms dict,
        *  computing aggregate statistics. */
       public Stats computeBlockStats() throws IOException {
@@ -1976,6 +1971,20 @@ public class BlockTreeTermsReader extend
       @Override
       public BytesRef next() throws IOException {
 
+        if (in == null) {
+          // Fresh TermsEnum; seek to first term:
+          final FST.Arc<BytesRef> arc;
+          if (index != null) {
+            arc = index.getFirstArc(arcs[0]);
+            // Empty string prefix must have an output in the index!
+            assert arc.isFinal();
+          } else {
+            arc = null;
+          }
+          currentFrame = pushFrame(arc, rootCode, 0);
+          currentFrame.loadBlock();
+        }
+
         targetBeforeCurrentLength = currentFrame.ord;
 
         assert !eof;
@@ -2243,6 +2252,11 @@ public class BlockTreeTermsReader extend
            use. */
         void loadBlock() throws IOException {
 
+          // Clone the IndexInput lazily, so that consumers
+          // that just pull a TermsEnum to
+          // seekExact(TermState) don't pay this cost:
+          initIndexInput();
+
           if (nextEnt != -1) {
             // Already loaded
             return;

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/Codec.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/Codec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/Codec.java Sun Nov 20 23:45:25 2011
@@ -46,6 +46,10 @@ public abstract class Codec implements N
   public void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException {
     postingsFormat().files(dir, info, "", files);
     storedFieldsFormat().files(dir, info, files);
+    termVectorsFormat().files(dir, info, files);
+    fieldInfosFormat().files(dir, info, files);
+    // TODO: segmentInfosFormat should be allowed to declare additional files
+    // if it wants, in addition to segments_N
     docValuesFormat().files(dir, info, files);
   }
   
@@ -58,6 +62,12 @@ public abstract class Codec implements N
   /** Encodes/decodes stored fields */
   public abstract StoredFieldsFormat storedFieldsFormat();
   
+  /** Encodes/decodes term vectors */
+  public abstract TermVectorsFormat termVectorsFormat();
+  
+  /** Encodes/decodes field infos file */
+  public abstract FieldInfosFormat fieldInfosFormat();
+  
   /** Encodes/decodes segments file */
   public abstract SegmentInfosFormat segmentInfosFormat();
   

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java Sun Nov 20 23:45:25 2011
@@ -18,6 +18,8 @@ package org.apache.lucene.index.codecs;
  */
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.IndexFormatTooOldException;
@@ -51,7 +53,7 @@ public class DefaultSegmentInfosReader e
       infos.setGlobalFieldMapVersion(input.readLong());
     }
     for (int i = input.readInt(); i > 0; i--) { // read segmentInfos
-      SegmentInfo si = new SegmentInfo(directory, format, input);
+      SegmentInfo si = readSegmentInfo(directory, format, input);
       if (si.getVersion() == null) {
         // Could be a 3.0 - try to open the doc stores - if it fails, it's a
         // 2.x segment, and an IndexFormatTooOldException will be thrown,
@@ -90,4 +92,100 @@ public class DefaultSegmentInfosReader e
       
     infos.userData = input.readStringStringMap();
   }
+  
+  // if we make a preflex impl we can remove a lot of this hair...
+  public SegmentInfo readSegmentInfo(Directory dir, int format, ChecksumIndexInput input) throws IOException {
+    final String version;
+    if (format <= DefaultSegmentInfosWriter.FORMAT_3_1) {
+      version = input.readString();
+    } else {
+      version = null;
+    }
+    final String name = input.readString();
+    final int docCount = input.readInt();
+    final long delGen = input.readLong();
+    final int docStoreOffset = input.readInt();
+    final String docStoreSegment;
+    final boolean docStoreIsCompoundFile;
+    if (docStoreOffset != -1) {
+      docStoreSegment = input.readString();
+      docStoreIsCompoundFile = input.readByte() == SegmentInfo.YES;
+    } else {
+      docStoreSegment = name;
+      docStoreIsCompoundFile = false;
+    }
+
+    if (format > DefaultSegmentInfosWriter.FORMAT_4_0) {
+      // pre-4.0 indexes write a byte if there is a single norms file
+      byte b = input.readByte();
+      assert 1 == b;
+    }
+
+    final int numNormGen = input.readInt();
+    final Map<Integer,Long> normGen;
+    if (numNormGen == SegmentInfo.NO) {
+      normGen = null;
+    } else {
+      normGen = new HashMap<Integer, Long>();
+      for(int j=0;j<numNormGen;j++) {
+        int fieldNumber = j;
+        if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) {
+          fieldNumber = input.readInt();
+        }
+
+        normGen.put(fieldNumber, input.readLong());
+      }
+    }
+    final boolean isCompoundFile = input.readByte() == SegmentInfo.YES;
+
+    final int delCount = input.readInt();
+    assert delCount <= docCount;
+
+    final int hasProx = input.readByte();
+
+    final Codec codec;
+    // note: if the codec is not available: Codec.forName will throw an exception.
+    if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) {
+      codec = Codec.forName(input.readString());
+    } else {
+      codec = Codec.forName("Lucene3x");
+    }
+    final Map<String,String> diagnostics = input.readStringStringMap();
+
+    final int hasVectors;
+    if (format <= DefaultSegmentInfosWriter.FORMAT_HAS_VECTORS) {
+      hasVectors = input.readByte();
+    } else {
+      final String storesSegment;
+      final String ext;
+      final boolean storeIsCompoundFile;
+      if (docStoreOffset != -1) {
+        storesSegment = docStoreSegment;
+        storeIsCompoundFile = docStoreIsCompoundFile;
+        ext = IndexFileNames.COMPOUND_FILE_STORE_EXTENSION;
+      } else {
+        storesSegment = name;
+        storeIsCompoundFile = isCompoundFile;
+        ext = IndexFileNames.COMPOUND_FILE_EXTENSION;
+      }
+      final Directory dirToTest;
+      if (storeIsCompoundFile) {
+        dirToTest = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(storesSegment, "", ext), IOContext.READONCE, false);
+      } else {
+        dirToTest = dir;
+      }
+      try {
+        // TODO: remove this manual file check or push to preflex codec
+        hasVectors = dirToTest.fileExists(IndexFileNames.segmentFileName(storesSegment, "", DefaultTermVectorsReader.VECTORS_INDEX_EXTENSION)) ? SegmentInfo.YES : SegmentInfo.NO;
+      } finally {
+        if (isCompoundFile) {
+          dirToTest.close();
+        }
+      }
+    }
+    
+    return new SegmentInfo(dir, version, name, docCount, delGen, docStoreOffset,
+      docStoreSegment, docStoreIsCompoundFile, normGen, isCompoundFile,
+      delCount, hasProx, codec, diagnostics, hasVectors);
+  }
 }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java Sun Nov 20 23:45:25 2011
@@ -18,6 +18,8 @@ package org.apache.lucene.index.codecs;
  */
 
 import java.io.IOException;
+import java.util.Map;
+import java.util.Map.Entry;
 
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentInfos;
@@ -50,6 +52,7 @@ public class DefaultSegmentInfosWriter e
 
   /** This must always point to the most recent file format.
    * whenever you add a new format, make it 1 smaller (negative version logic)! */
+  // TODO: move this, as its currently part of required preamble
   public static final int FORMAT_CURRENT = FORMAT_4_0;
   
   /** This must always point to the first supported file format. */
@@ -68,7 +71,7 @@ public class DefaultSegmentInfosWriter e
       out.writeLong(infos.getGlobalFieldMapVersion());
       out.writeInt(infos.size()); // write infos
       for (SegmentInfo si : infos) {
-        si.write(out);
+        writeInfo(out, si);
       }
       out.writeStringStringMap(infos.getUserData());
       success = true;
@@ -80,6 +83,40 @@ public class DefaultSegmentInfosWriter e
     }
   }
   
+  /** Save a single segment's info. */
+  private void writeInfo(IndexOutput output, SegmentInfo si) throws IOException {
+    assert si.getDelCount() <= si.docCount: "delCount=" + si.getDelCount() + " docCount=" + si.docCount + " segment=" + si.name;
+    // Write the Lucene version that created this segment, since 3.1
+    output.writeString(si.getVersion());
+    output.writeString(si.name);
+    output.writeInt(si.docCount);
+    output.writeLong(si.getDelGen());
+
+    output.writeInt(si.getDocStoreOffset());
+    if (si.getDocStoreOffset() != -1) {
+      output.writeString(si.getDocStoreSegment());
+      output.writeByte((byte) (si.getDocStoreIsCompoundFile() ? 1:0));
+    }
+
+    Map<Integer,Long> normGen = si.getNormGen();
+    if (normGen == null) {
+      output.writeInt(SegmentInfo.NO);
+    } else {
+      output.writeInt(normGen.size());
+      for (Entry<Integer,Long> entry : normGen.entrySet()) {
+        output.writeInt(entry.getKey());
+        output.writeLong(entry.getValue());
+      }
+    }
+
+    output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO));
+    output.writeInt(si.getDelCount());
+    output.writeByte((byte) (si.getHasProxInternal()));
+    output.writeString(si.getCodec().getName());
+    output.writeStringStringMap(si.getDiagnostics());
+    output.writeByte((byte) (si.getHasVectorsInternal()));
+  }
+  
   protected IndexOutput createOutput(Directory dir, String segmentFileName, IOContext context)
       throws IOException {
     IndexOutput plainOut = dir.createOutput(segmentFileName, context);

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultStoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultStoredFieldsReader.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultStoredFieldsReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultStoredFieldsReader.java Sun Nov 20 23:45:25 2011
@@ -32,7 +32,6 @@ import org.apache.lucene.store.AlreadyCl
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.CloseableThreadLocal;
 import org.apache.lucene.util.IOUtils;
 
 import java.io.Closeable;
@@ -50,16 +49,7 @@ public final class DefaultStoredFieldsRe
   private final static int FORMAT_SIZE = 4;
 
   private final FieldInfos fieldInfos;
-  private CloseableThreadLocal<IndexInput> fieldsStreamTL = new CloseableThreadLocal<IndexInput>();
-  
-  // The main fieldStream, used only for cloning.
-  private final IndexInput cloneableFieldsStream;
-
-  // This is a clone of cloneableFieldsStream used for reading documents.
-  // It should not be cloned outside of a synchronized context.
   private final IndexInput fieldsStream;
-
-  private final IndexInput cloneableIndexStream;
   private final IndexInput indexStream;
   private int numTotalDocs;
   private int size;
@@ -70,8 +60,6 @@ public final class DefaultStoredFieldsRe
   // file.  This will be 0 if we have our own private file.
   private int docStoreOffset;
 
-  private boolean isOriginal = false;
-
   /** Returns a cloned FieldsReader that shares open
    *  IndexInputs with the original one.  It is the caller's
    *  job not to close the original FieldsReader until all
@@ -80,7 +68,7 @@ public final class DefaultStoredFieldsRe
   @Override
   public DefaultStoredFieldsReader clone() {
     ensureOpen();
-    return new DefaultStoredFieldsReader(fieldInfos, numTotalDocs, size, format, docStoreOffset, cloneableFieldsStream, cloneableIndexStream);
+    return new DefaultStoredFieldsReader(fieldInfos, numTotalDocs, size, format, docStoreOffset, (IndexInput)fieldsStream.clone(), (IndexInput)indexStream.clone());
   }
 
   /** Verifies that the code version which wrote the segment is supported. */
@@ -102,16 +90,14 @@ public final class DefaultStoredFieldsRe
   
   // Used only by clone
   private DefaultStoredFieldsReader(FieldInfos fieldInfos, int numTotalDocs, int size, int format, int docStoreOffset,
-                       IndexInput cloneableFieldsStream, IndexInput cloneableIndexStream) {
+                       IndexInput fieldsStream, IndexInput indexStream) {
     this.fieldInfos = fieldInfos;
     this.numTotalDocs = numTotalDocs;
     this.size = size;
     this.format = format;
     this.docStoreOffset = docStoreOffset;
-    this.cloneableFieldsStream = cloneableFieldsStream;
-    this.cloneableIndexStream = cloneableIndexStream;
-    fieldsStream = (IndexInput) cloneableFieldsStream.clone();
-    indexStream = (IndexInput) cloneableIndexStream.clone();
+    this.fieldsStream = fieldsStream;
+    this.indexStream = indexStream;
   }
 
   public DefaultStoredFieldsReader(Directory d, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
@@ -119,24 +105,20 @@ public final class DefaultStoredFieldsRe
     final int docStoreOffset = si.getDocStoreOffset();
     final int size = si.docCount;
     boolean success = false;
-    isOriginal = true;
+    fieldInfos = fn;
     try {
-      fieldInfos = fn;
-
-      cloneableFieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", DefaultStoredFieldsWriter.FIELDS_EXTENSION), context);
+      fieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", DefaultStoredFieldsWriter.FIELDS_EXTENSION), context);
       final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", DefaultStoredFieldsWriter.FIELDS_INDEX_EXTENSION);
-      cloneableIndexStream = d.openInput(indexStreamFN, context);
+      indexStream = d.openInput(indexStreamFN, context);
       
-      format = cloneableIndexStream.readInt();
+      format = indexStream.readInt();
 
       if (format < DefaultStoredFieldsWriter.FORMAT_MINIMUM)
-        throw new IndexFormatTooOldException(cloneableIndexStream, format, DefaultStoredFieldsWriter.FORMAT_MINIMUM, DefaultStoredFieldsWriter.FORMAT_CURRENT);
+        throw new IndexFormatTooOldException(indexStream, format, DefaultStoredFieldsWriter.FORMAT_MINIMUM, DefaultStoredFieldsWriter.FORMAT_CURRENT);
       if (format > DefaultStoredFieldsWriter.FORMAT_CURRENT)
-        throw new IndexFormatTooNewException(cloneableIndexStream, format, DefaultStoredFieldsWriter.FORMAT_MINIMUM, DefaultStoredFieldsWriter.FORMAT_CURRENT);
+        throw new IndexFormatTooNewException(indexStream, format, DefaultStoredFieldsWriter.FORMAT_MINIMUM, DefaultStoredFieldsWriter.FORMAT_CURRENT);
 
-      fieldsStream = (IndexInput) cloneableFieldsStream.clone();
-
-      final long indexSize = cloneableIndexStream.length() - FORMAT_SIZE;
+      final long indexSize = indexStream.length() - FORMAT_SIZE;
       
       if (docStoreOffset != -1) {
         // We read only a slice out of this shared fields file
@@ -154,8 +136,6 @@ public final class DefaultStoredFieldsRe
           throw new CorruptIndexException("doc counts differ for segment " + segment + ": fieldsReader shows " + this.size + " but segmentInfo shows " + si.docCount);
         }
       }
-
-      indexStream = (IndexInput) cloneableIndexStream.clone();
       numTotalDocs = (int) (indexSize >> 3);
       success = true;
     } finally {
@@ -180,18 +160,14 @@ public final class DefaultStoredFieldsRe
   }
 
   /**
-   * Closes the underlying {@link org.apache.lucene.store.IndexInput} streams, including any ones associated with a
-   * lazy implementation of a Field.  This means that the Fields values will not be accessible.
+   * Closes the underlying {@link org.apache.lucene.store.IndexInput} streams.
+   * This means that the Fields values will not be accessible.
    *
    * @throws IOException
    */
   public final void close() throws IOException {
     if (!closed) {
-      if (isOriginal) {
-        IOUtils.close(fieldsStream, indexStream, fieldsStreamTL, cloneableFieldsStream, cloneableIndexStream);
-      } else {
-        IOUtils.close(fieldsStream, indexStream, fieldsStreamTL);
-      }
+      IOUtils.close(fieldsStream, indexStream);
       closed = true;
     }
   }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultStoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultStoredFieldsWriter.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultStoredFieldsWriter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultStoredFieldsWriter.java Sun Nov 20 23:45:25 2011
@@ -20,6 +20,7 @@ import java.io.IOException;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.MergeState;
@@ -72,8 +73,8 @@ public final class DefaultStoredFieldsWr
   /** Extension of stored fields index file */
   public static final String FIELDS_INDEX_EXTENSION = "fdx";
 
-  private Directory directory;
-  private String segment;
+  private final Directory directory;
+  private final String segment;
   private IndexOutput fieldsStream;
   private IndexOutput indexStream;
 
@@ -118,20 +119,19 @@ public final class DefaultStoredFieldsWr
   public void abort() {
     try {
       close();
-    } catch (IOException ignored) {
-    }
+    } catch (IOException ignored) {}
+    
     try {
       directory.deleteFile(IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION));
-    } catch (IOException ignored) {
-    }
+    } catch (IOException ignored) {}
+    
     try {
       directory.deleteFile(IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION));
-    } catch (IOException ignored) {
-    }
+    } catch (IOException ignored) {}
   }
 
-  public final void writeField(int fieldNumber, IndexableField field) throws IOException {
-    fieldsStream.writeVInt(fieldNumber);
+  public final void writeField(FieldInfo info, IndexableField field) throws IOException {
+    fieldsStream.writeVInt(info.number);
     int bits = 0;
     final BytesRef bytes;
     final String string;

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java Sun Nov 20 23:45:25 2011
@@ -17,14 +17,14 @@ package org.apache.lucene.index.codecs;
  * limitations under the License.
  */
 
+import java.io.Closeable;
+import java.io.IOException;
+
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.FieldsEnum;
 import org.apache.lucene.index.MergeState;
-import org.apache.lucene.index.TermsEnum;
-
-import java.io.IOException;
-import java.io.Closeable;
+import org.apache.lucene.index.Terms;
 
 /** Abstract API that consumes terms, doc, freq, prox and
  *  payloads postings.  Concrete implementations of this
@@ -48,10 +48,10 @@ public abstract class FieldsConsumer imp
     while((field = fieldsEnum.next()) != null) {
       mergeState.fieldInfo = mergeState.fieldInfos.fieldInfo(field);
       assert mergeState.fieldInfo != null : "FieldInfo for field is null: "+ field;
-      TermsEnum terms = fieldsEnum.terms();
+      Terms terms = fieldsEnum.terms();
       if (terms != null) {
         final TermsConsumer termsConsumer = addField(mergeState.fieldInfo);
-        termsConsumer.merge(mergeState, terms);
+        termsConsumer.merge(mergeState, terms.iterator(null));
       }
     }
   }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/StoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/StoredFieldsReader.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/StoredFieldsReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/StoredFieldsReader.java Sun Nov 20 23:45:25 2011
@@ -1,11 +1,5 @@
 package org.apache.lucene.index.codecs;
 
-import java.io.Closeable;
-import java.io.IOException;
-
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.StoredFieldVisitor;
-
 /**
  * Copyright 2004 The Apache Software Foundation
  *
@@ -22,6 +16,12 @@ import org.apache.lucene.index.StoredFie
  * the License.
  */
 
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.StoredFieldVisitor;
+
 /**
  * Codec API for reading stored fields:
  * 

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/StoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/StoredFieldsWriter.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/StoredFieldsWriter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/StoredFieldsWriter.java Sun Nov 20 23:45:25 2011
@@ -4,6 +4,7 @@ import java.io.Closeable;
 import java.io.IOException;
 
 import org.apache.lucene.document.Document;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.MergeState;
@@ -31,7 +32,7 @@ import org.apache.lucene.util.Bits;
  * <ol>
  *   <li>For every document, {@link #startDocument(int)} is called,
  *       informing the Codec how many fields will be written.
- *   <li>{@link #writeField(int, IndexableField)} is called for 
+ *   <li>{@link #writeField(FieldInfo, IndexableField)} is called for 
  *       each field in the document.
  *   <li>After all documents have been written, {@link #finish(int)} 
  *       is called for verification/sanity-checks.
@@ -43,14 +44,14 @@ import org.apache.lucene.util.Bits;
 public abstract class StoredFieldsWriter implements Closeable {
   
   /** Called before writing the stored fields of the document.
-   *  {@link #writeField(int, IndexableField)} will be called
+   *  {@link #writeField(FieldInfo, IndexableField)} will be called
    *  <code>numStoredFields</code> times. Note that this is
    *  called even if the document has no stored fields, in
    *  this case <code>numStoredFields</code> will be zero. */
   public abstract void startDocument(int numStoredFields) throws IOException;
   
   /** Writes a single stored field. */
-  public abstract void writeField(int fieldNumber, IndexableField field) throws IOException;
+  public abstract void writeField(FieldInfo info, IndexableField field) throws IOException;
 
   /** Aborts writing entirely, implementation should remove
    *  any partially-written files, etc. */
@@ -67,7 +68,7 @@ public abstract class StoredFieldsWriter
   /** Merges in the stored fields from the readers in 
    *  <code>mergeState</code>. The default implementation skips
    *  over deleted documents, and uses {@link #startDocument(int)},
-   *  {@link #writeField(int, IndexableField)}, and {@link #finish(int)},
+   *  {@link #writeField(FieldInfo, IndexableField)}, and {@link #finish(int)},
    *  returning the number of documents that were written.
    *  Implementations can override this method for more sophisticated
    *  merging (bulk-byte copying, etc). */
@@ -110,7 +111,7 @@ public abstract class StoredFieldsWriter
 
     for (IndexableField field : doc) {
       if (field.fieldType().stored()) {
-        writeField(fieldInfos.fieldNumber(field.name()), field);
+        writeField(fieldInfos.fieldInfo(field.name()), field);
       }
     }
   }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java Sun Nov 20 23:45:25 2011
@@ -24,14 +24,18 @@ import org.apache.lucene.index.PerDocWri
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.DefaultFieldInfosFormat;
 import org.apache.lucene.index.codecs.DefaultStoredFieldsFormat;
 import org.apache.lucene.index.codecs.DefaultSegmentInfosFormat;
+import org.apache.lucene.index.codecs.DefaultTermVectorsFormat;
 import org.apache.lucene.index.codecs.DocValuesFormat;
+import org.apache.lucene.index.codecs.FieldInfosFormat;
 import org.apache.lucene.index.codecs.StoredFieldsFormat;
 import org.apache.lucene.index.codecs.PerDocConsumer;
 import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.codecs.PostingsFormat;
 import org.apache.lucene.index.codecs.SegmentInfosFormat;
+import org.apache.lucene.index.codecs.TermVectorsFormat;
 import org.apache.lucene.store.Directory;
 
 /**
@@ -48,6 +52,12 @@ public class Lucene3xCodec extends Codec
   private final StoredFieldsFormat fieldsFormat = new DefaultStoredFieldsFormat();
   
   // TODO: this should really be a different impl
+  private final TermVectorsFormat vectorsFormat = new DefaultTermVectorsFormat();
+  
+  // TODO: this should really be a different impl
+  private final FieldInfosFormat fieldInfosFormat = new DefaultFieldInfosFormat();
+
+  // TODO: this should really be a different impl
   // also if we want preflex to *really* be read-only it should throw exception for the writer?
   // this way IR.commit fails on delete/undelete/setNorm/etc ?
   private final SegmentInfosFormat infosFormat = new DefaultSegmentInfosFormat();
@@ -82,6 +92,16 @@ public class Lucene3xCodec extends Codec
   public StoredFieldsFormat storedFieldsFormat() {
     return fieldsFormat;
   }
+  
+  @Override
+  public TermVectorsFormat termVectorsFormat() {
+    return vectorsFormat;
+  }
+  
+  @Override
+  public FieldInfosFormat fieldInfosFormat() {
+    return fieldInfosFormat;
+  }
 
   @Override
   public SegmentInfosFormat segmentInfosFormat() {

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java Sun Nov 20 23:45:25 2011
@@ -25,7 +25,6 @@ import java.util.Iterator;
 import java.util.Map;
 import java.util.TreeMap;
 
-import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.DocsAndPositionsEnum;
 import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.FieldInfo;
@@ -38,7 +37,6 @@ import org.apache.lucene.index.Term;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.codecs.FieldsProducer;
-import org.apache.lucene.store.CompoundFileDirectory;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
@@ -163,6 +161,11 @@ public class Lucene3xFields extends Fiel
   }
 
   @Override
+  public int getUniqueFieldCount() {
+    return preTerms.size();
+  }
+
+  @Override
   public long getUniqueTermCount() throws IOException {
     return getTermsDict().size();
   }
@@ -196,12 +199,10 @@ public class Lucene3xFields extends Fiel
 
   private class PreFlexFieldsEnum extends FieldsEnum {
     final Iterator<FieldInfo> it;
-    private final PreTermsEnum termsEnum;
     FieldInfo current;
 
     public PreFlexFieldsEnum() throws IOException {
       it = fields.values().iterator();
-      termsEnum = new PreTermsEnum();
     }
 
     @Override
@@ -215,9 +216,8 @@ public class Lucene3xFields extends Fiel
     }
 
     @Override
-    public TermsEnum terms() throws IOException {
-      termsEnum.reset(current);
-      return termsEnum;
+    public Terms terms() throws IOException {
+      return Lucene3xFields.this.terms(current.name);
     }
   }
   
@@ -228,7 +228,7 @@ public class Lucene3xFields extends Fiel
     }
 
     @Override
-    public TermsEnum iterator() throws IOException {    
+    public TermsEnum iterator(TermsEnum reuse) throws IOException {    
       PreTermsEnum termsEnum = new PreTermsEnum();
       termsEnum.reset(fieldInfo);
       return termsEnum;
@@ -994,6 +994,7 @@ public class Lucene3xFields extends Fiel
     public PreDocsEnum reset(SegmentTermEnum termEnum, Bits liveDocs) throws IOException {
       docs.setLiveDocs(liveDocs);
       docs.seek(termEnum);
+      docID = -1;
       return this;
     }
 
@@ -1050,6 +1051,7 @@ public class Lucene3xFields extends Fiel
     public DocsAndPositionsEnum reset(SegmentTermEnum termEnum, Bits liveDocs) throws IOException {
       pos.setLiveDocs(liveDocs);
       pos.seek(termEnum);
+      docID = -1;
       return this;
     }
 

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java Sun Nov 20 23:45:25 2011
@@ -19,12 +19,16 @@ package org.apache.lucene.index.codecs.l
 
 import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.index.codecs.DefaultDocValuesFormat;
+import org.apache.lucene.index.codecs.DefaultFieldInfosFormat;
 import org.apache.lucene.index.codecs.DefaultStoredFieldsFormat;
 import org.apache.lucene.index.codecs.DefaultSegmentInfosFormat;
+import org.apache.lucene.index.codecs.DefaultTermVectorsFormat;
 import org.apache.lucene.index.codecs.DocValuesFormat;
+import org.apache.lucene.index.codecs.FieldInfosFormat;
 import org.apache.lucene.index.codecs.StoredFieldsFormat;
 import org.apache.lucene.index.codecs.PostingsFormat;
 import org.apache.lucene.index.codecs.SegmentInfosFormat;
+import org.apache.lucene.index.codecs.TermVectorsFormat;
 import org.apache.lucene.index.codecs.perfield.PerFieldPostingsFormat;
 
 /**
@@ -39,7 +43,9 @@ import org.apache.lucene.index.codecs.pe
 // (it writes a minor version, etc).
 public class Lucene40Codec extends Codec {
   private final StoredFieldsFormat fieldsFormat = new DefaultStoredFieldsFormat();
+  private final TermVectorsFormat vectorsFormat = new DefaultTermVectorsFormat();
   private final DocValuesFormat docValuesFormat = new DefaultDocValuesFormat();
+  private final FieldInfosFormat fieldInfosFormat = new DefaultFieldInfosFormat();
   private final SegmentInfosFormat infosFormat = new DefaultSegmentInfosFormat();
   private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
     @Override
@@ -56,6 +62,11 @@ public class Lucene40Codec extends Codec
   public StoredFieldsFormat storedFieldsFormat() {
     return fieldsFormat;
   }
+  
+  @Override
+  public TermVectorsFormat termVectorsFormat() {
+    return vectorsFormat;
+  }
 
   @Override
   public DocValuesFormat docValuesFormat() {
@@ -68,6 +79,11 @@ public class Lucene40Codec extends Codec
   }
   
   @Override
+  public FieldInfosFormat fieldInfosFormat() {
+    return fieldInfosFormat;
+  }
+  
+  @Override
   public SegmentInfosFormat segmentInfosFormat() {
     return infosFormat;
   }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java Sun Nov 20 23:45:25 2011
@@ -273,7 +273,8 @@ public class Lucene40PostingsReader exte
 
     int limit;                                    // number of docs in this posting
     int ord;                                      // how many docs we've read
-    int doc;                                      // doc we last read
+    int doc = -1;                                 // doc we last read
+    int accum;                                    // accumulator for doc deltas
     int freq;                                     // freq we last read
 
     Bits liveDocs;
@@ -306,7 +307,8 @@ public class Lucene40PostingsReader exte
       limit = termState.docFreq;
       assert limit > 0;
       ord = 0;
-      doc = 0;
+      doc = -1;
+      accum = 0;
       // if (DEBUG) System.out.println("  sde limit=" + limit + " freqFP=" + freqOffset);
 
       skipped = false;
@@ -329,9 +331,9 @@ public class Lucene40PostingsReader exte
         final int code = freqIn.readVInt();
         // if (DEBUG) System.out.println("      code=" + code);
         if (omitTF) {
-          doc += code;
+          accum += code;
         } else {
-          doc += code >>> 1;              // shift off low bit
+          accum += code >>> 1;              // shift off low bit
           if ((code & 1) != 0) {          // if low bit is set
             freq = 1;                     // freq is one
           } else {
@@ -339,13 +341,13 @@ public class Lucene40PostingsReader exte
           }
         }
 
-        if (liveDocs == null || liveDocs.get(doc)) {
+        if (liveDocs == null || liveDocs.get(accum)) {
           break;
         }
       }
 
       //if (DEBUG) System.out.println("    stpr.nextDoc return doc=" + doc);
-      return doc;
+      return (doc = accum);
     }
 
     @Override
@@ -360,9 +362,9 @@ public class Lucene40PostingsReader exte
         // manually inlined call to next() for speed
         final int code = freqIn.readVInt();
         if (omitTF) {
-          doc += code;
+          accum += code;
         } else {
-          doc += code >>> 1;              // shift off low bit
+          accum += code >>> 1;              // shift off low bit
           if ((code & 1) != 0) {          // if low bit is set
             freq = 1;                     // freq is one
           } else {
@@ -370,8 +372,8 @@ public class Lucene40PostingsReader exte
           }
         }
 
-        if (liveDocs == null || liveDocs.get(doc)) {
-          docs[i] = doc;
+        if (liveDocs == null || liveDocs.get(accum)) {
+          docs[i] = doc = accum;
           freqs[i] = freq;
           ++i;
         }
@@ -422,7 +424,7 @@ public class Lucene40PostingsReader exte
           // Skipper moved
 
           ord = newOrd;
-          doc = skipper.getDoc();
+          doc = accum = skipper.getDoc();
           freqIn.seek(skipper.getFreqPointer());
         }
       }
@@ -444,7 +446,8 @@ public class Lucene40PostingsReader exte
 
     int limit;                                    // number of docs in this posting
     int ord;                                      // how many docs we've read
-    int doc;                                      // doc we last read
+    int doc = -1;                                 // doc we last read
+    int accum;                                    // accumulator for doc deltas
     int freq;                                     // freq we last read
     int position;
 
@@ -482,7 +485,8 @@ public class Lucene40PostingsReader exte
       assert limit > 0;
 
       ord = 0;
-      doc = 0;
+      doc = -1;
+      accum = 0;
       position = 0;
 
       skipped = false;
@@ -510,7 +514,7 @@ public class Lucene40PostingsReader exte
         // Decode next doc/freq pair
         final int code = freqIn.readVInt();
 
-        doc += code >>> 1;              // shift off low bit
+        accum += code >>> 1;              // shift off low bit
         if ((code & 1) != 0) {          // if low bit is set
           freq = 1;                     // freq is one
         } else {
@@ -518,7 +522,7 @@ public class Lucene40PostingsReader exte
         }
         posPendingCount += freq;
 
-        if (liveDocs == null || liveDocs.get(doc)) {
+        if (liveDocs == null || liveDocs.get(accum)) {
           break;
         }
       }
@@ -526,7 +530,7 @@ public class Lucene40PostingsReader exte
       position = 0;
 
       // if (DEBUG) System.out.println("  return doc=" + doc);
-      return doc;
+      return (doc = accum);
     }
 
     @Override
@@ -572,7 +576,7 @@ public class Lucene40PostingsReader exte
         if (newOrd > ord) {
           // Skipper moved
           ord = newOrd;
-          doc = skipper.getDoc();
+          doc = accum = skipper.getDoc();
           freqIn.seek(skipper.getFreqPointer());
           lazyProxPointer = skipper.getProxPointer();
           posPendingCount = 0;
@@ -636,7 +640,8 @@ public class Lucene40PostingsReader exte
 
     int limit;                                    // number of docs in this posting
     int ord;                                      // how many docs we've read
-    int doc;                                      // doc we last read
+    int doc = -1;                                 // doc we last read
+    int accum;                                    // accumulator for doc deltas
     int freq;                                     // freq we last read
     int position;
 
@@ -679,7 +684,8 @@ public class Lucene40PostingsReader exte
 
       limit = termState.docFreq;
       ord = 0;
-      doc = 0;
+      doc = -1;
+      accum = 0;
       position = 0;
 
       skipped = false;
@@ -707,7 +713,7 @@ public class Lucene40PostingsReader exte
         // Decode next doc/freq pair
         final int code = freqIn.readVInt();
 
-        doc += code >>> 1;              // shift off low bit
+        accum += code >>> 1;              // shift off low bit
         if ((code & 1) != 0) {          // if low bit is set
           freq = 1;                     // freq is one
         } else {
@@ -715,7 +721,7 @@ public class Lucene40PostingsReader exte
         }
         posPendingCount += freq;
 
-        if (liveDocs == null || liveDocs.get(doc)) {
+        if (liveDocs == null || liveDocs.get(accum)) {
           break;
         }
       }
@@ -723,7 +729,7 @@ public class Lucene40PostingsReader exte
       position = 0;
 
       //System.out.println("StandardR.D&PE nextDoc seg=" + segment + " return doc=" + doc);
-      return doc;
+      return (doc = accum);
     }
 
     @Override
@@ -769,7 +775,7 @@ public class Lucene40PostingsReader exte
         if (newOrd > ord) {
           // Skipper moved
           ord = newOrd;
-          doc = skipper.getDoc();
+          doc = accum = skipper.getDoc();
           freqIn.seek(skipper.getFreqPointer());
           lazyProxPointer = skipper.getProxPointer();
           posPendingCount = 0;

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsWriter.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsWriter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsWriter.java Sun Nov 20 23:45:25 2011
@@ -26,8 +26,8 @@ import java.util.List;
 
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.codecs.PostingsWriterBase;
@@ -36,6 +36,7 @@ import org.apache.lucene.store.IndexOutp
 import org.apache.lucene.store.RAMOutputStream;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CodecUtil;
+import org.apache.lucene.util.IOUtils;
 
 /** @lucene.experimental */
 public final class Lucene40PostingsWriter extends PostingsWriterBase {
@@ -92,14 +93,22 @@ public final class Lucene40PostingsWrite
     // this.segment = state.segmentName;
     String fileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, Lucene40PostingsFormat.FREQ_EXTENSION);
     freqOut = state.directory.createOutput(fileName, state.context);
-    if (state.fieldInfos.hasProx()) {
-      // At least one field does not omit TF, so create the
-      // prox file
-      fileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, Lucene40PostingsFormat.PROX_EXTENSION);
-      proxOut = state.directory.createOutput(fileName, state.context);
-    } else {
-      // Every field omits TF so we will write no prox file
-      proxOut = null;
+    boolean success = false;
+    try {
+      if (state.fieldInfos.hasProx()) {
+        // At least one field does not omit TF, so create the
+        // prox file
+        fileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, Lucene40PostingsFormat.PROX_EXTENSION);
+        proxOut = state.directory.createOutput(fileName, state.context);
+      } else {
+        // Every field omits TF so we will write no prox file
+        proxOut = null;
+      }
+      success = true;
+    } finally {
+      if (!success) {
+        IOUtils.closeWhileHandlingException(freqOut);
+      }
     }
 
     totalNumDocs = state.numDocs;

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java Sun Nov 20 23:45:25 2011
@@ -273,7 +273,8 @@ public class MemoryPostingsFormat extend
 
     private Bits liveDocs;
     private int docUpto;
-    private int docID;
+    private int docID = -1;
+    private int accum;
     private int freq;
     private int payloadLen;
     private int numDocs;
@@ -295,7 +296,8 @@ public class MemoryPostingsFormat extend
       in.reset(buffer, 0, bufferIn.length - bufferIn.offset);
       System.arraycopy(bufferIn.bytes, bufferIn.offset, buffer, 0, bufferIn.length - bufferIn.offset);
       this.liveDocs = liveDocs;
-      docID = 0;
+      docID = -1;
+      accum = 0;
       docUpto = 0;
       payloadLen = 0;
       this.numDocs = numDocs;
@@ -314,12 +316,12 @@ public class MemoryPostingsFormat extend
         }
         docUpto++;
         if (indexOptions == IndexOptions.DOCS_ONLY) {
-          docID += in.readVInt();
+          accum += in.readVInt();
           freq = 1;
         } else {
           final int code = in.readVInt();
-          docID += code >>> 1;
-          if (VERBOSE) System.out.println("  docID=" + docID + " code=" + code);
+          accum += code >>> 1;
+          if (VERBOSE) System.out.println("  docID=" + accum + " code=" + code);
           if ((code & 1) != 0) {
             freq = 1;
           } else {
@@ -343,9 +345,9 @@ public class MemoryPostingsFormat extend
           }
         }
 
-        if (liveDocs == null || liveDocs.get(docID)) {
-          if (VERBOSE) System.out.println("    return docID=" + docID + " freq=" + freq);
-          return docID;
+        if (liveDocs == null || liveDocs.get(accum)) {
+          if (VERBOSE) System.out.println("    return docID=" + accum + " freq=" + freq);
+          return (docID = accum);
         }
       }
     }
@@ -380,7 +382,8 @@ public class MemoryPostingsFormat extend
 
     private Bits liveDocs;
     private int docUpto;
-    private int docID;
+    private int docID = -1;
+    private int accum;
     private int freq;
     private int numDocs;
     private int posPending;
@@ -412,7 +415,8 @@ public class MemoryPostingsFormat extend
       in.reset(buffer, 0, bufferIn.length - bufferIn.offset);
       System.arraycopy(bufferIn.bytes, bufferIn.offset, buffer, 0, bufferIn.length - bufferIn.offset);
       this.liveDocs = liveDocs;
-      docID = 0;
+      docID = -1;
+      accum = 0;
       docUpto = 0;
       payload.bytes = buffer;
       payloadLength = 0;
@@ -436,7 +440,7 @@ public class MemoryPostingsFormat extend
         docUpto++;
         
         final int code = in.readVInt();
-        docID += code >>> 1;
+        accum += code >>> 1;
         if ((code & 1) != 0) {
           freq = 1;
         } else {
@@ -444,11 +448,11 @@ public class MemoryPostingsFormat extend
           assert freq > 0;
         }
 
-        if (liveDocs == null || liveDocs.get(docID)) {
+        if (liveDocs == null || liveDocs.get(accum)) {
           pos = 0;
           posPending = freq;
-          if (VERBOSE) System.out.println("    return docID=" + docID + " freq=" + freq);
-          return docID;
+          if (VERBOSE) System.out.println("    return docID=" + accum + " freq=" + freq);
+          return (docID = accum);
         }
 
         // Skip positions
@@ -723,7 +727,7 @@ public class MemoryPostingsFormat extend
     }
 
     @Override
-    public TermsEnum iterator() {
+    public TermsEnum iterator(TermsEnum reuse) {
       return new FSTTermsEnum(field, fst);
     }
 
@@ -768,8 +772,9 @@ public class MemoryPostingsFormat extend
             return current.field.name;
           }
 
-          public TermsEnum terms() {
-            return current.iterator();
+          @Override
+          public Terms terms() {
+            return current;
           }
         };
       }
@@ -780,6 +785,11 @@ public class MemoryPostingsFormat extend
       }
       
       @Override
+      public int getUniqueFieldCount() {
+        return fields.size();
+      }
+
+      @Override
       public void close() {
         // Drop ref to FST:
         for(TermsReader termsReader : fields.values()) {

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java Sun Nov 20 23:45:25 2011
@@ -34,7 +34,6 @@ import org.apache.lucene.index.SegmentIn
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
 import org.apache.lucene.index.codecs.PostingsFormat;
@@ -213,7 +212,7 @@ public abstract class PerFieldPostingsFo
       }
 
       @Override
-      public String next() {
+      public String next() throws IOException {
         if (it.hasNext()) {
           current = it.next();
         } else {
@@ -224,13 +223,8 @@ public abstract class PerFieldPostingsFo
       }
 
       @Override
-      public TermsEnum terms() throws IOException {
-        final Terms terms = fields.get(current).terms(current);
-        if (terms != null) {
-          return terms.iterator();
-        } else {
-          return TermsEnum.EMPTY;
-        }
+      public Terms terms() throws IOException {
+        return fields.get(current).terms(current);
       }
     }
 
@@ -246,6 +240,11 @@ public abstract class PerFieldPostingsFo
     }
     
     @Override
+    public int getUniqueFieldCount() {
+      return fields.size();
+    }
+
+    @Override
     public void close() throws IOException {
       IOUtils.close(formats.values());
     }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java Sun Nov 20 23:45:25 2011
@@ -257,7 +257,8 @@ public class PulsingPostingsReader exten
     private final IndexOptions indexOptions;
     private final boolean storePayloads;
     private Bits liveDocs;
-    private int docID;
+    private int docID = -1;
+    private int accum;
     private int freq;
     private int payloadLength;
 
@@ -279,7 +280,8 @@ public class PulsingPostingsReader exten
       }
       System.arraycopy(termState.postings, 0, postingsBytes, 0, termState.postingsSize);
       postings.reset(postingsBytes, 0, termState.postingsSize);
-      docID = 0;
+      docID = -1;
+      accum = 0;
       payloadLength = 0;
       freq = 1;
       this.liveDocs = liveDocs;
@@ -302,9 +304,9 @@ public class PulsingPostingsReader exten
         final int code = postings.readVInt();
         //System.out.println("  read code=" + code);
         if (indexOptions == IndexOptions.DOCS_ONLY) {
-          docID += code;
+          accum += code;
         } else {
-          docID += code >>> 1;              // shift off low bit
+          accum += code >>> 1;              // shift off low bit
           if ((code & 1) != 0) {          // if low bit is set
             freq = 1;                     // freq is one
           } else {
@@ -332,8 +334,8 @@ public class PulsingPostingsReader exten
           }
         }
 
-        if (liveDocs == null || liveDocs.get(docID)) {
-          return docID;
+        if (liveDocs == null || liveDocs.get(accum)) {
+          return (docID = accum);
         }
       }
     }
@@ -365,7 +367,8 @@ public class PulsingPostingsReader exten
     private final boolean storePayloads;
 
     private Bits liveDocs;
-    private int docID;
+    private int docID = -1;
+    private int accum;
     private int freq;
     private int posPending;
     private int position;
@@ -394,7 +397,8 @@ public class PulsingPostingsReader exten
       this.liveDocs = liveDocs;
       payloadLength = 0;
       posPending = 0;
-      docID = 0;
+      docID = -1;
+      accum = 0;
       //System.out.println("PR d&p reset storesPayloads=" + storePayloads + " bytes=" + bytes.length + " this=" + this);
       return this;
     }
@@ -414,7 +418,7 @@ public class PulsingPostingsReader exten
         }
 
         final int code = postings.readVInt();
-        docID += code >>> 1;            // shift off low bit
+        accum += code >>> 1;            // shift off low bit
         if ((code & 1) != 0) {          // if low bit is set
           freq = 1;                     // freq is one
         } else {
@@ -422,10 +426,10 @@ public class PulsingPostingsReader exten
         }
         posPending = freq;
 
-        if (liveDocs == null || liveDocs.get(docID)) {
+        if (liveDocs == null || liveDocs.get(accum)) {
           //System.out.println("  return docID=" + docID + " freq=" + freq);
           position = 0;
-          return docID;
+          return (docID = accum);
         }
       }
     }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java Sun Nov 20 23:45:25 2011
@@ -312,7 +312,8 @@ public class SepPostingsReader extends P
 
   class SepDocsEnum extends DocsEnum {
     int docFreq;
-    int doc;
+    int doc = -1;
+    int accum;
     int count;
     int freq;
     long freqStart;
@@ -376,7 +377,8 @@ public class SepPostingsReader extends P
       // NOTE: unused if docFreq < skipMinimum:
       skipFP = termState.skipFP;
       count = 0;
-      doc = 0;
+      doc = -1;
+      accum = 0;
       skipped = false;
 
       return this;
@@ -394,18 +396,18 @@ public class SepPostingsReader extends P
 
         // Decode next doc
         //System.out.println("decode docDelta:");
-        doc += docReader.next();
+        accum += docReader.next();
           
         if (!omitTF) {
           //System.out.println("decode freq:");
           freq = freqReader.next();
         }
 
-        if (liveDocs == null || liveDocs.get(doc)) {
+        if (liveDocs == null || liveDocs.get(accum)) {
           break;
         }
       }
-      return doc;
+      return (doc = accum);
     }
 
     @Override
@@ -420,14 +422,14 @@ public class SepPostingsReader extends P
         count++;
         // manually inlined call to next() for speed
         //System.out.println("decode doc");
-        doc += docReader.next();
+        accum += docReader.next();
         if (!omitTF) {
           //System.out.println("decode freq");
           freq = freqReader.next();
         }
 
-        if (liveDocs == null || liveDocs.get(doc)) {
-          docs[i] = doc;
+        if (liveDocs == null || liveDocs.get(accum)) {
+          docs[i] = doc = accum;
           freqs[i] = freq;
           //System.out.println("  docs[" + i + "]=" + doc + " count=" + count + " dF=" + docFreq);
           i++;
@@ -488,7 +490,7 @@ public class SepPostingsReader extends P
           }
           skipper.getDocIndex().seek(docReader);
           count = newCount;
-          doc = skipper.getDoc();
+          doc = accum = skipper.getDoc();
         }
       }
         
@@ -505,7 +507,8 @@ public class SepPostingsReader extends P
 
   class SepDocsAndPositionsEnum extends DocsAndPositionsEnum {
     int docFreq;
-    int doc;
+    int doc = -1;
+    int accum;
     int count;
     int freq;
     long freqStart;
@@ -572,7 +575,8 @@ public class SepPostingsReader extends P
 
       docFreq = termState.docFreq;
       count = 0;
-      doc = 0;
+      doc = -1;
+      accum = 0;
       pendingPosCount = 0;
       pendingPayloadBytes = 0;
       skipped = false;
@@ -595,20 +599,20 @@ public class SepPostingsReader extends P
 
         // Decode next doc
         //System.out.println("  sep d&p read doc");
-        doc += docReader.next();
+        accum += docReader.next();
 
         //System.out.println("  sep d&p read freq");
         freq = freqReader.next();
 
         pendingPosCount += freq;
 
-        if (liveDocs == null || liveDocs.get(doc)) {
+        if (liveDocs == null || liveDocs.get(accum)) {
           break;
         }
       }
 
       position = 0;
-      return doc;
+      return (doc = accum);
     }
 
     @Override
@@ -668,7 +672,7 @@ public class SepPostingsReader extends P
           posIndex.set(skipper.getPosIndex());
           posSeekPending = true;
           count = newCount;
-          doc = skipper.getDoc();
+          doc = accum = skipper.getDoc();
           //System.out.println("    moved to doc=" + doc);
           //payloadIn.seek(skipper.getPayloadPointer());
           payloadFP = skipper.getPayloadPointer();

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java Sun Nov 20 23:45:25 2011
@@ -151,7 +151,6 @@ public final class SepPostingsWriter ext
       if (!success) {
         IOUtils.closeWhileHandlingException(docOut, skipOut, freqOut, posOut, payloadOut);
       }
-
     }
   }
 

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java Sun Nov 20 23:45:25 2011
@@ -19,11 +19,12 @@ package org.apache.lucene.index.codecs.s
 
 import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.index.codecs.DefaultDocValuesFormat;
-import org.apache.lucene.index.codecs.DefaultSegmentInfosFormat;
 import org.apache.lucene.index.codecs.DocValuesFormat;
+import org.apache.lucene.index.codecs.FieldInfosFormat;
 import org.apache.lucene.index.codecs.PostingsFormat;
 import org.apache.lucene.index.codecs.SegmentInfosFormat;
 import org.apache.lucene.index.codecs.StoredFieldsFormat;
+import org.apache.lucene.index.codecs.TermVectorsFormat;
 
 /**
  * plain text index format.
@@ -34,11 +35,11 @@ import org.apache.lucene.index.codecs.St
 public final class SimpleTextCodec extends Codec {
   private final PostingsFormat postings = new SimpleTextPostingsFormat();
   private final StoredFieldsFormat storedFields = new SimpleTextStoredFieldsFormat();
-  
+  private final SegmentInfosFormat segmentInfos = new SimpleTextSegmentInfosFormat();
+  private final FieldInfosFormat fieldInfosFormat = new SimpleTextFieldInfosFormat();
+  private final TermVectorsFormat vectorsFormat = new SimpleTextTermVectorsFormat();
   // TODO: need a plain-text impl
   private final DocValuesFormat docValues = new DefaultDocValuesFormat();
-  // TODO: need a plain-text impl
-  private final SegmentInfosFormat segmentInfos = new DefaultSegmentInfosFormat();
   
   public SimpleTextCodec() {
     super("SimpleText");
@@ -58,6 +59,16 @@ public final class SimpleTextCodec exten
   public StoredFieldsFormat storedFieldsFormat() {
     return storedFields;
   }
+  
+  @Override
+  public TermVectorsFormat termVectorsFormat() {
+    return vectorsFormat;
+  }
+  
+  @Override
+  public FieldInfosFormat fieldInfosFormat() {
+    return fieldInfosFormat;
+  }
 
   @Override
   public SegmentInfosFormat segmentInfosFormat() {

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java Sun Nov 20 23:45:25 2011
@@ -87,8 +87,8 @@ class SimpleTextFieldsReader extends Fie
     }
 
     @Override
-    public TermsEnum terms() throws IOException {
-      return SimpleTextFieldsReader.this.terms(current).iterator();
+    public Terms terms() throws IOException {
+      return SimpleTextFieldsReader.this.terms(current);
     }
   }
 
@@ -314,7 +314,7 @@ class SimpleTextFieldsReader extends Fie
   private class SimpleTextDocsAndPositionsEnum extends DocsAndPositionsEnum {
     private final IndexInput inStart;
     private final IndexInput in;
-    private int docID;
+    private int docID = -1;
     private int tf;
     private Bits liveDocs;
     private final BytesRef scratch = new BytesRef(10);
@@ -336,6 +336,7 @@ class SimpleTextFieldsReader extends Fie
     public SimpleTextDocsAndPositionsEnum reset(long fp, Bits liveDocs) {
       this.liveDocs = liveDocs;
       nextDocStart = fp;
+      docID = -1;
       return this;
     }
 
@@ -523,7 +524,7 @@ class SimpleTextFieldsReader extends Fie
     }
 
     @Override
-    public TermsEnum iterator() throws IOException {
+    public TermsEnum iterator(TermsEnum reuse) throws IOException {
       if (fst != null) {
         return new SimpleTextTermsEnum(fst, indexOptions);
       } else {
@@ -582,6 +583,11 @@ class SimpleTextFieldsReader extends Fie
   }
 
   @Override
+  public int getUniqueFieldCount() {
+    return -1;
+  }
+
+  @Override
   public void close() throws IOException {
     in.close();
   }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextStoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextStoredFieldsWriter.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextStoredFieldsWriter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextStoredFieldsWriter.java Sun Nov 20 23:45:25 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.index.codecs.s
 
 import java.io.IOException;
 
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.codecs.StoredFieldsWriter;
@@ -87,9 +88,9 @@ public class SimpleTextStoredFieldsWrite
   }
 
   @Override
-  public void writeField(int fieldNumber, IndexableField field) throws IOException {
+  public void writeField(FieldInfo info, IndexableField field) throws IOException {
     write(FIELD);
-    write(Integer.toString(fieldNumber));
+    write(Integer.toString(info.number));
     newLine();
     
     write(NAME);

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java Sun Nov 20 23:45:25 2011
@@ -169,6 +169,7 @@ public class MultiIndexDocValues extends
     }
   }
 
+  // TODO: this is dup of IndexDocValues.getDefaultSource()?
   private static class EmptySource extends Source {
 
     public EmptySource(ValueType type) {

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java Sun Nov 20 23:45:25 2011
@@ -333,7 +333,7 @@ class FieldCacheImpl implements FieldCac
             setDocsWithField = false;
           }
         }
-        final TermsEnum termsEnum = terms.iterator();
+        final TermsEnum termsEnum = terms.iterator(null);
         DocsEnum docs = null;
         try {
           while(true) {
@@ -406,7 +406,7 @@ class FieldCacheImpl implements FieldCac
             setDocsWithField = false;
           }
         }
-        final TermsEnum termsEnum = terms.iterator();
+        final TermsEnum termsEnum = terms.iterator(null);
         DocsEnum docs = null;
         try {
           while(true) {
@@ -505,7 +505,7 @@ class FieldCacheImpl implements FieldCac
             setDocsWithField = false;
           }
         }
-        final TermsEnum termsEnum = terms.iterator();
+        final TermsEnum termsEnum = terms.iterator(null);
         DocsEnum docs = null;
         try {
           while(true) {
@@ -574,7 +574,7 @@ class FieldCacheImpl implements FieldCac
           // Fast case: all docs have this field:
           return new Bits.MatchAllBits(maxDoc);
         }
-        final TermsEnum termsEnum = terms.iterator();
+        final TermsEnum termsEnum = terms.iterator(null);
         DocsEnum docs = null;
         while(true) {
           final BytesRef term = termsEnum.next();
@@ -655,7 +655,7 @@ class FieldCacheImpl implements FieldCac
             setDocsWithField = false;
           }
         }
-        final TermsEnum termsEnum = terms.iterator();
+        final TermsEnum termsEnum = terms.iterator(null);
         DocsEnum docs = null;
         try {
           while(true) {
@@ -743,7 +743,7 @@ class FieldCacheImpl implements FieldCac
             setDocsWithField = false;
           }
         }
-        final TermsEnum termsEnum = terms.iterator();
+        final TermsEnum termsEnum = terms.iterator(null);
         DocsEnum docs = null;
         try {
           while(true) {
@@ -832,7 +832,7 @@ class FieldCacheImpl implements FieldCac
             setDocsWithField = false;
           }
         }
-        final TermsEnum termsEnum = terms.iterator();
+        final TermsEnum termsEnum = terms.iterator(null);
         DocsEnum docs = null;
         try {
           while(true) {
@@ -1128,7 +1128,7 @@ class FieldCacheImpl implements FieldCac
       int termOrd = 1;
 
       if (terms != null) {
-        final TermsEnum termsEnum = terms.iterator();
+        final TermsEnum termsEnum = terms.iterator(null);
         DocsEnum docs = null;
 
         while(true) {
@@ -1253,7 +1253,7 @@ class FieldCacheImpl implements FieldCac
 
       if (terms != null) {
         int termCount = 0;
-        final TermsEnum termsEnum = terms.iterator();
+        final TermsEnum termsEnum = terms.iterator(null);
         DocsEnum docs = null;
         while(true) {
           if (termCount++ == termCountHardLimit) {

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldComparator.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldComparator.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldComparator.java Sun Nov 20 23:45:25 2011
@@ -1670,11 +1670,17 @@ public abstract class FieldComparator<T>
 
       final IndexDocValues dv = context.reader.docValues(field);
       if (dv == null) {
+        // This may mean entire segment had no docs with
+        // this DV field; use default field value (empty
+        // byte[]) in this case:
         termsIndex = IndexDocValues.getDefaultSortedSource(ValueType.BYTES_VAR_SORTED, context.reader.maxDoc());
       } else {
         termsIndex = dv.getSource().asSortedSource();
         if (termsIndex == null) {
-          termsIndex = IndexDocValues.getDefaultSortedSource(ValueType.BYTES_VAR_SORTED, context.reader.maxDoc());
+          // This means segment has doc values, but they are
+          // not able to provide a sorted source; consider
+          // this a hard error:
+          throw new IllegalStateException("DocValues exist for field \"" + field + "\", but not as a sorted source: type=" + dv.getSource().type() + " reader=" + context.reader);
         }
       }
 

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyQuery.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyQuery.java Sun Nov 20 23:45:25 2011
@@ -139,7 +139,7 @@ public class FuzzyQuery extends MultiTer
   @Override
   protected TermsEnum getTermsEnum(Terms terms, AttributeSource atts) throws IOException {
     if (!termLongEnough) {  // can only match if it's exact
-      return new SingleTermsEnum(terms.iterator(), term.bytes());
+      return new SingleTermsEnum(terms.iterator(null), term.bytes());
     }
     return new FuzzyTermsEnum(terms, atts, getTerm(), minimumSimilarity, prefixLength);
   }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java Sun Nov 20 23:45:25 2011
@@ -406,7 +406,7 @@ public final class FuzzyTermsEnum extend
      * @throws IOException
      */
     public LinearFuzzyTermsEnum() throws IOException {
-      super(terms.iterator());
+      super(terms.iterator(null));
 
       this.text = new int[termLength - realPrefixLength];
       System.arraycopy(termText, realPrefixLength, text, 0, text.length);

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/IndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/IndexSearcher.java?rev=1204297&r1=1204296&r2=1204297&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/IndexSearcher.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/IndexSearcher.java Sun Nov 20 23:45:25 2011
@@ -41,7 +41,6 @@ import org.apache.lucene.index.Term;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
 import org.apache.lucene.search.similarities.SimilarityProvider;
-import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.NIOFSDirectory;    // javadoc
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
@@ -76,7 +75,6 @@ import org.apache.lucene.util.ThreadInte
  */
 public class IndexSearcher implements Closeable {
   final IndexReader reader; // package private for testing!
-  private boolean closeReader;
   
   // NOTE: these members might change in incompatible ways
   // in the next release
@@ -105,34 +103,9 @@ public class IndexSearcher implements Cl
   /** The SimilarityProvider implementation used by this searcher. */
   private SimilarityProvider similarityProvider = defaultProvider;
 
-  /** Creates a searcher searching the index in the named
-   *  directory, with readOnly=true
-   * @param path directory where IndexReader will be opened
-   * @throws CorruptIndexException if the index is corrupt
-   * @throws IOException if there is a low-level IO error
-   */
-  public IndexSearcher(Directory path) throws CorruptIndexException, IOException {
-    this(IndexReader.open(path, true), true, null);
-  }
-
-  /** Creates a searcher searching the index in the named
-   *  directory.  You should pass readOnly=true, since it
-   *  gives much better concurrent performance, unless you
-   *  intend to do write operations (delete documents or
-   *  change norms) with the underlying IndexReader.
-   * @param path directory where IndexReader will be opened
-   * @param readOnly if true, the underlying IndexReader
-   * will be opened readOnly
-   * @throws CorruptIndexException if the index is corrupt
-   * @throws IOException if there is a low-level IO error
-   */
-  public IndexSearcher(Directory path, boolean readOnly) throws CorruptIndexException, IOException {
-    this(IndexReader.open(path, readOnly), true, null);
-  }
-
   /** Creates a searcher searching the provided index. */
   public IndexSearcher(IndexReader r) {
-    this(r, false, null);
+    this(r, null);
   }
 
   /** Runs searches for each segment separately, using the
@@ -147,7 +120,7 @@ public class IndexSearcher implements Cl
    * 
    * @lucene.experimental */
   public IndexSearcher(IndexReader r, ExecutorService executor) {
-    this(r, false, executor);
+    this(r.getTopReaderContext(), executor);
   }
 
   /**
@@ -167,7 +140,12 @@ public class IndexSearcher implements Cl
    * @lucene.experimental
    */
   public IndexSearcher(ReaderContext context, ExecutorService executor) {
-    this(context, false, executor);
+    assert context.isTopLevel: "IndexSearcher's ReaderContext must be topLevel for reader" + context.reader;
+    reader = context.reader;
+    this.executor = executor;
+    this.readerContext = context;
+    leafContexts = ReaderUtil.leaves(context);
+    this.leafSlices = executor == null ? null : slices(leafContexts);
   }
 
   /**
@@ -178,22 +156,7 @@ public class IndexSearcher implements Cl
    * @lucene.experimental
    */
   public IndexSearcher(ReaderContext context) {
-    this(context, (ExecutorService) null);
-  }
-  
-  // convenience ctor for other IR based ctors
-  private IndexSearcher(IndexReader reader, boolean closeReader, ExecutorService executor) {
-    this(reader.getTopReaderContext(), closeReader, executor);
-  }
-
-  private IndexSearcher(ReaderContext context, boolean closeReader, ExecutorService executor) {
-    assert context.isTopLevel: "IndexSearcher's ReaderContext must be topLevel for reader" + context.reader;
-    reader = context.reader;
-    this.executor = executor;
-    this.closeReader = closeReader;
-    this.readerContext = context;
-    leafContexts = ReaderUtil.leaves(context);
-    this.leafSlices = executor == null ? null : slices(leafContexts);
+    this(context, null);
   }
   
   /**
@@ -236,17 +199,8 @@ public class IndexSearcher implements Cl
     return similarityProvider;
   }
 
-  /**
-   * Note that the underlying IndexReader is not closed, if
-   * IndexSearcher was constructed with IndexSearcher(IndexReader r).
-   * If the IndexReader was supplied implicitly by specifying a directory, then
-   * the IndexReader is closed.
-   */
   @Override
   public void close() throws IOException {
-    if (closeReader) {
-      reader.close();
-    }
   }
   
   /** @lucene.internal */



Mime
View raw message