lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From uschind...@apache.org
Subject svn commit: r931278 [4/10] - in /lucene/dev/trunk: lucene/ lucene/backwards/src/ lucene/backwards/src/java/org/apache/lucene/index/ lucene/backwards/src/java/org/apache/lucene/index/codecs/ lucene/backwards/src/java/org/apache/lucene/search/ lucene/bac...
Date Tue, 06 Apr 2010 19:19:36 GMT
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java Tue Apr  6 19:19:27 2010
@@ -25,7 +25,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -35,6 +35,11 @@ import org.apache.lucene.search.Similari
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.ReaderUtil;
+import org.apache.lucene.util.BytesRef;
+
 import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
 
 /** 
@@ -43,13 +48,14 @@ import org.apache.lucene.search.FieldCac
 class DirectoryReader extends IndexReader implements Cloneable {
   protected Directory directory;
   protected boolean readOnly;
+  
+  protected CodecProvider codecs;
 
   IndexWriter writer;
 
   private IndexDeletionPolicy deletionPolicy;
   private Lock writeLock;
   private SegmentInfos segmentInfos;
-  private SegmentInfos segmentInfosStart;
   private boolean stale;
   private final int termInfosIndexDivisor;
 
@@ -58,34 +64,57 @@ class DirectoryReader extends IndexReade
 
   private SegmentReader[] subReaders;
   private int[] starts;                           // 1st docno for each segment
+  private final Map<SegmentReader,ReaderUtil.Slice> subReaderToSlice = new HashMap<SegmentReader,ReaderUtil.Slice>();
   private Map<String,byte[]> normsCache = new HashMap<String,byte[]>();
   private int maxDoc = 0;
   private int numDocs = -1;
   private boolean hasDeletions = false;
 
+//  static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly,
+//      final int termInfosIndexDivisor) throws CorruptIndexException, IOException {
+//    return open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor, null);
+//  }
+  
   static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly,
-                          final int termInfosIndexDivisor) throws CorruptIndexException, IOException {
+                          final int termInfosIndexDivisor, CodecProvider codecs) throws CorruptIndexException, IOException {
+    final CodecProvider codecs2;
+    if (codecs == null) {
+      codecs2 = CodecProvider.getDefault();
+    } else {
+      codecs2 = codecs;
+    }
     return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) {
       @Override
       protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
         SegmentInfos infos = new SegmentInfos();
-        infos.read(directory, segmentFileName);
+        infos.read(directory, segmentFileName, codecs2);
         if (readOnly)
-          return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor);
+          return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor, codecs2);
         else
-          return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor);
+          return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor, codecs2);
       }
     }.run(commit);
   }
 
   /** Construct reading the named set of readers. */
-  DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws IOException {
+//  DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws IOException {
+//    this(directory, sis, deletionPolicy, readOnly, termInfosIndexDivisor, null);
+//  }
+  
+  /** Construct reading the named set of readers. */
+  DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, CodecProvider codecs) throws IOException {
     this.directory = directory;
     this.readOnly = readOnly;
     this.segmentInfos = sis;
     this.deletionPolicy = deletionPolicy;
     this.termInfosIndexDivisor = termInfosIndexDivisor;
 
+    if (codecs == null) {
+      this.codecs = CodecProvider.getDefault();
+    } else {
+      this.codecs = codecs;
+    }
+
     // To reduce the chance of hitting FileNotFound
     // (and having to retry), we open segments in
     // reverse because IndexWriter merges & deletes
@@ -115,12 +144,16 @@ class DirectoryReader extends IndexReade
   }
 
   // Used by near real-time search
-  DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor) throws IOException {
+  DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor, CodecProvider codecs) throws IOException {
     this.directory = writer.getDirectory();
     this.readOnly = true;
     segmentInfos = infos;
-    segmentInfosStart = (SegmentInfos) infos.clone();
     this.termInfosIndexDivisor = termInfosIndexDivisor;
+    if (codecs == null) {
+      this.codecs = CodecProvider.getDefault();
+    } else {
+      this.codecs = codecs;
+    }
 
     // IndexWriter synchronizes externally before calling
     // us, which ensures infos will not change; so there's
@@ -166,11 +199,17 @@ class DirectoryReader extends IndexReade
 
   /** This constructor is only used for {@link #reopen()} */
   DirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, int[] oldStarts,
-                  Map<String,byte[]> oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor) throws IOException {
+                  Map<String,byte[]> oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor, CodecProvider codecs) throws IOException {
     this.directory = directory;
     this.readOnly = readOnly;
     this.segmentInfos = infos;
     this.termInfosIndexDivisor = termInfosIndexDivisor;
+    if (codecs == null) {
+      this.codecs = CodecProvider.getDefault();
+    } else {
+      this.codecs = codecs;
+    }
+    
 
     // we put the old SegmentReaders in a map, that allows us
     // to lookup a reader using its segment name
@@ -296,25 +335,45 @@ class DirectoryReader extends IndexReade
         buffer.append(' ');
       }
       buffer.append(subReaders[i]);
+      buffer.append(' ');
     }
     buffer.append(')');
     return buffer.toString();
   }
 
-  private void initialize(SegmentReader[] subReaders) {
+  private void initialize(SegmentReader[] subReaders) throws IOException {
     this.subReaders = subReaders;
     starts = new int[subReaders.length + 1];    // build starts array
+
+    final List<Fields> subFields = new ArrayList<Fields>();
+    final List<ReaderUtil.Slice> fieldSlices = new ArrayList<ReaderUtil.Slice>();
+
     for (int i = 0; i < subReaders.length; i++) {
       starts[i] = maxDoc;
       maxDoc += subReaders[i].maxDoc();      // compute maxDocs
 
-      if (subReaders[i].hasDeletions())
+      if (subReaders[i].hasDeletions()) {
         hasDeletions = true;
+      }
+
+      final ReaderUtil.Slice slice = new ReaderUtil.Slice(starts[i], subReaders[i].maxDoc(), i);
+      subReaderToSlice.put(subReaders[i], slice);
+
+      final Fields f = subReaders[i].fields();
+      if (f != null) {
+        subFields.add(f);
+        fieldSlices.add(slice);
+      }
     }
     starts[subReaders.length] = maxDoc;
   }
 
   @Override
+  public Bits getDeletedDocs() {
+    throw new UnsupportedOperationException("please use MultiFields.getDeletedDocs if you really need a top level Bits deletedDocs (NOTE that it's usually better to work per segment instead)");
+  }
+
+  @Override
   public final synchronized Object clone() {
     try {
       return clone(readOnly); // Preserve current readOnly
@@ -435,7 +494,7 @@ class DirectoryReader extends IndexReade
       @Override
       protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
         SegmentInfos infos = new SegmentInfos();
-        infos.read(directory, segmentFileName);
+        infos.read(directory, segmentFileName, codecs);
         return doReopen(infos, false, openReadOnly);
       }
     }.run(commit);
@@ -444,9 +503,9 @@ class DirectoryReader extends IndexReade
   private synchronized DirectoryReader doReopen(SegmentInfos infos, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException {
     DirectoryReader reader;
     if (openReadOnly) {
-      reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache, doClone, termInfosIndexDivisor);
+      reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache, doClone, termInfosIndexDivisor, null);
     } else {
-      reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false, doClone, termInfosIndexDivisor);
+      reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false, doClone, termInfosIndexDivisor, null);
     }
     return reader;
   }
@@ -640,7 +699,7 @@ class DirectoryReader extends IndexReade
       // Optimize single segment case:
       return subReaders[0].terms();
     } else {
-      return new MultiTermEnum(this, subReaders, starts, null);
+     return new MultiTermEnum(this, subReaders, starts, null);
     }
   }
 
@@ -665,6 +724,16 @@ class DirectoryReader extends IndexReade
   }
 
   @Override
+  public int docFreq(String field, BytesRef term) throws IOException {
+    ensureOpen();
+    int total = 0;          // sum freqs in segments
+    for (int i = 0; i < subReaders.length; i++) {
+      total += subReaders[i].docFreq(field, term);
+    }
+    return total;
+  }
+
+  @Override
   public TermDocs termDocs() throws IOException {
     ensureOpen();
     if (subReaders.length == 1) {
@@ -687,6 +756,11 @@ class DirectoryReader extends IndexReade
   }
 
   @Override
+  public Fields fields() throws IOException {
+    throw new UnsupportedOperationException("please use MultiFields.getFields if you really need a top level Fields (NOTE that it's usually better to work per segment instead)");
+  }
+
+  @Override
   public TermPositions termPositions() throws IOException {
     ensureOpen();
     if (subReaders.length == 1) {
@@ -731,7 +805,7 @@ class DirectoryReader extends IndexReade
 
         // we have to check whether index has changed since this reader was opened.
         // if so, this reader is no longer valid for deletion
-        if (SegmentInfos.readCurrentVersion(directory) > segmentInfos.getVersion()) {
+        if (SegmentInfos.readCurrentVersion(directory, codecs) > segmentInfos.getVersion()) {
           stale = true;
           this.writeLock.release();
           this.writeLock = null;
@@ -751,13 +825,18 @@ class DirectoryReader extends IndexReade
    */
   @Override
   protected void doCommit(Map<String,String> commitUserData) throws IOException {
+    // poll subreaders for changes
+    for (int i = 0; !hasChanges && i < subReaders.length; i++) {
+      hasChanges |= subReaders[i].hasChanges;
+    }
+    
     if (hasChanges) {
       segmentInfos.setUserData(commitUserData);
       // Default deleter (for backwards compatibility) is
       // KeepOnlyLastCommitDeleter:
       IndexFileDeleter deleter = new IndexFileDeleter(directory,
                                                       deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy,
-                                                      segmentInfos, null, null);
+                                                      segmentInfos, null, null, codecs);
 
       // Checkpoint the state we are about to change, in
       // case we have to roll back:
@@ -828,20 +907,30 @@ class DirectoryReader extends IndexReade
   }
 
   @Override
+  public long getUniqueTermCount() throws IOException {
+    throw new UnsupportedOperationException("");
+  }
+
+  @Override
   public Map<String,String> getCommitUserData() {
     ensureOpen();
     return segmentInfos.getUserData();
   }
 
+  /**
+   * Check whether this IndexReader is still using the current (i.e., most recently committed) version of the index.  If
+   * a writer has committed any changes to the index since this reader was opened, this will return <code>false</code>,
+   * in which case you must open a new IndexReader in order
+   * to see the changes.  Use {@link IndexWriter#commit} to
+   * commit changes to the index.
+   *
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws IOException           if there is a low-level IO error
+   */
   @Override
   public boolean isCurrent() throws CorruptIndexException, IOException {
     ensureOpen();
-    if (writer == null || writer.isClosed()) {
-      // we loaded SegmentInfos from the directory
-      return SegmentInfos.readCurrentVersion(directory) == segmentInfos.getVersion();
-    } else {
-      return writer.nrtIsCurrent(segmentInfosStart);
-    }
+    return SegmentInfos.readCurrentVersion(directory, codecs) == segmentInfos.getVersion();
   }
 
   @Override
@@ -893,6 +982,11 @@ class DirectoryReader extends IndexReade
     return subReaders;
   }
 
+  @Override
+  public int getSubReaderDocBase(IndexReader subReader) {
+    return subReaderToSlice.get(subReader).start;
+  }
+
   /** Returns the directory this index resides in. */
   @Override
   public Directory directory() {
@@ -919,12 +1013,17 @@ class DirectoryReader extends IndexReade
 
   /** @see org.apache.lucene.index.IndexReader#listCommits */
   public static Collection<IndexCommit> listCommits(Directory dir) throws IOException {
+    return listCommits(dir, CodecProvider.getDefault());
+  }
+
+  /** @see org.apache.lucene.index.IndexReader#listCommits */
+  public static Collection<IndexCommit> listCommits(Directory dir, CodecProvider codecs) throws IOException {
     final String[] files = dir.listAll();
 
     Collection<IndexCommit> commits = new ArrayList<IndexCommit>();
 
     SegmentInfos latest = new SegmentInfos();
-    latest.read(dir);
+    latest.read(dir, codecs);
     final long currentGen = latest.getGeneration();
 
     commits.add(new ReaderCommit(latest, dir));
@@ -941,7 +1040,7 @@ class DirectoryReader extends IndexReade
         try {
           // IOException allowed to throw there, in case
           // segments_N is corrupt
-          sis.read(dir, fileName);
+          sis.read(dir, fileName, codecs);
         } catch (FileNotFoundException fnfe) {
           // LUCENE-948: on NFS (and maybe others), if
           // you have writers switching back and forth
@@ -1020,30 +1119,34 @@ class DirectoryReader extends IndexReade
       return userData;
     }
   }
-
+  
+  // @deprecated This is pre-flex API
+  // Exposes pre-flex API by doing on-the-fly merging
+  // pre-flex API to each segment
   static class MultiTermEnum extends TermEnum {
     IndexReader topReader; // used for matching TermEnum to TermDocs
-    private SegmentMergeQueue queue;
+    private LegacySegmentMergeQueue queue;
   
     private Term term;
     private int docFreq;
-    final SegmentMergeInfo[] matchingSegments; // null terminated array of matching segments
+    final LegacySegmentMergeInfo[] matchingSegments; // null terminated array of matching segments
 
     public MultiTermEnum(IndexReader topReader, IndexReader[] readers, int[] starts, Term t)
       throws IOException {
       this.topReader = topReader;
-      queue = new SegmentMergeQueue(readers.length);
-      matchingSegments = new SegmentMergeInfo[readers.length+1];
+      queue = new LegacySegmentMergeQueue(readers.length);
+      matchingSegments = new LegacySegmentMergeInfo[readers.length+1];
       for (int i = 0; i < readers.length; i++) {
         IndexReader reader = readers[i];
         TermEnum termEnum;
   
         if (t != null) {
           termEnum = reader.terms(t);
-        } else
+        } else {
           termEnum = reader.terms();
+        }
   
-        SegmentMergeInfo smi = new SegmentMergeInfo(starts[i], termEnum, reader);
+        LegacySegmentMergeInfo smi = new LegacySegmentMergeInfo(starts[i], termEnum, reader);
         smi.ord = i;
         if (t == null ? smi.next() : termEnum.term() != null)
           queue.add(smi);          // initialize queue
@@ -1059,7 +1162,7 @@ class DirectoryReader extends IndexReade
     @Override
     public boolean next() throws IOException {
       for (int i=0; i<matchingSegments.length; i++) {
-        SegmentMergeInfo smi = matchingSegments[i];
+        LegacySegmentMergeInfo smi = matchingSegments[i];
         if (smi==null) break;
         if (smi.next())
           queue.add(smi);
@@ -1070,7 +1173,7 @@ class DirectoryReader extends IndexReade
       int numMatchingSegments = 0;
       matchingSegments[0] = null;
 
-      SegmentMergeInfo top = queue.top();
+      LegacySegmentMergeInfo top = queue.top();
 
       if (top == null) {
         term = null;
@@ -1107,6 +1210,9 @@ class DirectoryReader extends IndexReade
     }
   }
 
+  // @deprecated This is pre-flex API
+  // Exposes pre-flex API by doing on-the-fly merging
+  // pre-flex API to each segment
   static class MultiTermDocs implements TermDocs {
     IndexReader topReader;  // used for matching TermEnum to TermDocs
     protected IndexReader[] readers;
@@ -1121,7 +1227,7 @@ class DirectoryReader extends IndexReade
 
     private MultiTermEnum tenum;  // the term enum used for seeking... can be null
     int matchingSegmentPos;  // position into the matching segments from tenum
-    SegmentMergeInfo smi;     // current segment mere info... can be null
+    LegacySegmentMergeInfo smi;     // current segment mere info... can be null
 
     public MultiTermDocs(IndexReader topReader, IndexReader[] r, int[] s) {
       this.topReader = topReader;
@@ -1217,7 +1323,7 @@ class DirectoryReader extends IndexReade
           return true;
         } else if (pointer < readers.length) {
           if (tenum != null) {
-            SegmentMergeInfo smi = tenum.matchingSegments[matchingSegmentPos++];
+            LegacySegmentMergeInfo smi = tenum.matchingSegments[matchingSegmentPos++];
             if (smi==null) {
               pointer = readers.length;
               return false;
@@ -1258,6 +1364,9 @@ class DirectoryReader extends IndexReade
     }
   }
 
+  // @deprecated This is pre-flex API
+  // Exposes pre-flex API by doing on-the-fly merging
+  // pre-flex API to each segment
   static class MultiTermPositions extends MultiTermDocs implements TermPositions {
     public MultiTermPositions(IndexReader topReader, IndexReader[] r, int[] s) {
       super(topReader,r,s);

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java Tue Apr  6 19:19:27 2010
@@ -67,7 +67,7 @@ final class DocFieldProcessor extends Do
     // consumer can alter the FieldInfo* if necessary.  EG,
     // FreqProxTermsWriter does this with
     // FieldInfo.storePayload.
-    final String fileName = state.segmentFileName(IndexFileNames.FIELD_INFOS_EXTENSION);
+    final String fileName = IndexFileNames.segmentFileName(state.segmentName, IndexFileNames.FIELD_INFOS_EXTENSION);
     fieldInfos.write(state.directory, fileName);
     state.flushedFiles.add(fileName);
   }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java Tue Apr  6 19:19:27 2010
@@ -113,8 +113,9 @@ final class DocFieldProcessorPerThread e
           else
             lastPerField.next = perField.next;
 
-          if (state.docWriter.infoStream != null)
-            state.docWriter.infoStream.println("  purge field=" + perField.fieldInfo.name);
+          if (state.infoStream != null) {
+            state.infoStream.println("  purge field=" + perField.fieldInfo.name);
+          }
 
           totalFieldCount--;
 
@@ -247,7 +248,7 @@ final class DocFieldProcessorPerThread e
       fields[i].consumer.processFields(fields[i].fields, fields[i].fieldCount);
 
     if (docState.maxTermPrefix != null && docState.infoStream != null) {
-      docState.infoStream.println("WARNING: document contains at least one immense term (longer than the max length " + DocumentsWriter.MAX_TERM_LENGTH + "), all of which were skipped.  Please correct the analyzer to not produce such terms.  The prefix of the first immense term is: '" + docState.maxTermPrefix + "...'");
+      docState.infoStream.println("WARNING: document contains at least one immense term (whose UTF8 encoding is longer than the max length " + DocumentsWriter.MAX_TERM_LENGTH_UTF8 + "), all of which were skipped.  Please correct the analyzer to not produce such terms.  The prefix of the first immense term is: '" + docState.maxTermPrefix + "...'"); 
       docState.maxTermPrefix = null;
     }
 
@@ -389,4 +390,4 @@ final class DocFieldProcessorPerThread e
       }
     }
   }
-}
+}
\ No newline at end of file

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocInverterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocInverterPerField.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocInverterPerField.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocInverterPerField.java Tue Apr  6 19:19:27 2010
@@ -116,8 +116,9 @@ final class DocInverterPerField extends 
               reader = readerValue;
             else {
               String stringValue = field.stringValue();
-              if (stringValue == null)
+              if (stringValue == null) {
                 throw new IllegalArgumentException("field must have either TokenStream, String or Reader value");
+              }
               perThread.stringReader.init(stringValue);
               reader = perThread.stringReader;
             }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocInverterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocInverterPerThread.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocInverterPerThread.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocInverterPerThread.java Tue Apr  6 19:19:27 2010
@@ -21,7 +21,7 @@ import java.io.IOException;
 
 import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 /** This is a DocFieldConsumer that inverts each field,
  *  separately, from a Document, and accepts a
@@ -34,16 +34,16 @@ final class DocInverterPerThread extends
   final SingleTokenAttributeSource singleToken = new SingleTokenAttributeSource();
   
   static class SingleTokenAttributeSource extends AttributeSource {
-    final TermAttribute termAttribute;
+    final CharTermAttribute termAttribute;
     final OffsetAttribute offsetAttribute;
     
     private SingleTokenAttributeSource() {
-      termAttribute = addAttribute(TermAttribute.class);
+      termAttribute = addAttribute(CharTermAttribute.class);
       offsetAttribute = addAttribute(OffsetAttribute.class);
     }
     
     public void reinit(String stringValue, int startOffset,  int endOffset) {
-      termAttribute.setTermBuffer(stringValue);
+      termAttribute.setEmpty().append(stringValue);
       offsetAttribute.setOffset(startOffset, endOffset);
     }
   }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java Tue Apr  6 19:19:27 2010
@@ -30,6 +30,7 @@ import java.util.Map.Entry;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.document.Document;
+import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
@@ -41,6 +42,7 @@ import org.apache.lucene.store.RAMFile;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.ThreadInterruptedException;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.RamUsageEstimator;
 
 /**
@@ -282,7 +284,6 @@ final class DocumentsWriter {
 
   // If we've allocated 5% over our RAM budget, we then
   // free down to 95%
-  private long freeTrigger = (long) (IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB*1024*1024*1.05);
   private long freeLevel = (long) (IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB*1024*1024*0.95);
 
   // Flush @ this number of docs.  If ramBufferSize is
@@ -353,7 +354,6 @@ final class DocumentsWriter {
       ramBufferSize = (long) (mb*1024*1024);
       waitQueuePauseBytes = (long) (ramBufferSize*0.1);
       waitQueueResumeBytes = (long) (ramBufferSize*0.05);
-      freeTrigger = (long) (1.05 * ramBufferSize);
       freeLevel = (long) (0.95 * ramBufferSize);
     }
   }
@@ -550,7 +550,6 @@ final class DocumentsWriter {
     flushPending = false;
     for(int i=0;i<threadStates.length;i++)
       threadStates[i].doAfterFlush();
-    numBytesUsed = 0;
   }
 
   // Returns true if an abort is in progress
@@ -590,9 +589,16 @@ final class DocumentsWriter {
 
   synchronized private void initFlushState(boolean onlyDocStore) {
     initSegmentName(onlyDocStore);
-    flushState = new SegmentWriteState(this, directory, segment, docStoreSegment, numDocsInRAM, numDocsInStore, writer.getConfig().getTermIndexInterval());
+    flushState = new SegmentWriteState(infoStream, directory, segment, docFieldProcessor.fieldInfos,
+                                       docStoreSegment, numDocsInRAM, numDocsInStore, writer.getConfig().getTermIndexInterval(),
+                                       writer.codecs);
   }
 
+  /** Returns the codec used to flush the last segment */
+  Codec getCodec() {
+    return flushState.codec;
+  }
+  
   /** Flush all pending docs to a new segment */
   synchronized int flush(boolean closeDocStore) throws IOException {
 
@@ -628,9 +634,10 @@ final class DocumentsWriter {
       consumer.flush(threads, flushState);
 
       if (infoStream != null) {
-        SegmentInfo si = new SegmentInfo(flushState.segmentName, flushState.numDocs, directory);
+        SegmentInfo si = new SegmentInfo(flushState.segmentName, flushState.numDocs, directory, flushState.codec);
+        si.setHasProx(hasProx());
         final long newSegmentSize = si.sizeInBytes();
-        String message = "  oldRAMSize=" + numBytesUsed +
+        String message = "  ramUsed=" + nf.format(((double) numBytesUsed)/1024./1024.) + " MB" +
           " newFlushedSize=" + newSegmentSize +
           " docs/MB=" + nf.format(numDocsInRAM/(newSegmentSize/1024./1024.)) +
           " new/old=" + nf.format(100.0*newSegmentSize/numBytesUsed) + "%";
@@ -659,8 +666,9 @@ final class DocumentsWriter {
     
     CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, 
         IndexFileNames.segmentFileName(segment, IndexFileNames.COMPOUND_FILE_EXTENSION));
-    for (final String flushedFile : flushState.flushedFiles)
-      cfsWriter.addFile(flushedFile);
+    for(String fileName : flushState.flushedFiles) {
+      cfsWriter.addFile(fileName);
+    }
       
     // Perform the merge
     cfsWriter.close();
@@ -1032,28 +1040,58 @@ final class DocumentsWriter {
 
     // Delete by term
     if (deletesFlushed.terms.size() > 0) {
-      TermDocs docs = reader.termDocs();
       try {
+        Fields fields = reader.fields();
+        TermsEnum termsEnum = null;
+        
+        String currentField = null;
+        BytesRef termRef = new BytesRef();
+        DocsEnum docs = null;
+        
         for (Entry<Term, BufferedDeletes.Num> entry: deletesFlushed.terms.entrySet()) {
           Term term = entry.getKey();
-          // LUCENE-2086: we should be iterating a TreeMap,
-          // here, so terms better be in order:
+          // Since we visit terms sorted, we gain performance
+          // by re-using the same TermsEnum and seeking only
+          // forwards
+          if (term.field() != currentField) {
+            assert currentField == null || currentField.compareTo(term.field()) < 0;
+            currentField = term.field();
+            Terms terms = fields.terms(currentField);
+            if (terms != null) {
+              termsEnum = terms.iterator();
+            } else {
+              termsEnum = null;
+            }
+          }
+          
+          if (termsEnum == null) {
+            continue;
+          }
           assert checkDeleteTerm(term);
-          docs.seek(term);
-          int limit = entry.getValue().getNum();
-          while (docs.next()) {
-            int docID = docs.doc();
-            if (docIDStart+docID >= limit)
-              break;
-            reader.deleteDocument(docID);
-            any = true;
+          
+          termRef.copy(term.text());
+          
+          if (termsEnum.seek(termRef, false) == TermsEnum.SeekStatus.FOUND) {
+            DocsEnum docsEnum = termsEnum.docs(reader.getDeletedDocs(), docs);
+            
+            if (docsEnum != null) {
+              docs = docsEnum;
+              int limit = entry.getValue().getNum();
+              while (true) {
+                final int docID = docs.nextDoc();
+                if (docID == DocsEnum.NO_MORE_DOCS || docIDStart+docID >= limit) {
+                  break;
+                }
+                reader.deleteDocument(docID);
+                any = true;
+              }
+            }
           }
         }
       } finally {
-        docs.close();
+        //docs.close();
       }
     }
-
     // Delete by docID
     for (Integer docIdInt : deletesFlushed.docIDs) {
       int docID = docIdInt.intValue();
@@ -1118,7 +1156,7 @@ final class DocumentsWriter {
   }
 
   synchronized boolean doBalanceRAM() {
-    return ramBufferSize != IndexWriterConfig.DISABLE_AUTO_FLUSH && !bufferIsFull && (numBytesUsed+deletesInRAM.bytesUsed+deletesFlushed.bytesUsed >= ramBufferSize || numBytesAlloc >= freeTrigger);
+    return ramBufferSize != IndexWriterConfig.DISABLE_AUTO_FLUSH && !bufferIsFull && (numBytesUsed+deletesInRAM.bytesUsed+deletesFlushed.bytesUsed >= ramBufferSize);
   }
 
   /** Does the synchronized work to finish/flush the
@@ -1201,7 +1239,6 @@ final class DocumentsWriter {
     return numBytesUsed + deletesInRAM.bytesUsed + deletesFlushed.bytesUsed;
   }
 
-  long numBytesAlloc;
   long numBytesUsed;
 
   NumberFormat nf = NumberFormat.getInstance();
@@ -1243,6 +1280,8 @@ final class DocumentsWriter {
   final static int BYTE_BLOCK_MASK = BYTE_BLOCK_SIZE - 1;
   final static int BYTE_BLOCK_NOT_MASK = ~BYTE_BLOCK_MASK;
 
+  final static int MAX_TERM_LENGTH_UTF8 = BYTE_BLOCK_SIZE-2;
+
   private class ByteBlockAllocator extends ByteBlockPool.Allocator {
     final int blockSize;
 
@@ -1259,19 +1298,16 @@ final class DocumentsWriter {
         final int size = freeByteBlocks.size();
         final byte[] b;
         if (0 == size) {
+          b = new byte[blockSize];
           // Always record a block allocated, even if
           // trackAllocations is false.  This is necessary
           // because this block will be shared between
           // things that don't track allocations (term
           // vectors) and things that do (freq/prox
           // postings).
-          numBytesAlloc += blockSize;
-          b = new byte[blockSize];
+          numBytesUsed += blockSize;
         } else
           b = freeByteBlocks.remove(size-1);
-        if (trackAllocations)
-          numBytesUsed += blockSize;
-        assert numBytesUsed <= numBytesAlloc;
         return b;
       }
     }
@@ -1291,7 +1327,7 @@ final class DocumentsWriter {
         final int size = blocks.size();
         for(int i=0;i<size;i++)
           freeByteBlocks.add(blocks.get(i));
-      }
+  }
     }
   }
 
@@ -1308,30 +1344,21 @@ final class DocumentsWriter {
     final int size = freeIntBlocks.size();
     final int[] b;
     if (0 == size) {
+      b = new int[INT_BLOCK_SIZE];
       // Always record a block allocated, even if
       // trackAllocations is false.  This is necessary
       // because this block will be shared between
       // things that don't track allocations (term
       // vectors) and things that do (freq/prox
       // postings).
-      numBytesAlloc += INT_BLOCK_SIZE*INT_NUM_BYTE;
-      b = new int[INT_BLOCK_SIZE];
+      numBytesUsed += INT_BLOCK_SIZE*INT_NUM_BYTE;
     } else
       b = freeIntBlocks.remove(size-1);
-    if (trackAllocations)
-      numBytesUsed += INT_BLOCK_SIZE*INT_NUM_BYTE;
-    assert numBytesUsed <= numBytesAlloc;
     return b;
   }
 
-  synchronized void bytesAllocated(long numBytes) {
-    numBytesAlloc += numBytes;
-    assert numBytesUsed <= numBytesAlloc;
-  }
-
   synchronized void bytesUsed(long numBytes) {
     numBytesUsed += numBytes;
-    assert numBytesUsed <= numBytesAlloc;
   }
 
   /* Return int[]s to the pool */
@@ -1346,78 +1373,34 @@ final class DocumentsWriter {
 
   final ByteBlockAllocator perDocAllocator = new ByteBlockAllocator(PER_DOC_BLOCK_SIZE);
 
-
-  /* Initial chunk size of the shared char[] blocks used to
-     store term text */
-  final static int CHAR_BLOCK_SHIFT = 14;
-  final static int CHAR_BLOCK_SIZE = 1 << CHAR_BLOCK_SHIFT;
-  final static int CHAR_BLOCK_MASK = CHAR_BLOCK_SIZE - 1;
-
-  final static int MAX_TERM_LENGTH = CHAR_BLOCK_SIZE-1;
-
-  private ArrayList<char[]> freeCharBlocks = new ArrayList<char[]>();
-
-  /* Allocate another char[] from the shared pool */
-  synchronized char[] getCharBlock() {
-    final int size = freeCharBlocks.size();
-    final char[] c;
-    if (0 == size) {
-      numBytesAlloc += CHAR_BLOCK_SIZE * CHAR_NUM_BYTE;
-      c = new char[CHAR_BLOCK_SIZE];
-    } else
-      c = freeCharBlocks.remove(size-1);
-    // We always track allocations of char blocks, for now,
-    // because nothing that skips allocation tracking
-    // (currently only term vectors) uses its own char
-    // blocks.
-    numBytesUsed += CHAR_BLOCK_SIZE * CHAR_NUM_BYTE;
-    assert numBytesUsed <= numBytesAlloc;
-    return c;
-  }
-
-  /* Return char[]s to the pool */
-  synchronized void recycleCharBlocks(char[][] blocks, int numBlocks) {
-    for(int i=0;i<numBlocks;i++)
-      freeCharBlocks.add(blocks[i]);
-  }
-
   String toMB(long v) {
     return nf.format(v/1024./1024.);
   }
 
-  /* We have four pools of RAM: Postings, byte blocks
-   * (holds freq/prox posting data), char blocks (holds
-   * characters in the term) and per-doc buffers (stored fields/term vectors).  
-   * Different docs require varying amount of storage from 
-   * these four classes.
-   * 
-   * For example, docs with many unique single-occurrence
-   * short terms will use up the Postings RAM and hardly any
-   * of the other two.  Whereas docs with very large terms
-   * will use alot of char blocks RAM and relatively less of
-   * the other two.  This method just frees allocations from
-   * the pools once we are over-budget, which balances the
-   * pools to match the current docs. */
+  /* We have three pools of RAM: Postings, byte blocks
+   * (holds freq/prox posting data) and per-doc buffers
+   * (stored fields/term vectors).  Different docs require
+   * varying amount of storage from these classes.  For
+   * example, docs with many unique single-occurrence short
+   * terms will use up the Postings RAM and hardly any of
+   * the other two.  Whereas docs with very large terms will
+   * use alot of byte blocks RAM.  This method just frees
+   * allocations from the pools once we are over-budget,
+   * which balances the pools to match the current docs. */
   void balanceRAM() {
 
-    // We flush when we've used our target usage
-    final long flushTrigger = ramBufferSize;
-
     final long deletesRAMUsed = deletesInRAM.bytesUsed+deletesFlushed.bytesUsed;
 
-    if (numBytesAlloc+deletesRAMUsed > freeTrigger) {
+    if (numBytesUsed+deletesRAMUsed > ramBufferSize) {
 
       if (infoStream != null)
         message("  RAM: now balance allocations: usedMB=" + toMB(numBytesUsed) +
-                " vs trigger=" + toMB(flushTrigger) +
-                " allocMB=" + toMB(numBytesAlloc) +
+                " vs trigger=" + toMB(ramBufferSize) +
                 " deletesMB=" + toMB(deletesRAMUsed) +
-                " vs trigger=" + toMB(freeTrigger) +
                 " byteBlockFree=" + toMB(byteBlockAllocator.freeByteBlocks.size()*BYTE_BLOCK_SIZE) +
-                " perDocFree=" + toMB(perDocAllocator.freeByteBlocks.size()*PER_DOC_BLOCK_SIZE) +
-                " charBlockFree=" + toMB(freeCharBlocks.size()*CHAR_BLOCK_SIZE*CHAR_NUM_BYTE));
+                " perDocFree=" + toMB(perDocAllocator.freeByteBlocks.size()*PER_DOC_BLOCK_SIZE));
 
-      final long startBytesAlloc = numBytesAlloc + deletesRAMUsed;
+      final long startBytesUsed = numBytesUsed + deletesRAMUsed;
 
       int iter = 0;
 
@@ -1427,46 +1410,38 @@ final class DocumentsWriter {
 
       boolean any = true;
 
-      while(numBytesAlloc+deletesRAMUsed > freeLevel) {
+      while(numBytesUsed+deletesRAMUsed > freeLevel) {
       
         synchronized(this) {
-          if (0 == perDocAllocator.freeByteBlocks.size() 
-              && 0 == byteBlockAllocator.freeByteBlocks.size() 
-              && 0 == freeCharBlocks.size() 
-              && 0 == freeIntBlocks.size() 
-              && !any) {
+          if (0 == perDocAllocator.freeByteBlocks.size() &&
+              0 == byteBlockAllocator.freeByteBlocks.size() &&
+              0 == freeIntBlocks.size() && !any) {
             // Nothing else to free -- must flush now.
-            bufferIsFull = numBytesUsed+deletesRAMUsed > flushTrigger;
+            bufferIsFull = numBytesUsed+deletesRAMUsed > ramBufferSize;
             if (infoStream != null) {
-              if (numBytesUsed > flushTrigger)
+              if (numBytesUsed+deletesRAMUsed > ramBufferSize)
                 message("    nothing to free; now set bufferIsFull");
               else
                 message("    nothing to free");
             }
-            assert numBytesUsed <= numBytesAlloc;
             break;
           }
 
-          if ((0 == iter % 5) && byteBlockAllocator.freeByteBlocks.size() > 0) {
+          if ((0 == iter % 4) && byteBlockAllocator.freeByteBlocks.size() > 0) {
             byteBlockAllocator.freeByteBlocks.remove(byteBlockAllocator.freeByteBlocks.size()-1);
-            numBytesAlloc -= BYTE_BLOCK_SIZE;
+            numBytesUsed -= BYTE_BLOCK_SIZE;
           }
 
-          if ((1 == iter % 5) && freeCharBlocks.size() > 0) {
-            freeCharBlocks.remove(freeCharBlocks.size()-1);
-            numBytesAlloc -= CHAR_BLOCK_SIZE * CHAR_NUM_BYTE;
-          }
-
-          if ((2 == iter % 5) && freeIntBlocks.size() > 0) {
+          if ((1 == iter % 4) && freeIntBlocks.size() > 0) {
             freeIntBlocks.remove(freeIntBlocks.size()-1);
-            numBytesAlloc -= INT_BLOCK_SIZE * INT_NUM_BYTE;
+            numBytesUsed -= INT_BLOCK_SIZE * INT_NUM_BYTE;
           }
 
-          if ((3 == iter % 5) && perDocAllocator.freeByteBlocks.size() > 0) {
+          if ((2 == iter % 4) && perDocAllocator.freeByteBlocks.size() > 0) {
             // Remove upwards of 32 blocks (each block is 1K)
             for (int i = 0; i < 32; ++i) {
               perDocAllocator.freeByteBlocks.remove(perDocAllocator.freeByteBlocks.size() - 1);
-              numBytesAlloc -= PER_DOC_BLOCK_SIZE;
+              numBytesUsed -= PER_DOC_BLOCK_SIZE;
               if (perDocAllocator.freeByteBlocks.size() == 0) {
                 break;
               }
@@ -1474,7 +1449,7 @@ final class DocumentsWriter {
           }
         }
 
-        if ((4 == iter % 5) && any)
+        if ((3 == iter % 4) && any)
           // Ask consumer to free any recycled state
           any = consumer.freeRAM();
 
@@ -1482,26 +1457,7 @@ final class DocumentsWriter {
       }
 
       if (infoStream != null)
-        message("    after free: freedMB=" + nf.format((startBytesAlloc-numBytesAlloc-deletesRAMUsed)/1024./1024.) + " usedMB=" + nf.format((numBytesUsed+deletesRAMUsed)/1024./1024.) + " allocMB=" + nf.format(numBytesAlloc/1024./1024.));
-      
-    } else {
-      // If we have not crossed the 100% mark, but have
-      // crossed the 95% mark of RAM we are actually
-      // using, go ahead and flush.  This prevents
-      // over-allocating and then freeing, with every
-      // flush.
-      synchronized(this) {
-
-        if (numBytesUsed+deletesRAMUsed > flushTrigger) {
-          if (infoStream != null)
-            message("  RAM: now flush @ usedMB=" + nf.format(numBytesUsed/1024./1024.) +
-                    " allocMB=" + nf.format(numBytesAlloc/1024./1024.) +
-                    " deletesMB=" + nf.format(deletesRAMUsed/1024./1024.) +
-                    " triggerMB=" + nf.format(flushTrigger/1024./1024.));
-
-          bufferIsFull = true;
-        }
-      }
+        message("    after free: freedMB=" + nf.format((startBytesUsed-numBytesUsed-deletesRAMUsed)/1024./1024.) + " usedMB=" + nf.format((numBytesUsed+deletesRAMUsed)/1024./1024.));
     }
   }
 

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java Tue Apr  6 19:19:27 2010
@@ -17,20 +17,21 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
-final class FieldInfo {
-  String name;
-  boolean isIndexed;
-  int number;
+/** @lucene.experimental */
+public final class FieldInfo {
+  public String name;
+  public boolean isIndexed;
+  public int number;
 
   // true if term vector for this field should be stored
   boolean storeTermVector;
   boolean storeOffsetWithTermVector;
   boolean storePositionWithTermVector;
 
-  boolean omitNorms; // omit norms associated with indexed fields  
-  boolean omitTermFreqAndPositions;
-  
-  boolean storePayloads; // whether this field stores payloads together with term positions
+  public boolean omitNorms; // omit norms associated with indexed fields  
+  public boolean omitTermFreqAndPositions;
+
+  public boolean storePayloads; // whether this field stores payloads together with term positions
 
   FieldInfo(String na, boolean tk, int nu, boolean storeTermVector, 
             boolean storePositionWithTermVector,  boolean storeOffsetWithTermVector, 

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java Tue Apr  6 19:19:27 2010
@@ -32,8 +32,9 @@ import java.util.*;
  *  of this class are thread-safe for multiple readers, but only one thread can
  *  be adding documents at a time, with no other reader or writer threads
  *  accessing this object.
+ *  @lucene.experimental
  */
-final class FieldInfos {
+public final class FieldInfos {
 
   // Used internally (ie not written to *.fnm files) for pre-2.9 files
   public static final int FORMAT_PRE = -1;
@@ -120,7 +121,7 @@ final class FieldInfos {
   }
 
   /** Returns true if any fields do not omitTermFreqAndPositions */
-  boolean hasProx() {
+  public boolean hasProx() {
     final int numFields = byNumber.size();
     for(int i=0;i<numFields;i++) {
       final FieldInfo fi = fieldInfo(i);

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java Tue Apr  6 19:19:27 2010
@@ -20,7 +20,9 @@ package org.apache.lucene.index;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.FieldSelector;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
 import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
+import org.apache.lucene.util.BytesRef;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -116,6 +118,11 @@ public class FilterIndexReader extends I
   }
   
   @Override
+  public Bits getDeletedDocs() throws IOException {
+    return in.getDeletedDocs();
+  }
+  
+  @Override
   public TermFreqVector[] getTermFreqVectors(int docNumber)
           throws IOException {
     ensureOpen();
@@ -218,6 +225,12 @@ public class FilterIndexReader extends I
   }
 
   @Override
+  public int docFreq(String field, BytesRef t) throws IOException {
+    ensureOpen();
+    return in.docFreq(field, t);
+  }
+  
+  @Override
   public TermDocs termDocs() throws IOException {
     ensureOpen();
     return in.termDocs();
@@ -297,4 +310,4 @@ public class FilterIndexReader extends I
     buffer.append(')');
     return buffer.toString();
   }
-}
+}
\ No newline at end of file

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxFieldMergeState.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxFieldMergeState.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxFieldMergeState.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxFieldMergeState.java Tue Apr  6 19:19:27 2010
@@ -18,6 +18,8 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
+import java.util.Comparator;
+import org.apache.lucene.util.BytesRef;
 
 import org.apache.lucene.index.FreqProxTermsWriterPerField.FreqProxPostingsArray;
 
@@ -31,13 +33,12 @@ final class FreqProxFieldMergeState {
 
   final FreqProxTermsWriterPerField field;
   final int numPostings;
-  final CharBlockPool charPool;
+  private final ByteBlockPool bytePool;
   final int[] termIDs;
   final FreqProxPostingsArray postings;
   int currentTermID;
   
-  char[] text;
-  int textOffset;
+  final BytesRef text = new BytesRef();
 
   private int postingUpto = -1;
 
@@ -47,29 +48,31 @@ final class FreqProxFieldMergeState {
   int docID;
   int termFreq;
 
-  public FreqProxFieldMergeState(FreqProxTermsWriterPerField field) {
+  public FreqProxFieldMergeState(FreqProxTermsWriterPerField field, Comparator<BytesRef> termComp) {
     this.field = field;
-    this.charPool = field.perThread.termsHashPerThread.charPool;
     this.numPostings = field.termsHashPerField.numPostings;
-    this.termIDs = field.termsHashPerField.sortPostings();
+    this.bytePool = field.perThread.termsHashPerThread.bytePool;
+    this.termIDs = field.termsHashPerField.sortPostings(termComp);
     this.postings = (FreqProxPostingsArray) field.termsHashPerField.postingsArray;
   }
 
   boolean nextTerm() throws IOException {
     postingUpto++;
-    if (postingUpto == numPostings)
+    if (postingUpto == numPostings) {
       return false;
+    }
 
     currentTermID = termIDs[postingUpto];
     docID = 0;
 
+    // Get BytesRef
     final int textStart = postings.textStarts[currentTermID];
-    text = charPool.buffers[textStart >> DocumentsWriter.CHAR_BLOCK_SHIFT];
-    textOffset = textStart & DocumentsWriter.CHAR_BLOCK_MASK;
+    bytePool.setBytesRef(text, textStart);
 
     field.termsHashPerField.initReader(freq, currentTermID, 0);
-    if (!field.fieldInfo.omitTermFreqAndPositions)
+    if (!field.fieldInfo.omitTermFreqAndPositions) {
       field.termsHashPerField.initReader(prox, currentTermID, 1);
+    }
 
     // Should always be true
     boolean result = nextDoc();

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java Tue Apr  6 19:19:27 2010
@@ -17,14 +17,19 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
-import org.apache.lucene.util.UnicodeUtil;
-
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Map;
-import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
+import java.util.Comparator;
+
+import org.apache.lucene.index.codecs.PostingsConsumer;
+import org.apache.lucene.index.codecs.FieldsConsumer;
+import org.apache.lucene.index.codecs.TermsConsumer;
+import org.apache.lucene.util.BytesRef;
 
 final class FreqProxTermsWriter extends TermsHashConsumer {
 
@@ -33,27 +38,13 @@ final class FreqProxTermsWriter extends 
     return new FreqProxTermsWriterPerThread(perThread);
   }
 
-  private static int compareText(final char[] text1, int pos1, final char[] text2, int pos2) {
-    while(true) {
-      final char c1 = text1[pos1++];
-      final char c2 = text2[pos2++];
-      if (c1 != c2) {
-        if (0xffff == c2)
-          return 1;
-        else if (0xffff == c1)
-          return -1;
-        else
-          return c1-c2;
-      } else if (0xffff == c1)
-        return 0;
-    }
-  }
-
   @Override
   void closeDocStore(SegmentWriteState state) {}
+
   @Override
   void abort() {}
 
+  private int flushedDocCount;
 
   // TODO: would be nice to factor out more of this, eg the
   // FreqProxFieldMergeState, and code to visit all Fields
@@ -66,6 +57,8 @@ final class FreqProxTermsWriter extends 
     // Gather all FieldData's that have postings, across all
     // ThreadStates
     List<FreqProxTermsWriterPerField> allFields = new ArrayList<FreqProxTermsWriterPerField>();
+    
+    flushedDocCount = state.numDocs;
 
     for (Map.Entry<TermsHashConsumerPerThread,Collection<TermsHashConsumerPerField>> entry : threadsAndFields.entrySet()) {
 
@@ -79,21 +72,23 @@ final class FreqProxTermsWriter extends 
       }
     }
 
+    final int numAllFields = allFields.size();
+
     // Sort by field name
     Collections.sort(allFields);
-    final int numAllFields = allFields.size();
 
-    // TODO: allow Lucene user to customize this consumer:
-    final FormatPostingsFieldsConsumer consumer = new FormatPostingsFieldsWriter(state, fieldInfos);
+    // TODO: allow Lucene user to customize this codec:
+    final FieldsConsumer consumer = state.codec.fieldsConsumer(state);
+
     /*
     Current writer chain:
-      FormatPostingsFieldsConsumer
-        -> IMPL: FormatPostingsFieldsWriter
-          -> FormatPostingsTermsConsumer
-            -> IMPL: FormatPostingsTermsWriter
-              -> FormatPostingsDocConsumer
-                -> IMPL: FormatPostingsDocWriter
-                  -> FormatPostingsPositionsConsumer
+      FieldsConsumer
+        -> IMPL: FormatPostingsTermsDictWriter
+          -> TermsConsumer
+            -> IMPL: FormatPostingsTermsDictWriter.TermsWriter
+              -> DocsConsumer
+                -> IMPL: FormatPostingsDocsWriter
+                  -> PositionsConsumer
                     -> IMPL: FormatPostingsPositionsWriter
     */
 
@@ -134,25 +129,29 @@ final class FreqProxTermsWriter extends 
       FreqProxTermsWriterPerThread perThread = (FreqProxTermsWriterPerThread) entry.getKey();
       perThread.termsHashPerThread.reset(true);
     }
-
-    consumer.finish();
+    consumer.close();
   }
 
-  private byte[] payloadBuffer;
+  BytesRef payload;
 
   /* Walk through all unique text tokens (Posting
    * instances) found in this field and serialize them
    * into a single RAM segment. */
   void appendPostings(FreqProxTermsWriterPerField[] fields,
-                      FormatPostingsFieldsConsumer consumer)
+                      FieldsConsumer consumer)
     throws CorruptIndexException, IOException {
 
     int numFields = fields.length;
 
+    final BytesRef text = new BytesRef();
+
     final FreqProxFieldMergeState[] mergeStates = new FreqProxFieldMergeState[numFields];
 
+    final TermsConsumer termsConsumer = consumer.addField(fields[0].fieldInfo);
+    final Comparator<BytesRef> termComp = termsConsumer.getComparator();
+
     for(int i=0;i<numFields;i++) {
-      FreqProxFieldMergeState fms = mergeStates[i] = new FreqProxFieldMergeState(fields[i]);
+      FreqProxFieldMergeState fms = mergeStates[i] = new FreqProxFieldMergeState(fields[i], termComp);
 
       assert fms.field.fieldInfo == fields[0].fieldInfo;
 
@@ -161,45 +160,63 @@ final class FreqProxTermsWriter extends 
       assert result;
     }
 
-    final FormatPostingsTermsConsumer termsConsumer = consumer.addField(fields[0].fieldInfo);
-
     FreqProxFieldMergeState[] termStates = new FreqProxFieldMergeState[numFields];
 
     final boolean currentFieldOmitTermFreqAndPositions = fields[0].fieldInfo.omitTermFreqAndPositions;
+    //System.out.println("flush terms field=" + fields[0].fieldInfo.name);
 
+    // TODO: really TermsHashPerField should take over most
+    // of this loop, including merge sort of terms from
+    // multiple threads and interacting with the
+    // TermsConsumer, only calling out to us (passing us the
+    // DocsConsumer) to handle delivery of docs/positions
     while(numFields > 0) {
 
       // Get the next term to merge
       termStates[0] = mergeStates[0];
       int numToMerge = 1;
 
+      // TODO: pqueue
       for(int i=1;i<numFields;i++) {
-        final char[] text = mergeStates[i].text;
-        final int textOffset = mergeStates[i].textOffset;
-        final int cmp = compareText(text, textOffset, termStates[0].text, termStates[0].textOffset);
-
+        final int cmp = termComp.compare(mergeStates[i].text, termStates[0].text);
         if (cmp < 0) {
           termStates[0] = mergeStates[i];
           numToMerge = 1;
-        } else if (cmp == 0)
+        } else if (cmp == 0) {
           termStates[numToMerge++] = mergeStates[i];
+        }
       }
 
-      final FormatPostingsDocsConsumer docConsumer = termsConsumer.addTerm(termStates[0].text, termStates[0].textOffset);
+      // Need shallow copy here because termStates[0].text
+      // changes by the time we call finishTerm
+      text.bytes = termStates[0].text.bytes;
+      text.offset = termStates[0].text.offset;
+      text.length = termStates[0].text.length;  
+
+      //System.out.println("  term=" + text.toUnicodeString());
+      //System.out.println("  term=" + text.toString());
+
+      final PostingsConsumer postingsConsumer = termsConsumer.startTerm(text);
 
       // Now termStates has numToMerge FieldMergeStates
       // which all share the same term.  Now we must
       // interleave the docID streams.
+      int numDocs = 0;
       while(numToMerge > 0) {
         
         FreqProxFieldMergeState minState = termStates[0];
-        for(int i=1;i<numToMerge;i++)
-          if (termStates[i].docID < minState.docID)
+        for(int i=1;i<numToMerge;i++) {
+          if (termStates[i].docID < minState.docID) {
             minState = termStates[i];
+          }
+        }
 
         final int termDocFreq = minState.termFreq;
+        numDocs++;
+
+        assert minState.docID < flushedDocCount: "doc=" + minState.docID + " maxDoc=" + flushedDocCount;
 
-        final FormatPostingsPositionsConsumer posConsumer = docConsumer.addDoc(minState.docID, termDocFreq);
+        postingsConsumer.startDoc(minState.docID, termDocFreq);
 
         final ByteSliceReader prox = minState.prox;
 
@@ -213,33 +230,48 @@ final class FreqProxTermsWriter extends 
           for(int j=0;j<termDocFreq;j++) {
             final int code = prox.readVInt();
             position += code >> 1;
+            //System.out.println("    pos=" + position);
 
             final int payloadLength;
+            final BytesRef thisPayload;
+
             if ((code & 1) != 0) {
               // This position has a payload
-              payloadLength = prox.readVInt();
-
-              if (payloadBuffer == null || payloadBuffer.length < payloadLength)
-                payloadBuffer = new byte[payloadLength];
+              payloadLength = prox.readVInt();  
+              
+              if (payload == null) {
+                payload = new BytesRef();
+                payload.bytes = new byte[payloadLength];
+              } else if (payload.bytes.length < payloadLength) {
+                payload.grow(payloadLength);
+              }
+
+              prox.readBytes(payload.bytes, 0, payloadLength);
+              payload.length = payloadLength;
+              thisPayload = payload;
 
-              prox.readBytes(payloadBuffer, 0, payloadLength);
-
-            } else
+            } else {
               payloadLength = 0;
+              thisPayload = null;
+            }
 
-            posConsumer.addPosition(position, payloadBuffer, 0, payloadLength);
+            postingsConsumer.addPosition(position, thisPayload);
           } //End for
 
-          posConsumer.finish();
+          postingsConsumer.finishDoc();
         }
 
         if (!minState.nextDoc()) {
 
           // Remove from termStates
           int upto = 0;
-          for(int i=0;i<numToMerge;i++)
-            if (termStates[i] != minState)
+          // TODO: inefficient O(N) where N = number of
+          // threads that had seen this term:
+          for(int i=0;i<numToMerge;i++) {
+            if (termStates[i] != minState) {
               termStates[upto++] = termStates[i];
+            }
+          }
           numToMerge--;
           assert upto == numToMerge;
 
@@ -258,11 +290,10 @@ final class FreqProxTermsWriter extends 
         }
       }
 
-      docConsumer.finish();
+      assert numDocs > 0;
+      termsConsumer.finishTerm(text, numDocs);
     }
 
     termsConsumer.finish();
   }
-
-  final UnicodeUtil.UTF8Result termsUTF8 = new UnicodeUtil.UTF8Result();
 }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java Tue Apr  6 19:19:27 2010
@@ -185,27 +185,28 @@ final class FreqProxTermsWriterPerField 
     int lastDocIDs[];                                  // Last docID where this term occurred
     int lastDocCodes[];                                // Code for prior doc
     int lastPositions[];                               // Last position where this term occurred
-    
+
     @Override
-    ParallelPostingsArray resize(int newSize) {
-      FreqProxPostingsArray newArray = new FreqProxPostingsArray(newSize);
-      copy(this, newArray);
-      return newArray;
+    ParallelPostingsArray newInstance(int size) {
+      return new FreqProxPostingsArray(size);
     }
-    
-    void copy(FreqProxPostingsArray fromArray, FreqProxPostingsArray toArray) {
-      super.copy(fromArray, toArray);
-      System.arraycopy(fromArray.docFreqs, 0, toArray.docFreqs, 0, fromArray.docFreqs.length);
-      System.arraycopy(fromArray.lastDocIDs, 0, toArray.lastDocIDs, 0, fromArray.lastDocIDs.length);
-      System.arraycopy(fromArray.lastDocCodes, 0, toArray.lastDocCodes, 0, fromArray.lastDocCodes.length);
-      System.arraycopy(fromArray.lastPositions, 0, toArray.lastPositions, 0, fromArray.lastPositions.length);
+
+    void copyTo(ParallelPostingsArray toArray, int numToCopy) {
+      assert toArray instanceof FreqProxPostingsArray;
+      FreqProxPostingsArray to = (FreqProxPostingsArray) toArray;
+
+      super.copyTo(toArray, numToCopy);
+
+      System.arraycopy(docFreqs, 0, to.docFreqs, 0, numToCopy);
+      System.arraycopy(lastDocIDs, 0, to.lastDocIDs, 0, numToCopy);
+      System.arraycopy(lastDocCodes, 0, to.lastDocCodes, 0, numToCopy);
+      System.arraycopy(lastPositions, 0, to.lastPositions, 0, numToCopy);
+    }
+
+    @Override
+    int bytesPerPosting() {
+      return ParallelPostingsArray.BYTES_PER_POSTING + 4 * DocumentsWriter.INT_NUM_BYTE;
     }
-    
-  }
-  
-  @Override
-  int bytesPerPosting() {
-    return ParallelPostingsArray.BYTES_PER_POSTING + 4 * DocumentsWriter.INT_NUM_BYTE;
   }
   
   public void abort() {}

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java Tue Apr  6 19:19:27 2010
@@ -17,18 +17,20 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
-import org.apache.lucene.store.Directory;
-
-import java.io.IOException;
 import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import java.io.IOException;
 import java.io.PrintStream;
-import java.util.Map;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 
 import java.util.List;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Collection;
+import java.util.Map;
+
+import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.store.Directory;
 
 /*
  * This class keeps track of each SegmentInfos instance that
@@ -114,6 +116,8 @@ final class IndexFileDeleter {
     infoStream.println("IFD [" + Thread.currentThread().getName() + "]: " + message);
   }
 
+  private final FilenameFilter indexFilenameFilter;
+
   /**
    * Initialize the deleter: find all previous commits in
    * the Directory, incref the files they reference, call
@@ -122,7 +126,8 @@ final class IndexFileDeleter {
    * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
    */
-  public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream, DocumentsWriter docWriter)
+  public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream, DocumentsWriter docWriter,
+                          CodecProvider codecs)
     throws CorruptIndexException, IOException {
 
     this.docWriter = docWriter;
@@ -137,8 +142,8 @@ final class IndexFileDeleter {
     // First pass: walk the files and initialize our ref
     // counts:
     long currentGen = segmentInfos.getGeneration();
-    IndexFileNameFilter filter = IndexFileNameFilter.getFilter();
-
+    indexFilenameFilter = new IndexFileNameFilter(codecs);
+    
     String[] files = directory.listAll();
 
     CommitPoint currentCommitPoint = null;
@@ -147,7 +152,7 @@ final class IndexFileDeleter {
 
       String fileName = files[i];
 
-      if (filter.accept(null, fileName) && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
+      if ((indexFilenameFilter.accept(null, fileName)) && !fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
 
         // Add this file to refCounts with initial count 0:
         getRefCount(fileName);
@@ -163,7 +168,7 @@ final class IndexFileDeleter {
             }
             SegmentInfos sis = new SegmentInfos();
             try {
-              sis.read(directory, fileName);
+              sis.read(directory, fileName, codecs);
             } catch (FileNotFoundException e) {
               // LUCENE-948: on NFS (and maybe others), if
               // you have writers switching back and forth
@@ -200,7 +205,7 @@ final class IndexFileDeleter {
       // try now to explicitly open this commit point:
       SegmentInfos sis = new SegmentInfos();
       try {
-        sis.read(directory, segmentInfos.getCurrentSegmentFileName());
+        sis.read(directory, segmentInfos.getCurrentSegmentFileName(), codecs);
       } catch (IOException e) {
         throw new CorruptIndexException("failed to locate current segments_N file");
       }
@@ -296,7 +301,6 @@ final class IndexFileDeleter {
    */
   public void refresh(String segmentName) throws IOException {
     String[] files = directory.listAll();
-    IndexFileNameFilter filter = IndexFileNameFilter.getFilter();
     String segmentPrefix1;
     String segmentPrefix2;
     if (segmentName != null) {
@@ -309,8 +313,8 @@ final class IndexFileDeleter {
     
     for(int i=0;i<files.length;i++) {
       String fileName = files[i];
-      if (filter.accept(null, fileName) &&
-          (segmentName == null || fileName.startsWith(segmentPrefix1) || fileName.startsWith(segmentPrefix2)) &&
+      if ((segmentName == null || fileName.startsWith(segmentPrefix1) || fileName.startsWith(segmentPrefix2)) &&
+          indexFilenameFilter.accept(null, fileName) &&
           !refCounts.containsKey(fileName) &&
           !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
         // Unreferenced file, so remove it

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexFileNames.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexFileNames.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexFileNames.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexFileNames.java Tue Apr  6 19:19:27 2010
@@ -17,6 +17,8 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
+import org.apache.lucene.index.codecs.Codec;  // for javadocs
+
 /**
  * This class contains useful constants representing filenames and extensions
  * used by lucene, as well as convenience methods for querying whether a file
@@ -25,16 +27,24 @@ package org.apache.lucene.index;
  * generation and extension (
  * {@link #fileNameFromGeneration(String, String, long) fileNameFromGeneration},
  * {@link #segmentFileName(String, String) segmentFileName}).
- * 
+ *
+ * <p><b>NOTE</b>: extensions used by codecs are not
+ * listed here.  You must interact with the {@link Codec}
+ * directly.
+ *
  * @lucene.internal
  */
+
 public final class IndexFileNames {
 
   /** Name of the index segment file */
   public static final String SEGMENTS = "segments";
 
+  /** Extension of gen file */
+  public static final String GEN_EXTENSION = "gen";
+  
   /** Name of the generation reference file name */
-  public static final String SEGMENTS_GEN = "segments.gen";
+  public static final String SEGMENTS_GEN = "segments." +  GEN_EXTENSION;
   
   /** Name of the index deletable file (only used in
    * pre-lockless indices) */
@@ -43,18 +53,6 @@ public final class IndexFileNames {
   /** Extension of norms file */
   public static final String NORMS_EXTENSION = "nrm";
 
-  /** Extension of freq postings file */
-  public static final String FREQ_EXTENSION = "frq";
-
-  /** Extension of prox postings file */
-  public static final String PROX_EXTENSION = "prx";
-
-  /** Extension of terms file */
-  public static final String TERMS_EXTENSION = "tis";
-
-  /** Extension of terms index file */
-  public static final String TERMS_INDEX_EXTENSION = "tii";
-
   /** Extension of stored fields index file */
   public static final String FIELDS_INDEX_EXTENSION = "fdx";
 
@@ -88,9 +86,6 @@ public final class IndexFileNames {
   /** Extension of separate norms */
   public static final String SEPARATE_NORMS_EXTENSION = "s";
 
-  /** Extension of gen file */
-  public static final String GEN_EXTENSION = "gen";
-
   /**
    * This array contains all filename extensions used by
    * Lucene's index files, with two exceptions, namely the
@@ -104,10 +99,6 @@ public final class IndexFileNames {
     FIELD_INFOS_EXTENSION,
     FIELDS_INDEX_EXTENSION,
     FIELDS_EXTENSION,
-    TERMS_INDEX_EXTENSION,
-    TERMS_EXTENSION,
-    FREQ_EXTENSION,
-    PROX_EXTENSION,
     DELETES_EXTENSION,
     VECTORS_INDEX_EXTENSION,
     VECTORS_DOCUMENTS_EXTENSION,
@@ -117,22 +108,6 @@ public final class IndexFileNames {
     COMPOUND_FILE_STORE_EXTENSION,
   };
 
-  /** File extensions that are added to a compound file
-   * (same as above, minus "del", "gen", "cfs"). */
-  public static final String[] INDEX_EXTENSIONS_IN_COMPOUND_FILE = new String[] {
-    FIELD_INFOS_EXTENSION,
-    FIELDS_INDEX_EXTENSION,
-    FIELDS_EXTENSION,
-    TERMS_INDEX_EXTENSION,
-    TERMS_EXTENSION,
-    FREQ_EXTENSION,
-    PROX_EXTENSION,
-    VECTORS_INDEX_EXTENSION,
-    VECTORS_DOCUMENTS_EXTENSION,
-    VECTORS_FIELDS_EXTENSION,
-    NORMS_EXTENSION
-  };
-
   public static final String[] STORE_INDEX_EXTENSIONS = new String[] {
     VECTORS_INDEX_EXTENSION,
     VECTORS_FIELDS_EXTENSION,
@@ -143,22 +118,13 @@ public final class IndexFileNames {
 
   public static final String[] NON_STORE_INDEX_EXTENSIONS = new String[] {
     FIELD_INFOS_EXTENSION,
-    FREQ_EXTENSION,
-    PROX_EXTENSION,
-    TERMS_EXTENSION,
-    TERMS_INDEX_EXTENSION,
     NORMS_EXTENSION
   };
   
-  /** File extensions of old-style index files */
-  public static final String COMPOUND_EXTENSIONS[] = new String[] {
+  static final String COMPOUND_EXTENSIONS_NOT_CODEC[] = new String[] {
     FIELD_INFOS_EXTENSION,
-    FREQ_EXTENSION,
-    PROX_EXTENSION,
     FIELDS_INDEX_EXTENSION,
     FIELDS_EXTENSION,
-    TERMS_INDEX_EXTENSION,
-    TERMS_EXTENSION
   };
   
   /** File extensions for term vector support */
@@ -222,6 +188,7 @@ public final class IndexFileNames {
    */
   public static final String segmentFileName(String segmentName, String ext) {
     if (ext.length() > 0) {
+      assert !ext.startsWith(".");
       return new StringBuilder(segmentName.length() + 1 + ext.length()).append(
           segmentName).append('.').append(ext).toString();
     } else {



Mime
View raw message