lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r824918 [2/11] - in /lucene/java/branches/flex_1458: contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/ contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/ contrib/benchmark/src/test/org/apache/lucene/benc...
Date Tue, 13 Oct 2009 20:44:59 GMT
Modified: lucene/java/branches/flex_1458/contrib/surround/src/java/org/apache/lucene/queryParser/surround/query/SrndTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/surround/src/java/org/apache/lucene/queryParser/surround/query/SrndTermQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/surround/src/java/org/apache/lucene/queryParser/surround/query/SrndTermQuery.java (original)
+++ lucene/java/branches/flex_1458/contrib/surround/src/java/org/apache/lucene/queryParser/surround/query/SrndTermQuery.java Tue Oct 13 20:44:51 2009
@@ -20,7 +20,9 @@
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermRef;
 
  
 public class SrndTermQuery extends SimpleTerm {
@@ -44,18 +46,16 @@
     MatchingTermVisitor mtv) throws IOException
   {
     /* check term presence in index here for symmetry with other SimpleTerm's */
-    TermEnum enumerator = reader.terms(getLuceneTerm(fieldName));
-    try {
-      Term it= enumerator.term(); /* same or following index term */
-      if ((it != null)
-          && it.text().equals(getTermText())
-          && it.field().equals(fieldName)) {
-        mtv.visitMatchingTerm(it);
+    Terms terms = reader.fields().terms(fieldName);
+    if (terms != null) {
+      TermsEnum termsEnum = terms.iterator();
+
+      TermsEnum.SeekStatus status = termsEnum.seek(new TermRef(getTermText()));
+      if (status == TermsEnum.SeekStatus.FOUND) {
+        mtv.visitMatchingTerm(getLuceneTerm(fieldName));
       } else {
         System.out.println("No term in " + fieldName + " field for: " + toString());
       }
-    } finally {
-      enumerator.close();
     }
   }
 }

Modified: lucene/java/branches/flex_1458/contrib/surround/src/java/org/apache/lucene/queryParser/surround/query/SrndTruncQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/surround/src/java/org/apache/lucene/queryParser/surround/query/SrndTruncQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/surround/src/java/org/apache/lucene/queryParser/surround/query/SrndTruncQuery.java (original)
+++ lucene/java/branches/flex_1458/contrib/surround/src/java/org/apache/lucene/queryParser/surround/query/SrndTruncQuery.java Tue Oct 13 20:44:51 2009
@@ -17,7 +17,9 @@
  */
 
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermRef;
 import org.apache.lucene.index.IndexReader;
 
 import java.io.IOException;
@@ -40,6 +42,7 @@
   private final char mask;
   
   private String prefix;
+  private TermRef prefixRef;
   private Pattern pattern;
   
   
@@ -67,6 +70,7 @@
       i++;
     }
     prefix = truncated.substring(0, i);
+    prefixRef = new TermRef(prefix);
     
     StringBuilder re = new StringBuilder();
     while (i < truncated.length()) {
@@ -83,28 +87,40 @@
   {
     boolean expanded = false;
     int prefixLength = prefix.length();
-    TermEnum enumerator = reader.terms(new Term(fieldName, prefix));
-    Matcher matcher = pattern.matcher("");
-    try {
-      do {
-        Term term = enumerator.term();
-        if (term != null) {
-          String text = term.text();
-          if ((! text.startsWith(prefix)) || (! term.field().equals(fieldName))) {
-            break;
-          } else {
-            matcher.reset( text.substring(prefixLength));
+    Terms terms = reader.fields().terms(fieldName);
+    if (terms != null) {
+      Matcher matcher = pattern.matcher("");
+      try {
+        TermsEnum termsEnum = terms.iterator();
+
+        TermsEnum.SeekStatus status = termsEnum.seek(prefixRef);
+        TermRef text;
+        if (status == TermsEnum.SeekStatus.FOUND) {
+          text = prefixRef;
+        } else if (status == TermsEnum.SeekStatus.NOT_FOUND) {
+          text = termsEnum.term();
+        } else {
+          text = null;
+        }
+
+        while(text != null) {
+          if (text != null && text.startsWith(prefixRef)) {
+            String textString = text.toString();
+            matcher.reset(textString.substring(prefixLength));
             if (matcher.matches()) {
-              mtv.visitMatchingTerm(term);
+              mtv.visitMatchingTerm(new Term(fieldName, textString));
               expanded = true;
             }
+          } else {
+            break;
           }
+          text = termsEnum.next();
         }
-      } while (enumerator.next());
-    } finally {
-      enumerator.close();
-      matcher.reset();
+      } finally {
+        matcher.reset();
+      }
     }
+      
     if (! expanded) {
       System.out.println("No terms in " + fieldName + " field for: " + toString());
     }

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllDocsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllDocsEnum.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllDocsEnum.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllDocsEnum.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.index;
+
+import org.apache.lucene.util.Bits;
+import java.io.IOException;
+
+class AllDocsEnum extends DocsEnum {
+  protected final Bits skipDocs;
+  protected final int maxDoc;
+  protected final IndexReader reader;
+  protected int doc = -1;
+
+  protected AllDocsEnum(IndexReader reader, Bits skipDocs) {
+    this.skipDocs = skipDocs;
+    this.maxDoc = reader.maxDoc();
+    this.reader = reader;
+  }
+
+  public int freq() {
+    return 1;
+  }
+
+  public int next() throws IOException {
+    return advance(doc+1);
+  }
+
+  public int read(int[] docs, int[] freqs) throws IOException {
+    final int length = docs.length;
+    int i = 0;
+    while (i < length && doc < maxDoc) {
+      if (skipDocs == null || !skipDocs.get(doc)) {
+        docs[i] = doc;
+        freqs[i] = 1;
+        ++i;
+      }
+      doc++;
+    }
+    return i;
+  }
+
+  public int advance(int target) throws IOException {
+    doc = target;
+    while (doc < maxDoc) {
+      if (skipDocs == null || !skipDocs.get(doc)) {
+        return doc;
+      }
+      doc++;
+    }
+    doc = NO_MORE_DOCS;
+    return doc;
+  }
+
+  public PositionsEnum positions() {
+    throw new UnsupportedOperationException();
+  }
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllDocsEnum.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllTermDocs.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllTermDocs.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllTermDocs.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllTermDocs.java Tue Oct 13 20:44:51 2009
@@ -20,6 +20,7 @@
 import org.apache.lucene.util.BitVector;
 import java.io.IOException;
 
+/** @deprecated Switch to AllDocsEnum */
 class AllTermDocs implements TermDocs {
   protected BitVector deletedDocs;
   protected int maxDoc;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CheckIndex.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CheckIndex.java Tue Oct 13 20:44:51 2009
@@ -22,6 +22,8 @@
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.document.AbstractField;  // for javadocs
 import org.apache.lucene.document.Document;
+import org.apache.lucene.index.codecs.Codecs;
+import org.apache.lucene.util.Bits;
 
 import java.text.NumberFormat;
 import java.io.PrintStream;
@@ -271,24 +273,6 @@
       infoStream.println(msg);
   }
 
-  private static class MySegmentTermDocs extends SegmentTermDocs {
-
-    int delCount;
-
-    MySegmentTermDocs(SegmentReader p) {    
-      super(p);
-    }
-
-    public void seek(Term term) throws IOException {
-      super.seek(term);
-      delCount = 0;
-    }
-
-    protected void skippingDoc() throws IOException {
-      delCount++;
-    }
-  }
-
   /** Returns true if index is clean, else false. 
    *  @deprecated Please instantiate a CheckIndex and then use {@link #checkIndex()} instead */
   public static boolean check(Directory dir, boolean doFix) throws IOException {
@@ -319,6 +303,10 @@
     return checkIndex(null);
   }
 
+  protected Status checkIndex(List onlySegments) throws IOException {
+    return checkIndex(onlySegments, Codecs.getDefault());
+  }
+  
   /** Returns a {@link Status} instance detailing
    *  the state of the index.
    * 
@@ -331,13 +319,13 @@
    *  <p><b>WARNING</b>: make sure
    *  you only call this when the index is not opened by any
    *  writer. */
-  public Status checkIndex(List onlySegments) throws IOException {
+  protected Status checkIndex(List onlySegments, Codecs codecs) throws IOException {
     NumberFormat nf = NumberFormat.getInstance();
     SegmentInfos sis = new SegmentInfos();
     Status result = new Status();
     result.dir = dir;
     try {
-      sis.read(dir);
+      sis.read(dir, codecs);
     } catch (Throwable t) {
       msg("ERROR: could not read any segments file in directory");
       result.missingSegments = true;
@@ -394,6 +382,8 @@
         sFormat = "FORMAT_USER_DATA [Lucene 2.9]";
       else if (format == SegmentInfos.FORMAT_DIAGNOSTICS)
         sFormat = "FORMAT_DIAGNOSTICS [Lucene 2.9]";
+      else if (format == SegmentInfos.FORMAT_FLEX_POSTINGS)
+        sFormat = "FORMAT_FLEX_POSTINGS [Lucene 2.9]";
       else if (format < SegmentInfos.CURRENT_FORMAT) {
         sFormat = "int=" + format + " [newer version of Lucene than this tool]";
         skip = true;
@@ -615,66 +605,87 @@
   private Status.TermIndexStatus testTermIndex(SegmentInfo info, SegmentReader reader) {
     final Status.TermIndexStatus status = new Status.TermIndexStatus();
 
+    final int maxDoc = reader.maxDoc();
+    final Bits delDocs = reader.getDeletedDocs();
+
     try {
+
       if (infoStream != null) {
         infoStream.print("    test: terms, freq, prox...");
       }
+      
+      final FieldsEnum fields = reader.fields().iterator();
+      while(true) {
+        final String field = fields.next();
+        if (field == null) {
+          break;
+        }
+        
+        final TermsEnum terms = fields.terms();
+        while(true) {
 
-      final TermEnum termEnum = reader.terms();
-      final TermPositions termPositions = reader.termPositions();
-
-      // Used only to count up # deleted docs for this term
-      final MySegmentTermDocs myTermDocs = new MySegmentTermDocs(reader);
+          final TermRef term = terms.next();
+          if (term == null) {
+            break;
+          }
+          final int docFreq = terms.docFreq();
+          status.totFreq += docFreq;
 
-      final int maxDoc = reader.maxDoc();
+          final DocsEnum docs = terms.docs(delDocs);
+          status.termCount++;
 
-      while (termEnum.next()) {
-        status.termCount++;
-        final Term term = termEnum.term();
-        final int docFreq = termEnum.docFreq();
-        termPositions.seek(term);
-        int lastDoc = -1;
-        int freq0 = 0;
-        status.totFreq += docFreq;
-        while (termPositions.next()) {
-          freq0++;
-          final int doc = termPositions.doc();
-          final int freq = termPositions.freq();
-          if (doc <= lastDoc)
-            throw new RuntimeException("term " + term + ": doc " + doc + " <= lastDoc " + lastDoc);
-          if (doc >= maxDoc)
-            throw new RuntimeException("term " + term + ": doc " + doc + " >= maxDoc " + maxDoc);
-
-          lastDoc = doc;
-          if (freq <= 0)
-            throw new RuntimeException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds");
+          int lastDoc = -1;
+          int freq0 = 0;
+          while(true) {
+            final int doc = docs.next();
+            if (doc == DocsEnum.NO_MORE_DOCS) {
+              break;
+            }
+            final int freq = docs.freq();
+            status.totPos += freq;
+
+            freq0++;
+            if (doc <= lastDoc) {
+              throw new RuntimeException("term " + term + ": doc " + doc + " <= lastDoc " + lastDoc);
+            }
+            if (doc >= maxDoc) {
+              throw new RuntimeException("term " + term + ": doc " + doc + " >= maxDoc " + maxDoc);
+            }
+
+            lastDoc = doc;
+            if (freq <= 0) {
+              throw new RuntimeException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds");
+            }
             
-          int lastPos = -1;
-          status.totPos += freq;
-          for(int j=0;j<freq;j++) {
-            final int pos = termPositions.nextPosition();
-            if (pos < -1)
-              throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " is out of bounds");
-            if (pos < lastPos)
-              throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " < lastPos " + lastPos);
-            lastPos = pos;
+            int lastPos = -1;
+            final PositionsEnum positions = docs.positions();
+            if (positions != null) {
+              for(int j=0;j<freq;j++) {
+                final int pos = positions.next();
+                if (pos < -1) {
+                  throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " is out of bounds");
+                }
+                if (pos < lastPos) {
+                  throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " < lastPos " + lastPos);
+                }
+                lastPos = pos;
+              }
+            }
           }
-        }
 
-        // Now count how many deleted docs occurred in
-        // this term:
-        final int delCount;
-        if (reader.hasDeletions()) {
-          myTermDocs.seek(term);
-          while(myTermDocs.next()) { }
-          delCount = myTermDocs.delCount;
-        } else {
-          delCount = 0; 
-        }
+          // Now count how many deleted docs occurred in
+          // this term:
 
-        if (freq0 + delCount != docFreq) {
-          throw new RuntimeException("term " + term + " docFreq=" + 
-                                     docFreq + " != num docs seen " + freq0 + " + num docs deleted " + delCount);
+          if (reader.hasDeletions()) {
+            final DocsEnum docsNoDel = terms.docs(null);
+            int count = 0;
+            while(docsNoDel.next() != DocsEnum.NO_MORE_DOCS) {
+              count++;
+            }
+            if (count != docFreq) {
+              throw new RuntimeException("term " + term + " docFreq=" + docFreq + " != tot docs w/o deletions " + count);
+            }
+          }
         }
       }
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java Tue Oct 13 20:44:51 2009
@@ -17,25 +17,30 @@
  * limitations under the License.
  */
 
-import java.io.IOException;
 import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.Collections;
-import java.util.ArrayList;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.FieldSelector;
 import org.apache.lucene.search.DefaultSimilarity;
+import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.index.codecs.Codecs;
+import org.apache.lucene.util.PriorityQueue;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.ReaderUtil;
 
 /** 
  * An IndexReader which reads indexes with multiple segments.
@@ -43,6 +48,8 @@
 class DirectoryReader extends IndexReader implements Cloneable {
   protected Directory directory;
   protected boolean readOnly;
+  
+  protected Codecs codecs;
 
   IndexWriter writer;
 
@@ -63,28 +70,52 @@
   private int numDocs = -1;
   private boolean hasDeletions = false;
 
+  private MultiFields fields;
+
+//  static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly,
+//      final int termInfosIndexDivisor) throws CorruptIndexException, IOException {
+//    return open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor, null);
+//  }
+  
   static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly,
-                          final int termInfosIndexDivisor) throws CorruptIndexException, IOException {
+                          final int termInfosIndexDivisor, Codecs codecs) throws CorruptIndexException, IOException {
+    final Codecs codecs2;
+    if (codecs == null) {
+      codecs2 = Codecs.getDefault();
+    } else {
+      codecs2 = codecs;
+    }
     return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) {
       protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
         SegmentInfos infos = new SegmentInfos();
-        infos.read(directory, segmentFileName);
+        infos.read(directory, segmentFileName, codecs2);
         if (readOnly)
-          return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor);
+          return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor, codecs2);
         else
-          return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor);
+          return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor, codecs2);
       }
     }.run(commit);
   }
 
   /** Construct reading the named set of readers. */
-  DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws IOException {
+//  DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws IOException {
+//    this(directory, sis, deletionPolicy, readOnly, termInfosIndexDivisor, null);
+//  }
+  
+  /** Construct reading the named set of readers. */
+  DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, Codecs codecs) throws IOException {
     this.directory = directory;
     this.readOnly = readOnly;
     this.segmentInfos = sis;
     this.deletionPolicy = deletionPolicy;
     this.termInfosIndexDivisor = termInfosIndexDivisor;
 
+    if (codecs == null) {
+      this.codecs = Codecs.getDefault();
+    } else {
+      this.codecs = codecs;
+    }
+    
     if (!readOnly) {
       // We assume that this segments_N was previously
       // properly sync'd:
@@ -120,11 +151,18 @@
   }
 
   // Used by near real-time search
-  DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor) throws IOException {
+  DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor, Codecs codecs) throws IOException {
     this.directory = writer.getDirectory();
     this.readOnly = true;
     this.segmentInfos = infos;
     this.termInfosIndexDivisor = termInfosIndexDivisor;
+    if (codecs == null) {
+      this.codecs = Codecs.getDefault();
+    } else {
+      this.codecs = codecs;
+    }
+
+    
     if (!readOnly) {
       // We assume that this segments_N was previously
       // properly sync'd:
@@ -175,11 +213,17 @@
 
   /** This constructor is only used for {@link #reopen()} */
   DirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, int[] oldStarts,
-                  Map oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor) throws IOException {
+                  Map oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor, Codecs codecs) throws IOException {
     this.directory = directory;
     this.readOnly = readOnly;
     this.segmentInfos = infos;
     this.termInfosIndexDivisor = termInfosIndexDivisor;
+    if (codecs == null) {
+      this.codecs = Codecs.getDefault();
+    } else {
+      this.codecs = codecs;
+    }
+    
     if (!readOnly) {
       // We assume that this segments_N was previously
       // properly sync'd:
@@ -301,14 +345,75 @@
   private void initialize(SegmentReader[] subReaders) {
     this.subReaders = subReaders;
     starts = new int[subReaders.length + 1];    // build starts array
+    Bits[] subs = new Bits[subReaders.length];
     for (int i = 0; i < subReaders.length; i++) {
       starts[i] = maxDoc;
       maxDoc += subReaders[i].maxDoc();      // compute maxDocs
 
-      if (subReaders[i].hasDeletions())
+      if (subReaders[i].hasDeletions()) {
         hasDeletions = true;
+      }
+      subs[i] = subReaders[i].getDeletedDocs();
     }
     starts[subReaders.length] = maxDoc;
+
+    if (hasDeletions) {
+      deletedDocs = new MultiBits(subs, starts);
+    } else {
+      deletedDocs = null;
+    }
+
+    fields = new MultiFields(subReaders, starts);
+  }
+
+  private MultiBits deletedDocs;
+
+  // Exposes a slice of an existing Bits as a new Bits
+  final static class SubBits implements Bits {
+    private final Bits parent;
+    private final int start;
+    private final int length;
+
+    // start is inclusive; end is exclusive (length = end-start)
+    public SubBits(Bits parent, int start, int end) {
+      this.parent = parent;
+      this.start = start;
+      this.length = end - start;
+    }
+    
+    public boolean get(int doc) {
+      if (doc >= length) {
+        throw new RuntimeException("doc " + doc + " is out of bounds 0 .. " + (length-1));
+      }
+      return parent.get(doc-start);
+    }
+  }
+    
+  // Concatenates multiple Bits together
+  // nocommit -- if none of the subs have deletions we
+  // should return null from getDeletedDocs:
+  static final class MultiBits implements Bits {
+    private final Bits[] subs;
+    final int[] starts;
+
+    public MultiBits(Bits[] subs, int[] starts) {
+      this.subs = subs;
+      this.starts = starts;
+    }
+
+    public boolean get(int doc) {
+      final int reader = ReaderUtil.subIndex(doc, starts);
+      final Bits bits = subs[reader];
+      if (bits == null) {
+        return false;
+      } else {
+        return bits.get(doc-starts[reader]);
+      }
+    }
+  }
+
+  public Bits getDeletedDocs() {
+    return deletedDocs;
   }
 
   public final synchronized Object clone() {
@@ -423,7 +528,7 @@
     return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) {
       protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
         SegmentInfos infos = new SegmentInfos();
-        infos.read(directory, segmentFileName);
+        infos.read(directory, segmentFileName, codecs);
         return doReopen(infos, false, openReadOnly);
       }
     }.run(commit);
@@ -432,9 +537,9 @@
   private synchronized DirectoryReader doReopen(SegmentInfos infos, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException {
     DirectoryReader reader;
     if (openReadOnly) {
-      reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache, doClone, termInfosIndexDivisor);
+      reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache, doClone, termInfosIndexDivisor, null);
     } else {
-      reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false, doClone, termInfosIndexDivisor);
+      reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false, doClone, termInfosIndexDivisor, null);
     }
     reader.setDisableFakeNorms(getDisableFakeNorms());
     return reader;
@@ -626,10 +731,23 @@
     return total;
   }
 
+  public int docFreq(String field, TermRef term) throws IOException {
+    ensureOpen();
+    int total = 0;          // sum freqs in segments
+    for (int i = 0; i < subReaders.length; i++) {
+      total += subReaders[i].docFreq(field, term);
+    }
+    return total;
+  }
+
   public TermDocs termDocs() throws IOException {
     ensureOpen();
     return new MultiTermDocs(this, subReaders, starts);
   }
+  
+  public Fields fields() throws IOException {
+    return fields;
+  }
 
   public TermPositions termPositions() throws IOException {
     ensureOpen();
@@ -669,7 +787,7 @@
 
         // we have to check whether index has changed since this reader was opened.
         // if so, this reader is no longer valid for deletion
-        if (SegmentInfos.readCurrentVersion(directory) > segmentInfos.getVersion()) {
+        if (SegmentInfos.readCurrentVersion(directory, codecs) > segmentInfos.getVersion()) {
           stale = true;
           this.writeLock.release();
           this.writeLock = null;
@@ -699,7 +817,7 @@
       // KeepOnlyLastCommitDeleter:
       IndexFileDeleter deleter = new IndexFileDeleter(directory,
                                                       deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy,
-                                                      segmentInfos, null, null);
+                                                      segmentInfos, null, null, codecs);
 
       // Checkpoint the state we are about to change, in
       // case we have to roll back:
@@ -794,7 +912,7 @@
    */
   public boolean isCurrent() throws CorruptIndexException, IOException {
     ensureOpen();
-    return SegmentInfos.readCurrentVersion(directory) == segmentInfos.getVersion();
+    return SegmentInfos.readCurrentVersion(directory, codecs) == segmentInfos.getVersion();
   }
 
   protected synchronized void doClose() throws IOException {
@@ -861,12 +979,17 @@
 
   /** @see org.apache.lucene.index.IndexReader#listCommits */
   public static Collection listCommits(Directory dir) throws IOException {
+    return listCommits(dir, Codecs.getDefault());
+  }
+
+  /** @see org.apache.lucene.index.IndexReader#listCommits */
+  public static Collection listCommits(Directory dir, Codecs codecs) throws IOException {
     final String[] files = dir.listAll();
 
     Collection commits = new ArrayList();
 
     SegmentInfos latest = new SegmentInfos();
-    latest.read(dir);
+    latest.read(dir, codecs);
     final long currentGen = latest.getGeneration();
 
     commits.add(new ReaderCommit(latest, dir));
@@ -883,7 +1006,7 @@
         try {
           // IOException allowed to throw there, in case
           // segments_N is corrupt
-          sis.read(dir, fileName);
+          sis.read(dir, fileName, codecs);
         } catch (FileNotFoundException fnfe) {
           // LUCENE-948: on NFS (and maybe others), if
           // you have writers switching back and forth
@@ -954,20 +1077,496 @@
       return userData;
     }
   }
+  
+  private final static class TermsWithBase {
+    Terms terms;
+    int base;
+    int length;
+    Bits deletedDocs;
+
+    public TermsWithBase(IndexReader reader, int base, String field) throws IOException {
+      this.base = base;
+      length = reader.maxDoc();
+      deletedDocs = reader.getDeletedDocs();
+      terms = reader.fields().terms(field);
+    }
+  }
+
+  private final static class FieldsEnumWithBase {
+    FieldsEnum fields;
+    String current;
+    int base;
+    int length;
+    Bits deletedDocs;
+
+    public FieldsEnumWithBase(IndexReader reader, int base) throws IOException {
+      this.base = base;
+      length = reader.maxDoc();
+      deletedDocs = reader.getDeletedDocs();
+      fields = reader.fields().iterator();
+    }
+  }
+
+  private final static class TermsEnumWithBase {
+    TermsEnum terms;
+    int base;
+    int length;
+    TermRef current;
+    Bits deletedDocs;
+
+    public TermsEnumWithBase(FieldsEnumWithBase start, TermsEnum terms, TermRef term) {
+      this.terms = terms;
+      current = term;
+      deletedDocs = start.deletedDocs;
+      base = start.base;
+      length = start.length;
+    }
+
+    public TermsEnumWithBase(TermsWithBase start, TermsEnum terms, TermRef term) {
+      this.terms = terms;
+      current = term;
+      deletedDocs = start.deletedDocs;
+      base = start.base;
+      length = start.length;
+    }
+  }
+
+  private final static class DocsEnumWithBase {
+    DocsEnum docs;
+    int base;
+  }
+
+  private final static class FieldMergeQueue extends PriorityQueue {
+    FieldMergeQueue(int size) {
+      initialize(size);
+    }
+
+    protected final boolean lessThan(Object a, Object b) {
+      FieldsEnumWithBase fieldsA = (FieldsEnumWithBase) a;
+      FieldsEnumWithBase fieldsB = (FieldsEnumWithBase) b;
+      return fieldsA.current.compareTo(fieldsB.current) < 0;
+    }
+  }
+
+  private final static class TermMergeQueue extends PriorityQueue {
+    TermMergeQueue(int size) {
+      initialize(size);
+    }
+
+    protected final boolean lessThan(Object a, Object b) {
+      TermsEnumWithBase termsA = (TermsEnumWithBase) a;
+      TermsEnumWithBase termsB = (TermsEnumWithBase) b;
+      final int cmp = termsA.current.compareTerm(termsB.current);
+      if (cmp != 0) {
+        return cmp < 0;
+      } else {
+        return termsA.base < termsB.base;
+      }
+    }
+  }
+
+  final static class MultiFields extends Fields {
+    private final IndexReader[] readers;
+    private final int[] starts;
+    private final HashMap<String,MultiTerms> terms = new HashMap<String,MultiTerms>();
+
+    public MultiFields(IndexReader[] readers, int[] starts) {
+      this.readers = readers;
+      this.starts = starts;
+    }
+
+    public FieldsEnum iterator() throws IOException {
+      FieldsEnumWithBase[] subs = new FieldsEnumWithBase[readers.length];
+      for(int i=0;i<subs.length;i++) {
+        subs[i] = new FieldsEnumWithBase(readers[i], starts[i]);
+      }
+      return new MultiFieldsEnum(subs);
+    }
+
+    public Terms terms(String field) throws IOException {
+      MultiTerms result = terms.get(field);
+      if (result == null) {
+
+        List<TermsWithBase> subs = new ArrayList<TermsWithBase>();
+
+        // Gather all sub-readers that have this field
+        for(int i=0;i<readers.length;i++) {
+          Terms subTerms = readers[i].fields().terms(field);
+          if (subTerms != null) {
+            subs.add(new TermsWithBase(readers[i], starts[i], field));
+          }
+        }
+        result = new MultiTerms(subs.toArray(new TermsWithBase[]{}));
+        terms.put(field, result);
+      }
+      return result;
+    }
+
+    public void close() {
+    }
+  }
+    
+  private final static class MultiTerms extends Terms {
+    private final TermsWithBase[] subs;
+    
+    public MultiTerms(TermsWithBase[] subs) {
+      this.subs = subs;
+    }
+
+    public TermsEnum iterator() throws IOException {
+      return new MultiTermsEnum(subs.length).reset(subs);
+    }
+  }
+
+  private final static class MultiFieldsEnum extends FieldsEnum {
+    private final FieldMergeQueue queue;
+
+    private String currentField;
+
+    private final FieldsEnumWithBase[] top;
+    private int numTop;
+
+    private final MultiTermsEnum terms;
+
+    MultiFieldsEnum(FieldsEnumWithBase[] subs) throws IOException {
+      terms = new MultiTermsEnum(subs.length);
+      queue = new FieldMergeQueue(subs.length);
+      top = new FieldsEnumWithBase[subs.length];
+      for(int i=0;i<subs.length;i++) {
+        subs[i].current = subs[i].fields.next();
+        if (subs[i].current != null) {
+          queue.add(subs[i]);
+        }
+      }
+    }
+
+    public String field() {
+      assert currentField != null;
+      return currentField;
+    }
+
+    public String next() throws IOException {
+
+      // restore queue
+      for(int i=0;i<numTop;i++) {
+        top[i].current = top[i].fields.next();
+        if (top[i].current != null) {
+          queue.add(top[i]);
+        } else {
+          // no more fields in this reader
+        }
+      }
+
+      numTop = 0;
+
+      // gather equal top fields
+      if (queue.size() > 0) {
+        while(true) {
+          top[numTop++] = (FieldsEnumWithBase) queue.pop();
+          if (queue.size() == 0 || ((FieldsEnumWithBase) queue.top()).current != top[0].current) {
+            break;
+          }
+        }
+        currentField = top[0].current;
+      } else {
+        currentField = null;
+      }
+
+      return currentField;
+    }
+
+    public TermsEnum terms() throws IOException {
+      return terms.reset(top, numTop);
+    }
+  }
+
+  private static final class MultiTermsEnum extends TermsEnum {
+    
+    private final TermMergeQueue queue;
+    private final TermsEnumWithBase[] subs;
+    private final TermsEnumWithBase[] top;
+    int numTop;
+    int numSubs;
+    private TermRef current;
+    private final MultiDocsEnum docs;
+
+    MultiTermsEnum(int size) {
+      queue = new TermMergeQueue(size);
+      top = new TermsEnumWithBase[size];
+      subs = new TermsEnumWithBase[size];
+      docs = new MultiDocsEnum(size);
+    }
+
+    public TermRef term() {
+      return current;
+    }
+
+    MultiTermsEnum reset(TermsWithBase[] terms) throws IOException {
+      assert terms.length <= top.length;
+      numSubs = 0;
+      numTop = 0;
+      for(int i=0;i<terms.length;i++) {
+        final TermsEnum termsEnum = terms[i].terms.iterator();
+        if (termsEnum != null) {
+          final TermRef term = termsEnum.next();
+          if (term != null) {
+            subs[numSubs] = new TermsEnumWithBase(terms[i], termsEnum, term);
+            queue.add(subs[numSubs]);
+            numSubs++;
+          } else {
+            // field has no terms
+          }
+        }
+      }
+
+      return this;
+    }
+
+    MultiTermsEnum reset(FieldsEnumWithBase[] fields, int numFields) throws IOException {
+      assert numFields <= top.length;
+      numSubs = 0;
+      numTop = 0;
+      for(int i=0;i<numFields;i++) {
+        final TermsEnum terms = fields[i].fields.terms();
+        if (terms != null) {
+          final TermRef term = terms.next();
+          if (term != null) {
+            subs[numSubs] = new TermsEnumWithBase(fields[i], terms, term);
+            queue.add(subs[numSubs]);
+            numSubs++;
+          } else {
+            // field has no terms
+          }
+        }
+      }
+
+      return this;
+    }
+
+    public SeekStatus seek(TermRef term) throws IOException {
+      queue.clear();
+      numTop = 0;
+      for(int i=0;i<numSubs;i++) {
+        final SeekStatus status = subs[i].terms.seek(term);
+        if (status == SeekStatus.FOUND) {
+          top[numTop++] = subs[i];
+          current = subs[i].current = term;
+        } else if (status == SeekStatus.NOT_FOUND) {
+          queue.add(subs[i]);
+          current = subs[i].current = subs[i].terms.term();
+        } else {
+          // enum exhausted
+        }
+      }
+
+      if (numTop > 0) {
+        return SeekStatus.FOUND;
+      } else if (queue.size() > 0) {
+        pullTop();
+        return SeekStatus.NOT_FOUND;
+      } else {
+        return SeekStatus.END;
+      }
+    }
+
+    public SeekStatus seek(long ord) throws IOException {
+      throw new UnsupportedOperationException();
+    }
+
+    public long ord() throws IOException {
+      throw new UnsupportedOperationException();
+    }
+
+    private final void pullTop() {
+      assert numTop == 0;
+      while(true) {
+        top[numTop++] = (TermsEnumWithBase) queue.pop();
+        if (queue.size() == 0 || !((TermsEnumWithBase) queue.top()).current.termEquals(top[0].current)) {
+          break;
+        }
+      } 
+      current = top[0].current;
+    }
+
+    private final void pushTop() throws IOException {
+      for(int i=0;i<numTop;i++) {
+        top[i].current = top[i].terms.next();
+        if (top[i].current != null) {
+          queue.add(top[i]);
+        } else {
+          // no more fields in this reader
+        }
+      }
+
+      numTop = 0;
+    }
+
+    public TermRef next() throws IOException {
+      // restore queue
+      pushTop();
+
+      // gather equal top fields
+      if (queue.size() > 0) {
+        pullTop();
+      } else {
+        current = null;
+      }
+
+      return current;
+    }
+
+    public int docFreq() {
+      int sum = 0;
+      for(int i=0;i<numTop;i++) {
+        sum += top[i].terms.docFreq();
+      }
+      return sum;
+    }
+
+    public DocsEnum docs(Bits skipDocs) throws IOException {
+      return docs.reset(top, numTop, skipDocs);
+    }
+  }
+
+  private static final class MultiDocsEnum extends DocsEnum {
+    final DocsEnumWithBase[] subs;
+    int numSubs;
+    int upto;
+    DocsEnum currentDocs;
+    int currentBase;
+    Bits skipDocs;
+
+    MultiDocsEnum(int count) {
+      subs = new DocsEnumWithBase[count];
+    }
+
+    MultiDocsEnum reset(TermsEnumWithBase[] subs, final int numSubs, final Bits skipDocs) throws IOException {
+      this.numSubs = 0;
+      this.skipDocs = skipDocs;
+      for(int i=0;i<numSubs;i++) {
+        Bits bits = null;
+        boolean handled = false;
+
+        // Optimize for common case: requested skip docs is simply our
+        // deleted docs
+        if (skipDocs instanceof MultiBits) {
+          MultiBits multiBits = (MultiBits) skipDocs;
+          int reader = ReaderUtil.subIndex(subs[i].base, multiBits.starts);
+          // System.out.println("bits=" + multiBits + " starts=" + multiBits.starts + " subs=" + subs + " subs[i]=" + subs[i] + " subs[1+i]=" + subs[1+i] + " i=" + i + " numSubs=" + numSubs);
+          if (multiBits.starts[reader] == subs[i].base &&
+              (i == numSubs-1 ||
+               reader == multiBits.starts.length-1 ||
+               multiBits.starts[1+reader] == subs[1+i].base)) {
+            bits = multiBits.subs[reader];
+            handled = true;
+          }
+        }
+
+        if (!handled && skipDocs != null) {
+          // custom case: requested skip docs is foreign
+          bits = new SubBits(skipDocs, subs[i].base, subs[i].length);
+        }
+
+        final DocsEnum docs = subs[i].terms.docs(bits);
+        if (docs != null) {
+          this.subs[this.numSubs] = new DocsEnumWithBase();
+          this.subs[this.numSubs].docs = docs;
+          this.subs[this.numSubs].base = subs[i].base;
+          this.numSubs++;
+        }
+      }
+      upto = -1;
+      currentDocs = null;
+      return this;
+    }
+
+    public int freq() {
+      return currentDocs.freq();
+    }
+
+    public int read(final int docs[], final int freqs[]) throws IOException {
+      while (true) {
+        while (currentDocs == null) {
+          if (upto == numSubs-1) {
+            return 0;
+          } else {
+            upto++;
+            currentDocs = subs[upto].docs;
+            currentBase = subs[upto].base;
+          }
+        }
+        final int end = currentDocs.read(docs, freqs);
+        if (end == 0) {          // none left in segment
+          currentDocs = null;
+        } else {            // got some
+          for (int i = 0; i < end; i++) {
+            docs[i] += currentBase;
+          }
+          return end;
+        }
+      }
+    }
+
+    public int advance(int target) throws IOException {
+      while(true) {
+        if (currentDocs != null) {
+          final int doc = currentDocs.advance(target-currentBase);
+          if (doc == NO_MORE_DOCS) {
+            currentDocs = null;
+          } else {
+            return doc + currentBase;
+          }
+        } else if (upto == numSubs-1) {
+          return NO_MORE_DOCS;
+        } else {
+          upto++;
+          currentDocs = subs[upto].docs;
+          currentBase = subs[upto].base;
+        }
+      }
+    }
+
+    public int next() throws IOException {
+      while(true) {
+        if (currentDocs == null) {
+          if (upto == numSubs-1) {
+            return NO_MORE_DOCS;
+          } else {
+            upto++;
+            currentDocs = subs[upto].docs;
+            currentBase = subs[upto].base;
+          }
+        }
+
+        final int doc = currentDocs.next();
+        if (doc != NO_MORE_DOCS) {
+          return currentBase + doc;
+        } else {
+          currentDocs = null;
+        }
+      }
+    }
+
+    public PositionsEnum positions() throws IOException {
+      return currentDocs.positions();
+    }
+  }
+
 
+  // Legacy API
   static class MultiTermEnum extends TermEnum {
     IndexReader topReader; // used for matching TermEnum to TermDocs
-    private SegmentMergeQueue queue;
+    private LegacySegmentMergeQueue queue;
   
     private Term term;
     private int docFreq;
-    final SegmentMergeInfo[] matchingSegments; // null terminated array of matching segments
+    final LegacySegmentMergeInfo[] matchingSegments; // null terminated array of matching segments
 
     public MultiTermEnum(IndexReader topReader, IndexReader[] readers, int[] starts, Term t)
       throws IOException {
       this.topReader = topReader;
-      queue = new SegmentMergeQueue(readers.length);
-      matchingSegments = new SegmentMergeInfo[readers.length+1];
+      queue = new LegacySegmentMergeQueue(readers.length);
+      matchingSegments = new LegacySegmentMergeInfo[readers.length+1];
       for (int i = 0; i < readers.length; i++) {
         IndexReader reader = readers[i];
         TermEnum termEnum;
@@ -977,7 +1576,7 @@
         } else
           termEnum = reader.terms();
   
-        SegmentMergeInfo smi = new SegmentMergeInfo(starts[i], termEnum, reader);
+        LegacySegmentMergeInfo smi = new LegacySegmentMergeInfo(starts[i], termEnum, reader);
         smi.ord = i;
         if (t == null ? smi.next() : termEnum.term() != null)
           queue.add(smi);          // initialize queue
@@ -992,7 +1591,7 @@
   
     public boolean next() throws IOException {
       for (int i=0; i<matchingSegments.length; i++) {
-        SegmentMergeInfo smi = matchingSegments[i];
+        LegacySegmentMergeInfo smi = matchingSegments[i];
         if (smi==null) break;
         if (smi.next())
           queue.add(smi);
@@ -1003,7 +1602,7 @@
       int numMatchingSegments = 0;
       matchingSegments[0] = null;
 
-      SegmentMergeInfo top = (SegmentMergeInfo)queue.top();
+      LegacySegmentMergeInfo top = (LegacySegmentMergeInfo)queue.top();
 
       if (top == null) {
         term = null;
@@ -1017,7 +1616,7 @@
         matchingSegments[numMatchingSegments++] = top;
         queue.pop();
         docFreq += top.termEnum.docFreq();    // increment freq
-        top = (SegmentMergeInfo)queue.top();
+        top = (LegacySegmentMergeInfo)queue.top();
       }
 
       matchingSegments[numMatchingSegments] = null;
@@ -1037,6 +1636,7 @@
     }
   }
 
+  // Legacy API
   static class MultiTermDocs implements TermDocs {
     IndexReader topReader;  // used for matching TermEnum to TermDocs
     protected IndexReader[] readers;
@@ -1051,7 +1651,7 @@
 
     private MultiTermEnum tenum;  // the term enum used for seeking... can be null
     int matchingSegmentPos;  // position into the matching segments from tenum
-    SegmentMergeInfo smi;     // current segment mere info... can be null
+    LegacySegmentMergeInfo smi;     // current segment mere info... can be null
 
     public MultiTermDocs(IndexReader topReader, IndexReader[] r, int[] s) {
       this.topReader = topReader;
@@ -1147,7 +1747,7 @@
           return true;
         } else if (pointer < readers.length) {
           if (tenum != null) {
-            SegmentMergeInfo smi = tenum.matchingSegments[matchingSegmentPos++];
+            LegacySegmentMergeInfo smi = tenum.matchingSegments[matchingSegmentPos++];
             if (smi==null) {
               pointer = readers.length;
               return false;
@@ -1188,6 +1788,7 @@
     }
   }
 
+  // Legacy API
   static class MultiTermPositions extends MultiTermDocs implements TermPositions {
     public MultiTermPositions(IndexReader topReader, IndexReader[] r, int[] s) {
       super(topReader,r,s);

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,67 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.util.AttributeSource;
+
+/** On obtaining a DocsEnum, you must first call next() */
+
+public abstract class DocsEnum extends AttributeSource {
+  // nocommit
+  public String desc;
+
+  public final static int NO_MORE_DOCS = Integer.MAX_VALUE;
+
+  /** Moves forward to the doc id >= target */
+  public abstract int advance(int target) throws IOException;
+
+  /** Returns the next docID, {@link #NO_MORE_DOCS} at the end. */
+  public abstract int next() throws IOException;
+
+  public abstract int freq();
+  
+  // nocommit -- fix this API so that intblock codecs are
+  // able to return their own int arrays, to save a copy
+  /** Bulk read: returns number of docs read.  Subclass may
+   * do this more efficiently. */
+  public int read(int[] docs, int[] freqs) throws IOException {
+    int count = 0;
+    while(count < docs.length) {
+      final int doc = next();
+      if (doc != NO_MORE_DOCS) {
+        docs[count] = doc;
+        freqs[count] = freq();
+        count++;
+      } else {
+        break;
+      }
+    }
+    return count;
+  }
+
+  // nocommit -- maybe move this up to TermsEnum?  that
+  // would disallow changing positions format/reader of each
+  // doc, though
+  // nocommit - doc whether this returns null if there are
+  // no positions, or a faker
+  /** Don't call next() or skipTo() or read() until you're
+   *  done consuming the positions */
+  public abstract PositionsEnum positions() throws IOException;
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocumentsWriter.java Tue Oct 13 20:44:51 2009
@@ -40,6 +40,8 @@
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.Constants;
 
+import org.apache.lucene.index.codecs.Codec;
+
 /**
  * This class accepts multiple added documents and directly
  * writes a single segment file.  It does this more
@@ -545,9 +547,16 @@
 
   synchronized private void initFlushState(boolean onlyDocStore) {
     initSegmentName(onlyDocStore);
-    flushState = new SegmentWriteState(this, directory, segment, docStoreSegment, numDocsInRAM, numDocsInStore, writer.getTermIndexInterval());
+    flushState = new SegmentWriteState(this, directory, segment, docFieldProcessor.fieldInfos,
+                                       docStoreSegment, numDocsInRAM, numDocsInStore, writer.getTermIndexInterval(),
+                                       writer.codecs);
   }
 
+  /** Returns the codec used to flush the last segment */
+  Codec getCodec() {
+    return flushState.codec;
+  }
+  
   /** Flush all pending docs to a new segment */
   synchronized int flush(boolean closeDocStore) throws IOException {
 
@@ -583,7 +592,8 @@
       consumer.flush(threads, flushState);
 
       if (infoStream != null) {
-        final long newSegmentSize = segmentSize(flushState.segmentName);
+        SegmentInfo si = new SegmentInfo(flushState.segmentName, flushState.numDocs, directory, flushState.codec);
+        final long newSegmentSize = si.sizeInBytes();
         String message = "  oldRAMSize=" + numBytesUsed +
           " newFlushedSize=" + newSegmentSize +
           " docs/MB=" + nf.format(numDocsInRAM/(newSegmentSize/1024./1024.)) +
@@ -613,8 +623,12 @@
     
     CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION);
     Iterator it = flushState.flushedFiles.iterator();
-    while(it.hasNext())
-      cfsWriter.addFile((String) it.next());
+    while(it.hasNext()) {
+      final String fileName = (String) it.next();
+      if (Codec.DEBUG)
+        System.out.println("make cfs " + fileName);
+      cfsWriter.addFile(fileName);
+    }
       
     // Perform the merge
     cfsWriter.close();
@@ -970,24 +984,27 @@
 
     // Delete by term
     Iterator iter = deletesFlushed.terms.entrySet().iterator();
-    TermDocs docs = reader.termDocs();
+    
     try {
       while (iter.hasNext()) {
         Entry entry = (Entry) iter.next();
         Term term = (Term) entry.getKey();
 
-        docs.seek(term);
-        int limit = ((BufferedDeletes.Num) entry.getValue()).getNum();
-        while (docs.next()) {
-          int docID = docs.doc();
-          if (docIDStart+docID >= limit)
-            break;
-          reader.deleteDocument(docID);
-          any = true;
+        DocsEnum docs = reader.termDocsEnum(reader.getDeletedDocs(), term.field, new TermRef(term.text));
+        if (docs != null) {
+          int limit = ((BufferedDeletes.Num) entry.getValue()).getNum();
+          while (true) {
+            final int docID = docs.next();
+            if (docID == DocsEnum.NO_MORE_DOCS || docIDStart+docID >= limit) {
+              break;
+            }
+            reader.deleteDocument(docID);
+            any = true;
+          }
         }
       }
     } finally {
-      docs.close();
+      //docs.close();
     }
 
     // Delete by docID
@@ -1140,24 +1157,6 @@
 
   NumberFormat nf = NumberFormat.getInstance();
 
-  // TODO FI: this is not flexible -- we can't hardwire
-  // extensions in here:
-  private long segmentSize(String segmentName) throws IOException {
-    // Used only when infoStream != null
-    assert infoStream != null;
-    
-    long size = directory.fileLength(segmentName + ".tii") +
-      directory.fileLength(segmentName + ".tis") +
-      directory.fileLength(segmentName + ".frq") +
-      directory.fileLength(segmentName + ".prx");
-
-    final String normFileName = segmentName + ".nrm";
-    if (directory.fileExists(normFileName))
-      size += directory.fileLength(normFileName);
-
-    return size;
-  }
-
   // Coarse estimates used to measure RAM usage of buffered deletes
   final static int OBJECT_HEADER_BYTES = 8;
   final static int POINTER_NUM_BYTE = Constants.JRE_IS_64BIT ? 8 : 4;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldInfo.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldInfo.java Tue Oct 13 20:44:51 2009
@@ -17,20 +17,27 @@
  * limitations under the License.
  */
 
-final class FieldInfo {
-  String name;
-  boolean isIndexed;
-  int number;
+// nocommit -- made this public:
+public final class FieldInfo {
+  // nocommit -- made this public
+  public String name;
+  // nocommit -- made this public
+  public boolean isIndexed;
+  // nocommit -- made this public
+  public int number;
 
   // true if term vector for this field should be stored
   boolean storeTermVector;
   boolean storeOffsetWithTermVector;
   boolean storePositionWithTermVector;
 
-  boolean omitNorms; // omit norms associated with indexed fields  
-  boolean omitTermFreqAndPositions;
-  
-  boolean storePayloads; // whether this field stores payloads together with term positions
+  // nocommit -- made this public
+  public boolean omitNorms; // omit norms associated with indexed fields  
+  // nocommit -- made this public
+  public boolean omitTermFreqAndPositions;
+
+  // nocommit -- made public
+  public boolean storePayloads; // whether this field stores payloads together with term positions
 
   FieldInfo(String na, boolean tk, int nu, boolean storeTermVector, 
             boolean storePositionWithTermVector,  boolean storeOffsetWithTermVector, 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldInfos.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldInfos.java Tue Oct 13 20:44:51 2009
@@ -33,7 +33,8 @@
  *  be adding documents at a time, with no other reader or writer threads
  *  accessing this object.
  */
-final class FieldInfos {
+// nocommit -- made this public:
+public final class FieldInfos {
 
   // Used internally (ie not written to *.fnm files) for pre-2.9 files
   public static final int FORMAT_PRE = -1;
@@ -121,14 +122,19 @@
   }
 
   /** Returns true if any fields do not omitTermFreqAndPositions */
-  boolean hasProx() {
+  // nocommit -- made public
+  public boolean hasProx() {
     final int numFields = byNumber.size();
     for(int i=0;i<numFields;i++) {
       final FieldInfo fi = fieldInfo(i);
       if (fi.isIndexed && !fi.omitTermFreqAndPositions) {
+        // mxx
+        //        System.out.println(Thread.currentThread().getName() + ": fieldInfos: hasProx=true");
         return true;
       }
     }
+    // mxx
+    //System.out.println(Thread.currentThread().getName() + ": fieldInfos: hasProx=false");
     return false;
   }
   

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Fields.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Fields.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Fields.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Fields.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,46 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+// TODO: split out an "iterator" api from the terms(String
+// field) API?
+
+// nocommit -- intended to be forward only?  eg no "reset"?
+
+/** Access to fields and terms
+ *
+ * NOTE: this API is experimental and will likely change */
+
+// TODO: someday expose public version of FieldInfos here
+public abstract class Fields {
+
+  // nocommit -- clarify if this is forwards only.  should
+  // this be "skipTo"?
+  // nocommit -- clarify: when this returns false, what is
+  // its internal state?  eg if i call field() after getting
+  // false back?
+  /** Returns an iterator that will step through all fields
+   *  names */
+  public abstract FieldsEnum iterator() throws IOException;
+
+  /** Get the {@link Terms} for this field */
+  public abstract Terms terms(String field) throws IOException;
+}
+

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Fields.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsEnum.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsEnum.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsEnum.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,41 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.util.AttributeSource;
+
+/** Enumerates indexed fields.
+ *
+ * NOTE: this API is experimental and will likely change */
+
+public abstract class FieldsEnum extends AttributeSource {
+
+  // nocommit -- do we need seek?
+
+  /** Increments the enumeration to the next field.
+   *  Returns null when there are no more fields.*/
+  public abstract String next() throws IOException;
+
+  /** Get TermsEnum for the current field.  You should not
+   *  call {@link #next()} until you're done using this
+   *  TermsEnum. */
+  public abstract TermsEnum terms() throws IOException;
+}
+

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsEnum.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FilterIndexReader.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FilterIndexReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FilterIndexReader.java Tue Oct 13 20:44:51 2009
@@ -20,6 +20,7 @@
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.FieldSelector;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -109,6 +110,10 @@
     return in.directory();
   }
   
+  public Bits getDeletedDocs() throws IOException {
+    return in.getDeletedDocs();
+  }
+  
   public TermFreqVector[] getTermFreqVectors(int docNumber)
           throws IOException {
     ensureOpen();
@@ -194,6 +199,11 @@
     return in.docFreq(t);
   }
 
+  public int docFreq(String field, TermRef t) throws IOException {
+    ensureOpen();
+    return in.docFreq(field, t);
+  }
+
   public TermDocs termDocs() throws IOException {
     ensureOpen();
     return in.termDocs();

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java Tue Oct 13 20:44:51 2009
@@ -17,17 +17,19 @@
  * limitations under the License.
  */
 
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.UnicodeUtil;
-
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Map;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.index.codecs.DocsConsumer;
+import org.apache.lucene.index.codecs.FieldsConsumer;
+import org.apache.lucene.index.codecs.PositionsConsumer;
+import org.apache.lucene.index.codecs.TermsConsumer;
+import org.apache.lucene.util.UnicodeUtil;
 
 final class FreqProxTermsWriter extends TermsHashConsumer {
 
@@ -60,6 +62,7 @@
   void closeDocStore(SegmentWriteState state) {}
   void abort() {}
 
+  private int flushedDocCount;
 
   // TODO: would be nice to factor out more of this, eg the
   // FreqProxFieldMergeState, and code to visit all Fields
@@ -71,6 +74,8 @@
     // Gather all FieldData's that have postings, across all
     // ThreadStates
     List allFields = new ArrayList();
+    
+    flushedDocCount = state.numDocs;
 
     Iterator it = threadsAndFields.entrySet().iterator();
     while(it.hasNext()) {
@@ -88,21 +93,23 @@
       }
     }
 
+    final int numAllFields = allFields.size();
+
     // Sort by field name
     Collections.sort(allFields);
-    final int numAllFields = allFields.size();
 
-    // TODO: allow Lucene user to customize this consumer:
-    final FormatPostingsFieldsConsumer consumer = new FormatPostingsFieldsWriter(state, fieldInfos);
+    // TODO: allow Lucene user to customize this codec:
+    final FieldsConsumer consumer = state.codec.fieldsConsumer(state);
+
     /*
     Current writer chain:
-      FormatPostingsFieldsConsumer
-        -> IMPL: FormatPostingsFieldsWriter
-          -> FormatPostingsTermsConsumer
-            -> IMPL: FormatPostingsTermsWriter
-              -> FormatPostingsDocConsumer
-                -> IMPL: FormatPostingsDocWriter
-                  -> FormatPostingsPositionsConsumer
+      FieldsConsumer
+        -> IMPL: FormatPostingsTermsDictWriter
+          -> TermsConsumer
+            -> IMPL: FormatPostingsTermsDictWriter.TermsWriter
+              -> DocsConsumer
+                -> IMPL: FormatPostingsDocsWriter
+                  -> PositionsConsumer
                     -> IMPL: FormatPostingsPositionsWriter
     */
 
@@ -145,8 +152,7 @@
       FreqProxTermsWriterPerThread perThread = (FreqProxTermsWriterPerThread) entry.getKey();
       perThread.termsHashPerThread.reset(true);
     }
-
-    consumer.finish();
+    consumer.close();
   }
 
   private byte[] payloadBuffer;
@@ -155,7 +161,7 @@
    * instances) found in this field and serialize them
    * into a single RAM segment. */
   void appendPostings(FreqProxTermsWriterPerField[] fields,
-                      FormatPostingsFieldsConsumer consumer)
+                      FieldsConsumer consumer)
     throws CorruptIndexException, IOException {
 
     int numFields = fields.length;
@@ -172,7 +178,7 @@
       assert result;
     }
 
-    final FormatPostingsTermsConsumer termsConsumer = consumer.addField(fields[0].fieldInfo);
+    final TermsConsumer termsConsumer = consumer.addField(fields[0].fieldInfo);
 
     FreqProxFieldMergeState[] termStates = new FreqProxFieldMergeState[numFields];
 
@@ -196,11 +202,18 @@
           termStates[numToMerge++] = mergeStates[i];
       }
 
-      final FormatPostingsDocsConsumer docConsumer = termsConsumer.addTerm(termStates[0].text, termStates[0].textOffset);
+      final char[] termText = termStates[0].text;
+      final int termTextOffset = termStates[0].textOffset;
+
+      // nocommit
+      //System.out.println("FLUSH term=" + new String(termText, termTextOffset, 10));
+
+      final DocsConsumer docConsumer = termsConsumer.startTerm(termText, termTextOffset);
 
       // Now termStates has numToMerge FieldMergeStates
       // which all share the same term.  Now we must
       // interleave the docID streams.
+      int numDocs = 0;
       while(numToMerge > 0) {
         
         FreqProxFieldMergeState minState = termStates[0];
@@ -209,8 +222,12 @@
             minState = termStates[i];
 
         final int termDocFreq = minState.termFreq;
+        numDocs++;
 
-        final FormatPostingsPositionsConsumer posConsumer = docConsumer.addDoc(minState.docID, termDocFreq);
+        assert minState.docID < flushedDocCount: "doc=" + minState.docID + " maxDoc=" + flushedDocCount;
+
+        //System.out.println("  docID=" + minState.docID);
+        final PositionsConsumer posConsumer = docConsumer.addDoc(minState.docID, termDocFreq);
 
         final ByteSliceReader prox = minState.prox;
 
@@ -224,6 +241,7 @@
           for(int j=0;j<termDocFreq;j++) {
             final int code = prox.readVInt();
             position += code >> 1;
+            //System.out.println("    pos=" + position);
 
             final int payloadLength;
             if ((code & 1) != 0) {
@@ -241,7 +259,7 @@
             posConsumer.addPosition(position, payloadBuffer, 0, payloadLength);
           } //End for
 
-          posConsumer.finish();
+          posConsumer.finishDoc();
         }
 
         if (!minState.nextDoc()) {
@@ -269,14 +287,12 @@
         }
       }
 
-      docConsumer.finish();
+      termsConsumer.finishTerm(termText, termTextOffset, numDocs);
     }
 
     termsConsumer.finish();
   }
 
-  private final TermInfo termInfo = new TermInfo(); // minimize consing
-
   final UnicodeUtil.UTF8Result termsUTF8 = new UnicodeUtil.UTF8Result();
 
   void files(Collection files) {}

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java Tue Oct 13 20:44:51 2009
@@ -17,18 +17,21 @@
  * limitations under the License.
  */
 
-import org.apache.lucene.store.Directory;
-
-import java.io.IOException;
+import java.io.File;
 import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import java.io.IOException;
 import java.io.PrintStream;
-import java.util.Map;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Collection;
+import java.util.Map;
+
+import org.apache.lucene.index.codecs.Codecs;
+import org.apache.lucene.store.Directory;
 
 /*
  * This class keeps track of each SegmentInfos instance that
@@ -114,6 +117,8 @@
     infoStream.println("IFD [" + Thread.currentThread().getName() + "]: " + message);
   }
 
+  private final FilenameFilter indexFilenameFilter;
+
   /**
    * Initialize the deleter: find all previous commits in
    * the Directory, incref the files they reference, call
@@ -122,7 +127,8 @@
    * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
    */
-  public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream, DocumentsWriter docWriter)
+  public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream, DocumentsWriter docWriter,
+                          Codecs codecs)
     throws CorruptIndexException, IOException {
 
     this.docWriter = docWriter;
@@ -137,8 +143,28 @@
     // First pass: walk the files and initialize our ref
     // counts:
     long currentGen = segmentInfos.getGeneration();
-    IndexFileNameFilter filter = IndexFileNameFilter.getFilter();
+    final Collection codecsExtensions = codecs.getAllExtensions();
+    final FilenameFilter mainFilter = IndexFileNameFilter.getFilter();
 
+    indexFilenameFilter = new FilenameFilter() {
+        public boolean accept(File dir, String name) {
+          if (mainFilter.accept(dir, name)) {
+            return true;
+          } else {
+            // See if any of the codecs claim this
+            // extension:
+            int i = name.lastIndexOf('.');
+            if (i != -1) {
+              String extension = name.substring(1+i);
+              if (codecsExtensions.contains(extension)) {
+                return true;
+              }
+            }
+            return false;
+        }
+      }
+      };
+    
     String[] files = directory.listAll();
 
     CommitPoint currentCommitPoint = null;
@@ -147,7 +173,7 @@
 
       String fileName = files[i];
 
-      if (filter.accept(null, fileName) && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
+      if ((indexFilenameFilter.accept(null, fileName)) && !fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
 
         // Add this file to refCounts with initial count 0:
         getRefCount(fileName);
@@ -163,7 +189,7 @@
             }
             SegmentInfos sis = new SegmentInfos();
             try {
-              sis.read(directory, fileName);
+              sis.read(directory, fileName, codecs);
             } catch (FileNotFoundException e) {
               // LUCENE-948: on NFS (and maybe others), if
               // you have writers switching back and forth
@@ -200,7 +226,7 @@
       // try now to explicitly open this commit point:
       SegmentInfos sis = new SegmentInfos();
       try {
-        sis.read(directory, segmentInfos.getCurrentSegmentFileName());
+        sis.read(directory, segmentInfos.getCurrentSegmentFileName(), codecs);
       } catch (IOException e) {
         throw new CorruptIndexException("failed to locate current segments_N file");
       }
@@ -298,7 +324,6 @@
    */
   public void refresh(String segmentName) throws IOException {
     String[] files = directory.listAll();
-    IndexFileNameFilter filter = IndexFileNameFilter.getFilter();
     String segmentPrefix1;
     String segmentPrefix2;
     if (segmentName != null) {
@@ -311,8 +336,8 @@
     
     for(int i=0;i<files.length;i++) {
       String fileName = files[i];
-      if (filter.accept(null, fileName) &&
-          (segmentName == null || fileName.startsWith(segmentPrefix1) || fileName.startsWith(segmentPrefix2)) &&
+      if ((segmentName == null || fileName.startsWith(segmentPrefix1) || fileName.startsWith(segmentPrefix2)) &&
+          indexFilenameFilter.accept(null, fileName) &&
           !refCounts.containsKey(fileName) &&
           !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
         // Unreferenced file, so remove it

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileNames.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileNames.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileNames.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileNames.java Tue Oct 13 20:44:51 2009
@@ -20,7 +20,8 @@
 /**
  * Useful constants representing filenames and extensions used by lucene
  */
-final class IndexFileNames {
+// nocommit -- made public
+public final class IndexFileNames {
 
   /** Name of the index segment file */
   static final String SEGMENTS = "segments";
@@ -36,16 +37,16 @@
   static final String NORMS_EXTENSION = "nrm";
 
   /** Extension of freq postings file */
-  static final String FREQ_EXTENSION = "frq";
+  //static final String FREQ_EXTENSION = "frq";
 
   /** Extension of prox postings file */
-  static final String PROX_EXTENSION = "prx";
+  //static final String PROX_EXTENSION = "prx";
 
   /** Extension of terms file */
-  static final String TERMS_EXTENSION = "tis";
+  //static final String TERMS_EXTENSION = "tis";
 
   /** Extension of terms index file */
-  static final String TERMS_INDEX_EXTENSION = "tii";
+  //static final String TERMS_INDEX_EXTENSION = "tii";
 
   /** Extension of stored fields index file */
   static final String FIELDS_INDEX_EXTENSION = "fdx";
@@ -96,10 +97,10 @@
     FIELD_INFOS_EXTENSION,
     FIELDS_INDEX_EXTENSION,
     FIELDS_EXTENSION,
-    TERMS_INDEX_EXTENSION,
-    TERMS_EXTENSION,
-    FREQ_EXTENSION,
-    PROX_EXTENSION,
+    //TERMS_INDEX_EXTENSION,
+    //TERMS_EXTENSION,
+    //FREQ_EXTENSION,
+    //PROX_EXTENSION,
     DELETES_EXTENSION,
     VECTORS_INDEX_EXTENSION,
     VECTORS_DOCUMENTS_EXTENSION,
@@ -107,6 +108,11 @@
     GEN_EXTENSION,
     NORMS_EXTENSION,
     COMPOUND_FILE_STORE_EXTENSION,
+    // nocommit -- need cleaner way!
+    "doc",
+    "pos",
+    "pyl",
+    "skp"
   };
 
   /** File extensions that are added to a compound file
@@ -115,10 +121,10 @@
     FIELD_INFOS_EXTENSION,
     FIELDS_INDEX_EXTENSION,
     FIELDS_EXTENSION,
-    TERMS_INDEX_EXTENSION,
-    TERMS_EXTENSION,
-    FREQ_EXTENSION,
-    PROX_EXTENSION,
+    //TERMS_INDEX_EXTENSION,
+    //TERMS_EXTENSION,
+    //FREQ_EXTENSION,
+    //PROX_EXTENSION,
     VECTORS_INDEX_EXTENSION,
     VECTORS_DOCUMENTS_EXTENSION,
     VECTORS_FIELDS_EXTENSION,
@@ -135,22 +141,28 @@
 
   static final String[] NON_STORE_INDEX_EXTENSIONS = new String[] {
     FIELD_INFOS_EXTENSION,
-    FREQ_EXTENSION,
-    PROX_EXTENSION,
-    TERMS_EXTENSION,
-    TERMS_INDEX_EXTENSION,
+    //FREQ_EXTENSION,
+    //PROX_EXTENSION,
+    //TERMS_EXTENSION,
+    //TERMS_INDEX_EXTENSION,
     NORMS_EXTENSION
   };
   
   /** File extensions of old-style index files */
   static final String COMPOUND_EXTENSIONS[] = new String[] {
     FIELD_INFOS_EXTENSION,
-    FREQ_EXTENSION,
-    PROX_EXTENSION,
+    //FREQ_EXTENSION,
+    //PROX_EXTENSION,
+    FIELDS_INDEX_EXTENSION,
+    FIELDS_EXTENSION,
+    //TERMS_INDEX_EXTENSION,
+    //TERMS_EXTENSION
+  };
+
+  static final String COMPOUND_EXTENSIONS_NOT_CODEC[] = new String[] {
+    FIELD_INFOS_EXTENSION,
     FIELDS_INDEX_EXTENSION,
     FIELDS_EXTENSION,
-    TERMS_INDEX_EXTENSION,
-    TERMS_EXTENSION
   };
   
   /** File extensions for term vector support */
@@ -194,7 +206,8 @@
     return false;
   }
 
-  static String segmentFileName(String segmentName, String ext) {
+  // nocommit -- made public
+  public static String segmentFileName(String segmentName, String ext) {
     return segmentName + "." + ext;
   }
 }



Mime
View raw message