lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r892930 [1/2] - in /lucene/java/branches/flex_1458: contrib/misc/src/java/org/apache/lucene/index/ src/java/org/apache/lucene/index/ src/java/org/apache/lucene/index/codecs/ src/java/org/apache/lucene/index/codecs/preflex/ src/java/org/apac...
Date Mon, 21 Dec 2009 19:10:31 GMT
Author: mikemccand
Date: Mon Dec 21 19:10:29 2009
New Revision: 892930

URL: http://svn.apache.org/viewvc?rev=892930&view=rev
Log:
LUCENE-2111 (flex branch): strengthen back compat layers

Added:
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/FlexTestUtil.java   (with props)
Modified:
    lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Fields.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsEnum.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFields.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFieldsEnum.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MultiReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermDocs.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermPositions.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsEnum.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/DocsConsumer.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/TermsConsumer.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardDocsReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPositionsReader.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestCodecs.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestFlexExternalReader.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestOmitTf.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestPayloads.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestStressIndexing.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestStressIndexing2.java

Modified: lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java (original)
+++ lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java Mon Dec 21 19:10:29 2009
@@ -238,5 +238,29 @@
         }        
       };
     }
+
+    @Override
+    public TermDocs termDocs() throws IOException {
+      return new FilterTermDocs(in.termDocs()) {
+
+        @Override
+        public boolean next() throws IOException {
+          boolean res;
+          while ((res = super.next())) {
+            if (!dels.get(doc())) {
+              break;
+            }
+          }
+          return res;
+        }        
+      };
+    }
+
+    @Override
+    public TermDocs termDocs(Term term) throws IOException {
+      TermDocs termDocs = termDocs();
+      termDocs.seek(term);
+      return termDocs;
+    }
   }
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java Mon Dec 21 19:10:29 2009
@@ -366,24 +366,29 @@
 
   private MultiBits deletedDocs;
 
-  // Exposes a slice of an existing Bits as a new Bits
-  final static class SubBits implements Bits {
+  // Exposes a slice of an existing Bits as a new Bits.
+  // Only used when one provides an external skipDocs (ie,
+  // not the del docs from this DirectoryReader), to pull
+  // the DocsEnum of the sub readers
+  private final static class SubBits implements Bits {
     private final Bits parent;
     private final int start;
     private final int length;
 
     // start is inclusive; end is exclusive (length = end-start)
-    public SubBits(Bits parent, int start, int end) {
+    public SubBits(Bits parent, int start, int length) {
       this.parent = parent;
       this.start = start;
-      this.length = end - start;
+      this.length = length;
+      assert length >= 0: "length=" + length;
     }
     
     public boolean get(int doc) {
       if (doc >= length) {
         throw new RuntimeException("doc " + doc + " is out of bounds 0 .. " + (length-1));
       }
-      return parent.get(doc-start);
+      assert doc < length: "doc=" + doc + " length=" + length;
+      return parent.get(doc+start);
     }
   }
     
@@ -392,6 +397,7 @@
   // should return null from getDeletedDocs:
   static final class MultiBits implements Bits {
     private final Bits[] subs;
+    // this is 1+subs.length, ie the last entry has the maxDoc
     final int[] starts;
 
     public MultiBits(Bits[] subs, int[] starts) {
@@ -405,6 +411,8 @@
       if (bits == null) {
         return false;
       } else {
+        final int length = starts[1+reader]-starts[reader];
+        assert doc - starts[reader] < length: "doc=" + doc + " reader=" + reader + " starts[reader]=" + starts[reader] + " length=" + length;
         return bits.get(doc-starts[reader]);
       }
     }
@@ -1144,12 +1152,13 @@
     Terms terms;
     int base;
     int length;
-    Bits deletedDocs;
+    Bits skipDocs;
 
     public TermsWithBase(IndexReader reader, int base, String field) throws IOException {
       this.base = base;
       length = reader.maxDoc();
-      deletedDocs = reader.getDeletedDocs();
+      assert length >= 0: "length=" + length;
+      skipDocs = reader.getDeletedDocs();
       terms = reader.fields().terms(field);
     }
   }
@@ -1159,37 +1168,40 @@
     String current;
     int base;
     int length;
-    Bits deletedDocs;
+    Bits skipDocs;
 
     public FieldsEnumWithBase(IndexReader reader, int base) throws IOException {
       this.base = base;
       length = reader.maxDoc();
-      deletedDocs = reader.getDeletedDocs(); 
-     fields = reader.fields().iterator();
+      assert length >= 0: "length=" + length;
+      skipDocs = reader.getDeletedDocs(); 
+      fields = reader.fields().iterator();
     }
   }
 
   private final static class TermsEnumWithBase {
-    TermsEnum terms;
-    int base;
-    int length;
+    final TermsEnum terms;
+    final int base;
+    final int length;
     TermRef current;
-    Bits deletedDocs;
+    final Bits skipDocs;
 
     public TermsEnumWithBase(FieldsEnumWithBase start, TermsEnum terms, TermRef term) {
       this.terms = terms;
       current = term;
-      deletedDocs = start.deletedDocs;
+      skipDocs = start.skipDocs;
       base = start.base;
       length = start.length;
+      assert length >= 0: "length=" + length;
     }
 
     public TermsEnumWithBase(TermsWithBase start, TermsEnum terms, TermRef term) {
       this.terms = terms;
       current = term;
-      deletedDocs = start.deletedDocs;
+      skipDocs = start.skipDocs;
       base = start.base;
       length = start.length;
+      assert length >= 0: "length=" + length;
     }
   }
 
@@ -1226,6 +1238,8 @@
     }
   }
 
+  // Exposes flex API, merged from flex API of
+  // sub-segments.
   final static class MultiFields extends Fields {
     private final IndexReader[] readers;
     private final int[] starts;
@@ -1250,9 +1264,10 @@
       MultiTerms result = terms.get(field);
       if (result == null) {
 
+        // First time this field is requested, we create & add to terms:
         List<TermsWithBase> subs = new ArrayList<TermsWithBase>();
 
-        // Gather all sub-readers that have this field
+        // Gather all sub-readers that share this field
         for(int i=0;i<readers.length;i++) {
           Terms subTerms = readers[i].fields().terms(field);
           if (subTerms != null) {
@@ -1264,11 +1279,10 @@
       }
       return result;
     }
-
-    public void close() {
-    }
   }
-    
+
+  // Exposes flex API, merged from flex API of
+  // sub-segments.
   private final static class MultiTerms extends Terms {
     private final TermsWithBase[] subs;
     private final TermRef.Comparator termComp;
@@ -1281,7 +1295,12 @@
         if (_termComp == null) {
           _termComp = subs[i].terms.getTermComparator();
         } else {
-          assert subs[i].terms.getTermComparator() == null || _termComp.equals(subs[i].terms.getTermComparator());
+          // We cannot merge sub-readers that have
+          // different TermComps
+          final TermRef.Comparator subTermComp = subs[i].terms.getTermComparator();
+          if (subTermComp != null && !subTermComp.equals(_termComp)) {
+            throw new IllegalStateException("sub-readers have different TermRef.Comparators; cannot merge");
+          }
         }
       }
       termComp = _termComp;
@@ -1298,21 +1317,28 @@
     }
   }
 
-  // Exposes flex API, merged from flex API of sub-segments
+  // Exposes flex API, merged from flex API of
+  // sub-segments.  This does a merge sort, by field name,
+  // of the sub-readers.
   private final static class MultiFieldsEnum extends FieldsEnum {
     private final FieldMergeQueue queue;
 
-    private String currentField;
-
+    // Holds sub-readers containing field we are currently
+    // on, popped from queue.
     private final FieldsEnumWithBase[] top;
     private int numTop;
 
+    // Re-used TermsEnum
     private final MultiTermsEnum terms;
 
+    private String currentField;
+    
     MultiFieldsEnum(FieldsEnumWithBase[] subs) throws IOException {
       terms = new MultiTermsEnum(subs.length);
       queue = new FieldMergeQueue(subs.length);
       top = new FieldsEnumWithBase[subs.length];
+
+      // Init q
       for(int i=0;i<subs.length;i++) {
         subs[i].current = subs[i].fields.next();
         if (subs[i].current != null) {
@@ -1323,6 +1349,7 @@
 
     public String field() {
       assert currentField != null;
+      assert numTop > 0;
       return currentField;
     }
 
@@ -1335,7 +1362,7 @@
         if (top[i].current != null) {
           queue.add(top[i]);
         } else {
-          // no more fields in this reader
+          // no more fields in this sub-reader
         }
       }
 
@@ -1363,7 +1390,9 @@
     }
   }
 
-  // Exposes flex API, merged from flex API of sub-segments
+  // Exposes flex API, merged from flex API of
+  // sub-segments.  This does a merge sort, by term text, of
+  // the sub-readers.
   private static final class MultiTermsEnum extends TermsEnum {
     
     private final TermMergeQueue queue;
@@ -1404,7 +1433,12 @@
           if (termComp == null) {
             queue.termComp = termComp = termsEnum.getTermComparator();
           } else {
-            assert termsEnum.getTermComparator() == null || termComp.equals(termsEnum.getTermComparator());
+            // We cannot merge sub-readers that have
+            // different TermComps
+            final TermRef.Comparator subTermComp = termsEnum.getTermComparator();
+            if (subTermComp != null && !subTermComp.equals(termComp)) {
+              throw new IllegalStateException("sub-readers have different TermRef.Comparators; cannot merge");
+            }
           }
           final TermRef term = termsEnum.next();
           if (term != null) {
@@ -1488,6 +1522,8 @@
     }
 
     private final void pullTop() {
+      // extract all subs from the queue that have the same
+      // top term
       assert numTop == 0;
       while(true) {
         top[numTop++] = queue.pop();
@@ -1499,6 +1535,7 @@
     }
 
     private final void pushTop() throws IOException {
+      // call next() on each top, and put back into queue
       for(int i=0;i<numTop;i++) {
         top[i].current = top[i].terms.next();
         if (top[i].current != null) {
@@ -1507,7 +1544,6 @@
           // no more fields in this reader
         }
       }
-
       numTop = 0;
     }
 
@@ -1561,16 +1597,20 @@
         Bits bits = null;
         boolean handled = false;
 
-        // Optimize for common case: requested skip docs is simply our
-        // deleted docs
+        assert subs[i].length >= 0: "subs[" + i + " of " + numSubs + "].length=" + subs[i].length;
+
+        // Optimize for common case: requested skip docs is
+        // simply our (DiretoryReader's) deleted docs.  In
+        // this case, we just pull the skipDocs from the sub
+        // reader, rather than making the inefficient
+        // Sub(Multi(sub-readers)):
         if (skipDocs instanceof MultiBits) {
           MultiBits multiBits = (MultiBits) skipDocs;
           int reader = ReaderUtil.subIndex(subs[i].base, multiBits.starts);
-          // System.out.println("bits=" + multiBits + " starts=" + multiBits.starts + " subs=" + subs + " subs[i]=" + subs[i] + " subs[1+i]=" + subs[1+i] + " i=" + i + " numSubs=" + numSubs);
+          assert reader < multiBits.starts.length-1: " reader=" + reader + " multiBits.starts.length=" + multiBits.starts.length;
+          final int length = multiBits.starts[reader+1] - multiBits.starts[reader];
           if (multiBits.starts[reader] == subs[i].base &&
-              (i == numSubs-1 ||
-               reader == multiBits.starts.length-1 ||
-               multiBits.starts[1+reader] == subs[1+i].base)) {
+              length == subs[i].length) {
             bits = multiBits.subs[reader];
             handled = true;
           }
@@ -1698,8 +1738,9 @@
   
         if (t != null) {
           termEnum = reader.terms(t);
-        } else
+        } else {
           termEnum = reader.terms();
+        }
   
         LegacySegmentMergeInfo smi = new LegacySegmentMergeInfo(starts[i], termEnum, reader);
         smi.ord = i;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java Mon Dec 21 19:10:29 2009
@@ -41,6 +41,8 @@
     return atts;
   }
   
+  // nocommit -- state in API that doc/freq are undefined
+  // (defined?) after this?
   // nocommit -- fix this API so that intblock codecs are
   // able to return their own int arrays, to save a copy
   /** Bulk read: returns number of docs read.  Subclass may

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Fields.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Fields.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Fields.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Fields.java Mon Dec 21 19:10:29 2009
@@ -19,23 +19,12 @@
 
 import java.io.IOException;
 
-// TODO: split out an "iterator" api from the terms(String
-// field) API?
-
-// nocommit -- intended to be forward only?  eg no "reset"?
-
 /** Access to fields and terms
  *
  * NOTE: this API is experimental and will likely change */
 
-// TODO: someday expose public version of FieldInfos here
 public abstract class Fields {
 
-  // nocommit -- clarify if this is forwards only.  should
-  // this be "skipTo"?
-  // nocommit -- clarify: when this returns false, what is
-  // its internal state?  eg if i call field() after getting
-  // false back?
   /** Returns an iterator that will step through all fields
    *  names */
   public abstract FieldsEnum iterator() throws IOException;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsEnum.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsEnum.java Mon Dec 21 19:10:29 2009
@@ -21,7 +21,8 @@
 
 import org.apache.lucene.util.AttributeSource;
 
-/** Enumerates indexed fields.
+/** Enumerates indexed fields.  You must first call {@link
+ * #next} before calling {@link #terms}.
  *
  * NOTE: this API is experimental and will likely change */
 
@@ -38,14 +39,20 @@
   }
   
   // nocommit -- do we need seek?
-
-  /** Increments the enumeration to the next field.
-   *  Returns null when there are no more fields.*/
+  // nocommit -- should this return FieldInfo?
+  /** Increments the enumeration to the next field.  The
+   *  returned field is always interned, so simple ==
+   *  comparison is allowed.  Returns null when there are no
+   *  more fields.*/
   public abstract String next() throws IOException;
 
-  /** Get TermsEnum for the current field.  You should not
-   *  call {@link #next()} until you're done using this
-   *  TermsEnum. */
+  // nocommit should we add a field()?  fieldInfo()?
+  // mirrors TermsEnum
+
+  /** Get {@link TermsEnum} for the current field.  You
+   *  should not call {@link #next()} until you're done
+   *  using this {@link TermsEnum}.  After {@link #next}
+   *  returns null, this method should not be called. */
   public abstract TermsEnum terms() throws IOException;
 }
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java Mon Dec 21 19:10:29 2009
@@ -1171,17 +1171,28 @@
    */
   public abstract Collection<String> getFieldNames(FieldOption fldOption);
 
+  // Only used by external subclasses of IndexReader; all
+  // internal classes should implement Bits more
+  // efficiently:
   private final class DeletedDocsBits implements Bits {
     public boolean get(int docID) {
       return isDeleted(docID);
     }
   }
 
+  /**
+   * Returns the {@link Bits} representing deleted docs.  A
+   * set bit indicates the doc ID has been deleted.  This
+   * method should return null when there are no deleted
+   * docs. */
+  private Bits deletedDocsBits;
   public Bits getDeletedDocs() throws IOException {
-    return new DeletedDocsBits();
+    if (deletedDocsBits == null) {
+      deletedDocsBits = new DeletedDocsBits();
+    }
+    return deletedDocsBits;
   }
 
-
   /**
    * Forcibly unlocks the index in the named directory.
    * <P>

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFields.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFields.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFields.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFields.java Mon Dec 21 19:10:29 2009
@@ -19,9 +19,9 @@
 
 import java.io.IOException;
 
-/** Implements new API (FieldsEnum/TermsEnum) on top of old
- *  API.  Used only for IndexReader impls outside Lucene's
- *  core. */
+/** Implements flex API (FieldsEnum/TermsEnum) on top of
+ *  non-flex API.  Used only for IndexReader impls outside
+ *  Lucene's core. */
 class LegacyFields extends Fields {
   private final IndexReader r;
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFieldsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFieldsEnum.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFieldsEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFieldsEnum.java Mon Dec 21 19:10:29 2009
@@ -22,32 +22,40 @@
 
 /** Implements flex API (FieldsEnum/TermsEnum) on top of
  *  pre-flex API.  Used only for IndexReader impls outside
- *  Lucene's core. */
+ *  Lucene's core.
+ *
+ *  @deprecated Migrate the external reader to the flex API */
+@Deprecated
 class LegacyFieldsEnum extends FieldsEnum {
   private final IndexReader r;
   private TermEnum terms;
   private String field;
+  private boolean init;
 
   public LegacyFieldsEnum(IndexReader r) throws IOException {
     this.r = r;
     terms = r.terms();
-  }
-
-  private void doSeek(Term t) throws IOException {
-    terms.close();
-    terms = r.terms(t);
+    init = true;
   }
 
   @Override
   public String next() throws IOException {
 
     if (field != null) {
-      final Term seekTo = new Term(field, "\uFFFF");
-      doSeek(seekTo);
+      terms.close();
+      // jump to end of the current field:
+      terms = r.terms(new Term(field, "\uFFFF"));
+      assert terms.term() == null || !terms.term().field.equals(field);
+    }
+    if (init) {
+      init = false;
+      if (!terms.next()) {
+        return null;
+      }
     }
     if (terms.term() != null) {
       String newField = terms.term().field;
-      assert !newField.equals(field);
+      assert field == null || !newField.equals(field);
       field = newField;
       return field;
     } else {
@@ -66,11 +74,12 @@
     private TermEnum terms;
     private TermRef current;
     private final TermRef tr = new TermRef();
+    private final LegacyDocsEnum docsEnum;
 
     LegacyTermsEnum(IndexReader r, String field) throws IOException {
       this.r = r;
       this.field = field;
-      this.terms = r.terms(new Term(field, ""));
+      docsEnum = new LegacyDocsEnum(r, field);
     }
 
     @Override
@@ -81,11 +90,9 @@
 
     @Override
     public SeekStatus seek(TermRef text) throws IOException {
-      
-      // nocommit -- should we optimize for "silly seek"
-      // cases, here?  ie seek to term you're already on, to
-      // very next term , etc.
-      terms.close();
+      if (terms != null) {
+        terms.close();
+      }
       terms = r.terms(new Term(field, text.toString()));
 
       final Term t = terms.term();
@@ -117,16 +124,23 @@
 
     @Override
     public TermRef next() throws IOException {
-      if (terms.next()) {
+      if (terms == null) {
+        // first next -- seek to start of field
+        terms = r.terms(new Term(field, ""));
+        if (terms.term() == null) {
+          return null;
+        } else {
+          tr.copy(terms.term().text());
+          return current = tr;
+        }
+      } else if (terms.next()) {
         if (terms.term().field == field) {
           tr.copy(terms.term().text());
-          current = tr;
+          return current = tr;
         } else {
-          current = null;
+          return null;
         }
-        return current;
       } else {
-        current = null;
         return null;
       }
     }
@@ -136,12 +150,6 @@
       return current;
     }
 
-    /*
-    public String text() {
-      return terms.term().text;
-    }
-    */
-
     @Override
     public int docFreq() {
       return terms.docFreq();
@@ -149,7 +157,8 @@
 
     @Override
     public DocsEnum docs(Bits skipDocs) throws IOException {
-      return new LegacyDocsEnum(r, field, terms.term(), skipDocs);
+      docsEnum.reset(terms.term(), skipDocs);
+      return docsEnum;
     }
 
     public void close() throws IOException {
@@ -159,30 +168,39 @@
 
   // Emulates flex on top of legacy API
   private static class LegacyDocsEnum extends DocsEnum {
-    final TermDocs td;
-    final Term term;
-    final IndexReader r;
-    final String field;
-    final Bits skipDocs;
+    private final IndexReader r;
+    private final String field;
+    private final TermPositions tp;
+    private final LegacyPositionsEnum posEnum;
+
+    private Term term;
 
-    TermPositions tp;
-    int doc = -1;
+    private int doc = -1;
 
-    LegacyDocsEnum(IndexReader r, String field, Term term, Bits skipDocs) throws IOException {
+    LegacyDocsEnum(IndexReader r, String field) throws IOException {
       this.r = r;
       this.field = field;
+      tp = r.termPositions();
+      posEnum = new LegacyPositionsEnum(tp);
+    }
+
+    public void reset(Term term, Bits skipDocs) throws IOException {
       this.term = term;
-      td = r.termDocs(term);
-      this.skipDocs = skipDocs;
+      tp.seek(term);
+
+      if (skipDocs != r.getDeletedDocs()) {
+        // An external reader's TermDocs/Positions will
+        // silently skip deleted docs, so, we can't allow
+        // arbitrary skipDocs here:
+        //System.out.println("skipDocs=" + skipDocs + " vs " + r.getDeletedDocs());
+        throw new IllegalStateException("external IndexReader requires skipDocs == IndexReader.getDeletedDocs()");
+      }
     }
 
-    // nocommit -- must enforce skipDocs... but old API will
-    // always secretly skip deleted docs, and we can't work
-    // around that for external readers?
     @Override
     public int nextDoc() throws IOException {
-      if (td.next()) {
-        return doc = td.doc();
+      if (tp.next()) {
+        return doc = tp.doc();
       } else {
         return doc = NO_MORE_DOCS;
       }
@@ -190,8 +208,8 @@
 
     @Override
     public int advance(int target) throws IOException {
-      if (td.skipTo(target)) {
-        return doc = td.doc();
+      if (tp.skipTo(target)) {
+        return doc = tp.doc();
       } else {
         return doc = NO_MORE_DOCS;
       }
@@ -199,7 +217,7 @@
 
     @Override
     public int freq() {
-      return td.freq();
+      return tp.freq();
     }
 
     @Override
@@ -207,27 +225,18 @@
       return doc;
     }
 
-    @Override
-    public int read(int[] docs, int[] freqs) throws IOException {
-      return td.read(docs, freqs);
-    }
-
     public void close() throws IOException {
-      td.close();
+      tp.close();
     }
 
-    LegacyPositionsEnum lpe;
-
     @Override
     public PositionsEnum positions() throws IOException {
-      if (tp == null) {
-        tp = r.termPositions(term);
-        lpe = new LegacyPositionsEnum(tp);
-      } else {
-        tp.seek(term);
-      }
-      return lpe;
+      return posEnum;
     }
+
+    // NOTE: we don't override bulk-read (docs & freqs) API
+    // -- leave it to base class, because TermPositions
+    // can't do bulk read
   }
 
   // Emulates flex on top of legacy API

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MultiReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MultiReader.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MultiReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MultiReader.java Mon Dec 21 19:10:29 2009
@@ -92,6 +92,7 @@
       }
       subs[i] = subReaders[i].getDeletedDocs();
     }
+
     starts[subReaders.length] = maxDoc;
     if (hasDeletions) {
       deletedDocs = new MultiBits(subs, starts);
@@ -154,8 +155,12 @@
   }
   
   @Override
-  public Bits getDeletedDocs() {
-    return deletedDocs;
+  public Bits getDeletedDocs() throws IOException {
+    if (subReaders.length == 1) {
+      return subReaders[0].getDeletedDocs();
+    } else {
+      return deletedDocs;
+    }
   }
 
   /**

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java Mon Dec 21 19:10:29 2009
@@ -813,7 +813,7 @@
       // direct access to old:
       return ((PreFlexFields) core.fields).tis.terms();
     } else {
-      // Emulate old API on top of new index
+      // Emulate pre-flex API on top of flex index
       return new LegacyTermEnum(null);
     }
   }
@@ -829,7 +829,7 @@
       // direct access to old:
       return ((PreFlexFields) core.fields).tis.terms(t);
     } else {
-      // Emulate old API on top of new index
+      // Emulate pre-flex API on top of flex index
       return new LegacyTermEnum(t);
     }
   }
@@ -875,7 +875,9 @@
       // converting old API -> new API -> old API, just give
       // direct access to old:
       final PreFlexFields pre = (PreFlexFields) core.fields;
-      return new SegmentTermDocs(pre.freqStream, deletedDocs, pre.tis, core.fieldInfos);
+      SegmentTermDocs std = new SegmentTermDocs(pre.freqStream, pre.tis, core.fieldInfos);
+      std.setSkipDocs(deletedDocs);
+      return std;
     } else {
       // Emulate old API
       return new LegacyTermDocs();
@@ -892,7 +894,9 @@
       // converting old API -> new API -> old API, just give
       // direct access to old:
       final PreFlexFields pre = (PreFlexFields) core.fields;
-      return new SegmentTermPositions(pre.freqStream, pre.proxStream, deletedDocs, pre.tis, core.fieldInfos);
+      SegmentTermPositions stp = new SegmentTermPositions(pre.freqStream, pre.proxStream, pre.tis, core.fieldInfos);
+      stp.setSkipDocs(deletedDocs);
+      return stp;
     } else
       // Emulate old API
       return new LegacyTermPositions();
@@ -1295,7 +1299,7 @@
     return core.termsIndexDivisor;
   }
   
-  // Back compat: legacy TermEnum API over flex API
+  // Back compat: pre-flex TermEnum API over flex API
   final private class LegacyTermEnum extends TermEnum {
     FieldsEnum fields;
     TermsEnum terms;
@@ -1304,7 +1308,6 @@
     TermRef currentTerm;
 
     public LegacyTermEnum(Term t) throws IOException {
-      // System.out.println("sr.lte.init: term=" + t);
       fields = core.fields.iterator();
       currentField = fields.next();
       if (currentField == null) {
@@ -1319,7 +1322,7 @@
         while(currentField.compareTo(t.field) < 0) {
           currentField = fields.next();
           if (currentField == null) {
-            // Didn't find the field
+            // Hit end of fields
             done = true;
             break;
           }
@@ -1329,10 +1332,12 @@
           // We found some field -- get its terms:
           terms = fields.terms();
 
-          if (currentField.equals(t.field)) {
+          // nocommit: confirm inlining is working!
+          if (currentField == t.field) {
             // We found exactly the requested field; now
             // seek the term text:
             String text = t.text();
+
             // this is only for backwards compatibility.
             // previously you could supply a term with unpaired surrogates,
             // and it would return the next Term.
@@ -1340,6 +1345,7 @@
             // this emulates the old behavior, and forms "valid UTF-8" unicode.
             TermRef tr = new TermRef(UnicodeUtil.nextValidUTF16String(text));
             TermsEnum.SeekStatus status = terms.seek(tr);
+
             if (status == TermsEnum.SeekStatus.END) {
               // Rollover to the next field
               terms = null;
@@ -1379,8 +1385,9 @@
           // Advance to the next field
           currentField = fields.next();
           if (currentField == null) {
-            if (Codec.DEBUG)
+            if (Codec.DEBUG) {
               System.out.println("  fields.next returned false");
+            }
             done = true;
             return false;
           }
@@ -1569,6 +1576,4 @@
       return positions.hasPayload();
     }
   }
-
-
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermDocs.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermDocs.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermDocs.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermDocs.java Mon Dec 21 19:10:29 2009
@@ -30,6 +30,7 @@
  @deprecated Use {@link DocsEnum} instead
 */
 
+@Deprecated
 public interface TermDocs extends Closeable {
   /** Sets this to the data for a term.
    * The enumeration is reset to the start of the data for this term.

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermPositions.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermPositions.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermPositions.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermPositions.java Mon Dec 21 19:10:29 2009
@@ -28,7 +28,7 @@
  * @see IndexReader#termPositions()
  * @deprecated Use {@link PositionsEnum} instead 
  */
-
+@Deprecated
 public interface TermPositions
     extends TermDocs
 {

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsEnum.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsEnum.java Mon Dec 21 19:10:29 2009
@@ -98,10 +98,10 @@
    *  are done using the DocsEnum. */
   public abstract DocsEnum docs(Bits skipDocs) throws IOException;
 
-  /** Return the TermRef Comparator used to sort terms
-   *  provided by the iterator.  NOTE: this may return null
-   *  if there are no terms.  This method may be invoked
-   *  many times; it's best to cache a single instance &
-   *  reuse it. */
+  /** Return the {@link TermRef} Comparator used to sort
+   *  terms provided by the iterator.  NOTE: this may return
+   *  null if there are no terms.  Callers may invoke this
+   *  method many times, so it's best to cache a single
+   *  instance & reuse it. */
   public abstract TermRef.Comparator getTermComparator() throws IOException;
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/DocsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/DocsConsumer.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/DocsConsumer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/DocsConsumer.java Mon Dec 21 19:10:29 2009
@@ -69,7 +69,7 @@
         if (docMap != null) {
           // map around deletions
           doc = docMap[startDoc];
-          assert doc != -1: "postings enum returned deleted docID " + startDoc + " freq=" + docs.freq() + " df=" + df;
+          assert doc != -1: "docs enum returned deleted docID " + startDoc + " freq=" + docs.freq() + " df=" + df + " de=" + docs;
         } else {
           doc = startDoc;
         }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/TermsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/TermsConsumer.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/TermsConsumer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/TermsConsumer.java Mon Dec 21 19:10:29 2009
@@ -110,6 +110,7 @@
 
       while(true) {
         TermMergeState state = pending[pendingCount++] = queue.pop();
+        
         DocsEnum docsEnum = state.termsEnum.docs(mergeState.readers.get(state.readerIndex).getDeletedDocs());
         if (docsEnum != null) {
           match[matchCount].docsEnum = docsEnum;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java Mon Dec 21 19:10:29 2009
@@ -30,7 +30,12 @@
 
 /** Codec that reads the pre-flex-indexing postings
  *  format.  It does not provide a writer because newly
- *  written segments should use StandardCodec. */
+ *  written segments should use StandardCodec.
+ *
+ * @deprecated This is only used to read indexes created
+ * before 3.1.
+ */
+@Deprecated
 public class PreFlexCodec extends Codec {
 
   /** Extension of terms file */

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java Mon Dec 21 19:10:29 2009
@@ -116,8 +116,8 @@
   }
 
   @Override
-  public FieldsEnum iterator() {
-    return new Fields();
+  public FieldsEnum iterator() throws IOException {
+    return new PreFlexFieldsEnum();
   }
 
   @Override
@@ -177,42 +177,32 @@
     }
   }
 
-  private class Fields extends FieldsEnum {
-    Iterator<FieldInfo> it;
+  private class PreFlexFieldsEnum extends FieldsEnum {
+    final Iterator<FieldInfo> it;
+    private final PreTermsEnum termsEnum;
+    private int count;
     FieldInfo current;
-    private SegmentTermEnum lastTermEnum;
-    private int fieldCount;
 
-    public Fields() {
+    public PreFlexFieldsEnum() throws IOException {
       it = fields.values().iterator();
+      termsEnum = new PreTermsEnum();
     }
 
     @Override
     public String next() {
       if (it.hasNext()) {
-        fieldCount++;
+        count++;
         current = it.next();
         return current.name;
       } else {
         return null;
       }
     }
-    
+
     @Override
     public TermsEnum terms() throws IOException {
-      final PreTermsEnum terms;
-      if (lastTermEnum != null) {
-        // Re-use SegmentTermEnum to avoid seeking for
-        // linear scan (done by merging)
-        terms = new PreTermsEnum(current, lastTermEnum);
-      } else {
-        // If fieldCount is 1 then the terms enum can simply
-        // start at the start of the index (need not seek to
-        // the current field):
-        terms = new PreTermsEnum(current, fieldCount != 1);
-        lastTermEnum = terms.terms;
-      }
-      return terms;
+      termsEnum.reset(current, count == 1);
+      return termsEnum;
     }
   }
   
@@ -223,9 +213,10 @@
     }
 
     @Override
-    public TermsEnum iterator() throws IOException {
-      //System.out.println("pff.init create no context");
-      return new PreTermsEnum(fieldInfo, true);
+    public TermsEnum iterator() throws IOException {    
+      PreTermsEnum termsEnum = new PreTermsEnum();
+      termsEnum.reset(fieldInfo, false);
+      return termsEnum;
     }
 
     @Override
@@ -236,41 +227,38 @@
   }
 
   private class PreTermsEnum extends TermsEnum {
-    private SegmentTermEnum terms;
-    private final FieldInfo fieldInfo;
-    private PreDocsEnum docsEnum; // nocommit -- unused
+    private SegmentTermEnum termEnum;
+    private FieldInfo fieldInfo;
+    private final PreDocsEnum docsEnum;
     private boolean skipNext;
     private TermRef current;
     private final TermRef scratchTermRef = new TermRef();
 
-    // Pass needsSeek=false if the field is the very first
-    // field in the index -- this is used for linear scan of
-    // the index, eg when merging segments:
-    PreTermsEnum(FieldInfo fieldInfo, boolean needsSeek) throws IOException {
-      this.fieldInfo = fieldInfo;
-      if (!needsSeek) {
-        terms = getTermsDict().terms();
-      } else {
-        terms = getTermsDict().terms(new Term(fieldInfo.name, ""));
-        skipNext = true;
-      }
+    public PreTermsEnum() throws IOException {
+      docsEnum = new PreDocsEnum();
     }
 
-    PreTermsEnum(FieldInfo fieldInfo, SegmentTermEnum terms) throws IOException {
+    void reset(FieldInfo fieldInfo, boolean isFirstField) throws IOException {
       this.fieldInfo = fieldInfo;
-      if (terms.term() == null || terms.term().field() != fieldInfo.name) {
-        terms = getTermsDict().terms(new Term(fieldInfo.name, ""));
+      if (termEnum == null) {
+        // First time reset is called
+        if (isFirstField) {
+          termEnum = getTermsDict().terms();
+          skipNext = false;
+        } else {
+          termEnum = getTermsDict().terms(new Term(fieldInfo.name, ""));
+          skipNext = true;
+        }
       } else {
-        // Carefully avoid seeking in the linear-scan case,
-        // because segment doesn't load/need the terms dict
-        // index during merging.  If the terms is already on
-        // our field, it must be because it had seeked to
-        // exhaustion on the last field
-        this.terms = terms;
-      }
-      skipNext = true;
-      if (Codec.DEBUG) {
-        System.out.println("pff.terms.init field=" + fieldInfo.name);
+        final Term t = termEnum.term();
+        if (t != null && t.field() == fieldInfo.name) {
+          // No need to seek -- we have already advanced onto
+          // this field
+        } else {
+          assert t == null || !t.field().equals(fieldInfo.name);  // make sure field name is interned
+          termEnum = getTermsDict().terms(new Term(fieldInfo.name, ""));
+        }
+        skipNext = true;
       }
     }
 
@@ -295,8 +283,9 @@
       if (Codec.DEBUG) {
         System.out.println("pff.seek term=" + term);
       }
-      terms = getTermsDict().terms(new Term(fieldInfo.name, term.toString()));
-      final Term t = terms.term();
+      skipNext = false;
+      termEnum = getTermsDict().terms(new Term(fieldInfo.name, term.toString()));
+      final Term t = termEnum.term();
 
       final TermRef tr;
       if (t != null) {
@@ -321,14 +310,16 @@
     @Override
     public TermRef next() throws IOException {
       if (skipNext) {
-        // nocommit -- is there a cleaner way?
         skipNext = false;
-        scratchTermRef.copy(terms.term().text());
-        current = scratchTermRef;
-        return current;
+        if (termEnum.term() == null) {
+          return null;
+        } else {
+          scratchTermRef.copy(termEnum.term().text());
+          return current = scratchTermRef;
+        }
       }
-      if (terms.next()) {
-        final Term t = terms.term();
+      if (termEnum.next()) {
+        final Term t = termEnum.term();
         if (Codec.DEBUG) {
           System.out.println("pff.next term=" + t);
         }
@@ -340,15 +331,14 @@
           current = scratchTermRef;
           return current;
         } else {
+          assert !t.field().equals(fieldInfo.name);  // make sure field name is interned
           // Crossed into new field
           if (Codec.DEBUG) {
             System.out.println("  stop (new field " + t.field());
           }
-          current = null;
           return null;
         }
       } else {
-        current = null;
         return null;
       }
     }
@@ -360,36 +350,29 @@
 
     @Override
     public int docFreq() {
-      return terms.docFreq();
+      return termEnum.docFreq();
     }
 
     @Override
     public DocsEnum docs(Bits skipDocs) throws IOException {
-      // nocommit -- reuse?
-      return new PreDocsEnum(skipDocs, terms);
+      docsEnum.reset(termEnum, skipDocs);        
+      return docsEnum;
     }
   }
 
   private final class PreDocsEnum extends DocsEnum {
-    final private SegmentTermDocs docs;
     final private SegmentTermPositions pos;
-    private SegmentTermDocs current;
     final private PrePositionsEnum prePos;
+    private Bits skipDocs;
 
-    PreDocsEnum(Bits skipDocs, Term t) throws IOException {
-      current = docs = new SegmentTermDocs(freqStream, skipDocs, getTermsDict(), fieldInfos);
-      pos = new SegmentTermPositions(freqStream, proxStream, skipDocs, getTermsDict(), fieldInfos);
+    PreDocsEnum() throws IOException {
+      pos = new SegmentTermPositions(freqStream, proxStream, getTermsDict(), fieldInfos);
       prePos = new PrePositionsEnum(pos);
-      docs.seek(t);
-      pos.seek(t);
     }
 
-    PreDocsEnum(Bits skipDocs, SegmentTermEnum te) throws IOException {
-      current = docs = new SegmentTermDocs(freqStream, skipDocs, getTermsDict(), fieldInfos);
-      pos = new SegmentTermPositions(freqStream, proxStream, skipDocs, getTermsDict(), fieldInfos);
-      prePos = new PrePositionsEnum(pos);
-      docs.seek(te);
-      pos.seek(te);
+    public void reset(SegmentTermEnum termEnum, Bits skipDocs) throws IOException {
+      pos.setSkipDocs(skipDocs);
+      pos.seek(termEnum);
     }
 
     @Override
@@ -397,8 +380,8 @@
       if (Codec.DEBUG) {
         System.out.println("pff.docs.next");
       }
-      if (current.next()) {
-        return current.doc();
+      if (pos.next()) {
+        return pos.doc();
       } else {
         return NO_MORE_DOCS;
       }
@@ -406,8 +389,8 @@
 
     @Override
     public int advance(int target) throws IOException {
-      if (current.skipTo(target)) {
-        return current.doc();
+      if (pos.skipTo(target)) {
+        return pos.doc();
       } else {
         return NO_MORE_DOCS;
       }
@@ -415,31 +398,22 @@
 
     @Override
     public int freq() {
-      return current.freq();
+      return pos.freq();
     }
 
     @Override
     public int docID() {
-      return current.doc();
-    }
-
-    @Override
-    public int read(int[] docIDs, int[] freqs) throws IOException {
-      if (current != docs) {
-        docs.skipTo(current.doc());
-        current = docs;
-      }
-      return current.read(docIDs, freqs);
+      return pos.doc();
     }
 
     @Override
     public PositionsEnum positions() throws IOException {
-      if (current != pos) {
-        pos.skipTo(docs.doc());
-        current = pos;
-      }
       return prePos;
     }
+
+    // NOTE: we don't override bulk-read (docs & freqs) API
+    // -- leave it to base class, because TermPositions
+    // can't do bulk read
   }
 
   private final class PrePositionsEnum extends PositionsEnum {

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java Mon Dec 21 19:10:29 2009
@@ -31,9 +31,9 @@
 /** @deprecated */
 public class SegmentTermDocs implements TermDocs {
   //protected SegmentReader parent;
-  protected final Bits skipDocs;
   private final FieldInfos fieldInfos;
   private final TermInfosReader tis;
+  protected Bits skipDocs;
   protected IndexInput freqStream;
   protected int count;
   protected int df;
@@ -66,9 +66,8 @@
   */
 
   // nocommit -- SR needs public
-  public SegmentTermDocs(IndexInput freqStream, Bits skipDocs, TermInfosReader tis, FieldInfos fieldInfos) {
+  public SegmentTermDocs(IndexInput freqStream, TermInfosReader tis, FieldInfos fieldInfos) {
     this.freqStream = (IndexInput) freqStream.clone();
-    this.skipDocs = skipDocs;
     this.tis = tis;
     this.fieldInfos = fieldInfos;
     skipInterval = tis.getSkipInterval();
@@ -80,6 +79,10 @@
     seek(ti, term);
   }
 
+  public void setSkipDocs(Bits skipDocs) {
+    this.skipDocs = skipDocs;
+  }
+
   public void seek(TermEnum termEnum) throws IOException {
     TermInfo ti;
     Term term;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java Mon Dec 21 19:10:29 2009
@@ -51,8 +51,8 @@
   */
 
   // nocommit -- public
-  public SegmentTermPositions(IndexInput freqStream, IndexInput proxStream, Bits skipDocs, TermInfosReader tis, FieldInfos fieldInfos) {
-    super(freqStream, skipDocs, tis, fieldInfos);
+  public SegmentTermPositions(IndexInput freqStream, IndexInput proxStream, TermInfosReader tis, FieldInfos fieldInfos) {
+    super(freqStream, tis, fieldInfos);
     this.proxStreamOrig = proxStream;  // the proxStream will be cloned lazily when nextPosition() is called for the first time
   }
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardDocsReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardDocsReader.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardDocsReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardDocsReader.java Mon Dec 21 19:10:29 2009
@@ -32,7 +32,7 @@
 import org.apache.lucene.index.codecs.standard.StandardTermsDictReader.CacheEntry;
 
 /** Concrete class that reads the current doc/freq/skip
- *  postings format */
+ *  postings format. */
 
 // nocommit -- should we switch "hasProx" higher up?  and
 // create two separate docs readers, one that also reads
@@ -237,8 +237,6 @@
       final boolean omitTF;
       private Bits skipDocs;
 
-      // nocommit -- should we do hasProx with 2 different enum classes?
-
       boolean skipped;
       DefaultSkipListReader skipper;
 
@@ -336,6 +334,7 @@
         if (Codec.DEBUG) {
           System.out.println("  result doc=" + doc);
         }
+
         return doc;
       }
 
@@ -406,8 +405,9 @@
             // indexing, which means we pretend termFreq is
             // always 1 with that 1 occurrence having
             // position 0
-            if (fakePositions == null)
+            if (fakePositions == null) {
               fakePositions = new FormatPostingsFakePositionsEnum();
+            }
             return fakePositions;
           } else {
             // TODO: abstraction violation

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPositionsReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPositionsReader.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPositionsReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPositionsReader.java Mon Dec 21 19:10:29 2009
@@ -112,9 +112,10 @@
     @Override
     public PositionsEnum positions() throws IOException {
 
-      if (positions == null)
+      if (positions == null) {
         // Lazy init
         positions = new SegmentPositionsEnum();
+      }
 
       return positions;
     }
@@ -161,8 +162,9 @@
 
       void skip(int numPositions) {
         skipPosCount += numPositions;
-        if (Codec.DEBUG)
+        if (Codec.DEBUG) {
           System.out.println("pr [" + desc + "] skip " + numPositions + " positions; now " + skipPosCount);
+        }
       }
 
       void catchUp(int currentCount) throws IOException { 
@@ -194,14 +196,16 @@
       @Override
       public int next() throws IOException {
 
-        if (Codec.DEBUG)
+        if (Codec.DEBUG) {
           System.out.println("    pr.next [" + desc + "]: fp=" + proxIn.getFilePointer() + " return pos=" + position);
+        }
 
         if (storePayloads) {
 
           if (payloadPending && payloadLength > 0) {
-            if (Codec.DEBUG)
+            if (Codec.DEBUG) {
               System.out.println("      payload pending: skip " + payloadLength + " bytes");
+            }
             proxIn.seek(proxIn.getFilePointer()+payloadLength);
           }
 
@@ -210,8 +214,9 @@
             // Payload length has changed
             payloadLength = proxIn.readVInt();
             assert payloadLength >= 0;
-            if (Codec.DEBUG)
+            if (Codec.DEBUG) {
               System.out.println("      new payloadLen=" + payloadLength);
+            }
           }
           assert payloadLength != -1;
           
@@ -223,10 +228,11 @@
         skipPosCount--;
 
         // NOTE: the old API actually allowed this...
-        assert skipPosCount >= 0: "next() was called too many times (more than FormatPostingsDocsEnum.freq() times)";
+        assert skipPosCount >= 0: "next() was called too many times (more than FormatPostingsDocsEnum.freq() times) skipPosCount=" + skipPosCount;
 
-        if (Codec.DEBUG)
+        if (Codec.DEBUG) {
           System.out.println("   proxFP=" + proxIn.getFilePointer() + " return pos=" + position);
+        }
         return position;
       }
 

Added: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/FlexTestUtil.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/FlexTestUtil.java?rev=892930&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/FlexTestUtil.java (added)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/FlexTestUtil.java Mon Dec 21 19:10:29 2009
@@ -0,0 +1,537 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.store.*;
+import org.apache.lucene.util.*;
+import org.apache.lucene.document.*;
+import org.apache.lucene.analysis.*;
+import static org.junit.Assert.*;
+
+import java.io.*;
+import java.util.*;
+
+public class FlexTestUtil {
+
+  // nocommit:
+  // index variations
+  //   need del docs
+  //   need payloads
+  //   lots of randomness
+  //   surrogate pairs
+  //   
+
+  // need more diverse index
+  //    with omitTFAP
+  //    with payloads
+  //    bigger index
+
+  // test advanceTo, mixed with getting or not getting positions
+  // test that custom/null skipDocs work
+  // test: direct flex compared to flex on non flex on flex
+  // test with flex on one index and non-flex on the other
+  //   (two dirs above)
+  // temporarily force pre-flex emulation on flex emulation
+  //   on pre-flex segment (in back compat test)
+  // a sub-reader that has nonzero doc count but 100% are deleted
+  // foreign bitdocs
+  // advancing in DocsEnum, then next'ing, nextPosition'ing
+  // mutlti-reader w/ empty sub-reader
+  //   - eg from merge that had all del docs
+  //   - eg from empty flush
+  // make sure we test empty field (no terms)
+  //  created from empty field in doc, but also, from
+  //  removing all docs that had the field
+  // test 1 vs many segment index
+  // test multireader vs dir reader
+  // test bulk read api
+  // wrap reader as "external" reader -- double emulation test
+  public static void verifyFlexVsPreFlex(Random rand, Directory d) throws Exception {
+    IndexReader r = IndexReader.open(d);
+    verifyFlexVsPreFlex(rand, r);
+    r.close();
+  }
+
+  public static void verifyFlexVsPreFlex(Random rand, IndexWriter w) throws Exception {
+    IndexReader r = w.getReader();
+    verifyFlexVsPreFlex(rand, r);
+    r.close();
+  }
+                                
+  public static void verifyFlexVsPreFlex(Random rand, IndexReader r) throws Exception {
+    // First test on DirReader
+
+    // nocommit turn back on
+    // verifyFlexVsPreFlexSingle(rand, r);
+
+    // Then on each individual sub reader
+    IndexReader[] subReaders = r.getSequentialSubReaders();
+    IndexReader[] forcedSubReaders = new IndexReader[subReaders.length];
+    for(int i=0;i<subReaders.length;i++) {
+      forcedSubReaders[i] = new ForcedExternalReader(subReaders[i]);
+      verifyFlexVsPreFlexSingle(rand, forcedSubReaders[i]);
+      verifyFlexVsPreFlexSingle(rand, subReaders[i]);
+    }
+
+    // Then on a new MultiReader
+    // nocommit -- back on:
+    if (false) {
+      IndexReader m = new MultiReader(subReaders, false);
+      verifyFlexVsPreFlexSingle(rand, m);
+      m.close();
+    }
+
+    // Then on a forced-external reader (forced flex to
+    // emulate API on pre-flex API, which in turn is
+    // emulating pre-flex on flex -- twisted, but, better
+    // work):
+    // nocommit back on
+    if (false) {
+      verifyFlexVsPreFlexSingle(rand, new ForcedExternalReader(r));
+      IndexReader m = new MultiReader(forcedSubReaders, false);
+      verifyFlexVsPreFlexSingle(rand, m);
+      m.close();
+    }
+  }
+
+  private static void verifyFlexVsPreFlexSingle(Random rand, IndexReader r) throws Exception {
+
+    //List<Term> allTerms = new ArrayList<Term>();
+    //System.out.println("TEST: now verify!!");
+    testStraightEnum(r);
+    testRandomSkips(rand, r);
+    testRandomSeeks(rand, r);
+  }
+
+  private static void testStraightEnum(IndexReader r) throws Exception {
+
+    // straight enum of fields/terms/docs/positions
+    TermEnum termEnum = r.terms();
+    FieldsEnum fields = r.fields().iterator();
+    while(true) {
+      final String field = fields.next();
+      if (field == null) {
+        boolean result = termEnum.next();
+        if (result) {
+          System.out.println("got unexpected term=" + termEnum.term() + " termEnum=" + termEnum);
+        }
+        assertFalse(result);
+        break;
+      }
+      TermsEnum terms = fields.terms();
+      final TermPositions termPos = r.termPositions();
+      while(true) {
+        final TermRef termRef = terms.next();
+        if (termRef == null) {
+          break;
+        } else {
+          assertTrue(termEnum.next());
+          Term t = termEnum.term();
+          assertEquals(t.field(), field);
+          assertEquals(t.text(), termRef.toString());
+          assertEquals(termEnum.docFreq(), terms.docFreq());
+          //allTerms.add(t);
+
+          DocsEnum docs = terms.docs(r.getDeletedDocs());
+          termPos.seek(t);
+          while(true) {
+            final int doc = docs.nextDoc();
+            if (doc == DocsEnum.NO_MORE_DOCS) {
+              assertFalse(termPos.next());
+              break;
+            } else {
+              assertTrue(termPos.next());
+              assertEquals(termPos.doc(), doc);
+              assertEquals(termPos.freq(), docs.freq());
+              //System.out.println("TEST:     doc=" + doc + " freq=" + docs.freq());
+              final int freq = docs.freq();
+              PositionsEnum pos = docs.positions();
+              for(int i=0;i<freq;i++) {
+                final int position = pos.next();
+                //System.out.println("TEST:       pos=" + position);
+                assertEquals(position, termPos.nextPosition());
+                assertEquals(pos.hasPayload(), termPos.isPayloadAvailable());
+                if (pos.hasPayload()) {
+                  assertEquals(pos.getPayloadLength(), termPos.getPayloadLength());
+                  byte[] b1 = pos.getPayload(null, 0);
+                  byte[] b2 = termPos.getPayload(null, 0);
+                  assertNotNull(b1);
+                  assertNotNull(b2);
+                  assertTrue(Arrays.equals(b1, b2));
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  private static void testRandomSkips(Random rand, IndexReader r) throws Exception {
+
+    TermEnum termEnum = r.terms();
+    FieldsEnum fields = r.fields().iterator();
+    boolean skipNext = false;
+    int[] docs1 = new int[16];
+    int[] freqs1 = new int[16];
+    int[] docs2 = new int[16];
+    int[] freqs2 = new int[16];
+    while(true) {
+      final String field = fields.next();
+      //System.out.println("TEST: enum field=" + field);
+      if (field == null) {
+        boolean result = termEnum.next();
+        if (result) {
+          System.out.println("got unexpected term=" + termEnum.term() + " termEnum=" + termEnum);
+        }
+        assertFalse(result);
+        break;
+      }
+      if (rand.nextInt(3) <= 1) {
+        // Enum the terms
+        //System.out.println("TEST:   get terms");
+        TermsEnum terms = fields.terms();
+        final TermPositions termPos = r.termPositions();
+        final TermDocs termDocs = r.termDocs();
+        while(true) {
+          final TermRef termRef = terms.next();
+          //System.out.println("TEST:   enum term=" + termRef);
+          if (termRef == null) {
+            break;
+          } else {
+            if (skipNext) {
+              skipNext = false;
+            } else {
+              assertTrue(termEnum.next());
+            }
+            Term t = termEnum.term();
+            assertEquals(t.field(), field);
+            assertEquals(t.text(), termRef.toString());
+            assertEquals(termEnum.docFreq(), terms.docFreq());
+            //allTerms.add(t);
+
+            if (rand.nextInt(3) <= 1) {
+              DocsEnum docs = terms.docs(r.getDeletedDocs());
+              if (rand.nextBoolean()) {
+                // use bulk read API
+                termDocs.seek(t);
+                while(true) {
+                  final int count1 = docs.read(docs1, freqs1);
+                  final int count2 = termDocs.read(docs2, freqs2);
+                  assertEquals(count1, count2);
+                  if (count1 == 0) {
+                    break;
+                  }
+                  for(int i=0;i<count1;i++) {
+                    assertEquals(docs1[i], docs2[i]);
+                    assertEquals(freqs1[i], freqs2[i]);
+                  }
+                }
+              } else {
+                // Enum the docs one by one
+                //System.out.println("TEST:      get docs");
+                termPos.seek(t);
+                while(true) {
+                  final int doc = docs.nextDoc();
+                  if (doc == DocsEnum.NO_MORE_DOCS) {
+                    assertFalse(termPos.next());
+                    break;
+                  } else {
+                    assertTrue(termPos.next());
+                    assertEquals(termPos.doc(), doc);
+                    assertEquals(termPos.freq(), docs.freq());
+                    //System.out.println("TEST:     doc=" + doc + " freq=" + docs.freq());
+                    if (rand.nextInt(3) <= 1) {
+                      // enum the positions
+                      final int freq = docs.freq();
+                      PositionsEnum pos = docs.positions();
+                      for(int i=0;i<freq;i++) {
+                        final int position = pos.next();
+                        //System.out.println("TEST:       pos=" + position);
+                        assertEquals(position, termPos.nextPosition());
+                        assertEquals(pos.hasPayload(), termPos.isPayloadAvailable());
+                        if (pos.hasPayload()) {
+                          assertEquals(pos.getPayloadLength(), termPos.getPayloadLength());
+                          if (rand.nextInt(3) <= 1) {
+                            byte[] b1 = pos.getPayload(null, 0);
+                            byte[] b2 = termPos.getPayload(null, 0);
+                            assertNotNull(b1);
+                            assertNotNull(b2);
+                            assertTrue(Arrays.equals(b1, b2));
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            } else {
+              //System.out.println("TEST:      skip docs");
+            }
+          }
+        }
+      } else {
+        // Skip terms for this field
+        termEnum = r.terms(new Term(field, "\uFFFF"));
+        skipNext = true;
+        //System.out.println("TEST:   skip terms; now=" + termEnum.term());
+      }
+    }
+
+    // seek to before first term in a field
+    // seek to after last term in a field
+    // seek to random terms
+    // enum docs, sometimes skipping
+    // enum positions, sometimes skipping payloads
+  }
+
+  public static int nextInt(Random rand, int min, int max) {
+    return min + rand.nextInt(max-min);
+  }
+
+  public static int nextInt(Random rand, int max) {
+    return rand.nextInt(max);
+  }
+
+  public static String getRandomText(Random rand, int minLen, int maxLen, boolean doUnpairedSurr) {
+    final int len = nextInt(rand, minLen, maxLen);
+    char[] buffer = new char[len];
+    
+    for(int i=0;i<len;i++) {
+      int t = rand.nextInt(doUnpairedSurr ? 6 : 5);
+      if (0 == t && i < len-1) {
+        // Make a surrogate pair
+        // High surrogate
+        buffer[i++] = (char) nextInt(rand, 0xd800, 0xdc00);
+        // Low surrogate
+        buffer[i] = (char) nextInt(rand, 0xdc00, 0xe000);
+      } else if (t <= 1) {
+        buffer[i] = (char) nextInt(rand, 0x80);
+      } else if (2 == t) {
+        buffer[i] = (char) nextInt(rand, 0x80, 0x800);
+      } else if (3 == t) {
+        buffer[i] = (char) nextInt(rand, 0x800, 0xd800);
+      } else if (4 == t) {
+        buffer[i] = (char) nextInt(rand, 0xe000, 0xffff);
+      } else if (5 == t) {
+        // Illegal unpaired surrogate
+        if (rand.nextBoolean()) {
+          buffer[i] = (char) nextInt(rand, 0xd800, 0xdc00);
+        } else {
+          buffer[i] = (char) nextInt(rand, 0xdc00, 0xe000);
+        }
+      }
+    }
+
+    return new String(buffer);
+  }
+
+  private static void testRandomSeeks(Random rand, IndexReader r) throws Exception {
+    final int ITER = 100;
+    List<String> allFields = new ArrayList<String>();
+    FieldsEnum fields = r.fields().iterator();
+    while(true) {
+      String f = fields.next();
+      if (f == null) {
+        break;
+      }
+      allFields.add(f);
+    }
+    final int fieldCount = allFields.size();
+    if (fieldCount == 0) {
+      return;
+    }
+    
+    final TermPositions termPositions = r.termPositions();
+    for(int i=0;i<ITER;i++) {
+      // Random field:
+      String f = allFields.get(rand.nextInt(fieldCount));
+
+      String text = getRandomText(rand, 1, 3, false);
+      final TermsEnum termsEnum = r.fields().terms(f).iterator();
+
+      final TermsEnum.SeekStatus seekStatus = termsEnum.seek(new TermRef(text));
+      Term t = new Term(f, text);
+      //System.out.println("seek to " + t);
+
+      final TermEnum termEnum = r.terms(t);
+      
+      if (seekStatus == TermsEnum.SeekStatus.END) {
+        //System.out.println("found end");
+        assertTrue(termEnum.term() == null || termEnum.term().field() != f);
+        continue;
+      } else if (seekStatus == TermsEnum.SeekStatus.FOUND) {
+        //System.out.println("found exact");
+        assertEquals(t, termEnum.term());
+      } else {
+        //System.out.println("found other");
+        assertEquals(termsEnum.term().toString(), termEnum.term().text());
+      }
+
+      assertEquals(termsEnum.docFreq(), termEnum.docFreq());
+
+      final DocsEnum docs = termsEnum.docs(r.getDeletedDocs());
+      termPositions.seek(termEnum.term());
+
+      int doc = 0;
+      for(int j=0;j<20;j++) {
+        final int inc = nextInt(rand, 1, Math.max(10, r.maxDoc()/15));
+        int newDoc1 = docs.advance(doc+inc);
+        boolean found = termPositions.skipTo(doc+inc);
+        int newDoc2;
+
+        if (newDoc1 == DocsEnum.NO_MORE_DOCS) {
+          assertFalse(found);
+          break;
+        } else {
+          assertTrue(found);
+          newDoc2 = termPositions.doc();
+        }
+        
+        assertEquals(newDoc1, newDoc2);
+        assertEquals(docs.freq(), termPositions.freq());
+
+        doc = newDoc1;
+
+        PositionsEnum posEnum = docs.positions();
+        for(int k=0;k<docs.freq();k++) {
+          int pos1 = posEnum.next();
+          int pos2 = termPositions.nextPosition();
+          assertEquals(pos1, pos2);
+          assertEquals(posEnum.hasPayload(), termPositions.isPayloadAvailable());
+          if (posEnum.hasPayload()) {
+            assertEquals(posEnum.getPayloadLength(), termPositions.getPayloadLength());
+            byte[] b1 = posEnum.getPayload(null, 0);
+            byte[] b2 = termPositions.getPayload(null, 0);
+            assertNotNull(b1);
+            assertNotNull(b2);
+            assertTrue(Arrays.equals(b1, b2));
+          }
+        }
+      }
+    }
+  }
+
+  // Delegates to a "normal" IndexReader, making it look
+  // "external", to force testing of the "flex API on
+  // external reader" layer
+  public final static class ForcedExternalReader extends IndexReader {
+    private final IndexReader r;
+    public ForcedExternalReader(IndexReader r) {
+      this.r = r;
+    }
+
+    public TermFreqVector[] getTermFreqVectors(int docNumber) throws IOException {
+      return r.getTermFreqVectors(docNumber);
+    }
+
+    public TermFreqVector getTermFreqVector(int docNumber, String field) throws IOException {
+      return r.getTermFreqVector(docNumber, field);
+    }
+
+    public void getTermFreqVector(int docNumber, String field, TermVectorMapper mapper) throws IOException {
+      r.getTermFreqVector(docNumber, field, mapper);
+    }
+
+    public void getTermFreqVector(int docNumber, TermVectorMapper mapper) throws IOException {
+      r.getTermFreqVector(docNumber, mapper);
+    }
+
+    public Bits getDeletedDocs() throws IOException {
+      return r.getDeletedDocs();
+    }
+
+    public int numDocs() {
+      return r.numDocs();
+    }
+
+    public int maxDoc() {
+      return r.maxDoc();
+    }
+
+    public Document document(int n, FieldSelector fieldSelector) throws CorruptIndexException, IOException {
+      return r.document(n, fieldSelector);
+    }
+
+    public boolean isDeleted(int n) {
+      return r.isDeleted(n);
+    }
+
+    public boolean hasDeletions() {
+      return r.hasDeletions();
+    }
+
+    public byte[] norms(String field) throws IOException {
+      return r.norms(field);
+    }
+
+    public void norms(String field, byte[] bytes, int offset) 
+      throws IOException {
+      r.norms(field, bytes, offset);
+    }
+    
+    protected  void doSetNorm(int doc, String field, byte value)
+      throws CorruptIndexException, IOException {
+      r.doSetNorm(doc, field, value);
+    }
+
+    public TermEnum terms() throws IOException {
+      return r.terms();
+    }
+
+    public TermEnum terms(Term t) throws IOException {
+      return r.terms(t);
+    }
+
+    public int docFreq(Term t) throws IOException {
+      return r.docFreq(t);
+    }
+
+    public TermDocs termDocs() throws IOException {
+      return r.termDocs();
+    }
+
+    public TermPositions termPositions() throws IOException {
+      return r.termPositions();
+    }
+
+    public void doDelete(int docID) throws IOException {
+      r.doDelete(docID);
+    }
+
+    public void doUndeleteAll() throws IOException {
+      r.doUndeleteAll();
+    }
+
+    protected void doCommit(Map<String, String> commitUserData) throws IOException {
+      r.doCommit(commitUserData);
+    }
+
+    protected void doClose() throws IOException {
+      r.doClose();
+    }
+
+    public Collection<String> getFieldNames(FieldOption fldOption) {
+      return r.getFieldNames(fldOption);
+    }
+  }
+
+  public static void main(String[] args) throws Exception {
+    //Directory dir = FSDirectory.open(new File("/x/lucene/wiki.5M/index"));
+    Directory dir = FSDirectory.open(new File("/x/lucene/flex.wiki.1M/index"));
+    verifyFlexVsPreFlex(new Random(), dir);
+    dir.close();
+  }
+}
\ No newline at end of file

Propchange: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/FlexTestUtil.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Mon Dec 21 19:10:29 2009
@@ -26,6 +26,7 @@
 import java.io.DataInputStream;
 import java.io.OutputStream;
 import java.util.Arrays;
+import java.util.Random;
 import java.util.Enumeration;
 import java.util.List;
 import java.util.ArrayList;
@@ -65,11 +66,11 @@
   // oldNames array.
 
   /*
-  public void testCreatePreLocklessCFS() throws IOException {
+  public void xxxtestCreatePreLocklessCFS() throws IOException {
     createIndex("index.cfs", true);
   }
 
-  public void testCreatePreLocklessNoCFS() throws IOException {
+  public void xxxtestCreatePreLocklessNoCFS() throws IOException {
     createIndex("index.nocfs", false);
   }
   */
@@ -110,13 +111,13 @@
     zipFile.close();
   }
 
-  public void testCreateCFS() throws IOException {
+  public void xxxtestCreateCFS() throws IOException {
     String dirName = "testindex.cfs";
     createIndex(dirName, true);
     rmDir(dirName);
   }
 
-  public void testCreateNoCFS() throws IOException {
+  public void xxxtestCreateNoCFS() throws IOException {
     String dirName = "testindex.nocfs";
     createIndex(dirName, true);
     rmDir(dirName);
@@ -203,15 +204,20 @@
     }
   }
 
-  public void testOptimizeOldIndex() throws IOException {
+  public void testOptimizeOldIndex() throws Exception {
     int hasTested29 = 0;
+
+    Random rand = newRandom();
     
     for(int i=0;i<oldNames.length;i++) {
       String dirName = "src/test/org/apache/lucene/index/index." + oldNames[i];
       unzip(dirName, oldNames[i]);
+
       String fullPath = fullDir(oldNames[i]);
       Directory dir = FSDirectory.open(new File(fullPath));
 
+      FlexTestUtil.verifyFlexVsPreFlex(rand, dir);
+
       if (oldNames[i].startsWith("29.")) {
         assertCompressedFields29(dir, true);
         hasTested29++;
@@ -219,6 +225,7 @@
 
       IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
       w.optimize();
+      FlexTestUtil.verifyFlexVsPreFlex(rand, w);
       w.close();
 
       _TestUtil.checkIndex(dir);
@@ -235,7 +242,7 @@
     assertEquals("test for compressed field should have run 4 times", 4, hasTested29);
   }
 
-  public void testSearchOldIndex() throws IOException {
+  public void xxxtestSearchOldIndex() throws IOException {
     for(int i=0;i<oldNames.length;i++) {
       String dirName = "src/test/org/apache/lucene/index/index." + oldNames[i];
       unzip(dirName, oldNames[i]);
@@ -244,7 +251,7 @@
     }
   }
 
-  public void testIndexOldIndexNoAdds() throws IOException {
+  public void xxxtestIndexOldIndexNoAdds() throws IOException {
     for(int i=0;i<oldNames.length;i++) {
       String dirName = "src/test/org/apache/lucene/index/index." + oldNames[i];
       unzip(dirName, oldNames[i]);
@@ -253,7 +260,7 @@
     }
   }
 
-  public void testIndexOldIndex() throws IOException {
+  public void xxxtestIndexOldIndex() throws IOException {
     for(int i=0;i<oldNames.length;i++) {
       String dirName = "src/test/org/apache/lucene/index/index." + oldNames[i];
       unzip(dirName, oldNames[i]);
@@ -503,7 +510,7 @@
 
   /* Verifies that the expected file names were produced */
 
-  public void testExactFileNames() throws IOException {
+  public void xxxtestExactFileNames() throws IOException {
 
     String outputDir = "lucene.backwardscompat0.index";
     rmDir(outputDir);
@@ -655,7 +662,7 @@
   }
 
   // flex: test basics of TermsEnum api on non-flex index
-  public void testNextIntoWrongField() throws Exception {
+  public void xxxtestNextIntoWrongField() throws Exception {
     for(int i=0;i<oldNames.length;i++) {
       String dirName = "src/test/org/apache/lucene/index/index." + oldNames[i];
       unzip(dirName, oldNames[i]);

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestCodecs.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestCodecs.java Mon Dec 21 19:10:29 2009
@@ -26,6 +26,7 @@
 // nocommit -- test multiple codecs here?
 
 // TODO
+//   - test across fields
 //   - fix this test to run once for all codecs
 //   - make more docs per term, to test > 1 level skipping
 //   - test all combinations of payloads/not and omitTF/not

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestFlexExternalReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestFlexExternalReader.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestFlexExternalReader.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestFlexExternalReader.java Mon Dec 21 19:10:29 2009
@@ -27,106 +27,6 @@
 
 public class TestFlexExternalReader extends LuceneTestCase {
 
-  // Delegates to a "normal" IndexReader, making it look
-  // "external", to force testing of the "flex API on
-  // external reader" layer
-  private final static class ExternalReader extends IndexReader {
-    private final IndexReader r;
-    public ExternalReader(IndexReader r) {
-      this.r = r;
-    }
-
-    public TermFreqVector[] getTermFreqVectors(int docNumber) throws IOException {
-      return r.getTermFreqVectors(docNumber);
-    }
-
-    public TermFreqVector getTermFreqVector(int docNumber, String field) throws IOException {
-      return r.getTermFreqVector(docNumber, field);
-    }
-
-    public void getTermFreqVector(int docNumber, String field, TermVectorMapper mapper) throws IOException {
-      r.getTermFreqVector(docNumber, field, mapper);
-    }
-
-    public void getTermFreqVector(int docNumber, TermVectorMapper mapper) throws IOException {
-      r.getTermFreqVector(docNumber, mapper);
-    }
-
-    public int numDocs() {
-      return r.numDocs();
-    }
-
-    public int maxDoc() {
-      return r.maxDoc();
-    }
-
-    public Document document(int n, FieldSelector fieldSelector) throws CorruptIndexException, IOException {
-      return r.document(n, fieldSelector);
-    }
-
-    public boolean isDeleted(int n) {
-      return r.isDeleted(n);
-    }
-
-    public boolean hasDeletions() {
-      return r.hasDeletions();
-    }
-
-    public byte[] norms(String field) throws IOException {
-      return r.norms(field);
-    }
-
-    public void norms(String field, byte[] bytes, int offset) 
-      throws IOException {
-      r.norms(field, bytes, offset);
-    }
-    
-    protected  void doSetNorm(int doc, String field, byte value)
-      throws CorruptIndexException, IOException {
-      r.doSetNorm(doc, field, value);
-    }
-
-    public TermEnum terms() throws IOException {
-      return r.terms();
-    }
-
-    public TermEnum terms(Term t) throws IOException {
-      return r.terms(t);
-    }
-
-    public int docFreq(Term t) throws IOException {
-      return r.docFreq(t);
-    }
-
-    public TermDocs termDocs() throws IOException {
-      return r.termDocs();
-    }
-
-    public TermPositions termPositions() throws IOException {
-      return r.termPositions();
-    }
-
-    public void doDelete(int docID) throws IOException {
-      r.doDelete(docID);
-    }
-
-    public void doUndeleteAll() throws IOException {
-      r.doUndeleteAll();
-    }
-
-    protected void doCommit(Map<String, String> commitUserData) throws IOException {
-      r.doCommit(commitUserData);
-    }
-
-    protected void doClose() throws IOException {
-      r.doClose();
-    }
-
-    public Collection<String> getFieldNames(FieldOption fldOption) {
-      return r.getFieldNames(fldOption);
-    }
-  }
-
   public void testExternalReader() throws Exception {
     Directory d = new MockRAMDirectory();
 
@@ -144,7 +44,7 @@
       w.addDocument(doc);
     }
 
-    IndexReader r = new ExternalReader(w.getReader());
+    IndexReader r = new FlexTestUtil.ForcedExternalReader(w.getReader());
 
     TermRef field1Term = new TermRef("field1");
     TermRef field2Term = new TermRef("field2");

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestIndexReaderReopen.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestIndexReaderReopen.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestIndexReaderReopen.java Mon Dec 21 19:10:29 2009
@@ -860,6 +860,8 @@
     assertReaderClosed(reader, true, true);
     assertReaderClosed(firstReader, true, true);
 
+    FlexTestUtil.verifyFlexVsPreFlex(rnd, dir);
+
     dir.close();
   }
   

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestOmitTf.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestOmitTf.java Mon Dec 21 19:10:29 2009
@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Random;
 
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
@@ -92,13 +93,18 @@
     f2.setOmitTermFreqAndPositions(false);        
     d.add(f2);
         
+    Random rnd = newRandom();
+
     writer.addDocument(d);
     // force merge
+    FlexTestUtil.verifyFlexVsPreFlex(rnd, writer);
     writer.optimize();
     // flush
     writer.close();
     _TestUtil.checkIndex(ram);
 
+    FlexTestUtil.verifyFlexVsPreFlex(rnd, ram);
+
     SegmentReader reader = SegmentReader.getOnlySegmentReader(ram);
     FieldInfos fi = reader.fieldInfos();
     assertTrue("OmitTermFreqAndPositions field bit should be set.", fi.fieldInfo("f1").omitTermFreqAndPositions);
@@ -144,8 +150,12 @@
     for(int i=0;i<30;i++)
       writer.addDocument(d);
         
+    Random rnd = newRandom();
+    FlexTestUtil.verifyFlexVsPreFlex(rnd, writer);
+
     // force merge
     writer.optimize();
+    FlexTestUtil.verifyFlexVsPreFlex(rnd, writer);
     // flush
     writer.close();
 

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestPayloads.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestPayloads.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestPayloads.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestPayloads.java Mon Dec 21 19:10:29 2009
@@ -137,6 +137,9 @@
         analyzer.setPayloadData("f2", "somedata".getBytes(), 0, 1);
         analyzer.setPayloadData("f3", "somedata".getBytes(), 0, 3);
         writer.addDocument(d);
+
+        FlexTestUtil.verifyFlexVsPreFlex(rnd, writer);
+
         // force merge
         writer.optimize();
         // flush
@@ -147,7 +150,8 @@
         assertFalse("Payload field bit should not be set.", fi.fieldInfo("f1").storePayloads);
         assertTrue("Payload field bit should be set.", fi.fieldInfo("f2").storePayloads);
         assertTrue("Payload field bit should be set.", fi.fieldInfo("f3").storePayloads);
-        reader.close();        
+        reader.close();
+        FlexTestUtil.verifyFlexVsPreFlex(rnd, ram);
     }
 
     // Tests if payloads are correctly stored and loaded using both RamDirectory and FSDirectory
@@ -211,7 +215,9 @@
             writer.addDocument(d);
         }
         
+        FlexTestUtil.verifyFlexVsPreFlex(rnd, writer);
         writer.optimize();
+        FlexTestUtil.verifyFlexVsPreFlex(rnd, writer);
         // flush
         writer.close();
         
@@ -322,7 +328,9 @@
         writer.addDocument(d);
 
         
+        FlexTestUtil.verifyFlexVsPreFlex(rnd, writer);
         writer.optimize();
+        FlexTestUtil.verifyFlexVsPreFlex(rnd, writer);
         // flush
         writer.close();
         

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestStressIndexing.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestStressIndexing.java?rev=892930&r1=892929&r2=892930&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestStressIndexing.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestStressIndexing.java Mon Dec 21 19:10:29 2009
@@ -153,6 +153,8 @@
 
     modifier.close();
 
+    FlexTestUtil.verifyFlexVsPreFlex(RANDOM, directory);
+
     for(int i=0;i<numThread;i++)
       assertTrue(! threads[i].failed);
 



Mime
View raw message