lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r904750 [1/4] - in /lucene/java/branches/flex_1458: ./ contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/ contrib/misc/src/java/org/apache/lucene/index/ contrib/queries/src/java/org/apache/lucene/search/ src/java/org/apache/luce...
Date Sat, 30 Jan 2010 10:18:25 GMT
Author: mikemccand
Date: Sat Jan 30 10:16:35 2010
New Revision: 904750

URL: http://svn.apache.org/viewvc?rev=904750&view=rev
Log:
LUCENE-2111 (flex branch): simplify how terms dict interacts with the postings readers/writers impls

Added:
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java   (with props)
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java   (with props)
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java   (with props)
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsWriterImpl.java   (with props)
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java   (with props)
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriterImpl.java   (with props)
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java   (with props)
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReaderImpl.java   (with props)
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java   (with props)
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriterImpl.java   (with props)
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/TermState.java   (with props)
Removed:
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/PositionsEnum.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/DocsConsumer.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PositionsConsumer.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingDocsReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingDocsWriter.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepDocsReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepDocsWriter.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPositionsReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPositionsWriter.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardDocsConsumer.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardDocsProducer.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardDocsReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardDocsWriter.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPositionsConsumer.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPositionsProducer.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPositionsReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPositionsWriter.java
Modified:
    lucene/java/branches/flex_1458/common-build.xml
    lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
    lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java
    lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/TermVectorAccessor.java
    lucene/java/branches/flex_1458/contrib/queries/src/java/org/apache/lucene/search/DuplicateFilter.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllDocsEnum.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CheckIndex.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocumentsWriter.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Fields.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsEnum.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFieldsEnum.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MultiReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Terms.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsEnum.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codec.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/FieldsProducer.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/TermsConsumer.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexInput.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/IntIndexInput.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/IntIndexOutput.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepCodec.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepSkipListReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepSkipListWriter.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SingleIntIndexInput.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SingleIntIndexOutput.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListWriter.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexWriter.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictWriter.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/ExactPhraseScorer.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FilteredTermsEnum.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/IndexSearcher.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiPhraseQuery.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhrasePositions.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseQuery.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseScorer.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/SloppyPhraseScorer.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermScorer.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/TermSpans.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/store/Directory.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/BytesRef.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestExternalCodecs.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/collation/CollationTestBase.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/FlexTestUtil.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestCodecs.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestIndexWriter.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestOmitTf.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/CheckHits.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/JustCompileSearch.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestFilteredSearch.java

Modified: lucene/java/branches/flex_1458/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/common-build.xml?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/common-build.xml (original)
+++ lucene/java/branches/flex_1458/common-build.xml Sat Jan 30 10:16:35 2010
@@ -43,7 +43,7 @@
   <property name="dev.version" value="3.1-dev"/>
   <property name="version" value="${dev.version}"/>
   <property name="backwards.branch" value="flex_1458_3_0_back_compat_tests"/>
-  <property name="backwards.rev" value="900186"/>
+  <property name="backwards.rev" value="904748"/>
   <property name="spec.version" value="${version}"/>	
   <property name="year" value="2000-${current.year}"/>
   <property name="final.name" value="lucene-${name}-${version}"/>

Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (original)
+++ lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java Sat Jan 30 10:16:35 2010
@@ -23,7 +23,6 @@
 import java.io.BufferedReader;
 import java.text.Collator;
 import java.util.List;
-import java.util.Iterator;
 import java.util.Locale;
 
 import org.apache.lucene.analysis.Analyzer;
@@ -476,8 +475,9 @@
       if (fieldName == DocMaker.ID_FIELD)
         continue;
       TermsEnum terms = fields.terms();
+      DocsEnum docs = null;
       while(terms.next() != null) {
-        DocsEnum docs = terms.docs(reader.getDeletedDocs());
+        docs = terms.docs(reader.getDeletedDocs(), docs);
         while(docs.nextDoc() != docs.NO_MORE_DOCS) {
           totalTokenCount2 += docs.freq();
         }

Modified: lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java (original)
+++ lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java Sat Jan 30 10:16:35 2010
@@ -117,8 +117,9 @@
         Terms terms = reader.fields().terms(field);
         if (terms != null) {
           TermsEnum termsEnum = terms.iterator();
+          DocsEnum docs = null;
           while(termsEnum.next() != null) {
-            DocsEnum docs = termsEnum.docs(delDocs);
+            docs = termsEnum.docs(delDocs, docs);
             while(true) {
               int docID = docs.nextDoc();
               if (docID != docs.NO_MORE_DOCS) {

Modified: lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/TermVectorAccessor.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/TermVectorAccessor.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/TermVectorAccessor.java (original)
+++ lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/TermVectorAccessor.java Sat Jan 30 10:16:35 2010
@@ -106,11 +106,18 @@
     boolean anyTerms = false;
     if (terms != null) {
       TermsEnum termsEnum = terms.iterator();
+      DocsEnum docs = null;
+      DocsAndPositionsEnum postings = null;
       while(true) {
         BytesRef text = termsEnum.next();
         if (text != null) {
           anyTerms = true;
-          DocsEnum docs = termsEnum.docs(delDocs);
+          if (!mapper.isIgnoringPositions()) {
+            docs = postings = termsEnum.docsAndPositions(delDocs, postings);
+          } else {
+            docs = termsEnum.docs(delDocs, docs);
+          }
+
           int docID = docs.advance(documentNumber);
           if (docID == documentNumber) {
 
@@ -119,9 +126,8 @@
 
             if (!mapper.isIgnoringPositions()) {
               int[] positions = new int[docs.freq()];
-              PositionsEnum posEnum = docs.positions();
               for (int i = 0; i < positions.length; i++) {
-                positions[i] = posEnum.next();
+                positions[i] = postings.nextPosition();
               }
               this.positions.add(positions);
             } else {

Modified: lucene/java/branches/flex_1458/contrib/queries/src/java/org/apache/lucene/search/DuplicateFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/queries/src/java/org/apache/lucene/search/DuplicateFilter.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/queries/src/java/org/apache/lucene/search/DuplicateFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/queries/src/java/org/apache/lucene/search/DuplicateFilter.java Sat Jan 30 10:16:35 2010
@@ -87,12 +87,13 @@
     Terms terms = reader.fields().terms(fieldName);
     if (terms != null) {
       TermsEnum termsEnum = terms.iterator();
+      DocsEnum docs = null;
       while(true) {
         BytesRef currTerm = termsEnum.next();
         if (currTerm == null) {
           break;
         } else {
-          DocsEnum docs = termsEnum.docs(delDocs);
+          docs = termsEnum.docs(delDocs, docs);
           int doc = docs.nextDoc();
           if (doc != docs.NO_MORE_DOCS) {
             if (keepMode == KM_USE_FIRST_OCCURRENCE) {
@@ -124,6 +125,7 @@
     Terms terms = reader.fields().terms(fieldName);
     if (terms != null) {
       TermsEnum termsEnum = terms.iterator();
+      DocsEnum docs = null;
       while(true) {
         BytesRef currTerm = termsEnum.next();
         if (currTerm == null) {
@@ -131,7 +133,7 @@
         } else {
           if (termsEnum.docFreq() > 1) {
             // unset potential duplicates
-            DocsEnum docs = termsEnum.docs(delDocs);
+            docs = termsEnum.docs(delDocs, docs);
             int doc = docs.nextDoc();
             if (doc != docs.NO_MORE_DOCS) {
               if (keepMode == KM_USE_FIRST_OCCURRENCE) {

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllDocsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllDocsEnum.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllDocsEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllDocsEnum.java Sat Jan 30 10:16:35 2010
@@ -74,9 +74,4 @@
     doc = NO_MORE_DOCS;
     return doc;
   }
-
-  @Override
-  public PositionsEnum positions() {
-    throw new UnsupportedOperationException();
-  }
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CheckIndex.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CheckIndex.java Sat Jan 30 10:16:35 2010
@@ -598,6 +598,10 @@
         }
         
         final TermsEnum terms = fields.terms();
+
+        DocsEnum docs = null;
+        DocsAndPositionsEnum postings = null;
+
         while(true) {
 
           final BytesRef term = terms.next();
@@ -607,16 +611,25 @@
           final int docFreq = terms.docFreq();
           status.totFreq += docFreq;
 
-          final DocsEnum docs = terms.docs(delDocs);
+          docs = terms.docs(delDocs, docs);
+          postings = terms.docsAndPositions(delDocs, postings);
+
+          final DocsEnum docs2;
+          if (postings != null) {
+            docs2 = postings;
+          } else {
+            docs2 = docs;
+          }
+
           status.termCount++;
 
           int lastDoc = -1;
           while(true) {
-            final int doc = docs.nextDoc();
+            final int doc = docs2.nextDoc();
             if (doc == DocsEnum.NO_MORE_DOCS) {
               break;
             }
-            final int freq = docs.freq();
+            final int freq = docs2.freq();
             status.totPos += freq;
 
             if (doc <= lastDoc) {
@@ -632,10 +645,9 @@
             }
             
             int lastPos = -1;
-            final PositionsEnum positions = docs.positions();
-            if (positions != null) {
+            if (postings != null) {
               for(int j=0;j<freq;j++) {
-                final int pos = positions.next();
+                final int pos = postings.nextPosition();
                 if (pos < -1) {
                   throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " is out of bounds");
                 }
@@ -643,6 +655,9 @@
                   throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " < lastPos " + lastPos);
                 }
                 lastPos = pos;
+                if (postings.getPayloadLength() != 0) {
+                  postings.getPayload();
+                }
               }
             }
           }
@@ -651,7 +666,7 @@
           // this term:
 
           if (reader.hasDeletions()) {
-            final DocsEnum docsNoDel = terms.docs(null);
+            final DocsEnum docsNoDel = terms.docs(null, docs);
             int count = 0;
             while(docsNoDel.nextDoc() != DocsEnum.NO_MORE_DOCS) {
               count++;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java Sat Jan 30 10:16:35 2010
@@ -66,6 +66,7 @@
 
   private SegmentReader[] subReaders;
   private int[] starts;                           // 1st docno for each segment
+  private final Map<SegmentReader,Integer> subReaderToDocBase = new HashMap<SegmentReader,Integer>();
   private Map<String,byte[]> normsCache = new HashMap<String,byte[]>();
   private int maxDoc = 0;
   private int numDocs = -1;
@@ -370,6 +371,7 @@
         hasDeletions = true;
       }
       subs[i] = subReaders[i].getDeletedDocs();
+      subReaderToDocBase.put(subReaders[i], Integer.valueOf(starts[i]));
     }
     starts[subReaders.length] = maxDoc;
 
@@ -1047,6 +1049,11 @@
     return subReaders;
   }
 
+  @Override
+  public int getSubReaderDocBase(IndexReader subReader) {
+    return subReaderToDocBase.get(subReader).intValue();
+  }
+
   /** Returns the directory this index resides in. */
   @Override
   public Directory directory() {
@@ -1237,8 +1244,8 @@
     }
   }
 
-  private final static class DocsEnumWithBase {
-    DocsEnum docs;
+  private final static class PostingsEnumWithBase {
+    DocsAndPositionsEnum postings;
     int base;
   }
 
@@ -1433,14 +1440,12 @@
     int numTop;
     int numSubs;
     private BytesRef current;
-    private final MultiDocsEnum docs;
     private BytesRef.Comparator termComp;
 
     MultiTermsEnum(int size) {
       queue = new TermMergeQueue(size);
       top = new TermsEnumWithBase[size];
       subs = new TermsEnumWithBase[size];
-      docs = new MultiDocsEnum(size);
     }
 
     @Override
@@ -1604,25 +1609,34 @@
     }
 
     @Override
-    public DocsEnum docs(Bits skipDocs) throws IOException {
-      return docs.reset(top, numTop, skipDocs);
+    public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException {
+      return docsAndPositions(skipDocs, (DocsAndPositionsEnum) reuse);
+    }
+
+    @Override
+    public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException {
+      if (reuse != null) {
+        return ((MultiDocsAndPositionsEnum) reuse).reset(top, numTop, skipDocs);
+      } else {
+        return new MultiDocsAndPositionsEnum(subs.length).reset(top, numTop, skipDocs);
+      }
     }
   }
 
-  private static final class MultiDocsEnum extends DocsEnum {
-    final DocsEnumWithBase[] subs;
+  private static final class MultiDocsAndPositionsEnum extends DocsAndPositionsEnum {
+    final PostingsEnumWithBase[] subs;
     int numSubs;
     int upto;
-    DocsEnum currentDocs;
+    DocsAndPositionsEnum currentDocs;
     int currentBase;
     Bits skipDocs;
     int doc = -1;
 
-    MultiDocsEnum(int count) {
-      subs = new DocsEnumWithBase[count];
+    MultiDocsAndPositionsEnum(int count) {
+      subs = new PostingsEnumWithBase[count];
     }
 
-    MultiDocsEnum reset(TermsEnumWithBase[] subs, final int numSubs, final Bits skipDocs) throws IOException {
+    MultiDocsAndPositionsEnum reset(TermsEnumWithBase[] subs, final int numSubs, final Bits skipDocs) throws IOException {
       this.numSubs = 0;
       this.skipDocs = skipDocs;
       for(int i=0;i<numSubs;i++) {
@@ -1653,10 +1667,10 @@
           bits = new SubBits(skipDocs, subs[i].base, subs[i].length);
         }
 
-        final DocsEnum docs = subs[i].terms.docs(bits);
-        if (docs != null) {
-          this.subs[this.numSubs] = new DocsEnumWithBase();
-          this.subs[this.numSubs].docs = docs;
+        final DocsAndPositionsEnum postings = subs[i].terms.docsAndPositions(bits, null);
+        if (postings != null) {
+          this.subs[this.numSubs] = new PostingsEnumWithBase();
+          this.subs[this.numSubs].postings = postings;
           this.subs[this.numSubs].base = subs[i].base;
           this.numSubs++;
         }
@@ -1677,30 +1691,6 @@
     }
 
     @Override
-    public int read(final int docs[], final int freqs[]) throws IOException {
-      while (true) {
-        while (currentDocs == null) {
-          if (upto == numSubs-1) {
-            return 0;
-          } else {
-            upto++;
-            currentDocs = subs[upto].docs;
-            currentBase = subs[upto].base;
-          }
-        }
-        final int end = currentDocs.read(docs, freqs);
-        if (end == 0) {          // none left in segment
-          currentDocs = null;
-        } else {            // got some
-          for (int i = 0; i < end; i++) {
-            docs[i] += currentBase;
-          }
-          return end;
-        }
-      }
-    }
-
-    @Override
     public int advance(int target) throws IOException {
       while(true) {
         if (currentDocs != null) {
@@ -1714,7 +1704,7 @@
           return this.doc = NO_MORE_DOCS;
         } else {
           upto++;
-          currentDocs = subs[upto].docs;
+          currentDocs = subs[upto].postings;
           currentBase = subs[upto].base;
         }
       }
@@ -1728,7 +1718,7 @@
             return this.doc = NO_MORE_DOCS;
           } else {
             upto++;
-            currentDocs = subs[upto].docs;
+            currentDocs = subs[upto].postings;
             currentBase = subs[upto].base;
           }
         }
@@ -1743,8 +1733,23 @@
     }
 
     @Override
-    public PositionsEnum positions() throws IOException {
-      return currentDocs.positions();
+    public int nextPosition() throws IOException {
+      return currentDocs.nextPosition();
+    }
+
+    @Override
+    public int getPayloadLength() {
+      return currentDocs.getPayloadLength();
+    }
+
+    @Override
+    public BytesRef getPayload() throws IOException {
+      return currentDocs.getPayload();
+    }
+
+    @Override
+    public boolean hasPayload() {
+      return currentDocs.hasPayload();
     }
   }
 

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java?rev=904750&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java Sat Jan 30 10:16:35 2010
@@ -0,0 +1,44 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.util.BytesRef;
+
+/** Also iterates through positions. */
+public abstract class DocsAndPositionsEnum extends DocsEnum {
+
+  /** Returns the next position.  You should only call this
+   *  up to {@link FormatPostingsDocsEnum#freq()} times else
+   *  the behavior is not defined. */
+  public abstract int nextPosition() throws IOException;
+
+  /** Returns length of payload at current position */
+  public abstract int getPayloadLength();
+
+  /** Returns the payload at this position, or null if no
+   *  payload was indexed. */
+  public abstract BytesRef getPayload() throws IOException;
+
+  public abstract boolean hasPayload();
+
+  public final int read(int[] docs, int[] freqs) {
+    throw new UnsupportedOperationException();
+  }
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java Sat Jan 30 10:16:35 2010
@@ -22,8 +22,10 @@
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.util.AttributeSource;
 
-/** On obtaining a DocsEnum, you must first call next() */
-
+/** Iterates through the documents, term freq and positions.
+ *  NOTE: you must first call {@link #next}.
+ *
+ *  @lucene.experimental */
 public abstract class DocsEnum extends DocIdSetIterator {
 
   private AttributeSource atts = null;
@@ -31,16 +33,18 @@
   // nocommit
   public String desc;
 
+  /** Returns term frequency in the current document.  Do
+   *  not call this before {@link #next} is first called,
+   *  nor after {@link #next} returns NO_MORE_DOCS. */
   public abstract int freq();
   
-  /**
-   * Returns the related attributes.
-   */
+  /** Returns the related attributes. */
   public AttributeSource attributes() {
     if (atts == null) atts = new AttributeSource();
     return atts;
   }
-  
+
+  // nocommit -- bulk read makes no sense w/ positions enum..
   // nocommit -- state in API that doc/freq are undefined
   // (defined?) after this?
   // nocommit -- fix this API so that intblock codecs are
@@ -64,12 +68,4 @@
     }
     return count;
   }
-
-  /** Don't call next() or skipTo() or read() until you're
-   *  done consuming the positions.  NOTE: this method may
-   *  return null, if the index contains no positional
-   *  information for this document.  The standard codec
-   *  (default) does this today when the field was indexed
-   *  with {@link Field#setOmitTermFreqAndPositions}. */
-  public abstract PositionsEnum positions() throws IOException;
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocumentsWriter.java Sat Jan 30 10:16:35 2010
@@ -593,6 +593,7 @@
 
       if (infoStream != null) {
         SegmentInfo si = new SegmentInfo(flushState.segmentName, flushState.numDocs, directory, flushState.codec);
+        si.setHasProx(hasProx());
         final long newSegmentSize = si.sizeInBytes();
         String message = "  oldRAMSize=" + numBytesUsed +
           " newFlushedSize=" + newSegmentSize +
@@ -623,8 +624,9 @@
     
     CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION);
     for(String fileName : flushState.flushedFiles) {
-      if (Codec.DEBUG)
+      if (Codec.DEBUG) {
         System.out.println("make cfs " + fileName);
+      }
       cfsWriter.addFile(fileName);
     }
       
@@ -1003,6 +1005,8 @@
 
       String currentField = null;
       BytesRef termRef = new BytesRef();
+      DocsEnum docs = null;
+
       for (Entry<Term, BufferedDeletes.Num> entry: deletesFlushed.terms.entrySet()) {
         Term term = entry.getKey();
         // Since we visit terms sorted, we gain performance
@@ -1026,9 +1030,10 @@
 
         termRef.copy(term.text());
         if (termsEnum.seek(termRef) == TermsEnum.SeekStatus.FOUND) {
-          DocsEnum docs = termsEnum.docs(reader.getDeletedDocs());
+          DocsEnum docsEnum = termsEnum.docs(reader.getDeletedDocs(), docs);
 
-          if (docs != null) {
+          if (docsEnum != null) {
+            docs = docsEnum;
             int limit = entry.getValue().getNum();
             while (true) {
               final int docID = docs.nextDoc();

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Fields.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Fields.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Fields.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Fields.java Sat Jan 30 10:16:35 2010
@@ -20,8 +20,7 @@
 import java.io.IOException;
 
 /** Access to fields and terms
- *
- * NOTE: this API is experimental and will likely change */
+ *  @lucene.experimental */
 
 public abstract class Fields {
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsEnum.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsEnum.java Sat Jan 30 10:16:35 2010
@@ -22,9 +22,9 @@
 import org.apache.lucene.util.AttributeSource;
 
 /** Enumerates indexed fields.  You must first call {@link
- * #next} before calling {@link #terms}.
+ *  #next} before calling {@link #terms}.
  *
- * NOTE: this API is experimental and will likely change */
+ * @lucene.experimental */
 
 public abstract class FieldsEnum {
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java Sat Jan 30 10:16:35 2010
@@ -25,9 +25,8 @@
 import java.util.List;
 import java.util.Map;
 
-import org.apache.lucene.index.codecs.DocsConsumer;
+import org.apache.lucene.index.codecs.PostingsConsumer;
 import org.apache.lucene.index.codecs.FieldsConsumer;
-import org.apache.lucene.index.codecs.PositionsConsumer;
 import org.apache.lucene.index.codecs.TermsConsumer;
 import org.apache.lucene.util.UnicodeUtil;
 import org.apache.lucene.util.BytesRef;
@@ -144,7 +143,6 @@
     consumer.close();
   }
 
-  private byte[] payloadBuffer;
   BytesRef payload;
 
   /* Walk through all unique text tokens (Posting
@@ -189,6 +187,7 @@
       termStates[0] = mergeStates[0];
       int numToMerge = 1;
 
+      // TODO: pqueue
       for(int i=1;i<numFields;i++) {
         final int cmp = termComp.compare(mergeStates[i].text, termStates[0].text);
         if (cmp < 0) {
@@ -208,7 +207,7 @@
       //System.out.println("  term=" + text.toUnicodeString());
       //System.out.println("  term=" + text.toString());
 
-      final DocsConsumer docConsumer = termsConsumer.startTerm(text);
+      final PostingsConsumer postingsConsumer = termsConsumer.startTerm(text);
 
       // Now termStates has numToMerge FieldMergeStates
       // which all share the same term.  Now we must
@@ -228,7 +227,7 @@
 
         assert minState.docID < flushedDocCount: "doc=" + minState.docID + " maxDoc=" + flushedDocCount;
 
-        final PositionsConsumer posConsumer = docConsumer.addDoc(minState.docID, termDocFreq);
+        postingsConsumer.addDoc(minState.docID, termDocFreq);
 
         final ByteSliceReader prox = minState.prox;
 
@@ -267,10 +266,10 @@
               thisPayload = null;
             }
 
-            posConsumer.add(position, thisPayload);
+            postingsConsumer.addPosition(position, thisPayload);
           } //End for
 
-          posConsumer.finishDoc();
+          postingsConsumer.finishDoc();
         }
 
         if (!minState.nextDoc()) {

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java Sat Jan 30 10:16:35 2010
@@ -940,37 +940,9 @@
     return termDocs;
   }
 
-  private static class NullDocsEnum extends DocsEnum {
-    @Override
-    public int advance(int target) {
-      return NO_MORE_DOCS;
-    }
-    @Override
-    public int nextDoc() {
-      return NO_MORE_DOCS;
-    }
-    @Override
-    public int docID() {
-      return -1;
-    }
-    @Override
-    public int freq() {
-      return 1;
-    }
-    @Override
-    public int read(int[] docs, int[] freqs) {
-      return 0;
-    }
-    @Override
-    public PositionsEnum positions() {
-      return null;
-    }
-  }
-  private static final NullDocsEnum nullDocsEnum = new NullDocsEnum();
-
-  // nocommit -- tap into per-thread cache, here?
-  // nocommit -- should we return null or NullDocsEnum?
-  /** Returns DocsEnum for the specified field & term. */
+  /** Returns {@link DocsEnum} for the specified field &
+   *  term.  This may return null, for example if either the
+   *  field or term does not exist. */
   public DocsEnum termDocsEnum(Bits skipDocs, String field, BytesRef term) throws IOException {
 
     assert field != null;
@@ -979,19 +951,42 @@
     final Terms terms = fields().terms(field);
     if (terms != null) {
       if (Codec.DEBUG) {
-        System.out.println("ir.termDocsEnum field=" + field + " terms=" + terms + " this=" + this);
+        System.out.println("ir.termDocsEnum field=" + field + " term=" + term + " terms=" + terms + " this=" + this);
       }
-      final DocsEnum docs = terms.docs(skipDocs, term);
+      final DocsEnum docs = terms.docs(skipDocs, term, null);
       if (Codec.DEBUG) {
         System.out.println("ir.termDocsEnum field=" + field + " docs=" +docs);
       }
       if (docs != null) {
         return docs;
       } else {
-        return nullDocsEnum;
+        return null;
+      }
+    } else {
+      return null;
+    }
+  }
+
+  /** Returns {@link DocsAndPositionsEnum} for the specified
+   *  field & term.  This may return null, for example if
+   *  either the field or term does not exist. */
+  public DocsAndPositionsEnum termPositionsEnum(Bits skipDocs, String field, BytesRef term) throws IOException {
+
+    assert field != null;
+    assert term != null;
+
+    final Terms terms = fields().terms(field);
+    if (terms != null) {
+      if (Codec.DEBUG) {
+        System.out.println("ir.termPositionsEnum field=" + field + " term=" + term + " terms=" + terms + " this=" + this);
       }
+      final DocsAndPositionsEnum postings = terms.docsAndPositions(skipDocs, term, null);
+      if (Codec.DEBUG) {
+        System.out.println("ir.termPositionsEnum field=" + field + " postings=" +postings);
+      }
+      return postings;
     } else {
-      return nullDocsEnum;
+      return null;
     }
   }
 
@@ -1370,6 +1365,12 @@
     return null;
   }
 
+
+  /** Expert: returns the docID base for this subReader. */
+  public int getSubReaderDocBase(IndexReader subReader) {
+    throw new UnsupportedOperationException();
+  }
+
   /** Expert */
   public Object getFieldCacheKey() {
     return this;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java Sat Jan 30 10:16:35 2010
@@ -613,6 +613,8 @@
         // Returns a ref, which we xfer to readerMap:
         // nocommit: old api
         sr = SegmentReader.get(false, info.dir, info, readBufferSize, doOpenStores, termsIndexDivisor, null);
+        // nocommit -- if info is from external dir DO NOT
+        // cache it!
         readerMap.put(info, sr);
       } else {
         if (doOpenStores) {
@@ -3932,7 +3934,7 @@
           mergeInit(merge);
 
           if (infoStream != null)
-            message("now merge\n  merge=" + merge.segString(directory) + "\n  merge=" + merge + "\n  index=" + segString());
+            message("now merge\n  merge=" + merge.segString(directory) + "\n  index=" + segString());
 
           mergeMiddle(merge);
           mergeSuccess(merge);

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFieldsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFieldsEnum.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFieldsEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFieldsEnum.java Sat Jan 30 10:16:35 2010
@@ -75,12 +75,10 @@
     private TermEnum terms;
     private BytesRef current;
     private final BytesRef tr = new BytesRef();
-    private final LegacyDocsEnum docsEnum;
 
     LegacyTermsEnum(IndexReader r, String field) throws IOException {
       this.r = r;
       this.field = field;
-      docsEnum = new LegacyDocsEnum(r, field);
     }
 
     @Override
@@ -157,9 +155,21 @@
     }
 
     @Override
-    public DocsEnum docs(Bits skipDocs) throws IOException {
-      docsEnum.reset(terms.term(), skipDocs);
-      return docsEnum;
+    public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException {
+      if (reuse != null) {
+        return ((LegacyDocsEnum) reuse).reset(terms.term(), skipDocs);
+      } else {
+        return (new LegacyDocsEnum(r, field)).reset(terms.term(), skipDocs);
+      }
+    }
+
+    @Override
+    public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException {
+      if (reuse != null) {
+        return ((LegacyDocsAndPositionsEnum) reuse).reset(terms.term(), skipDocs);
+      } else {
+        return (new LegacyDocsAndPositionsEnum(r, field)).reset(terms.term(), skipDocs);
+      }
     }
 
     public void close() throws IOException {
@@ -171,8 +181,7 @@
   private static class LegacyDocsEnum extends DocsEnum {
     private final IndexReader r;
     private final String field;
-    private final TermPositions tp;
-    private final LegacyPositionsEnum posEnum;
+    private final TermDocs td;
 
     private Term term;
 
@@ -181,11 +190,69 @@
     LegacyDocsEnum(IndexReader r, String field) throws IOException {
       this.r = r;
       this.field = field;
+      td = r.termDocs();
+    }
+
+    public DocsEnum reset(Term term, Bits skipDocs) throws IOException {
+      this.term = term;
+      td.seek(term);
+
+      if (skipDocs != r.getDeletedDocs()) {
+        // An external reader's TermDocs/Positions will
+        // silently skip deleted docs, so, we can't allow
+        // arbitrary skipDocs here:
+        throw new IllegalStateException("external IndexReader requires skipDocs == IndexReader.getDeletedDocs()");
+      }
+
+      return this;
+    }
+
+    @Override
+    public int nextDoc() throws IOException {
+      if (td.next()) {
+        return doc = td.doc();
+      } else {
+        return doc = NO_MORE_DOCS;
+      }
+    }
+
+    @Override
+    public int advance(int target) throws IOException {
+      if (td.skipTo(target)) {
+        return doc = td.doc();
+      } else {
+        return doc = NO_MORE_DOCS;
+      }
+    }
+
+    @Override
+    public int freq() {
+      return td.freq();
+    }
+
+    @Override
+    public int docID() {
+      return doc;
+    }
+  }
+
+  // Emulates flex on top of legacy API
+  private static class LegacyDocsAndPositionsEnum extends DocsAndPositionsEnum {
+    private final IndexReader r;
+    private final String field;
+    private final TermPositions tp;
+
+    private Term term;
+
+    private int doc = -1;
+
+    LegacyDocsAndPositionsEnum(IndexReader r, String field) throws IOException {
+      this.r = r;
+      this.field = field;
       tp = r.termPositions();
-      posEnum = new LegacyPositionsEnum(tp);
     }
 
-    public void reset(Term term, Bits skipDocs) throws IOException {
+    public DocsAndPositionsEnum reset(Term term, Bits skipDocs) throws IOException {
       this.term = term;
       tp.seek(term);
 
@@ -193,9 +260,10 @@
         // An external reader's TermDocs/Positions will
         // silently skip deleted docs, so, we can't allow
         // arbitrary skipDocs here:
-        //System.out.println("skipDocs=" + skipDocs + " vs " + r.getDeletedDocs());
         throw new IllegalStateException("external IndexReader requires skipDocs == IndexReader.getDeletedDocs()");
       }
+
+      return this;
     }
 
     @Override
@@ -226,31 +294,12 @@
       return doc;
     }
 
-    public void close() throws IOException {
-      tp.close();
-    }
-
-    @Override
-    public PositionsEnum positions() throws IOException {
-      return posEnum;
-    }
-
     // NOTE: we don't override bulk-read (docs & freqs) API
     // -- leave it to base class, because TermPositions
     // can't do bulk read
-  }
-
-  // Emulates flex on top of legacy API
-  private static class LegacyPositionsEnum extends PositionsEnum {
-
-    final TermPositions tp;
-
-    LegacyPositionsEnum(TermPositions tp) {
-      this.tp = tp;
-    }
 
     @Override
-    public int next() throws IOException {
+    public int nextPosition() throws IOException {
       return tp.nextPosition();
     }
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MultiReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MultiReader.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MultiReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/MultiReader.java Sat Jan 30 10:16:35 2010
@@ -40,6 +40,7 @@
 public class MultiReader extends IndexReader implements Cloneable {
   protected IndexReader[] subReaders;
   private int[] starts;                           // 1st docno for each segment
+  private final Map<IndexReader,Integer> subReaderToDocBase = new HashMap<IndexReader,Integer>();
   private boolean[] decrefOnClose;                // remember which subreaders to decRef on close
   private Map<String,byte[]> normsCache = new HashMap<String,byte[]>();
   private int maxDoc = 0;
@@ -93,6 +94,7 @@
         hasDeletions = true;
       }
       subs[i] = subReaders[i].getDeletedDocs();
+      subReaderToDocBase.put(subReaders[i], Integer.valueOf(starts[i]));
     }
 
     starts[subReaders.length] = maxDoc;
@@ -105,6 +107,11 @@
   }
 
   @Override
+  public int getSubReaderDocBase(IndexReader subReader) {
+    return subReaderToDocBase.get(subReader).intValue();
+  }
+
+  @Override
   public Fields fields() throws IOException {
     if (subReaders.length == 1) {
       // Optimize the single reader case

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java Sat Jan 30 10:16:35 2010
@@ -1474,8 +1474,8 @@
     String currentField;
     final Fields fields;
     TermsEnum terms;
-    DocsEnum docs;
-    int doc;
+    DocsEnum docsEnum;
+    boolean any;
 
     LegacyTermDocs() throws IOException {
       fields = core.fields;
@@ -1489,16 +1489,11 @@
     }
 
     public boolean skipTo(int target) throws IOException {
-      if (docs == null) return false;
-      doc = docs.advance(target);
-      return doc != docs.NO_MORE_DOCS;
-    }
-
-    public int read(int[] docs, int[] freqs) throws IOException {
-      if (this.docs == null) {
-        return 0;
+      if (!any) {
+        return false;
+      } else {
+        return docsEnum.advance(target) != docsEnum.NO_MORE_DOCS;
       }
-      return this.docs.read(docs, freqs);
     }
 
     public void seek(Term term) throws IOException {
@@ -1507,20 +1502,19 @@
         System.out.println("\nwrapper termdocs.seek term=" + term);
       }
 
-      docs = null;
+      any = false;
 
       if (terms != null && !term.field.equals(currentField)) {
+        // new field
         if (Codec.DEBUG) {
           System.out.println("  switch field");
         }
-        if (terms != null) {
-          terms = null;
-        }
+        terms = null;
       }
 
       if (terms == null) {
         currentField = term.field;
-        Terms terms1 = fields.terms(term.field);
+        Terms terms1 = fields.terms(currentField);
         if (terms1 == null) {
           // no such field
           return;
@@ -1531,12 +1525,12 @@
 
       if (terms.seek(new BytesRef(term.text)) == TermsEnum.SeekStatus.FOUND) {
         // Term exists
-        docs = terms.docs(deletedDocs);
+        any = true;
+        docsEnum = terms.docs(deletedDocs, docsEnum);
         if (Codec.DEBUG) {
           System.out.println("  init docs enum");
         }
       } else {
-        docs = null;
         if (Codec.DEBUG) {
           System.out.println("  clear docs enum");
         }
@@ -1544,93 +1538,164 @@
     }
 
     public int doc() {
-      if (docs == null) return 0;
-      else return doc;
+      if (!any) {
+        return 0;
+      } else {
+        return docsEnum.docID();
+      }
+    }
+
+    public int read(int[] docs, int[] freqs) throws IOException {
+      if (!any) {
+        return 0;
+      } else {
+        return docsEnum.read(docs, freqs);
+      }
     }
 
     public int freq() {
-      if (docs == null) return 0;
-      return docs.freq();
+      if (!any) {
+        return 0;
+      } else {
+        return docsEnum.freq();
+      }
     }
 
     public boolean next() throws IOException {
-      if (docs == null) return false;
-      doc = docs.nextDoc();
-      return doc != DocsEnum.NO_MORE_DOCS;
+      if (!any) {
+        return false;
+      } else {
+        return docsEnum.nextDoc() != DocsEnum.NO_MORE_DOCS;
+      }
     }
   }
 
   // Back compat: implements legacy TermPositions API on top
   // of flex API
-  final private class LegacyTermPositions extends LegacyTermDocs implements TermPositions {
+  final private class LegacyTermPositions implements TermPositions {
 
-    PositionsEnum positions;
-    boolean didGetPositions;
+    String currentField;
+    final Fields fields;
+    TermsEnum terms;
+    DocsAndPositionsEnum postingsEnum;
+    DocsEnum docsEnum;
+    boolean any;
 
     LegacyTermPositions() throws IOException {
-      super();
+      fields = core.fields;
     }
 
-    @Override
+    public void close() {}
+
     public void seek(TermEnum termEnum) throws IOException {
-      super.seek(termEnum);
-      if (docs != null) {
-        positions = docs.positions();
-        didGetPositions = true;
+      // nocommit -- optimize for the special cases here
+      seek(termEnum.term());
+    }
+
+    public boolean skipTo(int target) throws IOException {
+      if (!any) {
+        return false;
       } else {
-        didGetPositions = false;
+        return docsEnum.advance(target) != docsEnum.NO_MORE_DOCS;
       }
     }
 
-    @Override
-    public boolean skipTo(int target) throws IOException {
-      boolean result = super.skipTo(target);
-      didGetPositions = false;
-      return result;
+    public void seek(Term term) throws IOException {
+
+      if (Codec.DEBUG) {
+        System.out.println("\nwrapper termdocs.seek term=" + term);
+      }
+
+      any = false;
+
+      if (terms != null && !term.field.equals(currentField)) {
+        // new field
+        if (Codec.DEBUG) {
+          System.out.println("  switch field");
+        }
+        terms = null;
+      }
+
+      if (terms == null) {
+        currentField = term.field;
+        Terms terms1 = fields.terms(currentField);
+        if (terms1 == null) {
+          // no such field
+          return;
+        } else {
+          terms = terms1.iterator();
+        }
+      }
+
+      if (terms.seek(new BytesRef(term.text)) == TermsEnum.SeekStatus.FOUND) {
+        // Term exists
+        any = true;
+        postingsEnum = terms.docsAndPositions(deletedDocs, postingsEnum);
+        if (postingsEnum == null) {
+          docsEnum = terms.docs(deletedDocs, postingsEnum);
+        } else {
+          docsEnum = postingsEnum;
+        }
+        if (Codec.DEBUG) {
+          System.out.println("  init docs enum");
+        }
+      } else {
+        if (Codec.DEBUG) {
+          System.out.println("  clear docs enum");
+        }
+      }
     }
 
-    @Override
-    public int read(int[] docs, int[] freqs) throws IOException {
-      throw new UnsupportedOperationException("TermPositions does not support processing multiple documents in one call. Use TermDocs instead.");
+    public int doc() {
+      if (!any) {
+        return 0;
+      } else {
+        return docsEnum.docID();
+      }
     }
 
-    @Override
-    public void seek(Term term) throws IOException {
-      super.seek(term);
-      didGetPositions = false;
+    public int freq() {
+      if (!any) {
+        return 0;
+      } else {
+        return docsEnum.freq();
+      }
     }
 
-    @Override
     public boolean next() throws IOException {
-      boolean result = super.next();
-      didGetPositions = false;
-      return result;
+      if (!any) {
+        return false;
+      } else {
+        return docsEnum.nextDoc() != DocsEnum.NO_MORE_DOCS;
+      }
+    }
+
+    @Override
+    public int read(int[] docs, int[] freqs) throws IOException {
+      throw new UnsupportedOperationException("TermPositions does not support processing multiple documents in one call. Use TermDocs instead.");
     }
 
     public int nextPosition() throws IOException {     
-      if (!didGetPositions) {
-        positions = docs.positions();
-        didGetPositions = true;
-      }
-        
-      if (positions == null) {
-        // With omitTFAP, pre-flex API pretended there was
-        // one occurrence of the term, at position 0:
+      if (!any || postingsEnum == null) {
         return 0;
       } else {
-        return positions.next();
+        return postingsEnum.nextPosition();
       }
     }
 
     public int getPayloadLength() {
-      if (positions == null) {
+      if (!any || postingsEnum == null) {
         return 0;
+      } else {
+        return postingsEnum.getPayloadLength();
       }
-      return positions.getPayloadLength();
     }
 
     public byte[] getPayload(byte[] bytes, int offset) throws IOException {
-      final BytesRef payload = positions.getPayload();
+      if (!any || postingsEnum == null) {
+        return null;
+      }
+      final BytesRef payload = postingsEnum.getPayload();
       // old API would always used passed in bytes if it
       // "fits", else allocate new:
       if (bytes != null && payload.length <= bytes.length - offset) {
@@ -1646,10 +1711,11 @@
     }
 
     public boolean isPayloadAvailable() {
-      if (positions == null) {
+      if (!any || postingsEnum == null) {
         return false;
+      } else {
+        return postingsEnum.hasPayload();
       }
-      return positions.hasPayload();
     }
   }
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Terms.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Terms.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Terms.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Terms.java Sat Jan 30 10:16:35 2010
@@ -20,13 +20,19 @@
 import java.io.IOException;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CloseableThreadLocal;
 
 /**
- * NOTE: this API is experimental and will likely change
+ * Access to the terms in a specific field.  See {@link #Fields}.
+ * @lucene.experimental
  */
 
 public abstract class Terms {
 
+  // Privately cache a TermsEnum per-thread for looking up
+  // docFreq and getting a private DocsEnum
+  private final CloseableThreadLocal<TermsEnum> threadEnums = new CloseableThreadLocal<TermsEnum>();
+
   /** Returns an iterator that will step through all terms */
   public abstract TermsEnum iterator() throws IOException;
   
@@ -37,25 +43,36 @@
    *  reuse it. */
   public abstract BytesRef.Comparator getComparator() throws IOException;
 
-  /** Returns the docFreq of the specified term text. */
+  /** Returns the number of documents containing the
+   *  specified term text.  Returns 0 if the term does not
+   *  exist. */
   public int docFreq(BytesRef text) throws IOException {
-    // nocommit -- make thread private cache so we share
-    // single enum
-    // NOTE: subclasses may have more efficient impl
-    final TermsEnum terms = iterator();
-    if (terms.seek(text) == TermsEnum.SeekStatus.FOUND) {
-      return terms.docFreq();
+    final TermsEnum termsEnum = getThreadTermsEnum();
+    if (termsEnum.seek(text) == TermsEnum.SeekStatus.FOUND) {
+      return termsEnum.docFreq();
     } else {
       return 0;
     }
   }
 
-  /** Get DocsEnum for the specified term. */
-  public DocsEnum docs(Bits skipDocs, BytesRef text) throws IOException {
-    // NOTE: subclasses may have more efficient impl
-    final TermsEnum terms = iterator();
-    if (terms.seek(text) == TermsEnum.SeekStatus.FOUND) {
-      return terms.docs(skipDocs);
+  // nocommit -- or maybe make a separate positions(...) method?
+  /** Get DocsEnum for the specified term.  Returns null if
+   *  the term does not exist. */
+  public DocsEnum docs(Bits skipDocs, BytesRef text, DocsEnum reuse) throws IOException {
+    final TermsEnum termsEnum = getThreadTermsEnum();
+    if (termsEnum.seek(text) == TermsEnum.SeekStatus.FOUND) {
+      return termsEnum.docs(skipDocs, reuse);
+    } else {
+      return null;
+    }
+  }
+
+  /** Get DocsEnum for the specified term.  Returns null if
+   *  the term does not exist. */
+  public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, BytesRef text, DocsAndPositionsEnum reuse) throws IOException {
+    final TermsEnum termsEnum = getThreadTermsEnum();
+    if (termsEnum.seek(text) == TermsEnum.SeekStatus.FOUND) {
+      return termsEnum.docsAndPositions(skipDocs, reuse);
     } else {
       return null;
     }
@@ -64,4 +81,18 @@
   public long getUniqueTermCount() throws IOException {
     throw new UnsupportedOperationException("this reader does not implement getUniqueTermCount()");
   }
+
+  protected TermsEnum getThreadTermsEnum() throws IOException {
+    TermsEnum termsEnum = (TermsEnum) threadEnums.get();
+    if (termsEnum == null) {
+      termsEnum = iterator();
+      threadEnums.set(termsEnum);
+    }
+    return termsEnum;
+  }
+
+  // subclass must close when done:
+  protected void close() {
+    threadEnums.close();
+  }
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsEnum.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsEnum.java Sat Jan 30 10:16:35 2010
@@ -23,10 +23,6 @@
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 
-/**
- * NOTE: this API is experimental and will likely change
- */
-
 /** Iterator to seek ({@link #seek}) or step through ({@link
  * #next} terms, obtain frequency information ({@link
  * #docFreq}), and obtain a {@link DocsEnum} for the current
@@ -37,14 +33,14 @@
  * greater than all that precede it.</p>
  *
  * <p>On obtaining a TermsEnum, you must first call
- * {@link #next} or {@link #seek}. */
+ * {@link #next} or {@link #seek}.
+ *
+ * @lucene.experimental */
 public abstract class TermsEnum {
 
   private AttributeSource atts = null;
 
-  /**
-   * Returns the related attributes.
-   */
+  /** Returns the related attributes. */
   public AttributeSource attributes() {
     if (atts == null) atts = new AttributeSource();
     return atts;
@@ -75,29 +71,39 @@
    *  to next. */
   public abstract BytesRef next() throws IOException;
 
-  /** Returns current term.  This is undefined after next()
-   *  returns null or seek returns {@link SeekStatus#END}. */
+  /** Returns current term. Do not call this before calling
+   *  next() for the first time, after next() returns null
+   *  or seek returns {@link SeekStatus#END}.*/
   public abstract BytesRef term() throws IOException;
 
-  /** Returns ordinal position for current term.  Not all
-   *  codecs implement this, so be prepared to catch an
-   *  {@link UnsupportedOperationException}.  This is
-   *  undefined after next() returns null or seek returns
-   *  {@link SeekStatus#END}. */
+  /** Returns ordinal position for current term.  This is an
+   *  optional method (the codec may throw {@link
+   *  UnsupportedOperationException}).  Do not call this
+   *  before calling next() for the first time, after next()
+   *  returns null or seek returns {@link
+   *  SeekStatus#END}. */
   public abstract long ord() throws IOException;
 
-  /** Returns the docFreq of the current term.  This is
-   *  undefined after next() returns null or seek returns
+  /** Returns the number of documents containing the current
+   *  term.  Do not call this before calling next() for the
+   *  first time, after next() returns null or seek returns
    *  {@link SeekStatus#END}.*/
   public abstract int docFreq();
 
   // nocommit -- clarify if this may return null
-  /** Get {@link DocsEnum} for the current term.  The
-   *  returned {@link DocsEnum} may share state with this
-   *  TermsEnum instance, so you should not call this
-   *  TermsEnum's {@link #seek} or {@link #next} until you
-   *  are done using the DocsEnum. */
-  public abstract DocsEnum docs(Bits skipDocs) throws IOException;
+  // nocommit -- maybe require up front boolean doPositions?
+  // nocommit -- or maybe make a separate positions(...) method?
+  /** Get {@link DocsEnum} for the current term.  Do not
+   *  call this before calling next() for the first time,
+   *  after next() returns null or seek returns {@link
+   *  SeekStatus#END}.
+   *  
+   * @param skipDocs set bits are documents that should not
+   * be returned
+   * @param reuse pass a prior DocsEnum for possible reuse */
+  public abstract DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException;
+
+  public abstract DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException;
 
   /** Return the {@link BytesRef} Comparator used to sort
    *  terms provided by the iterator.  NOTE: this may return

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codec.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codec.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codec.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codec.java Sat Jan 30 10:16:35 2010
@@ -41,6 +41,17 @@
   /** Writes a new segment */
   public abstract FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException;
 
+  public static void debug(String s, String desc) {
+    if (desc != null) {
+      System.out.println(Thread.currentThread().getName()+ " [" + desc + "]:" + s);
+    } else {
+      System.out.println(Thread.currentThread().getName() + ": " + s);
+    }
+  }
+  public static void debug(String s) {
+    debug(s, null);
+  }
+
   /** Reads a segment.  NOTE: by the time this call
    *  returns, it must hold open any files it will need to
    *  use; else, those files may be deleted. */

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/FieldsProducer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/FieldsProducer.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/FieldsProducer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/FieldsProducer.java Sat Jan 30 10:16:35 2010
@@ -20,6 +20,7 @@
 import org.apache.lucene.index.Fields;
 
 import java.io.IOException;
+import java.io.Closeable;
 
 /** Abstract API that consumes terms, doc, freq, prox and
  *  payloads postings.  Concrete implementations of this
@@ -28,7 +29,8 @@
  *
  * NOTE: this API is experimental and will likely change
  */
-public abstract class FieldsProducer extends Fields {
+
+public abstract class FieldsProducer extends Fields implements Closeable {
   public abstract void close() throws IOException;
   public abstract void loadTermsIndex(int indexDivisor) throws IOException;
 }

Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java?rev=904750&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java Sat Jan 30 10:16:35 2010
@@ -0,0 +1,129 @@
+package org.apache.lucene.index.codecs;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * NOTE: this API is experimental and will likely change
+ */
+
+public abstract class PostingsConsumer {
+
+  // nocommit
+  public String desc;
+  /*
+  public boolean setDesc(String desc) {
+    this.desc = desc;
+    return true;
+  }
+  */
+
+  // nocommit -- rename to startDoc?
+  /** Adds a new doc in this term.  Return null if this
+   *  consumer doesn't need to see the positions for this
+   *  doc. */
+  public abstract void addDoc(int docID, int termDocFreq) throws IOException;
+
+  public static class PostingsMergeState {
+    DocsEnum docsEnum;
+    int[] docMap;
+    int docBase;
+  }
+
+  /** Add a new position & payload.  A null payload means no
+   *  payload; a non-null payload with zero length also
+   *  means no payload.  Caller may reuse the {@link
+   *  BytesRef} for the payload between calls (method must
+   *  fully consume the payload). */
+  public abstract void addPosition(int position, BytesRef payload) throws IOException;
+
+  /** Called when we are done adding positions & payloads
+   * for each doc */
+  public abstract void finishDoc() throws IOException;
+
+  /** Default merge impl: append documents, mapping around
+   *  deletes */
+  public int merge(MergeState mergeState, PostingsMergeState[] toMerge, int count) throws IOException {
+
+    int df = 0;
+
+    // Append docs in order:
+    for(int i=0;i<count;i++) {
+      final DocsEnum docsEnum = toMerge[i].docsEnum;
+      final int[] docMap = toMerge[i].docMap;
+      final int base = toMerge[i].docBase;
+
+      final DocsAndPositionsEnum postingsEnum;
+
+      if (!mergeState.omitTermFreqAndPositions) {
+        postingsEnum = (DocsAndPositionsEnum) docsEnum;
+      } else {
+        postingsEnum = null;
+      }
+
+      while(true) {
+        final int startDoc = docsEnum.nextDoc();
+        if (startDoc == DocsAndPositionsEnum.NO_MORE_DOCS) {
+          break;
+        }
+        df++;
+
+        int doc;
+        if (docMap != null) {
+          // map around deletions
+          doc = docMap[startDoc];
+          assert doc != -1: "docs enum returned deleted docID " + startDoc + " freq=" + docsEnum.freq() + " df=" + df + " de=" + docsEnum;
+        } else {
+          doc = startDoc;
+        }
+
+        doc += base;                              // convert to merged space
+        assert doc < mergeState.mergedDocCount: "doc=" + doc + " maxDoc=" + mergeState.mergedDocCount;
+
+        final int freq = docsEnum.freq();
+
+        addDoc(doc, freq);
+
+        // nocommit -- omitTF should be "private", and this
+        // code (and FreqProxTermsWriter) should instead
+        // check if posConsumer is null?
+        if (!mergeState.omitTermFreqAndPositions) {
+          for(int j=0;j<freq;j++) {
+            final int position = postingsEnum.nextPosition();
+            final int payloadLength = postingsEnum.getPayloadLength();
+            final BytesRef payload;
+            if (payloadLength > 0) {
+              payload = postingsEnum.getPayload();
+            } else {
+              payload = null;
+            }
+            addPosition(position, payload);
+          }
+          finishDoc();
+        }
+      }
+    }
+
+    return df;
+  }
+}

Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/TermsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/TermsConsumer.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/TermsConsumer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/TermsConsumer.java Sat Jan 30 10:16:35 2010
@@ -31,7 +31,7 @@
 public abstract class TermsConsumer {
 
   /** Starts a new term in this field. */
-  public abstract DocsConsumer startTerm(BytesRef text) throws IOException;
+  public abstract PostingsConsumer startTerm(BytesRef text) throws IOException;
 
   /** Finishes the current term */
   public abstract void finishTerm(BytesRef text, int numDocs) throws IOException;
@@ -71,7 +71,7 @@
   }
 
   private MergeQueue queue;
-  private DocsConsumer.DocsMergeState[] match;
+  private PostingsConsumer.PostingsMergeState[] match;
   private TermMergeState[] pending;
 
   /** Default merge impl */
@@ -83,9 +83,9 @@
 
     if (queue == null) {
       queue = new MergeQueue(mergeState.readerCount, termComp);
-      match = new DocsConsumer.DocsMergeState[mergeState.readerCount];
+      match = new PostingsConsumer.PostingsMergeState[mergeState.readerCount];
       for(int i=0;i<mergeState.readerCount;i++) {
-        match[i] = new DocsConsumer.DocsMergeState();
+        match[i] = new PostingsConsumer.PostingsMergeState();
       }
       pending = new TermMergeState[mergeState.readerCount];
     } else if (!queue.termComp.equals(termComp)) {
@@ -111,7 +111,10 @@
       while(true) {
         TermMergeState state = pending[pendingCount++] = queue.pop();
         
-        DocsEnum docsEnum = state.termsEnum.docs(mergeState.readers.get(state.readerIndex).getDeletedDocs());
+        DocsEnum docsEnum = state.termsEnum.docsAndPositions(mergeState.readers.get(state.readerIndex).getDeletedDocs(), null);
+        if (docsEnum == null) {
+          docsEnum = state.termsEnum.docs(mergeState.readers.get(state.readerIndex).getDeletedDocs(), null);
+        }
         if (docsEnum != null) {
           match[matchCount].docsEnum = docsEnum;
           match[matchCount].docMap = mergeState.docMaps[state.readerIndex];
@@ -128,8 +131,8 @@
         // Merge one term
         final BytesRef term = pending[0].current;
         //System.out.println("  merge term=" + term);
-        final DocsConsumer docsConsumer = startTerm(term);
-        final int numDocs = docsConsumer.merge(mergeState, match, matchCount);
+        final PostingsConsumer postingsConsumer = startTerm(term);
+        final int numDocs = postingsConsumer.merge(mergeState, match, matchCount);
         finishTerm(term, numDocs);
       }
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexInput.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexInput.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/FixedIntBlockIndexInput.java Sat Jan 30 10:16:35 2010
@@ -184,26 +184,12 @@
       upto = idx.upto;
     }
 
-    public class State extends IndexState {
-      long fp;
-      int upto;
-    }
-
-    // nocommit handle with set and/or clone?
-    @Override
-    public IndexState captureState() {
-      final State state = new State();
-      state.fp = fp;
-      state.upto = upto;
-      return state;
-    }
-
-    // nocommit handle with set and/or clone?
     @Override
-    public void setState(final IndexState state) {
-      final State iState = (State) state;
-      this.fp = iState.fp;
-      this.upto = iState.upto;
+    public Object clone() {
+      Index other = new Index();
+      other.fp = fp;
+      other.upto = upto;
+      return other;
     }
   }
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java Sat Jan 30 10:16:35 2010
@@ -27,16 +27,17 @@
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
 import org.apache.lucene.index.codecs.sep.SepCodec;
-import org.apache.lucene.index.codecs.sep.SepDocsReader;
-import org.apache.lucene.index.codecs.sep.SepDocsWriter;
+import org.apache.lucene.index.codecs.sep.SepPostingsReaderImpl;
+import org.apache.lucene.index.codecs.sep.SepPostingsWriterImpl;
 import org.apache.lucene.index.codecs.standard.SimpleStandardTermsIndexReader;
 import org.apache.lucene.index.codecs.standard.SimpleStandardTermsIndexWriter;
-import org.apache.lucene.index.codecs.standard.StandardDocsConsumer;
-import org.apache.lucene.index.codecs.standard.StandardDocsProducer;
+import org.apache.lucene.index.codecs.standard.StandardPostingsWriter;
+import org.apache.lucene.index.codecs.standard.StandardPostingsReader;
 import org.apache.lucene.index.codecs.standard.StandardTermsDictReader;
 import org.apache.lucene.index.codecs.standard.StandardTermsDictWriter;
 import org.apache.lucene.index.codecs.standard.StandardTermsIndexReader;
 import org.apache.lucene.index.codecs.standard.StandardTermsIndexWriter;
+import org.apache.lucene.index.codecs.standard.StandardCodec;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 
@@ -51,7 +52,7 @@
 
   @Override
   public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
-    StandardDocsConsumer docsWriter = new SepDocsWriter(state, new SimpleIntBlockFactory(1024));
+    StandardPostingsWriter postingsWriter = new SepPostingsWriterImpl(state, new SimpleIntBlockFactory(1024));
 
     boolean success = false;
     StandardTermsIndexWriter indexWriter;
@@ -60,19 +61,19 @@
       success = true;
     } finally {
       if (!success) {
-        docsWriter.close();
+        postingsWriter.close();
       }
     }
 
     success = false;
     try {
-      FieldsConsumer ret = new StandardTermsDictWriter(indexWriter, state, docsWriter, BytesRef.getUTF8SortedAsUTF16Comparator());
+      FieldsConsumer ret = new StandardTermsDictWriter(indexWriter, state, postingsWriter, BytesRef.getUTF8SortedAsUTF16Comparator());
       success = true;
       return ret;
     } finally {
       if (!success) {
         try {
-          docsWriter.close();
+          postingsWriter.close();
         } finally {
           indexWriter.close();
         }
@@ -82,7 +83,7 @@
 
   @Override
   public FieldsProducer fieldsProducer(Directory dir, FieldInfos fieldInfos, SegmentInfo si, int readBufferSize, int indexDivisor) throws IOException {
-    StandardDocsProducer docsReader = new SepDocsReader(dir, si, readBufferSize, new SimpleIntBlockFactory(1024));
+    StandardPostingsReader postingsReader = new SepPostingsReaderImpl(dir, si, readBufferSize, new SimpleIntBlockFactory(1024));
 
     StandardTermsIndexReader indexReader;
     boolean success = false;
@@ -95,7 +96,7 @@
       success = true;
     } finally {
       if (!success) {
-        docsReader.close();
+        postingsReader.close();
       }
     }
 
@@ -103,15 +104,16 @@
     try {
       FieldsProducer ret = new StandardTermsDictReader(indexReader,
                                                        dir, fieldInfos, si.name,
-                                                       docsReader,
+                                                       postingsReader,
                                                        readBufferSize,
-                                                       BytesRef.getUTF8SortedAsUTF16Comparator());
+                                                       BytesRef.getUTF8SortedAsUTF16Comparator(),
+                                                       StandardCodec.TERMS_CACHE_SIZE);
       success = true;
       return ret;
     } finally {
       if (!success) {
         try {
-          docsReader.close();
+          postingsReader.close();
         } finally {
           indexReader.close();
         }
@@ -121,7 +123,7 @@
 
   @Override
   public void files(Directory dir, SegmentInfo segmentInfo, Collection<String> files) {
-    SepDocsReader.files(segmentInfo, files);
+    SepPostingsReaderImpl.files(segmentInfo, files);
     StandardTermsDictReader.files(dir, segmentInfo, files);
     SimpleStandardTermsIndexReader.files(dir, segmentInfo, files);
   }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java?rev=904750&r1=904749&r2=904750&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java Sat Jan 30 10:16:35 2010
@@ -23,11 +23,11 @@
 import java.util.TreeMap;
 
 import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.DocsAndPositionsEnum;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.FieldsEnum;
 import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.PositionsEnum;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.Terms;
@@ -229,15 +229,10 @@
   private class PreTermsEnum extends TermsEnum {
     private SegmentTermEnum termEnum;
     private FieldInfo fieldInfo;
-    private final PreDocsEnum docsEnum;
     private boolean skipNext;
     private BytesRef current;
     private final BytesRef scratchBytesRef = new BytesRef();
 
-    public PreTermsEnum() throws IOException {
-      docsEnum = new PreDocsEnum();
-    }
-
     void reset(FieldInfo fieldInfo, boolean isFirstField) throws IOException {
       this.fieldInfo = fieldInfo;
       if (termEnum == null) {
@@ -354,27 +349,89 @@
     }
 
     @Override
-    public DocsEnum docs(Bits skipDocs) throws IOException {
+    public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException {
+      // nocommit -- must assert that skipDocs "matches" the
+      // underlying deletedDocs?
+      if (reuse != null) {
+        return ((PreDocsEnum) reuse).reset(termEnum, skipDocs);        
+      } else {
+        return (new PreDocsEnum()).reset(termEnum, skipDocs);
+      }
+    }
+
+    @Override
+    public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException {
       // nocommit -- must assert that skipDocs "matches" the
       // underlying deletedDocs?
-      docsEnum.reset(termEnum, skipDocs);        
-      return docsEnum;
+      if (reuse != null) {
+        return ((PreDocsAndPositionsEnum) reuse).reset(termEnum, skipDocs);        
+      } else {
+        return (new PreDocsAndPositionsEnum()).reset(termEnum, skipDocs);
+      }
     }
   }
 
   private final class PreDocsEnum extends DocsEnum {
-    final private SegmentTermPositions pos;
-    final private PrePositionsEnum prePos;
-    private Bits skipDocs;
+    final private SegmentTermDocs docs;
 
     PreDocsEnum() throws IOException {
+      docs = new SegmentTermDocs(freqStream, getTermsDict(), fieldInfos);
+    }
+
+    public PreDocsEnum reset(SegmentTermEnum termEnum, Bits skipDocs) throws IOException {
+      docs.setSkipDocs(skipDocs);
+      docs.seek(termEnum);
+      return this;
+    }
+
+    @Override
+    public int nextDoc() throws IOException {
+      if (Codec.DEBUG) {
+        System.out.println("pff.docs.next");
+      }
+      if (docs.next()) {
+        return docs.doc();
+      } else {
+        return NO_MORE_DOCS;
+      }
+    }
+
+    @Override
+    public int advance(int target) throws IOException {
+      if (docs.skipTo(target)) {
+        return docs.doc();
+      } else {
+        return NO_MORE_DOCS;
+      }
+    }
+
+    @Override
+    public int freq() {
+      return docs.freq();
+    }
+
+    @Override
+    public int docID() {
+      return docs.doc();
+    }
+
+    @Override
+    public int read(int[] docs, int[] freqs) throws IOException {
+      return this.docs.read(docs, freqs);
+    }
+  }
+
+  private final class PreDocsAndPositionsEnum extends DocsAndPositionsEnum {
+    final private SegmentTermPositions pos;
+
+    PreDocsAndPositionsEnum() throws IOException {
       pos = new SegmentTermPositions(freqStream, proxStream, getTermsDict(), fieldInfos);
-      prePos = new PrePositionsEnum(pos);
     }
 
-    public void reset(SegmentTermEnum termEnum, Bits skipDocs) throws IOException {
+    public DocsAndPositionsEnum reset(SegmentTermEnum termEnum, Bits skipDocs) throws IOException {
       pos.setSkipDocs(skipDocs);
       pos.seek(termEnum);
+      return this;
     }
 
     @Override
@@ -409,23 +466,7 @@
     }
 
     @Override
-    public PositionsEnum positions() throws IOException {
-      return prePos;
-    }
-
-    // NOTE: we don't override bulk-read (docs & freqs) API
-    // -- leave it to base class, because TermPositions
-    // can't do bulk read
-  }
-
-  private final class PrePositionsEnum extends PositionsEnum {
-    final private SegmentTermPositions pos;
-    PrePositionsEnum(SegmentTermPositions pos) {
-      this.pos = pos;
-    }
-
-    @Override
-    public int next() throws IOException {
+    public int nextPosition() throws IOException {
       return pos.nextPosition();
     }
 



Mime
View raw message