lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From uschind...@apache.org
Subject svn commit: r1210184 [1/3] - in /lucene/dev/branches/lucene3606: ./ lucene/ lucene/contrib/memory/src/java/org/apache/lucene/index/memory/ lucene/contrib/memory/src/test/org/apache/lucene/index/memory/ lucene/contrib/misc/src/java/org/apache/lucene/mis...
Date Sun, 04 Dec 2011 19:18:20 GMT
Author: uschindler
Date: Sun Dec  4 19:18:17 2011
New Revision: 1210184

URL: http://svn.apache.org/viewvc?rev=1210184&view=rev
Log:
Merge in Mikes huge commit (r1210176)

Added:
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/MatchOnlyConjunctionTermsScorer.java
      - copied unchanged from r1210179, lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MatchOnlyConjunctionTermsScorer.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java
      - copied unchanged from r1210179, lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java
Modified:
    lucene/dev/branches/lucene3606/   (props changed)
    lucene/dev/branches/lucene3606/lucene/   (props changed)
    lucene/dev/branches/lucene3606/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
    lucene/dev/branches/lucene3606/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
    lucene/dev/branches/lucene3606/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java
    lucene/dev/branches/lucene3606/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java
    lucene/dev/branches/lucene3606/lucene/contrib/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java
    lucene/dev/branches/lucene3606/lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/DuplicateFilterTest.java
    lucene/dev/branches/lucene3606/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/CheckIndex.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocTermOrds.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocsEnum.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FieldInfo.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FilteredTermsEnum.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/IndexReader.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/MultiFields.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/TermsEnum.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/TermsConsumer.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40TermVectorsReader.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsReader.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/BooleanQuery.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/PhraseQuery.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/TermQuery.java
    lucene/dev/branches/lucene3606/lucene/src/test-framework/java/org/apache/lucene/index/codecs/ramonly/RAMOnlyPostingsFormat.java
    lucene/dev/branches/lucene3606/lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestCodecs.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestDocCount.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestLongPostings.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestMultiFields.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestOmitPositions.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestParallelTermEnum.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestStressAdvance.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestTermdocPerf.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/codecs/pulsing/Test10KPulsings.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/codecs/pulsing/TestPulsingReuse.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/search/TestTermVectors.java
    lucene/dev/branches/lucene3606/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestKeywordAnalyzer.java
    lucene/dev/branches/lucene3606/modules/benchmark/   (props changed)
    lucene/dev/branches/lucene3606/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
    lucene/dev/branches/lucene3606/modules/facet/   (props changed)
    lucene/dev/branches/lucene3606/modules/facet/src/java/org/apache/lucene/facet/search/sampling/TakmiSampleFixer.java
    lucene/dev/branches/lucene3606/modules/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java
    lucene/dev/branches/lucene3606/modules/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java
    lucene/dev/branches/lucene3606/modules/facet/src/test/org/apache/lucene/facet/FacetTestBase.java
    lucene/dev/branches/lucene3606/modules/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java
    lucene/dev/branches/lucene3606/modules/queries/src/java/org/apache/lucene/queries/TermsFilter.java
    lucene/dev/branches/lucene3606/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java
    lucene/dev/branches/lucene3606/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/TermFreqValueSource.java
    lucene/dev/branches/lucene3606/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
    lucene/dev/branches/lucene3606/solr/   (props changed)
    lucene/dev/branches/lucene3606/solr/core/   (props changed)
    lucene/dev/branches/lucene3606/solr/core/src/java/   (props changed)
    lucene/dev/branches/lucene3606/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
    lucene/dev/branches/lucene3606/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java
    lucene/dev/branches/lucene3606/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
    lucene/dev/branches/lucene3606/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java
    lucene/dev/branches/lucene3606/solr/core/src/test/   (props changed)
    lucene/dev/branches/lucene3606/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java

Modified: lucene/dev/branches/lucene3606/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/lucene3606/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Sun Dec  4 19:18:17 2011
@@ -936,7 +936,7 @@ public class MemoryIndex {
       }
 
       @Override
-      public DocsEnum docs(Bits liveDocs, DocsEnum reuse) {
+      public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) {
         if (reuse == null || !(reuse instanceof MemoryDocsEnum)) {
           reuse = new MemoryDocsEnum();
         }

Modified: lucene/dev/branches/lucene3606/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (original)
+++ lucene/dev/branches/lucene3606/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java Sun Dec  4 19:18:17 2011
@@ -188,7 +188,7 @@ public class MemoryIndexTest extends Bas
     MemoryIndex memory = new MemoryIndex();
     memory.addField("foo", "bar", analyzer);
     IndexReader reader = memory.createSearcher().getIndexReader();
-    DocsEnum disi = reader.termDocsEnum(null, "foo", new BytesRef("bar"));
+    DocsEnum disi = _TestUtil.docs(random, reader, "foo", new BytesRef("bar"), null, null, false);
     int docid = disi.docID();
     assertTrue(docid == -1 || docid == DocIdSetIterator.NO_MORE_DOCS);
     assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
@@ -196,7 +196,7 @@ public class MemoryIndexTest extends Bas
     // now reuse and check again
     TermsEnum te = reader.terms("foo").iterator(null);
     assertTrue(te.seekExact(new BytesRef("bar"), true));
-    disi = te.docs(null, disi);
+    disi = te.docs(null, disi, false);
     docid = disi.docID();
     assertTrue(docid == -1 || docid == DocIdSetIterator.NO_MORE_DOCS);
     assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);

Modified: lucene/dev/branches/lucene3606/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java (original)
+++ lucene/dev/branches/lucene3606/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java Sun Dec  4 19:18:17 2011
@@ -201,7 +201,7 @@ public class HighFreqTerms {
             return;
           }
         }
-        DocsEnum de = r.termDocsEnum(liveDocs, field, termText);
+        DocsEnum de = r.termDocsEnum(liveDocs, field, termText, true);
         if (de != null) {
           while (de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS)
             totalTF[0] += de.freq();

Modified: lucene/dev/branches/lucene3606/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java (original)
+++ lucene/dev/branches/lucene3606/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java Sun Dec  4 19:18:17 2011
@@ -29,11 +29,11 @@ import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.TieredMergePolicy;
 import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.TermsEnum.SeekStatus;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.TieredMergePolicy;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexOutput;
@@ -139,7 +139,7 @@ public class TestAppendingCodec extends 
     assertEquals(SeekStatus.FOUND, te.seekCeil(new BytesRef("lazy")));
     assertEquals(SeekStatus.FOUND, te.seekCeil(new BytesRef("dog")));
     assertEquals(SeekStatus.FOUND, te.seekCeil(new BytesRef("the")));
-    DocsEnum de = te.docs(null, null);
+    DocsEnum de = te.docs(null, null, true);
     assertTrue(de.advance(0) != DocsEnum.NO_MORE_DOCS);
     assertEquals(2, de.freq());
     assertTrue(de.advance(1) != DocsEnum.NO_MORE_DOCS);

Modified: lucene/dev/branches/lucene3606/lucene/contrib/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/contrib/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/contrib/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java (original)
+++ lucene/dev/branches/lucene3606/lucene/contrib/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java Sun Dec  4 19:18:17 2011
@@ -93,7 +93,7 @@ public class DuplicateFilter extends Fil
       if (currTerm == null) {
         break;
       } else {
-        docs = termsEnum.docs(acceptDocs, docs);
+        docs = termsEnum.docs(acceptDocs, docs, false);
         int doc = docs.nextDoc();
         if (doc != DocsEnum.NO_MORE_DOCS) {
           if (keepMode == KeepMode.KM_USE_FIRST_OCCURRENCE) {
@@ -133,7 +133,7 @@ public class DuplicateFilter extends Fil
       } else {
         if (termsEnum.docFreq() > 1) {
           // unset potential duplicates
-          docs = termsEnum.docs(acceptDocs, docs);
+          docs = termsEnum.docs(acceptDocs, docs, false);
           int doc = docs.nextDoc();
           if (doc != DocsEnum.NO_MORE_DOCS) {
             if (keepMode == KeepMode.KM_USE_FIRST_OCCURRENCE) {

Modified: lucene/dev/branches/lucene3606/lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/DuplicateFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/DuplicateFilterTest.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/DuplicateFilterTest.java (original)
+++ lucene/dev/branches/lucene3606/lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/DuplicateFilterTest.java Sun Dec  4 19:18:17 2011
@@ -17,6 +17,9 @@ package org.apache.lucene.sandbox.querie
  * limitations under the License.
  */
 
+import java.io.IOException;
+import java.util.HashSet;
+
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.StringField;
@@ -28,9 +31,7 @@ import org.apache.lucene.search.TermQuer
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
-
-import java.io.IOException;
-import java.util.HashSet;
+import org.apache.lucene.util._TestUtil;
 
 public class DuplicateFilterTest extends LuceneTestCase {
   private static final String KEY_FIELD = "url";
@@ -134,10 +135,13 @@ public class DuplicateFilterTest extends
     for (ScoreDoc hit : hits) {
       Document d = searcher.doc(hit.doc);
       String url = d.get(KEY_FIELD);
-      DocsEnum td = MultiFields.getTermDocsEnum(reader,
-          MultiFields.getLiveDocs(reader),
-          KEY_FIELD,
-          new BytesRef(url));
+      DocsEnum td = _TestUtil.docs(random, reader,
+                                   KEY_FIELD,
+                                   new BytesRef(url),
+                                   MultiFields.getLiveDocs(reader),
+                                   null,
+                                   false);
+
       int lastDoc = 0;
       while (td.nextDoc() != DocsEnum.NO_MORE_DOCS) {
         lastDoc = td.docID();
@@ -155,10 +159,13 @@ public class DuplicateFilterTest extends
     for (ScoreDoc hit : hits) {
       Document d = searcher.doc(hit.doc);
       String url = d.get(KEY_FIELD);
-      DocsEnum td = MultiFields.getTermDocsEnum(reader,
-          MultiFields.getLiveDocs(reader),
-          KEY_FIELD,
-          new BytesRef(url));
+      DocsEnum td = _TestUtil.docs(random, reader,
+                                   KEY_FIELD,
+                                   new BytesRef(url),
+                                   MultiFields.getLiveDocs(reader),
+                                   null,
+                                   false);
+
       int lastDoc = 0;
       td.nextDoc();
       lastDoc = td.docID();

Modified: lucene/dev/branches/lucene3606/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java (original)
+++ lucene/dev/branches/lucene3606/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java Sun Dec  4 19:18:17 2011
@@ -57,7 +57,7 @@ public class CartesianShapeFilter extend
       return new DocIdSet() {
         @Override
         public DocIdSetIterator iterator() throws IOException {
-          return context.reader.termDocsEnum(acceptDocs, fieldName, bytesRef);
+          return context.reader.termDocsEnum(acceptDocs, fieldName, bytesRef, false);
         }
         
         @Override
@@ -70,7 +70,7 @@ public class CartesianShapeFilter extend
       for (int i =0; i< sz; i++) {
         double boxId = area.get(i).doubleValue();
         NumericUtils.longToPrefixCoded(NumericUtils.doubleToSortableLong(boxId), 0, bytesRef);
-        final DocsEnum docsEnum = context.reader.termDocsEnum(acceptDocs, fieldName, bytesRef);
+        final DocsEnum docsEnum = context.reader.termDocsEnum(acceptDocs, fieldName, bytesRef, false);
         if (docsEnum == null) continue;
         // iterate through all documents
         // which have this boxId

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java Sun Dec  4 19:18:17 2011
@@ -387,7 +387,7 @@ class BufferedDeletesStream {
       // System.out.println("  term=" + term);
 
       if (termsEnum.seekExact(term.bytes(), false)) {
-        DocsEnum docsEnum = termsEnum.docs(reader.getLiveDocs(), docs);
+        DocsEnum docsEnum = termsEnum.docs(reader.getLiveDocs(), docs, false);
         //System.out.println("BDS: got docsEnum=" + docsEnum);
 
         if (docsEnum != null) {

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/CheckIndex.java Sun Dec  4 19:18:17 2011
@@ -683,6 +683,7 @@ public class CheckIndex {
       }
      
       DocsEnum docs = null;
+      DocsEnum docsAndFreqs = null;
       DocsAndPositionsEnum postings = null;
 
       final FieldsEnum fieldsEnum = fields.iterator();
@@ -740,7 +741,8 @@ public class CheckIndex {
           status.totFreq += docFreq;
           sumDocFreq += docFreq;
 
-          docs = termsEnum.docs(liveDocs, docs);
+          docs = termsEnum.docs(liveDocs, docs, false);
+          docsAndFreqs = termsEnum.docs(liveDocs, docsAndFreqs, true);
           postings = termsEnum.docsAndPositions(liveDocs, postings);
 
           if (hasOrd) {
@@ -762,13 +764,24 @@ public class CheckIndex {
           status.termCount++;
 
           final DocsEnum docs2;
+          final DocsEnum docsAndFreqs2;
           final boolean hasPositions;
+          final boolean hasFreqs;
           if (postings != null) {
             docs2 = postings;
+            docsAndFreqs2 = postings;
             hasPositions = true;
+            hasFreqs = true;
+          } else if (docsAndFreqs != null) {
+            docs2 = docsAndFreqs;
+            docsAndFreqs2 = docsAndFreqs;
+            hasPositions = false;
+            hasFreqs = true;
           } else {
             docs2 = docs;
+            docsAndFreqs2 = null;
             hasPositions = false;
+            hasFreqs = false;
           }
 
           int lastDoc = -1;
@@ -780,9 +793,15 @@ public class CheckIndex {
               break;
             }
             visitedDocs.set(doc);
-            final int freq = docs2.freq();
-            status.totPos += freq;
-            totalTermFreq += freq;
+            int freq = -1;
+            if (hasFreqs) {
+              freq = docsAndFreqs2.freq();
+              if (freq <= 0) {
+                throw new RuntimeException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds");
+              }
+              status.totPos += freq;
+              totalTermFreq += freq;
+            }
             docCount++;
 
             if (doc <= lastDoc) {
@@ -793,12 +812,9 @@ public class CheckIndex {
             }
 
             lastDoc = doc;
-            if (freq <= 0) {
-              throw new RuntimeException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds");
-            }
             
             int lastPos = -1;
-            if (postings != null) {
+            if (hasPositions) {
               for(int j=0;j<freq;j++) {
                 final int pos = postings.nextPosition();
                 if (pos < -1) {
@@ -820,13 +836,23 @@ public class CheckIndex {
 
           // Re-count if there are deleted docs:
           if (reader.hasDeletions()) {
-            final DocsEnum docsNoDel = termsEnum.docs(null, docs);
-            docCount = 0;
-            totalTermFreq = 0;
-            while(docsNoDel.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
-              visitedDocs.set(docsNoDel.docID());
-              docCount++;
-              totalTermFreq += docsNoDel.freq();
+            if (hasFreqs) {
+              final DocsEnum docsNoDel = termsEnum.docs(null, docsAndFreqs, true);
+              docCount = 0;
+              totalTermFreq = 0;
+              while(docsNoDel.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+                visitedDocs.set(docsNoDel.docID());
+                docCount++;
+                totalTermFreq += docsNoDel.freq();
+              }
+            } else {
+              final DocsEnum docsNoDel = termsEnum.docs(null, docs, false);
+              docCount = 0;
+              totalTermFreq = -1;
+              while(docsNoDel.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+                visitedDocs.set(docsNoDel.docID());
+                docCount++;
+              }
             }
           }
 
@@ -883,7 +909,7 @@ public class CheckIndex {
           } else {
             for(int idx=0;idx<7;idx++) {
               final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8);
-              docs = termsEnum.docs(liveDocs, docs);
+              docs = termsEnum.docs(liveDocs, docs, false);
               final int docID = docs.advance(skipDocID);
               if (docID == DocsEnum.NO_MORE_DOCS) {
                 break;
@@ -986,7 +1012,7 @@ public class CheckIndex {
                   throw new RuntimeException("seek to existing term " + seekTerms[i] + " failed");
                 }
               
-                docs = termsEnum.docs(liveDocs, docs);
+                docs = termsEnum.docs(liveDocs, docs, false);
                 if (docs == null) {
                   throw new RuntimeException("null DocsEnum from to existing term " + seekTerms[i]);
                 }
@@ -1168,6 +1194,7 @@ public class CheckIndex {
 
       // TODO: maybe we can factor out testTermIndex and reuse here?
       DocsEnum docs = null;
+      DocsEnum docsAndFreqs = null;
       DocsAndPositionsEnum postings = null;
       final Bits liveDocs = reader.getLiveDocs();
       for (int j = 0; j < info.docCount; ++j) {
@@ -1210,50 +1237,53 @@ public class CheckIndex {
                 if (totalTermFreq != -1 && totalTermFreq <= 0) {
                   throw new RuntimeException("totalTermFreq: " + totalTermFreq + " is out of bounds");
                 }
-                
-                DocsEnum docsEnum;
-                DocsAndPositionsEnum dp = termsEnum.docsAndPositions(null, postings);
-                if (dp == null) {
-                  DocsEnum d = termsEnum.docs(null, docs);
-                  docsEnum = docs = d;
+
+                postings = termsEnum.docsAndPositions(null, postings);
+                if (postings == null) {
+                  docsAndFreqs = termsEnum.docs(null, docsAndFreqs, true);
+                  if (docsAndFreqs == null) {
+                    docs = termsEnum.docs(null, docs, false);
+                  } else {
+                    docs = docsAndFreqs;
+                  }
                 } else {
-                  docsEnum = postings = dp;
+                  docs = docsAndFreqs = postings;
                 }
-                  
-                final int doc = docsEnum.nextDoc();
+
+                final int doc = docs.nextDoc();
                   
                 if (doc != 0) {
                   throw new RuntimeException("vector for doc " + j + " didn't return docID=0: got docID=" + doc);
                 }
-                  
-                final int tf = docsEnum.freq();
-                tfvComputedSumTotalTermFreq += tf;
-                
-                if (tf <= 0) {
-                  throw new RuntimeException("vector freq " + tf + " is out of bounds");
-                }
-                
-                if (totalTermFreq != -1 && totalTermFreq != tf) {
-                  throw new RuntimeException("vector totalTermFreq " + totalTermFreq + " != tf " + tf);
-                }
+
+                if (docsAndFreqs != null) {
+                  final int tf = docsAndFreqs.freq();
+                  if (tf <= 0) {
+                    throw new RuntimeException("vector freq " + tf + " is out of bounds");
+                  }
+                  if (totalTermFreq != -1 && totalTermFreq != tf) {
+                    throw new RuntimeException("vector totalTermFreq " + totalTermFreq + " != tf " + tf);
+                  }
+                  tfvComputedSumTotalTermFreq += tf;
                 
-                if (dp != null) {
-                  int lastPosition = -1;
-                  for (int i = 0; i < tf; i++) {
-                    int pos = dp.nextPosition();
-                    if (pos != -1 && pos < 0) {
-                      throw new RuntimeException("vector position " + pos + " is out of bounds");
-                    }
+                  if (postings != null) {
+                    int lastPosition = -1;
+                    for (int i = 0; i < tf; i++) {
+                      int pos = postings.nextPosition();
+                      if (pos != -1 && pos < 0) {
+                        throw new RuntimeException("vector position " + pos + " is out of bounds");
+                      }
                     
-                    if (pos < lastPosition) {
-                      throw new RuntimeException("vector position " + pos + " < lastPos " + lastPosition);
-                    }
+                      if (pos < lastPosition) {
+                        throw new RuntimeException("vector position " + pos + " < lastPos " + lastPosition);
+                      }
                     
-                    lastPosition = pos;
+                      lastPosition = pos;
+                    }
                   }
                 }
                   
-                if (docsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+                if (docs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
                   throw new RuntimeException("vector for doc " + j + " references multiple documents!");
                 }
               }

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocTermOrds.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocTermOrds.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocTermOrds.java Sun Dec  4 19:18:17 2011
@@ -314,7 +314,7 @@ public class DocTermOrds {
       final int df = te.docFreq();
       if (df <= maxTermDocFreq) {
 
-        docsEnum = te.docs(liveDocs, docsEnum);
+        docsEnum = te.docs(liveDocs, docsEnum, false);
 
         // dF, but takes deletions into account
         int actualDF = 0;
@@ -650,8 +650,8 @@ public class DocTermOrds {
     }
 
     @Override    
-    public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
-      return termsEnum.docs(liveDocs, reuse);
+    public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+      return termsEnum.docs(liveDocs, reuse, needsFreqs);
     }
 
     @Override    

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocsEnum.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocsEnum.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocsEnum.java Sun Dec  4 19:18:17 2011
@@ -20,7 +20,7 @@ package org.apache.lucene.index;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.util.AttributeSource;
 
-/** Iterates through the documents, term freq and positions.
+/** Iterates through the documents and term freqs.
  *  NOTE: you must first call {@link #nextDoc} before using
  *  any of the per-doc methods. */
 public abstract class DocsEnum extends DocIdSetIterator {

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FieldInfo.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FieldInfo.java Sun Dec  4 19:18:17 2011
@@ -44,6 +44,7 @@ public final class FieldInfo {
    */
   public static enum IndexOptions { 
     /** only documents are indexed: term frequencies and positions are omitted */
+    // TODO: maybe rename to just DOCS?
     DOCS_ONLY,
     /** only documents and term frequencies are indexed: positions are omitted */  
     DOCS_AND_FREQS,

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java Sun Dec  4 19:18:17 2011
@@ -175,8 +175,8 @@ public class FilterIndexReader extends I
     }
 
     @Override
-    public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
-      return in.docs(liveDocs, reuse);
+    public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+      return in.docs(liveDocs, reuse, needsFreqs);
     }
 
     @Override

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FilteredTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FilteredTermsEnum.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FilteredTermsEnum.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FilteredTermsEnum.java Sun Dec  4 19:18:17 2011
@@ -166,8 +166,8 @@ public abstract class FilteredTermsEnum 
   }
 
   @Override
-  public DocsEnum docs(Bits bits, DocsEnum reuse) throws IOException {
-    return tenum.docs(bits, reuse);
+  public DocsEnum docs(Bits bits, DocsEnum reuse, boolean needsFreqs) throws IOException {
+    return tenum.docs(bits, reuse, needsFreqs);
   }
     
   @Override

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/IndexReader.java Sun Dec  4 19:18:17 2011
@@ -997,7 +997,7 @@ public abstract class IndexReader implem
   /** Returns {@link DocsEnum} for the specified field &
    *  term.  This may return null, if either the field or
    *  term does not exist. */
-  public DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term) throws IOException {
+  public DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term, boolean needsFreqs) throws IOException {
     assert field != null;
     assert term != null;
     final Fields fields = fields();
@@ -1006,7 +1006,7 @@ public abstract class IndexReader implem
       if (terms != null) {
         final TermsEnum termsEnum = terms.iterator(null);
         if (termsEnum.seekExact(term, true)) {
-          return termsEnum.docs(liveDocs, null);
+          return termsEnum.docs(liveDocs, null, needsFreqs);
         }
       }
     }
@@ -1016,7 +1016,7 @@ public abstract class IndexReader implem
   /** Returns {@link DocsAndPositionsEnum} for the specified
    *  field & term.  This may return null, if either the
    *  field or term does not exist, or, positions were not
-   *  stored for this term. */
+   *  indexed for this field. */
   public DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term) throws IOException {
     assert field != null;
     assert term != null;
@@ -1038,7 +1038,7 @@ public abstract class IndexReader implem
    * {@link TermState}. This may return null, if either the field or the term
    * does not exists or the {@link TermState} is invalid for the underlying
    * implementation.*/
-  public DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term, TermState state) throws IOException {
+  public DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term, TermState state, boolean needsFreqs) throws IOException {
     assert state != null;
     assert field != null;
     final Fields fields = fields();
@@ -1047,7 +1047,7 @@ public abstract class IndexReader implem
       if (terms != null) {
         final TermsEnum termsEnum = terms.iterator(null);
         termsEnum.seekExact(term, state);
-        return termsEnum.docs(liveDocs, null);
+        return termsEnum.docs(liveDocs, null, needsFreqs);
       }
     }
     return null;
@@ -1057,7 +1057,7 @@ public abstract class IndexReader implem
    * Returns {@link DocsAndPositionsEnum} for the specified field and
    * {@link TermState}. This may return null, if either the field or the term
    * does not exists, the {@link TermState} is invalid for the underlying
-   * implementation, or positions were not stored for this term.*/
+   * implementation, or positions were not indexed for this field. */
   public DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term, TermState state) throws IOException {
     assert state != null;
     assert field != null;

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/MultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/MultiFields.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/MultiFields.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/MultiFields.java Sun Dec  4 19:18:17 2011
@@ -151,14 +151,14 @@ public final class MultiFields extends F
   /** Returns {@link DocsEnum} for the specified field &
    *  term.  This may return null if the term does not
    *  exist. */
-  public static DocsEnum getTermDocsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term) throws IOException {
+  public static DocsEnum getTermDocsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term, boolean needsFreqs) throws IOException {
     assert field != null;
     assert term != null;
     final Terms terms = getTerms(r, field);
     if (terms != null) {
       final TermsEnum termsEnum = terms.iterator(null);
       if (termsEnum.seekExact(term, true)) {
-        return termsEnum.docs(liveDocs, null);
+        return termsEnum.docs(liveDocs, null, needsFreqs);
       }
     }
     return null;

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java Sun Dec  4 19:18:17 2011
@@ -347,7 +347,7 @@ public final class MultiTermsEnum extend
   }
 
   @Override
-  public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+  public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
     MultiDocsEnum docsEnum;
     // Can only reuse if incoming enum is also a MultiDocsEnum
     if (reuse != null && reuse instanceof MultiDocsEnum) {
@@ -397,14 +397,16 @@ public final class MultiTermsEnum extend
       }
 
       assert entry.index < docsEnum.subDocsEnum.length: entry.index + " vs " + docsEnum.subDocsEnum.length + "; " + subs.length;
-      final DocsEnum subDocsEnum = entry.terms.docs(b, docsEnum.subDocsEnum[entry.index]);
-
+      final DocsEnum subDocsEnum = entry.terms.docs(b, docsEnum.subDocsEnum[entry.index], needsFreqs);
       if (subDocsEnum != null) {
         docsEnum.subDocsEnum[entry.index] = subDocsEnum;
         subDocs[upto].docsEnum = subDocsEnum;
         subDocs[upto].slice = entry.subSlice;
-
         upto++;
+      } else {
+        // One of our subs cannot provide freqs:
+        assert needsFreqs;
+        return null;
       }
     }
 
@@ -475,7 +477,7 @@ public final class MultiTermsEnum extend
         subDocsAndPositions[upto].slice = entry.subSlice;
         upto++;
       } else {
-        if (entry.terms.docs(b, null) != null) {
+        if (entry.terms.docs(b, null, false) != null) {
           // At least one of our subs does not store
           // positions -- we can't correctly produce a
           // MultiDocsAndPositions enum

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/TermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/TermsEnum.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/TermsEnum.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/TermsEnum.java Sun Dec  4 19:18:17 2011
@@ -147,12 +147,16 @@ public abstract class TermsEnum {
 
   /** Get {@link DocsEnum} for the current term.  Do not
    *  call this when the enum is unpositioned.  This method
-   *  will not return null.
+   *  may return null (if needsFreqs is true but freqs were
+   *  not indexed for this field).
    *  
    * @param liveDocs unset bits are documents that should not
    * be returned
-   * @param reuse pass a prior DocsEnum for possible reuse */
-  public abstract DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException;
+   * @param reuse pass a prior DocsEnum for possible reuse
+   * @param needsFreqs true if the caller intends to call
+   * {@link DocsEnum#freq}.  If you pass false you must not
+   * call {@link DocsEnum#freq} in the returned DocsEnum. */
+  public abstract DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException;
 
   /** Get {@link DocsAndPositionsEnum} for the current term.
    *  Do not call this when the enum is unpositioned.
@@ -229,7 +233,7 @@ public abstract class TermsEnum {
     }
 
     @Override
-    public DocsEnum docs(Bits liveDocs, DocsEnum reuse) {
+    public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) {
       throw new IllegalStateException("this method should never be called");
     }
       

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java Sun Dec  4 19:18:17 2011
@@ -25,8 +25,8 @@ import java.util.TreeMap;
 
 import org.apache.lucene.index.DocsAndPositionsEnum;
 import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.FieldsEnum;
 import org.apache.lucene.index.IndexFileNames;
@@ -689,14 +689,11 @@ public class BlockTermsReader extends Fi
       }
 
       @Override
-      public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+      public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
         //System.out.println("BTR.docs this=" + this);
         decodeMetaData();
         //System.out.println("BTR.docs:  state.docFreq=" + state.docFreq);
-        final DocsEnum docsEnum = postingsReader.docs(fieldInfo, state, liveDocs, reuse);
-        assert docsEnum != null;
-        //System.out.println("BTR.docs:  return docsEnum=" + docsEnum);
-        return docsEnum;
+        return postingsReader.docs(fieldInfo, state, liveDocs, reuse, needsFreqs);
       }
 
       @Override

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java Sun Dec  4 19:18:17 2011
@@ -875,9 +875,9 @@ public class BlockTreeTermsReader extend
       }
 
       @Override
-      public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException {
+      public DocsEnum docs(Bits skipDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
         currentFrame.decodeMetaData();
-        return postingsReader.docs(fieldInfo, currentFrame.termState, skipDocs, reuse);
+        return postingsReader.docs(fieldInfo, currentFrame.termState, skipDocs, reuse, needsFreqs);
       }
 
       @Override
@@ -2082,7 +2082,7 @@ public class BlockTreeTermsReader extend
       }
 
       @Override
-      public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException {
+      public DocsEnum docs(Bits skipDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
         assert !eof;
         //if (DEBUG) {
         //System.out.println("BTTR.docs seg=" + segment);
@@ -2091,10 +2091,7 @@ public class BlockTreeTermsReader extend
         //if (DEBUG) {
         //System.out.println("  state=" + currentFrame.state);
         //}
-        final DocsEnum docsEnum = postingsReader.docs(fieldInfo, currentFrame.state, skipDocs, reuse);
-
-        assert docsEnum != null;
-        return docsEnum;
+        return postingsReader.docs(fieldInfo, currentFrame.state, skipDocs, reuse, needsFreqs);
       }
 
       @Override

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java Sun Dec  4 19:18:17 2011
@@ -63,7 +63,19 @@ public abstract class PostingsConsumer {
     int df = 0;
     long totTF = 0;
 
-    if (mergeState.fieldInfo.indexOptions != IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
+    if (mergeState.fieldInfo.indexOptions == IndexOptions.DOCS_ONLY) {
+      while(true) {
+        final int doc = postings.nextDoc();
+        if (doc == DocIdSetIterator.NO_MORE_DOCS) {
+          break;
+        }
+        visitedDocs.set(doc);
+        this.startDoc(doc, 0);
+        this.finishDoc();
+        df++;
+      }
+      totTF = -1;
+    } else if (mergeState.fieldInfo.indexOptions == IndexOptions.DOCS_AND_FREQS) {
       while(true) {
         final int doc = postings.nextDoc();
         if (doc == DocIdSetIterator.NO_MORE_DOCS) {

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java Sun Dec  4 19:18:17 2011
@@ -51,7 +51,7 @@ public abstract class PostingsReaderBase
 
   /** Must fully consume state, since after this call that
    *  TermState may be reused. */
-  public abstract DocsEnum docs(FieldInfo fieldInfo, BlockTermState state, Bits skipDocs, DocsEnum reuse) throws IOException;
+  public abstract DocsEnum docs(FieldInfo fieldInfo, BlockTermState state, Bits skipDocs, DocsEnum reuse, boolean needsFreqs) throws IOException;
 
   /** Must fully consume state, since after this call that
    *  TermState may be reused. */

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/TermsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/TermsConsumer.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/TermsConsumer.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/TermsConsumer.java Sun Dec  4 19:18:17 2011
@@ -51,8 +51,9 @@ public abstract class TermsConsumer {
   public abstract Comparator<BytesRef> getComparator() throws IOException;
 
   /** Default merge impl */
-  private MappingMultiDocsEnum docsEnum = null;
-  private MappingMultiDocsAndPositionsEnum postingsEnum = null;
+  private MappingMultiDocsEnum docsEnum;
+  private MappingMultiDocsEnum docsAndFreqsEnum;
+  private MappingMultiDocsAndPositionsEnum postingsEnum;
 
   public void merge(MergeState mergeState, TermsEnum termsEnum) throws IOException {
 
@@ -63,7 +64,7 @@ public abstract class TermsConsumer {
     long sumDFsinceLastAbortCheck = 0;
     FixedBitSet visitedDocs = new FixedBitSet(mergeState.mergedDocCount);
 
-    if (mergeState.fieldInfo.indexOptions != IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
+    if (mergeState.fieldInfo.indexOptions == IndexOptions.DOCS_ONLY) {
       if (docsEnum == null) {
         docsEnum = new MappingMultiDocsEnum();
       }
@@ -74,14 +75,14 @@ public abstract class TermsConsumer {
       while((term = termsEnum.next()) != null) {
         // We can pass null for liveDocs, because the
         // mapping enum will skip the non-live docs:
-        docsEnumIn = (MultiDocsEnum) termsEnum.docs(null, docsEnumIn);
+        docsEnumIn = (MultiDocsEnum) termsEnum.docs(null, docsEnumIn, false);
         if (docsEnumIn != null) {
           docsEnum.reset(docsEnumIn);
           final PostingsConsumer postingsConsumer = startTerm(term);
           final TermStats stats = postingsConsumer.merge(mergeState, docsEnum, visitedDocs);
           if (stats.docFreq > 0) {
             finishTerm(term, stats);
-            sumTotalTermFreq += stats.totalTermFreq;
+            sumTotalTermFreq += stats.docFreq;
             sumDFsinceLastAbortCheck += stats.docFreq;
             sumDocFreq += stats.docFreq;
             if (sumDFsinceLastAbortCheck > 60000) {
@@ -91,7 +92,35 @@ public abstract class TermsConsumer {
           }
         }
       }
+    } else if (mergeState.fieldInfo.indexOptions == IndexOptions.DOCS_AND_FREQS) {
+      if (docsAndFreqsEnum == null) {
+        docsAndFreqsEnum = new MappingMultiDocsEnum();
+      }
+      docsAndFreqsEnum.setMergeState(mergeState);
+
+      MultiDocsEnum docsAndFreqsEnumIn = null;
+
+      while((term = termsEnum.next()) != null) {
+        // We can pass null for liveDocs, because the
+        // mapping enum will skip the non-live docs:
+        docsAndFreqsEnumIn = (MultiDocsEnum) termsEnum.docs(null, docsAndFreqsEnumIn, true);
+        assert docsAndFreqsEnumIn != null;
+        docsAndFreqsEnum.reset(docsAndFreqsEnumIn);
+        final PostingsConsumer postingsConsumer = startTerm(term);
+        final TermStats stats = postingsConsumer.merge(mergeState, docsAndFreqsEnum, visitedDocs);
+        if (stats.docFreq > 0) {
+          finishTerm(term, stats);
+          sumTotalTermFreq += stats.totalTermFreq;
+          sumDFsinceLastAbortCheck += stats.docFreq;
+          sumDocFreq += stats.docFreq;
+          if (sumDFsinceLastAbortCheck > 60000) {
+            mergeState.checkAbort.work(sumDFsinceLastAbortCheck/5.0);
+            sumDFsinceLastAbortCheck = 0;
+          }
+        }
+      }
     } else {
+      assert mergeState.fieldInfo.indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
       if (postingsEnum == null) {
         postingsEnum = new MappingMultiDocsAndPositionsEnum();
       }
@@ -101,27 +130,26 @@ public abstract class TermsConsumer {
         // We can pass null for liveDocs, because the
         // mapping enum will skip the non-live docs:
         postingsEnumIn = (MultiDocsAndPositionsEnum) termsEnum.docsAndPositions(null, postingsEnumIn);
-        if (postingsEnumIn != null) {
-          postingsEnum.reset(postingsEnumIn);
-          // set PayloadProcessor
-          if (mergeState.payloadProcessorProvider != null) {
-            for (int i = 0; i < mergeState.readers.size(); i++) {
-              if (mergeState.dirPayloadProcessor[i] != null) {
-                mergeState.currentPayloadProcessor[i] = mergeState.dirPayloadProcessor[i].getProcessor(mergeState.fieldInfo.name, term);
-              }
+        assert postingsEnumIn != null;
+        postingsEnum.reset(postingsEnumIn);
+        // set PayloadProcessor
+        if (mergeState.payloadProcessorProvider != null) {
+          for (int i = 0; i < mergeState.readers.size(); i++) {
+            if (mergeState.dirPayloadProcessor[i] != null) {
+              mergeState.currentPayloadProcessor[i] = mergeState.dirPayloadProcessor[i].getProcessor(mergeState.fieldInfo.name, term);
             }
           }
-          final PostingsConsumer postingsConsumer = startTerm(term);
-          final TermStats stats = postingsConsumer.merge(mergeState, postingsEnum, visitedDocs);
-          if (stats.docFreq > 0) {
-            finishTerm(term, stats);
-            sumTotalTermFreq += stats.totalTermFreq;
-            sumDFsinceLastAbortCheck += stats.docFreq;
-            sumDocFreq += stats.docFreq;
-            if (sumDFsinceLastAbortCheck > 60000) {
-              mergeState.checkAbort.work(sumDFsinceLastAbortCheck/5.0);
-              sumDFsinceLastAbortCheck = 0;
-            }
+        }
+        final PostingsConsumer postingsConsumer = startTerm(term);
+        final TermStats stats = postingsConsumer.merge(mergeState, postingsEnum, visitedDocs);
+        if (stats.docFreq > 0) {
+          finishTerm(term, stats);
+          sumTotalTermFreq += stats.totalTermFreq;
+          sumDFsinceLastAbortCheck += stats.docFreq;
+          sumDocFreq += stats.docFreq;
+          if (sumDFsinceLastAbortCheck > 60000) {
+            mergeState.checkAbort.work(sumDFsinceLastAbortCheck/5.0);
+            sumDFsinceLastAbortCheck = 0;
           }
         }
       }

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java Sun Dec  4 19:18:17 2011
@@ -27,8 +27,8 @@ import java.util.TreeMap;
 
 import org.apache.lucene.index.DocsAndPositionsEnum;
 import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.FieldsEnum;
 import org.apache.lucene.index.IndexFileNames;
@@ -950,9 +950,11 @@ public class Lucene3xFields extends Fiel
     }
 
     @Override
-    public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+    public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
       PreDocsEnum docsEnum;
-      if (reuse == null || !(reuse instanceof PreDocsEnum)) {
+      if (needsFreqs && fieldInfo.indexOptions == IndexOptions.DOCS_ONLY) {
+        return null;
+      } else if (reuse == null || !(reuse instanceof PreDocsEnum)) {
         docsEnum = new PreDocsEnum();
       } else {
         docsEnum = (PreDocsEnum) reuse;

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java Sun Dec  4 19:18:17 2011
@@ -112,7 +112,10 @@ public class SegmentTermDocs {
   }
 
   public final int doc() { return doc; }
-  public final int freq() { return freq; }
+  public final int freq() {
+    assert indexOptions != IndexOptions.DOCS_ONLY;
+    return freq;
+  }
 
   protected void skippingDoc() throws IOException {
   }
@@ -125,7 +128,6 @@ public class SegmentTermDocs {
       
       if (indexOptions == IndexOptions.DOCS_ONLY) {
         doc += docCode;
-        freq = 1;
       } else {
         doc += docCode >>> 1;       // shift off low bit
         if ((docCode & 1) != 0)       // if low bit is set

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java Sun Dec  4 19:18:17 2011
@@ -18,7 +18,6 @@ package org.apache.lucene.index.codecs.l
  */
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.Collection;
 
 import org.apache.lucene.index.DocsAndPositionsEnum;
@@ -209,9 +208,11 @@ public class Lucene40PostingsReader exte
   }
     
   @Override
-  public DocsEnum docs(FieldInfo fieldInfo, BlockTermState termState, Bits liveDocs, DocsEnum reuse) throws IOException {
+  public DocsEnum docs(FieldInfo fieldInfo, BlockTermState termState, Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
     SegmentDocsEnum docsEnum;
-    if (reuse == null || !(reuse instanceof SegmentDocsEnum)) {
+    if (needsFreqs && fieldInfo.indexOptions == IndexOptions.DOCS_ONLY) {
+      return null;
+    } else if (reuse == null || !(reuse instanceof SegmentDocsEnum)) {
       docsEnum = new SegmentDocsEnum(freqIn);
     } else {
       docsEnum = (SegmentDocsEnum) reuse;
@@ -277,7 +278,7 @@ public class Lucene40PostingsReader exte
     final IndexInput freqIn;
     final IndexInput startFreqIn;
 
-    boolean omitTF;                               // does current field omit term freq?
+    boolean indexOmitsTF;                               // does current field omit term freq?
     boolean storePayloads;                        // does current field store payloads?
 
     int limit;                                    // number of docs in this posting
@@ -300,12 +301,7 @@ public class Lucene40PostingsReader exte
     }
 
     public SegmentDocsEnum reset(FieldInfo fieldInfo, StandardTermState termState, Bits liveDocs) throws IOException {
-      omitTF = fieldInfo.indexOptions == IndexOptions.DOCS_ONLY;
-      if (omitTF) {
-        freq = 1;
-        Arrays.fill(freqs, 1);
-      }
-      
+      indexOmitsTF = fieldInfo.indexOptions == IndexOptions.DOCS_ONLY;
       storePayloads = fieldInfo.storePayloads;
       this.liveDocs = liveDocs;
       freqOffset = termState.freqOffset;
@@ -331,6 +327,7 @@ public class Lucene40PostingsReader exte
     
     @Override
     public int freq() {
+      assert !indexOmitsTF;
       return freq;
     }
 
@@ -389,7 +386,7 @@ public class Lucene40PostingsReader exte
       count = bufferSize;
       ord += bufferSize;
       
-      if (omitTF)
+      if (indexOmitsTF)
         fillDocs(bufferSize);
       else
         fillDocsAndFreqs(bufferSize);
@@ -400,7 +397,7 @@ public class Lucene40PostingsReader exte
     private int scanTo(int target) throws IOException {
       while (ord++ < limit) {
         int code = freqIn.readVInt();
-        if (omitTF) {
+        if (indexOmitsTF) {
           accum += code;
         } else {
           accum += code >>> 1;            // shift off low bit

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40TermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40TermVectorsReader.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40TermVectorsReader.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40TermVectorsReader.java Sun Dec  4 19:18:17 2011
@@ -505,7 +505,7 @@ public class Lucene40TermVectorsReader e
     }
 
     @Override
-    public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+    public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs /* ignored */) throws IOException {
       TVDocsEnum docsEnum;
       if (reuse != null && reuse instanceof TVDocsEnum) {
         docsEnum = (TVDocsEnum) reuse;

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java Sun Dec  4 19:18:17 2011
@@ -26,8 +26,8 @@ import java.util.TreeMap;
 
 import org.apache.lucene.index.DocsAndPositionsEnum;
 import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.FieldsEnum;
 import org.apache.lucene.index.IndexFileNames;
@@ -36,10 +36,10 @@ import org.apache.lucene.index.SegmentRe
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.codecs.PostingsFormat;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
 import org.apache.lucene.index.codecs.PostingsConsumer;
+import org.apache.lucene.index.codecs.PostingsFormat;
 import org.apache.lucene.index.codecs.TermStats;
 import org.apache.lucene.index.codecs.TermsConsumer;
 import org.apache.lucene.store.ByteArrayDataInput;
@@ -317,7 +317,6 @@ public class MemoryPostingsFormat extend
         docUpto++;
         if (indexOptions == IndexOptions.DOCS_ONLY) {
           accum += in.readVInt();
-          freq = 1;
         } else {
           final int code = in.readVInt();
           accum += code >>> 1;
@@ -371,6 +370,7 @@ public class MemoryPostingsFormat extend
 
     @Override
     public int freq() {
+      assert indexOptions != IndexOptions.DOCS_ONLY;
       return freq;
     }
   }
@@ -600,10 +600,13 @@ public class MemoryPostingsFormat extend
     }
     
     @Override
-    public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+    public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
       decodeMetaData();
       FSTDocsEnum docsEnum;
-      if (reuse == null || !(reuse instanceof FSTDocsEnum)) {
+
+      if (needsFreqs && field.indexOptions == IndexOptions.DOCS_ONLY) {
+        return null;
+      } else if (reuse == null || !(reuse instanceof FSTDocsEnum)) {
         docsEnum = new FSTDocsEnum(field.indexOptions, field.storePayloads);
       } else {
         docsEnum = (FSTDocsEnum) reuse;        

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java Sun Dec  4 19:18:17 2011
@@ -178,7 +178,10 @@ public class PulsingPostingsReader exten
   }
 
   @Override
-  public DocsEnum docs(FieldInfo field, BlockTermState _termState, Bits liveDocs, DocsEnum reuse) throws IOException {
+  public DocsEnum docs(FieldInfo field, BlockTermState _termState, Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+    if (needsFreqs && field.indexOptions == IndexOptions.DOCS_ONLY) {
+      return null;
+    }
     PulsingTermState termState = (PulsingTermState) _termState;
     if (termState.postingsSize != -1) {
       PulsingDocsEnum postings;
@@ -202,11 +205,11 @@ public class PulsingPostingsReader exten
       return postings.reset(liveDocs, termState);
     } else {
       if (reuse instanceof PulsingDocsEnum) {
-        DocsEnum wrapped = wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, getOther(reuse));
+        DocsEnum wrapped = wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, getOther(reuse), needsFreqs);
         setOther(wrapped, reuse); // wrapped.other = reuse
         return wrapped;
       } else {
-        return wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, reuse);
+        return wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, reuse, needsFreqs);
       }
     }
   }
@@ -283,7 +286,6 @@ public class PulsingPostingsReader exten
       docID = -1;
       accum = 0;
       payloadLength = 0;
-      freq = 1;
       this.liveDocs = liveDocs;
       return this;
     }
@@ -342,6 +344,7 @@ public class PulsingPostingsReader exten
 
     @Override
     public int freq() {
+      assert indexOptions != IndexOptions.DOCS_ONLY;
       return freq;
     }
 

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java Sun Dec  4 19:18:17 2011
@@ -272,7 +272,10 @@ public class SepPostingsReader extends P
   }
 
   @Override
-  public DocsEnum docs(FieldInfo fieldInfo, BlockTermState _termState, Bits liveDocs, DocsEnum reuse) throws IOException {
+  public DocsEnum docs(FieldInfo fieldInfo, BlockTermState _termState, Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+    if (needsFreqs && fieldInfo.indexOptions == IndexOptions.DOCS_ONLY) {
+      return null;
+    }
     final SepTermState termState = (SepTermState) _termState;
     SepDocsEnum docsEnum;
     if (reuse == null || !(reuse instanceof SepDocsEnum)) {
@@ -369,8 +372,6 @@ public class SepPostingsReader extends P
       if (!omitTF) {
         freqIndex.set(termState.freqIndex);
         freqIndex.seek(freqReader);
-      } else {
-        freq = 1;
       }
 
       docFreq = termState.docFreq;
@@ -412,6 +413,7 @@ public class SepPostingsReader extends P
 
     @Override
     public int freq() {
+      assert !omitTF;
       return freq;
     }
 

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java Sun Dec  4 19:18:17 2011
@@ -17,19 +17,24 @@ package org.apache.lucene.index.codecs.s
  * limitations under the License.
  */
 
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.index.codecs.FieldsProducer;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
-import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.FieldsEnum;
+import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.DocsAndPositionsEnum;
 import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.codecs.FieldsProducer;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CharsRef;
 import org.apache.lucene.util.OpenBitSet;
 import org.apache.lucene.util.StringHelper;
@@ -37,13 +42,8 @@ import org.apache.lucene.util.UnicodeUti
 import org.apache.lucene.util.fst.Builder;
 import org.apache.lucene.util.fst.BytesRefFSTEnum;
 import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.fst.PositiveIntOutputs;
 import org.apache.lucene.util.fst.PairOutputs;
-
-import java.io.IOException;
-import java.util.Comparator;
-import java.util.Map;
-import java.util.HashMap;
+import org.apache.lucene.util.fst.PositiveIntOutputs;
 
 class SimpleTextFieldsReader extends FieldsProducer {
 
@@ -190,14 +190,17 @@ class SimpleTextFieldsReader extends Fie
     }
  
     @Override
-    public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+    public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+      if (needsFreqs && indexOptions == IndexOptions.DOCS_ONLY) {
+        return null;
+      }
       SimpleTextDocsEnum docsEnum;
       if (reuse != null && reuse instanceof SimpleTextDocsEnum && ((SimpleTextDocsEnum) reuse).canReuse(SimpleTextFieldsReader.this.in)) {
         docsEnum = (SimpleTextDocsEnum) reuse;
       } else {
         docsEnum = new SimpleTextDocsEnum();
       }
-      return docsEnum.reset(docsStart, liveDocs, indexOptions == IndexOptions.DOCS_ONLY);
+      return docsEnum.reset(docsStart, liveDocs, !needsFreqs);
     }
 
     @Override
@@ -245,9 +248,6 @@ class SimpleTextFieldsReader extends Fie
       in.seek(fp);
       this.omitTF = omitTF;
       docID = -1;
-      if (omitTF) {
-        tf = 1;
-      }
       return this;
     }
 
@@ -258,6 +258,7 @@ class SimpleTextFieldsReader extends Fie
 
     @Override
     public int freq() {
+      assert !omitTF;
       return tf;
     }
 

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsReader.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsReader.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsReader.java Sun Dec  4 19:18:17 2011
@@ -365,10 +365,10 @@ public class SimpleTextTermVectorsReader
     }
 
     @Override
-    public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+    public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
       // TODO: reuse
       SimpleTVDocsEnum e = new SimpleTVDocsEnum();
-      e.reset(liveDocs, current.getValue().freq);
+      e.reset(liveDocs, needsFreqs ? current.getValue().freq : -1);
       return e;
     }
 
@@ -399,6 +399,7 @@ public class SimpleTextTermVectorsReader
 
     @Override
     public int freq() {
+      assert freq != -1;
       return freq;
     }
 

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/BooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/BooleanQuery.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/BooleanQuery.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/BooleanQuery.java Sun Dec  4 19:18:17 2011
@@ -17,20 +17,21 @@ package org.apache.lucene.search;
  * limitations under the License.
  */
 
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.TermsEnum;
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.ToStringUtils;
+import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.ConjunctionTermScorer.DocsAndFreqs;
-import org.apache.lucene.search.similarities.SimilarityProvider;
-import org.apache.lucene.search.similarities.Similarity.ExactDocScorer;
 import org.apache.lucene.search.TermQuery.TermWeight;
-
-import java.io.IOException;
-import java.util.*;
+import org.apache.lucene.search.similarities.Similarity.ExactDocScorer;
+import org.apache.lucene.search.similarities.SimilarityProvider;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.ToStringUtils;
 
 /** A Query that matches documents matching boolean combinations of other
   * queries, e.g. {@link TermQuery}s, {@link PhraseQuery}s or other
@@ -349,6 +350,11 @@ public class BooleanQuery extends Query 
 
     private Scorer createConjunctionTermScorer(AtomicReaderContext context, Bits acceptDocs)
         throws IOException {
+
+      // TODO: fix scorer API to specify "needsScores" up
+      // front, so we can do match-only if caller doesn't
+      // needs scores
+
       final DocsAndFreqs[] docsAndFreqs = new DocsAndFreqs[weights.size()];
       for (int i = 0; i < docsAndFreqs.length; i++) {
         final TermWeight weight = (TermWeight) weights.get(i);
@@ -357,12 +363,46 @@ public class BooleanQuery extends Query 
           return null;
         }
         final ExactDocScorer docScorer = weight.createDocScorer(context);
-        docsAndFreqs[i] = new DocsAndFreqs(termsEnum.docs(
-            acceptDocs, null), termsEnum.docFreq(), docScorer);
+        final DocsEnum docsAndFreqsEnum = termsEnum.docs(acceptDocs, null, true);
+        if (docsAndFreqsEnum == null) {
+          // TODO: we could carry over TermState from the
+          // terms we already seek'd to, to save re-seeking
+          // to make the match-only scorer, but it's likely
+          // rare that BQ mixes terms from omitTf and
+          // non-omitTF fields:
+
+          // At least one sub cannot provide freqs; abort
+          // and fallback to full match-only scorer:
+          return createMatchOnlyConjunctionTermScorer(context, acceptDocs);
+        }
+
+        docsAndFreqs[i] = new DocsAndFreqs(docsAndFreqsEnum,
+                                           docsAndFreqsEnum,
+                                           termsEnum.docFreq(), docScorer);
       }
       return new ConjunctionTermScorer(this, disableCoord ? 1.0f : coord(
           docsAndFreqs.length, docsAndFreqs.length), docsAndFreqs);
     }
+
+    private Scorer createMatchOnlyConjunctionTermScorer(AtomicReaderContext context, Bits acceptDocs)
+        throws IOException {
+
+      final DocsAndFreqs[] docsAndFreqs = new DocsAndFreqs[weights.size()];
+      for (int i = 0; i < docsAndFreqs.length; i++) {
+        final TermWeight weight = (TermWeight) weights.get(i);
+        final TermsEnum termsEnum = weight.getTermsEnum(context);
+        if (termsEnum == null) {
+          return null;
+        }
+        final ExactDocScorer docScorer = weight.createDocScorer(context);
+        docsAndFreqs[i] = new DocsAndFreqs(null,
+                                           termsEnum.docs(acceptDocs, null, false),
+                                           termsEnum.docFreq(), docScorer);
+      }
+
+      return new MatchOnlyConjunctionTermScorer(this, disableCoord ? 1.0f : coord(
+          docsAndFreqs.length, docsAndFreqs.length), docsAndFreqs);
+    }
     
     @Override
     public boolean scoresDocsOutOfOrder() {

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java Sun Dec  4 19:18:17 2011
@@ -17,17 +17,18 @@ package org.apache.lucene.search;
  * limitations under the License.
  */
 
+import java.io.IOException;
+import java.util.Comparator;
+
 import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.search.similarities.Similarity.ExactDocScorer;
 import org.apache.lucene.util.ArrayUtil;
-import java.io.IOException;
-import java.util.Comparator;
 
 /** Scorer for conjunctions, sets of terms, all of which are required. */
-final class ConjunctionTermScorer extends Scorer {
-  private final float coord;
-  private int lastDoc = -1;
-  private final DocsAndFreqs[] docsAndFreqs;
+class ConjunctionTermScorer extends Scorer {
+  protected final float coord;
+  protected int lastDoc = -1;
+  protected final DocsAndFreqs[] docsAndFreqs;
   private final DocsAndFreqs lead;
 
   ConjunctionTermScorer(Weight weight, float coord,
@@ -39,7 +40,7 @@ final class ConjunctionTermScorer extend
     // lead the matching.
     ArrayUtil.mergeSort(docsAndFreqs, new Comparator<DocsAndFreqs>() {
       public int compare(DocsAndFreqs o1, DocsAndFreqs o2) {
-        return o1.freq - o2.freq;
+        return o1.docFreq - o2.docFreq;
       }
     });
 
@@ -96,14 +97,16 @@ final class ConjunctionTermScorer extend
   }
 
   static final class DocsAndFreqs {
+    final DocsEnum docsAndFreqs;
     final DocsEnum docs;
-    final int freq;
+    final int docFreq;
     final ExactDocScorer docScorer;
     int doc = -1;
 
-    DocsAndFreqs(DocsEnum docs, int freq, ExactDocScorer docScorer) {
+    DocsAndFreqs(DocsEnum docsAndFreqs, DocsEnum docs, int docFreq, ExactDocScorer docScorer) {
+      this.docsAndFreqs = docsAndFreqs;
       this.docs = docs;
-      this.freq = freq;
+      this.docFreq = docFreq;
       this.docScorer = docScorer;
     }
   }

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java Sun Dec  4 19:18:17 2011
@@ -342,7 +342,7 @@ class FieldCacheImpl implements FieldCac
               break;
             }
             final byte termval = parser.parseByte(term);
-            docs = termsEnum.docs(null, docs);
+            docs = termsEnum.docs(null, docs, false);
             while (true) {
               final int docID = docs.nextDoc();
               if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -415,7 +415,7 @@ class FieldCacheImpl implements FieldCac
               break;
             }
             final short termval = parser.parseShort(term);
-            docs = termsEnum.docs(null, docs);
+            docs = termsEnum.docs(null, docs, false);
             while (true) {
               final int docID = docs.nextDoc();
               if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -519,7 +519,7 @@ class FieldCacheImpl implements FieldCac
               retArray = new int[maxDoc];
             }
 
-            docs = termsEnum.docs(null, docs);
+            docs = termsEnum.docs(null, docs, false);
             while (true) {
               final int docID = docs.nextDoc();
               if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -586,7 +586,7 @@ class FieldCacheImpl implements FieldCac
             res = new FixedBitSet(maxDoc);
           }
 
-          docs = termsEnum.docs(null, docs);
+          docs = termsEnum.docs(null, docs, false);
           // TODO: use bulk API
           while (true) {
             final int docID = docs.nextDoc();
@@ -669,7 +669,7 @@ class FieldCacheImpl implements FieldCac
               retArray = new float[maxDoc];
             }
             
-            docs = termsEnum.docs(null, docs);
+            docs = termsEnum.docs(null, docs, false);
             while (true) {
               final int docID = docs.nextDoc();
               if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -757,7 +757,7 @@ class FieldCacheImpl implements FieldCac
               retArray = new long[maxDoc];
             }
 
-            docs = termsEnum.docs(null, docs);
+            docs = termsEnum.docs(null, docs, false);
             while (true) {
               final int docID = docs.nextDoc();
               if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -846,7 +846,7 @@ class FieldCacheImpl implements FieldCac
               retArray = new double[maxDoc];
             }
 
-            docs = termsEnum.docs(null, docs);
+            docs = termsEnum.docs(null, docs, false);
             while (true) {
               final int docID = docs.nextDoc();
               if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -1020,7 +1020,7 @@ class FieldCacheImpl implements FieldCac
       }
 
       @Override
-      public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+      public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
         throw new UnsupportedOperationException();
       }
 
@@ -1147,7 +1147,7 @@ class FieldCacheImpl implements FieldCac
             termOrdToBytesOffset = termOrdToBytesOffset.resize(ArrayUtil.oversize(1+termOrd, 1));
           }
           termOrdToBytesOffset.set(termOrd, bytes.copyUsingLengthPrefix(term));
-          docs = termsEnum.docs(null, docs);
+          docs = termsEnum.docs(null, docs, false);
           while (true) {
             final int docID = docs.nextDoc();
             if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -1268,7 +1268,7 @@ class FieldCacheImpl implements FieldCac
             break;
           }
           final long pointer = bytes.copyUsingLengthPrefix(term);
-          docs = termsEnum.docs(null, docs);
+          docs = termsEnum.docs(null, docs, false);
           while (true) {
             final int docID = docs.nextDoc();
             if (docID == DocsEnum.NO_MORE_DOCS) {



Mime
View raw message