lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1231794 [3/3] - in /lucene/dev/trunk: lucene/ lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/ lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/ lucene/contrib/memory/src/java/org/apache/l...
Date Sun, 15 Jan 2012 23:17:47 GMT
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java Sun Jan 15 23:17:45 2012
@@ -128,7 +128,7 @@ public class TestDocumentWriter extends 
     SegmentReader reader = new SegmentReader(info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random));
 
     DocsAndPositionsEnum termPositions = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader),
-                                                                          "repeated", new BytesRef("repeated"));
+                                                                          "repeated", new BytesRef("repeated"), false);
     assertTrue(termPositions.nextDoc() != termPositions.NO_MORE_DOCS);
     int freq = termPositions.freq();
     assertEquals(2, freq);
@@ -199,7 +199,7 @@ public class TestDocumentWriter extends 
     writer.close();
     SegmentReader reader = new SegmentReader(info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random));
 
-    DocsAndPositionsEnum termPositions = MultiFields.getTermPositionsEnum(reader, reader.getLiveDocs(), "f1", new BytesRef("a"));
+    DocsAndPositionsEnum termPositions = MultiFields.getTermPositionsEnum(reader, reader.getLiveDocs(), "f1", new BytesRef("a"), false);
     assertTrue(termPositions.nextDoc() != termPositions.NO_MORE_DOCS);
     int freq = termPositions.freq();
     assertEquals(3, freq);
@@ -243,18 +243,18 @@ public class TestDocumentWriter extends 
     writer.close();
     SegmentReader reader = new SegmentReader(info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random));
 
-    DocsAndPositionsEnum termPositions = reader.termPositionsEnum(reader.getLiveDocs(), "preanalyzed", new BytesRef("term1"));
+    DocsAndPositionsEnum termPositions = reader.termPositionsEnum(reader.getLiveDocs(), "preanalyzed", new BytesRef("term1"), false);
     assertTrue(termPositions.nextDoc() != termPositions.NO_MORE_DOCS);
     assertEquals(1, termPositions.freq());
     assertEquals(0, termPositions.nextPosition());
 
-    termPositions = reader.termPositionsEnum(reader.getLiveDocs(), "preanalyzed", new BytesRef("term2"));
+    termPositions = reader.termPositionsEnum(reader.getLiveDocs(), "preanalyzed", new BytesRef("term2"), false);
     assertTrue(termPositions.nextDoc() != termPositions.NO_MORE_DOCS);
     assertEquals(2, termPositions.freq());
     assertEquals(1, termPositions.nextPosition());
     assertEquals(3, termPositions.nextPosition());
     
-    termPositions = reader.termPositionsEnum(reader.getLiveDocs(), "preanalyzed", new BytesRef("term3"));
+    termPositions = reader.termPositionsEnum(reader.getLiveDocs(), "preanalyzed", new BytesRef("term3"), false);
     assertTrue(termPositions.nextDoc() != termPositions.NO_MORE_DOCS);
     assertEquals(1, termPositions.freq());
     assertEquals(2, termPositions.nextPosition());

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java Sun Jan 15 23:17:45 2012
@@ -260,17 +260,17 @@ public class TestDuelingCodecs extends L
       assertEquals(info, term, rightTermsEnum.next());
       assertTermStats(leftTermsEnum, rightTermsEnum);
       if (deep) {
-        assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions),
-            rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions));
-        assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions),
-            rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions));
+        assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions, false),
+                                   rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions, false));
+        assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions, false),
+                                   rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions, false));
 
         assertPositionsSkipping(leftTermsEnum.docFreq(), 
-            leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions),
-            rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions));
+                                leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions, false),
+                                rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions, false));
         assertPositionsSkipping(leftTermsEnum.docFreq(), 
-            leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions),
-            rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions));
+                                leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions, false),
+                                rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions, false));
 
         // with freqs:
         assertDocsEnum(leftDocs = leftTermsEnum.docs(null, leftDocs, true),
@@ -341,6 +341,8 @@ public class TestDuelingCodecs extends L
       for (int i = 0; i < freq; i++) {
         assertEquals(info, leftDocs.nextPosition(), rightDocs.nextPosition());
         assertEquals(info, leftDocs.hasPayload(), rightDocs.hasPayload());
+        assertEquals(info, leftDocs.startOffset(), rightDocs.startOffset());
+        assertEquals(info, leftDocs.endOffset(), rightDocs.endOffset());
         if (leftDocs.hasPayload()) {
           assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload());
         }

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java Sun Jan 15 23:17:45 2012
@@ -90,8 +90,8 @@ public class TestFilterIndexReader exten
       }
 
       @Override
-      public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException {
-        return new TestPositions(super.docsAndPositions(liveDocs, reuse == null ? null : ((FilterDocsAndPositionsEnum) reuse).in));
+      public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
+        return new TestPositions(super.docsAndPositions(liveDocs, reuse == null ? null : ((FilterDocsAndPositionsEnum) reuse).in, needsOffsets));
       }
     }
 
@@ -166,7 +166,7 @@ public class TestFilterIndexReader exten
     assertEquals(TermsEnum.SeekStatus.FOUND, terms.seekCeil(new BytesRef("one")));
     
     DocsAndPositionsEnum positions = terms.docsAndPositions(MultiFields.getLiveDocs(reader),
-                                                            null);
+                                                            null, false);
     while (positions.nextDoc() != DocsEnum.NO_MORE_DOCS) {
       assertTrue((positions.docID() % 2) == 1);
     }

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReader.java Sun Jan 15 23:17:45 2012
@@ -603,8 +603,8 @@ public class TestIndexReader extends Luc
 
         while(enum1.next() != null) {
           assertEquals("Different terms", enum1.term(), enum2.next());
-          DocsAndPositionsEnum tp1 = enum1.docsAndPositions(liveDocs, null);
-          DocsAndPositionsEnum tp2 = enum2.docsAndPositions(liveDocs, null);
+          DocsAndPositionsEnum tp1 = enum1.docsAndPositions(liveDocs, null, false);
+          DocsAndPositionsEnum tp2 = enum2.docsAndPositions(liveDocs, null, false);
 
           while(tp1.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
             assertTrue(tp2.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Sun Jan 15 23:17:45 2012
@@ -39,6 +39,7 @@ import org.apache.lucene.document.Stored
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.FieldCache;
 import org.apache.lucene.search.IndexSearcher;
@@ -899,7 +900,8 @@ public class TestIndexWriter extends Luc
     DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(s.getIndexReader(),
                                                                 MultiFields.getLiveDocs(s.getIndexReader()),
                                                                 "field",
-                                                                new BytesRef("a"));
+                                                                new BytesRef("a"),
+                                                                false);
 
     assertTrue(tps.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
     assertEquals(1, tps.freq());
@@ -964,14 +966,14 @@ public class TestIndexWriter extends Luc
     Terms tpv = r.getTermVectors(0).terms("field");
     TermsEnum termsEnum = tpv.iterator(null);
     assertNotNull(termsEnum.next());
-    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null);
+    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, false);
     assertNotNull(dpEnum);
     assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
     assertEquals(1, dpEnum.freq());
     assertEquals(100, dpEnum.nextPosition());
 
     assertNotNull(termsEnum.next());
-    dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+    dpEnum = termsEnum.docsAndPositions(null, dpEnum, false);
     assertNotNull(dpEnum);
     assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
     assertEquals(1, dpEnum.freq());
@@ -1634,7 +1636,7 @@ public class TestIndexWriter extends Luc
 
     // Make sure position is still incremented when
     // massive term is skipped:
-    DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(reader, null, "content", new BytesRef("another"));
+    DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(reader, null, "content", new BytesRef("another"), false);
     assertEquals(0, tps.nextDoc());
     assertEquals(1, tps.freq());
     assertEquals(3, tps.nextPosition());
@@ -1761,4 +1763,27 @@ public class TestIndexWriter extends Luc
     w1.close();
     d.close();
   }
+
+  public void testChangeIndexOptions() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriter w = new IndexWriter(dir,
+                                    new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+
+    FieldType docsAndFreqs = new FieldType(TextField.TYPE_UNSTORED);
+    docsAndFreqs.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
+
+    FieldType docsOnly = new FieldType(TextField.TYPE_UNSTORED);
+    docsOnly.setIndexOptions(IndexOptions.DOCS_ONLY);
+
+    Document doc = new Document();
+    doc.add(new Field("field", "a b c", docsAndFreqs));
+    w.addDocument(doc);
+    w.addDocument(doc);
+
+    doc = new Document();
+    doc.add(new Field("field", "a b c", docsOnly));
+    w.addDocument(doc);
+    w.close();
+    dir.close();
+  }
 }

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexableField.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexableField.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexableField.java Sun Jan 15 23:17:45 2012
@@ -263,14 +263,14 @@ public class TestIndexableField extends 
             TermsEnum termsEnum = tfv.iterator(null);
             assertEquals(new BytesRef(""+counter), termsEnum.next());
             assertEquals(1, termsEnum.totalTermFreq());
-            DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null);
+            DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, false);
             assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
             assertEquals(1, dpEnum.freq());
             assertEquals(1, dpEnum.nextPosition());
 
             assertEquals(new BytesRef("text"), termsEnum.next());
             assertEquals(1, termsEnum.totalTermFreq());
-            dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+            dpEnum = termsEnum.docsAndPositions(null, dpEnum, false);
             assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
             assertEquals(1, dpEnum.freq());
             assertEquals(0, dpEnum.nextPosition());

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java Sun Jan 15 23:17:45 2012
@@ -156,7 +156,8 @@ public class TestLazyProxSkipping extend
         DocsAndPositionsEnum tp = MultiFields.getTermPositionsEnum(reader,
                                                                    MultiFields.getLiveDocs(reader),
                                                                    this.field,
-                                                                   new BytesRef("b"));
+                                                                   new BytesRef("b"),
+                                                                   false);
 
         for (int i = 0; i < 10; i++) {
             tp.nextDoc();
@@ -167,7 +168,8 @@ public class TestLazyProxSkipping extend
         tp = MultiFields.getTermPositionsEnum(reader,
                                               MultiFields.getLiveDocs(reader),
                                               this.field,
-                                              new BytesRef("a"));
+                                              new BytesRef("a"),
+                                              false);
 
         for (int i = 0; i < 10; i++) {
             tp.nextDoc();

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLongPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLongPostings.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLongPostings.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLongPostings.java Sun Jan 15 23:17:45 2012
@@ -173,7 +173,7 @@ public class TestLongPostings extends Lu
         System.out.println("\nTEST: iter=" + iter + " doS1=" + doS1);
       }
         
-      final DocsAndPositionsEnum postings = MultiFields.getTermPositionsEnum(r, null, "field", new BytesRef(term));
+      final DocsAndPositionsEnum postings = MultiFields.getTermPositionsEnum(r, null, "field", new BytesRef(term), false);
 
       int docID = -1;
       while(docID < DocsEnum.NO_MORE_DOCS) {

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java Sun Jan 15 23:17:45 2012
@@ -86,7 +86,8 @@ public class TestMultiLevelSkipList exte
       counter = 0;
       DocsAndPositionsEnum tp = reader.termPositionsEnum(reader.getLiveDocs(),
                                                          term.field(),
-                                                         new BytesRef(term.text()));
+                                                         new BytesRef(term.text()),
+                                                         false);
 
       checkSkipTo(tp, 14, 185); // no skips
       checkSkipTo(tp, 17, 190); // one skip on level 0

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestOmitPositions.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestOmitPositions.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestOmitPositions.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestOmitPositions.java Sun Jan 15 23:17:45 2012
@@ -51,7 +51,7 @@ public class TestOmitPositions extends L
     IndexReader reader = w.getReader();
     w.close();
     
-    assertNull(MultiFields.getTermPositionsEnum(reader, null, "foo", new BytesRef("test")));
+    assertNull(MultiFields.getTermPositionsEnum(reader, null, "foo", new BytesRef("test"), false));
     
     DocsEnum de = _TestUtil.docs(random, reader, "foo", new BytesRef("test"), null, null, true);
     while (de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java Sun Jan 15 23:17:45 2012
@@ -155,7 +155,7 @@ public class TestPayloadProcessorProvide
     IndexReader reader = IndexReader.open(dir);
     try {
       int numPayloads = 0;
-      DocsAndPositionsEnum tpe = MultiFields.getTermPositionsEnum(reader, null, field, text);
+      DocsAndPositionsEnum tpe = MultiFields.getTermPositionsEnum(reader, null, field, text, false);
       while (tpe.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
         tpe.nextPosition();
         if (tpe.hasPayload()) {

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPayloads.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPayloads.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPayloads.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPayloads.java Sun Jan 15 23:17:45 2012
@@ -222,7 +222,8 @@ public class TestPayloads extends Lucene
           tps[i] = MultiFields.getTermPositionsEnum(reader,
                                                     MultiFields.getLiveDocs(reader),
                                                     terms[i].field(),
-                                                    new BytesRef(terms[i].text()));
+                                                    new BytesRef(terms[i].text()),
+                                                    false);
         }
         
         while (tps[0].nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
@@ -259,7 +260,8 @@ public class TestPayloads extends Lucene
         DocsAndPositionsEnum tp = MultiFields.getTermPositionsEnum(reader,
                                                                    MultiFields.getLiveDocs(reader),
                                                                    terms[0].field(),
-                                                                   new BytesRef(terms[0].text()));
+                                                                   new BytesRef(terms[0].text()),
+                                                                   false);
         tp.nextDoc();
         tp.nextPosition();
         // NOTE: prior rev of this test was failing to first
@@ -287,7 +289,8 @@ public class TestPayloads extends Lucene
         tp = MultiFields.getTermPositionsEnum(reader,
                                               MultiFields.getLiveDocs(reader),
                                               terms[1].field(),
-                                              new BytesRef(terms[1].text()));
+                                              new BytesRef(terms[1].text()),
+                                              false);
         tp.nextDoc();
         tp.nextPosition();
         assertEquals("Wrong payload length.", 1, tp.getPayload().length);
@@ -330,7 +333,8 @@ public class TestPayloads extends Lucene
         tp = MultiFields.getTermPositionsEnum(reader,
                                               MultiFields.getLiveDocs(reader),
                                               fieldName,
-                                              new BytesRef(singleTerm));
+                                              new BytesRef(singleTerm),
+                                              false);
         tp.nextDoc();
         tp.nextPosition();
         
@@ -532,7 +536,7 @@ public class TestPayloads extends Lucene
         DocsAndPositionsEnum tp = null;
         while (terms.next() != null) {
           String termText = terms.term().utf8ToString();
-          tp = terms.docsAndPositions(liveDocs, tp);
+          tp = terms.docsAndPositions(liveDocs, tp, false);
           while(tp.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
             int freq = tp.freq();
             for (int i = 0; i < freq; i++) {

Added: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPostingsOffsets.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPostingsOffsets.java?rev=1231794&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPostingsOffsets.java (added)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPostingsOffsets.java Sun Jan 15 23:17:45 2012
@@ -0,0 +1,240 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.analysis.CannedAnalyzer;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.NumericField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Assume;
+
+public class TestPostingsOffsets extends LuceneTestCase {
+
+  public void testBasic() throws Exception {
+
+    // Currently only SimpleText can index offsets into postings:
+    Assume.assumeTrue(Codec.getDefault().getName().equals("SimpleText"));
+
+    Directory dir = newDirectory();
+    RandomIndexWriter w = new RandomIndexWriter(random, dir);
+    Document doc = new Document();
+
+    FieldType ft = new FieldType(TextField.TYPE_UNSTORED);
+    ft.setIndexOptions(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+    Token[] tokens = new Token[] {
+      makeToken("a", 1, 0, 6),
+      makeToken("b", 1, 8, 9),
+      makeToken("a", 1, 9, 17),
+      makeToken("c", 1, 19, 50),
+    };
+    doc.add(new Field("content", new CannedAnalyzer.CannedTokenizer(tokens), ft));
+
+    w.addDocument(doc, new CannedAnalyzer(tokens));
+    IndexReader r = w.getReader();
+    w.close();
+
+    DocsAndPositionsEnum dp = MultiFields.getTermPositionsEnum(r, null, "content", new BytesRef("a"), true);
+    assertNotNull(dp);
+    assertEquals(0, dp.nextDoc());
+    assertEquals(2, dp.freq());
+    assertEquals(0, dp.nextPosition());
+    assertEquals(0, dp.startOffset());
+    assertEquals(6, dp.endOffset());
+    assertEquals(2, dp.nextPosition());
+    assertEquals(9, dp.startOffset());
+    assertEquals(17, dp.endOffset());
+    assertEquals(DocsEnum.NO_MORE_DOCS, dp.nextDoc());
+
+    dp = MultiFields.getTermPositionsEnum(r, null, "content", new BytesRef("b"), true);
+    assertNotNull(dp);
+    assertEquals(0, dp.nextDoc());
+    assertEquals(1, dp.freq());
+    assertEquals(1, dp.nextPosition());
+    assertEquals(8, dp.startOffset());
+    assertEquals(9, dp.endOffset());
+    assertEquals(DocsEnum.NO_MORE_DOCS, dp.nextDoc());
+
+    dp = MultiFields.getTermPositionsEnum(r, null, "content", new BytesRef("c"), true);
+    assertNotNull(dp);
+    assertEquals(0, dp.nextDoc());
+    assertEquals(1, dp.freq());
+    assertEquals(3, dp.nextPosition());
+    assertEquals(19, dp.startOffset());
+    assertEquals(50, dp.endOffset());
+    assertEquals(DocsEnum.NO_MORE_DOCS, dp.nextDoc());
+
+    r.close();
+    dir.close();
+  }
+
+  public void testRandom() throws Exception {
+    // Currently only SimpleText can index offsets into postings:
+    Assume.assumeTrue(Codec.getDefault().getName().equals("SimpleText"));
+
+    // token -> docID -> tokens
+    final Map<String,Map<Integer,List<Token>>> actualTokens = new HashMap<String,Map<Integer,List<Token>>>();
+
+    Directory dir = newDirectory();
+    RandomIndexWriter w = new RandomIndexWriter(random, dir);
+
+    final int numDocs = atLeast(20);
+    //final int numDocs = atLeast(5);
+
+    FieldType ft = new FieldType(TextField.TYPE_UNSTORED);
+
+    // TODO: randomize what IndexOptions we use; also test
+    // changing this up in one IW buffered segment...:
+    ft.setIndexOptions(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+    if (random.nextBoolean()) {
+      ft.setStoreTermVectors(true);
+      ft.setStoreTermVectorOffsets(random.nextBoolean());
+      ft.setStoreTermVectorPositions(random.nextBoolean());
+    }
+
+    for(int docCount=0;docCount<numDocs;docCount++) {
+      Document doc = new Document();
+      doc.add(new NumericField("id", docCount));
+      List<Token> tokens = new ArrayList<Token>();
+      final int numTokens = atLeast(100);
+      //final int numTokens = atLeast(20);
+      int pos = -1;
+      int offset = 0;
+      //System.out.println("doc id=" + docCount);
+      for(int tokenCount=0;tokenCount<numTokens;tokenCount++) {
+        final String text;
+        if (random.nextBoolean()) {
+          text = "a";
+        } else if (random.nextBoolean()) {
+          text = "b";
+        } else if (random.nextBoolean()) {
+          text = "c";
+        } else {
+          text = "d";
+        }       
+        
+        int posIncr = random.nextBoolean() ? 1 : random.nextInt(5);
+        if (tokenCount == 0 && posIncr == 0) {
+          posIncr = 1;
+        }
+        final int offIncr = random.nextBoolean() ? 0 : random.nextInt(5);
+        final int tokenOffset = random.nextInt(5);
+
+        final Token token = makeToken(text, posIncr, offset+offIncr, offset+offIncr+tokenOffset);
+        if (!actualTokens.containsKey(text)) {
+          actualTokens.put(text, new HashMap<Integer,List<Token>>());
+        }
+        final Map<Integer,List<Token>> postingsByDoc = actualTokens.get(text);
+        if (!postingsByDoc.containsKey(docCount)) {
+          postingsByDoc.put(docCount, new ArrayList<Token>());
+        }
+        postingsByDoc.get(docCount).add(token);
+        tokens.add(token);
+        pos += posIncr;
+        // stuff abs position into type:
+        token.setType(""+pos);
+        offset += offIncr + tokenOffset;
+        //System.out.println("  " + token + " posIncr=" + token.getPositionIncrement() + " pos=" + pos + " off=" + token.startOffset() + "/" + token.endOffset() + " (freq=" + postingsByDoc.get(docCount).size() + ")");
+      }
+      doc.add(new Field("content", new CannedAnalyzer.CannedTokenizer(tokens.toArray(new Token[tokens.size()])), ft));
+      w.addDocument(doc);
+    }
+    final IndexReader r = w.getReader();
+    w.close();
+
+    final String[] terms = new String[] {"a", "b", "c", "d"};
+    for(IndexReader sub : r.getSequentialSubReaders()) {
+      //System.out.println("\nsub=" + sub);
+      final TermsEnum termsEnum = sub.fields().terms("content").iterator(null);
+      DocsEnum docs = null;
+      DocsAndPositionsEnum docsAndPositions = null;
+      DocsAndPositionsEnum docsAndPositionsAndOffsets = null;
+      final int docIDToID[] = FieldCache.DEFAULT.getInts(sub, "id", false);
+      for(String term : terms) {
+        //System.out.println("  term=" + term);
+        if (termsEnum.seekExact(new BytesRef(term), random.nextBoolean())) {
+          docs = termsEnum.docs(null, docs, true);
+          assertNotNull(docs);
+          int doc;
+          //System.out.println("    doc/freq");
+          while((doc = docs.nextDoc()) != DocsEnum.NO_MORE_DOCS) {
+            final List<Token> expected = actualTokens.get(term).get(docIDToID[doc]);
+            //System.out.println("      doc=" + docIDToID[doc] + " docID=" + doc + " " + expected.size() + " freq");
+            assertNotNull(expected);
+            assertEquals(expected.size(), docs.freq());
+          }
+
+          docsAndPositions = termsEnum.docsAndPositions(null, docsAndPositions, false);
+          assertNotNull(docsAndPositions);
+          //System.out.println("    doc/freq/pos");
+          while((doc = docsAndPositions.nextDoc()) != DocsEnum.NO_MORE_DOCS) {
+            final List<Token> expected = actualTokens.get(term).get(docIDToID[doc]);
+            //System.out.println("      doc=" + docIDToID[doc] + " " + expected.size() + " freq");
+            assertNotNull(expected);
+            assertEquals(expected.size(), docsAndPositions.freq());
+            for(Token token : expected) {
+              int pos = Integer.parseInt(token.type());
+              //System.out.println("        pos=" + pos);
+              assertEquals(pos, docsAndPositions.nextPosition());
+            }
+          }
+
+          docsAndPositionsAndOffsets = termsEnum.docsAndPositions(null, docsAndPositions, true);
+          assertNotNull(docsAndPositionsAndOffsets);
+          //System.out.println("    doc/freq/pos/offs");
+          while((doc = docsAndPositions.nextDoc()) != DocsEnum.NO_MORE_DOCS) {
+            final List<Token> expected = actualTokens.get(term).get(docIDToID[doc]);
+            //System.out.println("      doc=" + docIDToID[doc] + " " + expected.size() + " freq");
+            assertNotNull(expected);
+            assertEquals(expected.size(), docsAndPositions.freq());
+            for(Token token : expected) {
+              int pos = Integer.parseInt(token.type());
+              //System.out.println("        pos=" + pos);
+              assertEquals(pos, docsAndPositions.nextPosition());
+              assertEquals(token.startOffset(), docsAndPositions.startOffset());
+              assertEquals(token.endOffset(), docsAndPositions.endOffset());
+            }
+          }
+        }
+      }        
+      // TODO: test advance:
+    }
+    r.close();
+    dir.close();
+  }
+
+  private Token makeToken(String text, int posIncr, int startOffset, int endOffset) {
+    final Token t = new Token();
+    t.append(text);
+    t.setPositionIncrement(posIncr);
+    t.setOffset(startOffset, endOffset);
+    return t;
+  }
+}

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java Sun Jan 15 23:17:45 2012
@@ -148,7 +148,8 @@ public class TestSegmentReader extends L
     DocsAndPositionsEnum positions = MultiFields.getTermPositionsEnum(reader,
                                                                       MultiFields.getLiveDocs(reader),
                                                                       DocHelper.TEXT_FIELD_1_KEY,
-                                                                      new BytesRef("field"));
+                                                                      new BytesRef("field"),
+                                                                      false);
     // NOTE: prior rev of this test was failing to first
     // call next here:
     assertTrue(positions.nextDoc() != DocsEnum.NO_MORE_DOCS);

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java Sun Jan 15 23:17:45 2012
@@ -406,7 +406,7 @@ public class TestStressIndexing2 extends
             BytesRef term2;
             while((term2 = termsEnum3.next()) != null) {
               System.out.println("      " + term2.utf8ToString() + ": freq=" + termsEnum3.totalTermFreq());
-              dpEnum = termsEnum3.docsAndPositions(null, dpEnum);
+              dpEnum = termsEnum3.docsAndPositions(null, dpEnum, false);
               if (dpEnum != null) {
                 assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
                 final int freq = dpEnum.freq();
@@ -440,7 +440,7 @@ public class TestStressIndexing2 extends
             BytesRef term2;
             while((term2 = termsEnum3.next()) != null) {
               System.out.println("      " + term2.utf8ToString() + ": freq=" + termsEnum3.totalTermFreq());
-              dpEnum = termsEnum3.docsAndPositions(null, dpEnum);
+              dpEnum = termsEnum3.docsAndPositions(null, dpEnum, false);
               if (dpEnum != null) {
                 assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
                 final int freq = dpEnum.freq();
@@ -630,8 +630,8 @@ public class TestStressIndexing2 extends
         assertEquals(termsEnum1.totalTermFreq(),
                      termsEnum2.totalTermFreq());
         
-        dpEnum1 = termsEnum1.docsAndPositions(null, dpEnum1);
-        dpEnum2 = termsEnum2.docsAndPositions(null, dpEnum2);
+        dpEnum1 = termsEnum1.docsAndPositions(null, dpEnum1, false);
+        dpEnum2 = termsEnum2.docsAndPositions(null, dpEnum2, false);
         if (dpEnum1 != null) {
           assertNotNull(dpEnum2);
           int docID1 = dpEnum1.nextDoc();

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java Sun Jan 15 23:17:45 2012
@@ -259,7 +259,7 @@ public class TestTermVectorsReader exten
       //System.out.println("Term: " + term);
       assertEquals(testTerms[i], term);
 
-      dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+      dpEnum = termsEnum.docsAndPositions(null, dpEnum, false);
       assertNotNull(dpEnum);
       int doc = dpEnum.docID();
       assertTrue(doc == -1 || doc == DocIdSetIterator.NO_MORE_DOCS);
@@ -270,18 +270,16 @@ public class TestTermVectorsReader exten
       }
       assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
 
-      dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+      dpEnum = termsEnum.docsAndPositions(null, dpEnum, true);
       doc = dpEnum.docID();
       assertTrue(doc == -1 || doc == DocIdSetIterator.NO_MORE_DOCS);
       assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
       assertNotNull(dpEnum);
-      final OffsetAttribute offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
-      assertNotNull(offsetAtt);
       assertEquals(dpEnum.freq(), positions[i].length);
       for (int j = 0; j < positions[i].length; j++) {
         assertEquals(positions[i][j], dpEnum.nextPosition());
-        assertEquals(j*10, offsetAtt.startOffset());
-        assertEquals(j*10 + testTerms[i].length(), offsetAtt.endOffset());
+        assertEquals(j*10, dpEnum.startOffset());
+        assertEquals(j*10 + testTerms[i].length(), dpEnum.endOffset());
       }
       assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
     }
@@ -315,7 +313,7 @@ public class TestTermVectorsReader exten
       String term = text.utf8ToString();
       assertEquals(testTerms[i], term);
 
-      dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+      dpEnum = termsEnum.docsAndPositions(null, dpEnum, false);
       assertNotNull(dpEnum);
       assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
       assertEquals(dpEnum.freq(), positions[i].length);
@@ -324,16 +322,14 @@ public class TestTermVectorsReader exten
       }
       assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
 
-      dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+      dpEnum = termsEnum.docsAndPositions(null, dpEnum, true);
       assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
-      final OffsetAttribute offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
-      assertNotNull(offsetAtt);
       assertNotNull(dpEnum);
       assertEquals(dpEnum.freq(), positions[i].length);
       for (int j = 0; j < positions[i].length; j++) {
         assertEquals(positions[i][j], dpEnum.nextPosition());
-        assertEquals(j*10, offsetAtt.startOffset());
-        assertEquals(j*10 + testTerms[i].length(), offsetAtt.endOffset());
+        assertEquals(j*10, dpEnum.startOffset());
+        assertEquals(j*10 + testTerms[i].length(), dpEnum.endOffset());
       }
       assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
     }

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java Sun Jan 15 23:17:45 2012
@@ -26,7 +26,6 @@ import org.apache.lucene.analysis.MockAn
 import org.apache.lucene.analysis.MockTokenFilter;
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
@@ -69,34 +68,30 @@ public class TestTermVectorsWriter exten
     // Token "" occurred once
     assertEquals(1, termsEnum.totalTermFreq());
 
-    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null);
-    OffsetAttribute offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
-    assertNotNull(offsetAtt);
+    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, true);
     assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
     dpEnum.nextPosition();
-    assertEquals(8, offsetAtt.startOffset());
-    assertEquals(8, offsetAtt.endOffset());
+    assertEquals(8, dpEnum.startOffset());
+    assertEquals(8, dpEnum.endOffset());
     assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
 
     // Token "abcd" occurred three times
     assertEquals(new BytesRef("abcd"), termsEnum.next());
-    dpEnum = termsEnum.docsAndPositions(null, dpEnum);
-    offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
-    assertNotNull(offsetAtt);
+    dpEnum = termsEnum.docsAndPositions(null, dpEnum, true);
     assertEquals(3, termsEnum.totalTermFreq());
 
     assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
     dpEnum.nextPosition();
-    assertEquals(0, offsetAtt.startOffset());
-    assertEquals(4, offsetAtt.endOffset());
+    assertEquals(0, dpEnum.startOffset());
+    assertEquals(4, dpEnum.endOffset());
 
     dpEnum.nextPosition();
-    assertEquals(4, offsetAtt.startOffset());
-    assertEquals(8, offsetAtt.endOffset());
+    assertEquals(4, dpEnum.startOffset());
+    assertEquals(8, dpEnum.endOffset());
 
     dpEnum.nextPosition();
-    assertEquals(8, offsetAtt.startOffset());
-    assertEquals(12, offsetAtt.endOffset());
+    assertEquals(8, dpEnum.startOffset());
+    assertEquals(12, dpEnum.endOffset());
 
     assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
     assertNull(termsEnum.next());
@@ -122,19 +117,17 @@ public class TestTermVectorsWriter exten
     IndexReader r = IndexReader.open(dir);
     TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null);
     assertNotNull(termsEnum.next());
-    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null);
-    OffsetAttribute offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
-    assertNotNull(offsetAtt);
+    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, true);
     assertEquals(2, termsEnum.totalTermFreq());
 
     assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
     dpEnum.nextPosition();
-    assertEquals(0, offsetAtt.startOffset());
-    assertEquals(4, offsetAtt.endOffset());
+    assertEquals(0, dpEnum.startOffset());
+    assertEquals(4, dpEnum.endOffset());
 
     dpEnum.nextPosition();
-    assertEquals(5, offsetAtt.startOffset());
-    assertEquals(9, offsetAtt.endOffset());
+    assertEquals(5, dpEnum.startOffset());
+    assertEquals(9, dpEnum.endOffset());
     assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
 
     r.close();
@@ -159,19 +152,17 @@ public class TestTermVectorsWriter exten
     IndexReader r = IndexReader.open(dir);
     TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null);
     assertNotNull(termsEnum.next());
-    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null);
-    OffsetAttribute offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
-    assertNotNull(offsetAtt);
+    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, true);
     assertEquals(2, termsEnum.totalTermFreq());
 
     assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
     dpEnum.nextPosition();
-    assertEquals(0, offsetAtt.startOffset());
-    assertEquals(4, offsetAtt.endOffset());
+    assertEquals(0, dpEnum.startOffset());
+    assertEquals(4, dpEnum.endOffset());
 
     dpEnum.nextPosition();
-    assertEquals(8, offsetAtt.startOffset());
-    assertEquals(12, offsetAtt.endOffset());
+    assertEquals(8, dpEnum.startOffset());
+    assertEquals(12, dpEnum.endOffset());
     assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
 
     r.close();
@@ -200,19 +191,17 @@ public class TestTermVectorsWriter exten
     IndexReader r = IndexReader.open(dir);
     TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null);
     assertNotNull(termsEnum.next());
-    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null);
-    OffsetAttribute offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
-    assertNotNull(offsetAtt);
+    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, true);
     assertEquals(2, termsEnum.totalTermFreq());
 
     assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
     dpEnum.nextPosition();
-    assertEquals(0, offsetAtt.startOffset());
-    assertEquals(4, offsetAtt.endOffset());
+    assertEquals(0, dpEnum.startOffset());
+    assertEquals(4, dpEnum.endOffset());
 
     dpEnum.nextPosition();
-    assertEquals(8, offsetAtt.startOffset());
-    assertEquals(12, offsetAtt.endOffset());
+    assertEquals(8, dpEnum.startOffset());
+    assertEquals(12, dpEnum.endOffset());
     assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
 
     r.close();
@@ -238,19 +227,17 @@ public class TestTermVectorsWriter exten
     IndexReader r = IndexReader.open(dir);
     TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null);
     assertNotNull(termsEnum.next());
-    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null);
-    OffsetAttribute offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
-    assertNotNull(offsetAtt);
+    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, true);
     assertEquals(2, termsEnum.totalTermFreq());
 
     assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
     dpEnum.nextPosition();
-    assertEquals(0, offsetAtt.startOffset());
-    assertEquals(4, offsetAtt.endOffset());
+    assertEquals(0, dpEnum.startOffset());
+    assertEquals(4, dpEnum.endOffset());
 
     dpEnum.nextPosition();
-    assertEquals(9, offsetAtt.startOffset());
-    assertEquals(13, offsetAtt.endOffset());
+    assertEquals(9, dpEnum.startOffset());
+    assertEquals(13, dpEnum.endOffset());
     assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
 
     r.close();
@@ -277,32 +264,26 @@ public class TestTermVectorsWriter exten
     IndexReader r = IndexReader.open(dir);
     TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null);
     assertNotNull(termsEnum.next());
-    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null);
-    OffsetAttribute offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
-    assertNotNull(offsetAtt);
+    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, true);
 
     assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
     dpEnum.nextPosition();
-    assertEquals(0, offsetAtt.startOffset());
-    assertEquals(4, offsetAtt.endOffset());
+    assertEquals(0, dpEnum.startOffset());
+    assertEquals(4, dpEnum.endOffset());
 
     assertNotNull(termsEnum.next());
-    dpEnum = termsEnum.docsAndPositions(null, dpEnum);
-    offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
-    assertNotNull(offsetAtt);
+    dpEnum = termsEnum.docsAndPositions(null, dpEnum, true);
     assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
     dpEnum.nextPosition();
-    assertEquals(11, offsetAtt.startOffset());
-    assertEquals(17, offsetAtt.endOffset());
+    assertEquals(11, dpEnum.startOffset());
+    assertEquals(17, dpEnum.endOffset());
 
     assertNotNull(termsEnum.next());
-    dpEnum = termsEnum.docsAndPositions(null, dpEnum);
-    offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
-    assertNotNull(offsetAtt);
+    dpEnum = termsEnum.docsAndPositions(null, dpEnum, true);
     assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
     dpEnum.nextPosition();
-    assertEquals(18, offsetAtt.startOffset());
-    assertEquals(21, offsetAtt.endOffset());
+    assertEquals(18, dpEnum.startOffset());
+    assertEquals(21, dpEnum.endOffset());
 
     r.close();
     dir.close();
@@ -328,24 +309,20 @@ public class TestTermVectorsWriter exten
     IndexReader r = IndexReader.open(dir);
     TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null);
     assertNotNull(termsEnum.next());
-    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null);
-    OffsetAttribute offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
-    assertNotNull(offsetAtt);
+    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, true);
 
     assertEquals(1, (int) termsEnum.totalTermFreq());
     assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
     dpEnum.nextPosition();
-    assertEquals(1, offsetAtt.startOffset());
-    assertEquals(7, offsetAtt.endOffset());
+    assertEquals(1, dpEnum.startOffset());
+    assertEquals(7, dpEnum.endOffset());
 
     assertNotNull(termsEnum.next());
-    dpEnum = termsEnum.docsAndPositions(null, dpEnum);
-    offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
-    assertNotNull(offsetAtt);
+    dpEnum = termsEnum.docsAndPositions(null, dpEnum, true);
     assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
     dpEnum.nextPosition();
-    assertEquals(8, offsetAtt.startOffset());
-    assertEquals(11, offsetAtt.endOffset());
+    assertEquals(8, dpEnum.startOffset());
+    assertEquals(11, dpEnum.endOffset());
 
     r.close();
     dir.close();
@@ -375,24 +352,20 @@ public class TestTermVectorsWriter exten
     IndexReader r = IndexReader.open(dir);
     TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null);
     assertNotNull(termsEnum.next());
-    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null);
-    OffsetAttribute offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
-    assertNotNull(offsetAtt);
+    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, true);
 
     assertEquals(1, (int) termsEnum.totalTermFreq());
     assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
     dpEnum.nextPosition();
-    assertEquals(0, offsetAtt.startOffset());
-    assertEquals(4, offsetAtt.endOffset());
+    assertEquals(0, dpEnum.startOffset());
+    assertEquals(4, dpEnum.endOffset());
 
     assertNotNull(termsEnum.next());
-    dpEnum = termsEnum.docsAndPositions(null, dpEnum);
-    offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
-    assertNotNull(offsetAtt);
+    dpEnum = termsEnum.docsAndPositions(null, dpEnum, true);
     assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
     dpEnum.nextPosition();
-    assertEquals(6, offsetAtt.startOffset());
-    assertEquals(12, offsetAtt.endOffset());
+    assertEquals(6, dpEnum.startOffset());
+    assertEquals(12, dpEnum.endOffset());
 
 
     r.close();

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java Sun Jan 15 23:17:45 2012
@@ -17,36 +17,38 @@ package org.apache.lucene.search;
  * limitations under the License.
  */
 
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.MultiFields;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.TermContext;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.Collection;
+import java.util.LinkedList;
+
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.CannedAnalyzer;
+import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.similarities.DefaultSimilarity;
 import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
 import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.io.Reader;
+import org.apache.lucene.util.TermContext;
 
 /**
  * This class tests the MultiPhraseQuery class.
@@ -329,68 +331,18 @@ public class TestMultiPhraseQuery extend
     indexStore.close();
   }
 
-  private static class TokenAndPos {
-    public final String token;
-    public final int pos;
-    public TokenAndPos(String token, int pos) {
-      this.token = token;
-      this.pos = pos;
-    }
-  }
-
-  private static class CannedAnalyzer extends Analyzer {
-    private final TokenAndPos[] tokens;
-    
-    public CannedAnalyzer(TokenAndPos[] tokens) {
-      this.tokens = tokens;
-    }
-
-    @Override
-    public TokenStreamComponents createComponents(String fieldName, Reader reader) {
-      return new TokenStreamComponents(new CannedTokenizer(tokens));
-    }
-  }
-
-  private static class CannedTokenizer extends Tokenizer {
-    private final TokenAndPos[] tokens;
-    private int upto = 0;
-    private int lastPos = 0;
-    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
-    private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
-
-    public CannedTokenizer(TokenAndPos[] tokens) {
-      this.tokens = tokens;
-    }
-
-    @Override
-    public final boolean incrementToken() throws IOException {
-      clearAttributes();      
-      if (upto < tokens.length) {
-        final TokenAndPos token = tokens[upto++];
-        termAtt.setEmpty();
-        termAtt.append(token.token);
-        posIncrAtt.setPositionIncrement(token.pos - lastPos);
-        lastPos = token.pos;
-        return true;
-      } else {
-        return false;
-      }
-    }
-
-    @Override
-    public void reset() throws IOException {
-      super.reset();
-      this.upto = 0;
-      this.lastPos = 0;
-    }
-  }
-
   public void testZeroPosIncr() throws IOException {
     Directory dir = new RAMDirectory();
-    final TokenAndPos[] tokens = new TokenAndPos[3];
-    tokens[0] = new TokenAndPos("a", 0);
-    tokens[1] = new TokenAndPos("b", 0);
-    tokens[2] = new TokenAndPos("c", 0);
+    final Token[] tokens = new Token[3];
+    tokens[0] = new Token();
+    tokens[0].append("a");
+    tokens[0].setPositionIncrement(1);
+    tokens[1] = new Token();
+    tokens[1].append("b");
+    tokens[1].setPositionIncrement(0);
+    tokens[2] = new Token();
+    tokens[2].append("c");
+    tokens[2].setPositionIncrement(0);
 
     RandomIndexWriter writer = new RandomIndexWriter(random, dir, new CannedAnalyzer(tokens));
     Document doc = new Document();
@@ -429,40 +381,47 @@ public class TestMultiPhraseQuery extend
     dir.close();
   }
 
-  private final static TokenAndPos[] INCR_0_DOC_TOKENS = new TokenAndPos[] {
-      new TokenAndPos("x", 0),
-      new TokenAndPos("a", 1),
-      new TokenAndPos("1", 1),
-      new TokenAndPos("m", 2), // not existing, relying on slop=2
-      new TokenAndPos("b", 3),
-      new TokenAndPos("1", 3),
-      new TokenAndPos("n", 4), // not existing, relying on slop=2
-      new TokenAndPos("c", 5),
-      new TokenAndPos("y", 6)
+  private static Token makeToken(String text, int posIncr) {
+    final Token t = new Token();
+    t.append(text);
+    t.setPositionIncrement(posIncr);
+    return t;
+  }
+
+  private final static Token[] INCR_0_DOC_TOKENS = new Token[] {
+    makeToken("x", 1),
+    makeToken("a", 1),
+    makeToken("1", 0),
+    makeToken("m", 1),  // not existing, relying on slop=2
+    makeToken("b", 1),
+    makeToken("1", 0),
+    makeToken("n", 1), // not existing, relying on slop=2
+    makeToken("c", 1),
+    makeToken("y", 1)
   };
   
-  private final static TokenAndPos[] INCR_0_QUERY_TOKENS_AND = new TokenAndPos[] {
-      new TokenAndPos("a", 0),
-      new TokenAndPos("1", 0),
-      new TokenAndPos("b", 1),
-      new TokenAndPos("1", 1),
-      new TokenAndPos("c", 2)
+  private final static Token[] INCR_0_QUERY_TOKENS_AND = new Token[] {
+    makeToken("a", 1),
+    makeToken("1", 0),
+    makeToken("b", 1),
+    makeToken("1", 0),
+    makeToken("c", 1)
   };
   
-  private final static TokenAndPos[][] INCR_0_QUERY_TOKENS_AND_OR_MATCH = new TokenAndPos[][] {
-      { new TokenAndPos("a", 0) },
-      { new TokenAndPos("x", 0), new TokenAndPos("1", 0) },
-      { new TokenAndPos("b", 1) },
-      { new TokenAndPos("x", 1), new TokenAndPos("1", 1) },
-      { new TokenAndPos("c", 2) }
+  private final static Token[][] INCR_0_QUERY_TOKENS_AND_OR_MATCH = new Token[][] {
+    { makeToken("a", 1) },
+    { makeToken("x", 1), makeToken("1", 0) },
+    { makeToken("b", 2) },
+    { makeToken("x", 2), makeToken("1", 0) },
+    { makeToken("c", 3) }
   };
   
-  private final static TokenAndPos[][] INCR_0_QUERY_TOKENS_AND_OR_NO_MATCHN = new TokenAndPos[][] {
-      { new TokenAndPos("x", 0) },
-      { new TokenAndPos("a", 0), new TokenAndPos("1", 0) },
-      { new TokenAndPos("x", 1) },
-      { new TokenAndPos("b", 1), new TokenAndPos("1", 1) },
-      { new TokenAndPos("c", 2) }
+  private final static Token[][] INCR_0_QUERY_TOKENS_AND_OR_NO_MATCHN = new Token[][] {
+    { makeToken("x", 1) },
+    { makeToken("a", 1), makeToken("1", 0) },
+    { makeToken("x", 2) },
+    { makeToken("b", 2), makeToken("1", 0) },
+    { makeToken("c", 3) }
   };
   
   /**
@@ -515,8 +474,10 @@ public class TestMultiPhraseQuery extend
    */
   public void testZeroPosIncrSloppyPqAnd() throws IOException {
     final PhraseQuery pq = new PhraseQuery();
-    for (TokenAndPos tap : INCR_0_QUERY_TOKENS_AND) {
-      pq.add(new Term("field",tap.token), tap.pos);
+    int pos = -1;
+    for (Token tap : INCR_0_QUERY_TOKENS_AND) {
+      pos += tap.getPositionIncrement();
+      pq.add(new Term("field",tap.toString()), pos);
     }
     doTestZeroPosIncrSloppy(pq, 0);
     pq.setSlop(1);
@@ -530,8 +491,10 @@ public class TestMultiPhraseQuery extend
    */
   public void testZeroPosIncrSloppyMpqAnd() throws IOException {
     final MultiPhraseQuery mpq = new MultiPhraseQuery();
-    for (TokenAndPos tap : INCR_0_QUERY_TOKENS_AND) {
-      mpq.add(new Term[]{new Term("field",tap.token)}, tap.pos); //AND logic
+    int pos = -1;
+    for (Token tap : INCR_0_QUERY_TOKENS_AND) {
+      pos += tap.getPositionIncrement();
+      mpq.add(new Term[]{new Term("field",tap.toString())}, pos); //AND logic
     }
     doTestZeroPosIncrSloppy(mpq, 0);
     mpq.setSlop(1);
@@ -545,9 +508,9 @@ public class TestMultiPhraseQuery extend
    */
   public void testZeroPosIncrSloppyMpqAndOrMatch() throws IOException {
     final MultiPhraseQuery mpq = new MultiPhraseQuery();
-    for (TokenAndPos tap[] : INCR_0_QUERY_TOKENS_AND_OR_MATCH) {
+    for (Token tap[] : INCR_0_QUERY_TOKENS_AND_OR_MATCH) {
       Term[] terms = tapTerms(tap);
-      final int pos = tap[0].pos;
+      final int pos = tap[0].getPositionIncrement()-1;
       mpq.add(terms, pos); //AND logic in pos, OR across lines 
     }
     doTestZeroPosIncrSloppy(mpq, 0);
@@ -562,9 +525,9 @@ public class TestMultiPhraseQuery extend
    */
   public void testZeroPosIncrSloppyMpqAndOrNoMatch() throws IOException {
     final MultiPhraseQuery mpq = new MultiPhraseQuery();
-    for (TokenAndPos tap[] : INCR_0_QUERY_TOKENS_AND_OR_NO_MATCHN) {
+    for (Token tap[] : INCR_0_QUERY_TOKENS_AND_OR_NO_MATCHN) {
       Term[] terms = tapTerms(tap);
-      final int pos = tap[0].pos;
+      final int pos = tap[0].getPositionIncrement()-1;
       mpq.add(terms, pos); //AND logic in pos, OR across lines 
     }
     doTestZeroPosIncrSloppy(mpq, 0);
@@ -572,10 +535,10 @@ public class TestMultiPhraseQuery extend
     doTestZeroPosIncrSloppy(mpq, 0);
   }
 
-  private Term[] tapTerms(TokenAndPos[] tap) {
+  private Term[] tapTerms(Token[] tap) {
     Term[] terms = new Term[tap.length];
     for (int i=0; i<terms.length; i++) {
-      terms[i] = new Term("field",tap[i].token);
+      terms[i] = new Term("field",tap[i].toString());
     }
     return terms;
   }

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java Sun Jan 15 23:17:45 2012
@@ -42,8 +42,6 @@ import org.apache.lucene.search.spans.Sp
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.search.spans.Spans;
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.automaton.CharacterRunAutomaton;
-import org.apache.lucene.util.automaton.RegExp;
 import org.apache.lucene.util.BytesRef;
 
 /**
@@ -102,7 +100,8 @@ public class TestPositionIncrement exten
     DocsAndPositionsEnum pos = MultiFields.getTermPositionsEnum(searcher.getIndexReader(),
                                                                 MultiFields.getLiveDocs(searcher.getIndexReader()),
                                                                 "field",
-                                                                new BytesRef("1"));
+                                                                new BytesRef("1"),
+                                                                false);
     pos.nextDoc();
     // first token should be at position 0
     assertEquals(0, pos.nextPosition());
@@ -110,7 +109,8 @@ public class TestPositionIncrement exten
     pos = MultiFields.getTermPositionsEnum(searcher.getIndexReader(),
                                            MultiFields.getLiveDocs(searcher.getIndexReader()),
                                            "field",
-                                           new BytesRef("2"));
+                                           new BytesRef("2"),
+                                           false);
     pos.nextDoc();
     // second token should be at position 2
     assertEquals(2, pos.nextPosition());
@@ -200,10 +200,6 @@ public class TestPositionIncrement exten
     store.close();
   }
 
-  // stoplist that accepts case-insensitive "stop"
-  private static final CharacterRunAutomaton stopStopList = 
-    new CharacterRunAutomaton(new RegExp("[sS][tT][oO][pP]").toAutomaton());
-  
   public void testPayloadsPos0() throws Exception {
     Directory dir = newDirectory();
     RandomIndexWriter writer = new RandomIndexWriter(random, dir, new MockPayloadAnalyzer());
@@ -217,7 +213,8 @@ public class TestPositionIncrement exten
 
     DocsAndPositionsEnum tp = r.termPositionsEnum(r.getLiveDocs(),
                                                   "content",
-                                                  new BytesRef("a"));
+                                                  new BytesRef("a"),
+                                                  false);
     
     int count = 0;
     assertTrue(tp.nextDoc() != DocsAndPositionsEnum.NO_MORE_DOCS);

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestTermVectors.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestTermVectors.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestTermVectors.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestTermVectors.java Sun Jan 15 23:17:45 2012
@@ -23,7 +23,6 @@ import java.util.Map;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
@@ -135,19 +134,19 @@ public class TestTermVectors extends Luc
       assertNotNull(terms);
       TermsEnum termsEnum = terms.iterator(null);
       assertEquals("content", termsEnum.next().utf8ToString());
-      dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+      dpEnum = termsEnum.docsAndPositions(null, dpEnum, false);
       assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
       assertEquals(1, dpEnum.freq());
       assertEquals(expectedPositions[0], dpEnum.nextPosition());
 
       assertEquals("here", termsEnum.next().utf8ToString());
-      dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+      dpEnum = termsEnum.docsAndPositions(null, dpEnum, false);
       assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
       assertEquals(1, dpEnum.freq());
       assertEquals(expectedPositions[1], dpEnum.nextPosition());
 
       assertEquals("some", termsEnum.next().utf8ToString());
-      dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+      dpEnum = termsEnum.docsAndPositions(null, dpEnum, false);
       assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
       assertEquals(1, dpEnum.freq());
       assertEquals(expectedPositions[2], dpEnum.nextPosition());
@@ -171,31 +170,21 @@ public class TestTermVectors extends Luc
       
       TermsEnum termsEnum = vectors.terms("field").iterator(null);
       assertNotNull(termsEnum.next());
-      dpEnum = termsEnum.docsAndPositions(null, dpEnum);
-      OffsetAttribute offsetAtt = dpEnum == null ? null : dpEnum.attributes().getAttribute(OffsetAttribute.class);
 
       boolean shouldBePosVector = hits[i].doc % 2 == 0;
-      assertTrue(!shouldBePosVector
-                 || (shouldBePosVector && dpEnum != null));
-      
       boolean shouldBeOffVector = hits[i].doc % 3 == 0;
-      assertTrue(!shouldBeOffVector
-                 || (shouldBeOffVector && offsetAtt != null));
       
       if (shouldBePosVector || shouldBeOffVector) {
         while(true) {
-          dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+          dpEnum = termsEnum.docsAndPositions(null, dpEnum, shouldBeOffVector);
           assertNotNull(dpEnum);
-          offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
+          assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
+
+          dpEnum.nextPosition();
 
-          if (shouldBePosVector) {
-            assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
-          }
-          
           if (shouldBeOffVector) {
-            assertNotNull(offsetAtt);
-          } else {
-            assertNull(offsetAtt);
+            assertTrue(dpEnum.startOffset() != -1);
+            assertTrue(dpEnum.endOffset() != -1);
           }
 
           if (termsEnum.next() == null) {
@@ -437,7 +426,7 @@ public class TestTermVectors extends Luc
     assertNotNull(termsEnum.next());
     assertEquals("one", termsEnum.term().utf8ToString());
     assertEquals(5, termsEnum.totalTermFreq());
-    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null);
+    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, false);
     assertNotNull(dpEnum);
     assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
     assertEquals(5, dpEnum.freq());
@@ -445,16 +434,14 @@ public class TestTermVectors extends Luc
       assertEquals(i, dpEnum.nextPosition());
     }
 
-    dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+    dpEnum = termsEnum.docsAndPositions(null, dpEnum, true);
     assertNotNull(dpEnum);
-    OffsetAttribute offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
-    assertNotNull(offsetAtt);
     assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
     assertEquals(5, dpEnum.freq());
     for(int i=0;i<5;i++) {
       dpEnum.nextPosition();
-      assertEquals(4*i, offsetAtt.startOffset());
-      assertEquals(4*i+3, offsetAtt.endOffset());
+      assertEquals(4*i, dpEnum.startOffset());
+      assertEquals(4*i+3, dpEnum.endOffset());
     }
     reader.close();
   }

Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestClassicAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestClassicAnalyzer.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestClassicAnalyzer.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestClassicAnalyzer.java Sun Jan 15 23:17:45 2012
@@ -283,7 +283,8 @@ public class TestClassicAnalyzer extends
     DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(reader,
                                                                 MultiFields.getLiveDocs(reader),
                                                                 "content",
-                                                                new BytesRef("another"));
+                                                                new BytesRef("another"),
+                                                                false);
     assertTrue(tps.nextDoc() != DocsEnum.NO_MORE_DOCS);
     assertEquals(1, tps.freq());
     assertEquals(3, tps.nextPosition());

Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java Sun Jan 15 23:17:45 2012
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.core.L
 import org.apache.lucene.analysis.standard.StandardFilter;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
@@ -110,16 +109,15 @@ public class TestTeeSinkTokenFilter exte
     TermsEnum termsEnum = vector.iterator(null);
     termsEnum.next();
     assertEquals(2, termsEnum.totalTermFreq());
-    DocsAndPositionsEnum positions = termsEnum.docsAndPositions(null, null);
-    OffsetAttribute offsetAtt = positions.attributes().getAttribute(OffsetAttribute.class);
+    DocsAndPositionsEnum positions = termsEnum.docsAndPositions(null, null, true);
     assertTrue(positions.nextDoc() != DocsEnum.NO_MORE_DOCS);
     assertEquals(2, positions.freq());
     positions.nextPosition();
-    assertEquals(0, offsetAtt.startOffset());
-    assertEquals(4, offsetAtt.endOffset());
+    assertEquals(0, positions.startOffset());
+    assertEquals(4, positions.endOffset());
     positions.nextPosition();
-    assertEquals(8, offsetAtt.startOffset());
-    assertEquals(12, offsetAtt.endOffset());
+    assertEquals(8, positions.startOffset());
+    assertEquals(12, positions.endOffset());
     assertEquals(DocsEnum.NO_MORE_DOCS, positions.nextDoc());
     r.close();
     dir.close();

Modified: lucene/dev/trunk/modules/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/modules/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java (original)
+++ lucene/dev/trunk/modules/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java Sun Jan 15 23:17:45 2012
@@ -53,7 +53,7 @@ public class PayloadIterator {
     this.buffer = buffer;
     // TODO (Facet): avoid Multi*?
     Bits liveDocs = MultiFields.getLiveDocs(indexReader);
-    this.tp = MultiFields.getTermPositionsEnum(indexReader, liveDocs, term.field(), term.bytes());
+    this.tp = MultiFields.getTermPositionsEnum(indexReader, liveDocs, term.field(), term.bytes(), false);
   }
 
   /**

Modified: lucene/dev/trunk/modules/facet/src/java/org/apache/lucene/facet/taxonomy/directory/ParentArray.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/facet/src/java/org/apache/lucene/facet/taxonomy/directory/ParentArray.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/modules/facet/src/java/org/apache/lucene/facet/taxonomy/directory/ParentArray.java (original)
+++ lucene/dev/trunk/modules/facet/src/java/org/apache/lucene/facet/taxonomy/directory/ParentArray.java Sun Jan 15 23:17:45 2012
@@ -104,7 +104,8 @@ class ParentArray {
     // TODO (Facet): avoid Multi*?
     Bits liveDocs = MultiFields.getLiveDocs(indexReader);
     DocsAndPositionsEnum positions = MultiFields.getTermPositionsEnum(indexReader, liveDocs,
-        Consts.FIELD_PAYLOADS, new BytesRef(Consts.PAYLOAD_PARENT));
+                                                                      Consts.FIELD_PAYLOADS, new BytesRef(Consts.PAYLOAD_PARENT),
+                                                                      false);
       if ((positions == null || positions.advance(first) == DocsAndPositionsEnum.NO_MORE_DOCS) && first < num) {
         throw new CorruptIndexException("Missing parent data for category " + first);
       }

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java?rev=1231794&r1=1231793&r2=1231794&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java Sun Jan 15 23:17:45 2012
@@ -8,7 +8,6 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.index.DocsAndPositionsEnum;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.Fields;
@@ -283,18 +282,17 @@ public class TermVectorComponent extends
         termInfo.add("tf", freq);
       }
 
-      dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+      dpEnum = termsEnum.docsAndPositions(null, dpEnum, fieldOptions.offsets);
+      boolean useOffsets = fieldOptions.offsets;
+      if (dpEnum == null) {
+        useOffsets = false;
+        dpEnum = termsEnum.docsAndPositions(null, dpEnum, false);
+      }
 
       boolean usePositions = false;
-      boolean useOffsets = false;
-      OffsetAttribute offsetAtt = null;
       if (dpEnum != null) {
         dpEnum.nextDoc();
         usePositions = fieldOptions.positions;
-        if (fieldOptions.offsets && dpEnum.attributes().hasAttribute(OffsetAttribute.class)) {
-          useOffsets = true;
-          offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
-        }
       }
 
       NamedList<Number> theOffsets = null;
@@ -317,8 +315,8 @@ public class TermVectorComponent extends
           }
 
           if (theOffsets != null) {
-            theOffsets.add("start", offsetAtt.startOffset());
-            theOffsets.add("end", offsetAtt.endOffset());
+            theOffsets.add("start", dpEnum.startOffset());
+            theOffsets.add("end", dpEnum.endOffset());
           }
         }
       }



Mime
View raw message