lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1524840 [3/4] - in /lucene/dev/trunk/lucene: ./ codecs/src/java/org/apache/lucene/codecs/blockterms/ codecs/src/java/org/apache/lucene/codecs/bloom/ codecs/src/java/org/apache/lucene/codecs/memory/ codecs/src/java/org/apache/lucene/codecs/...
Date Thu, 19 Sep 2013 20:57:11 GMT
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java Thu Sep 19 20:57:09 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.search;
  */
 
 import java.io.IOException;
-import java.util.Comparator;
 import java.util.LinkedList;
 
 import org.apache.lucene.analysis.NumericTokenStream; // for javadocs
@@ -392,7 +391,6 @@ public final class NumericRangeQuery<T e
     private BytesRef currentLowerBound, currentUpperBound;
 
     private final LinkedList<BytesRef> rangeBounds = new LinkedList<BytesRef>();
-    private final Comparator<BytesRef> termComp;
 
     NumericRangeTermsEnum(final TermsEnum tenum) {
       super(tenum);
@@ -481,15 +479,13 @@ public final class NumericRangeQuery<T e
           // should never happen
           throw new IllegalArgumentException("Invalid NumericType");
       }
-
-      termComp = getComparator();
     }
     
     private void nextRange() {
       assert rangeBounds.size() % 2 == 0;
 
       currentLowerBound = rangeBounds.removeFirst();
-      assert currentUpperBound == null || termComp.compare(currentUpperBound, currentLowerBound) <= 0 :
+      assert currentUpperBound == null || currentUpperBound.compareTo(currentLowerBound) <= 0 :
         "The current upper bound must be <= the new lower bound";
       
       currentUpperBound = rangeBounds.removeFirst();
@@ -501,10 +497,10 @@ public final class NumericRangeQuery<T e
         nextRange();
         
         // if the new upper bound is before the term parameter, the sub-range is never a hit
-        if (term != null && termComp.compare(term, currentUpperBound) > 0)
+        if (term != null && term.compareTo(currentUpperBound) > 0)
           continue;
         // never seek backwards, so use current term if lower bound is smaller
-        return (term != null && termComp.compare(term, currentLowerBound) > 0) ?
+        return (term != null && term.compareTo(currentLowerBound) > 0) ?
           term : currentLowerBound;
       }
       
@@ -516,11 +512,11 @@ public final class NumericRangeQuery<T e
     
     @Override
     protected final AcceptStatus accept(BytesRef term) {
-      while (currentUpperBound == null || termComp.compare(term, currentUpperBound) > 0) {
+      while (currentUpperBound == null || term.compareTo(currentUpperBound) > 0) {
         if (rangeBounds.isEmpty())
           return AcceptStatus.END;
         // peek next sub-range, only seek if the current term is smaller than next lower bound
-        if (termComp.compare(term, rangeBounds.getFirst()) < 0)
+        if (term.compareTo(rangeBounds.getFirst()) < 0)
           return AcceptStatus.NO_AND_SEEK;
         // step forward to next range without seeking, as next lower range bound is less or equal current term
         nextRange();

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/PrefixTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/PrefixTermsEnum.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/PrefixTermsEnum.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/PrefixTermsEnum.java Thu Sep 19 20:57:09 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.util.StringHelp
  * Subclass of FilteredTermEnum for enumerating all terms that match the
  * specified prefix filter term.
  * <p>Term enumerations are always ordered by
- * {@link #getComparator}.  Each term in the enumeration is
+ * {@link BytesRef#compareTo}.  Each term in the enumeration is
  * greater than all that precede it.</p>
  */
 public class PrefixTermsEnum extends FilteredTermsEnum {

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/ScoringRewrite.java Thu Sep 19 20:57:09 2013
@@ -109,7 +109,7 @@ public abstract class ScoringRewrite<Q e
     
     final int size = col.terms.size();
     if (size > 0) {
-      final int sort[] = col.terms.sort(col.termsEnum.getComparator());
+      final int sort[] = col.terms.sort(BytesRef.getUTF8SortedAsUnicodeComparator());
       final float[] boost = col.array.boost;
       final TermContext[] termStates = col.array.termState;
       for (int i = 0; i < size; i++) {

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TermCollectingRewrite.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TermCollectingRewrite.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TermCollectingRewrite.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TermCollectingRewrite.java Thu Sep 19 20:57:09 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.search;
  */
 
 import java.io.IOException;
-import java.util.Comparator;
 
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.Fields;
@@ -47,7 +46,6 @@ abstract class TermCollectingRewrite<Q e
   
   final void collectTerms(IndexReader reader, MultiTermQuery query, TermCollector collector) throws IOException {
     IndexReaderContext topReaderContext = reader.getContext();
-    Comparator<BytesRef> lastTermComp = null;
     for (AtomicReaderContext context : topReaderContext.leaves()) {
       final Fields fields = context.reader().fields();
       if (fields == null) {
@@ -67,11 +65,6 @@ abstract class TermCollectingRewrite<Q e
       if (termsEnum == TermsEnum.EMPTY)
         continue;
       
-      // Check comparator compatibility:
-      final Comparator<BytesRef> newTermComp = termsEnum.getComparator();
-      if (lastTermComp != null && newTermComp != null && newTermComp != lastTermComp)
-        throw new RuntimeException("term comparator should not change between segments: "+lastTermComp+" != "+newTermComp);
-      lastTermComp = newTermComp;
       collector.setReaderContext(topReaderContext, context);
       collector.setNextEnum(termsEnum);
       BytesRef bytes;

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TermRangeTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TermRangeTermsEnum.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TermRangeTermsEnum.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TermRangeTermsEnum.java Thu Sep 19 20:57:09 2013
@@ -17,18 +17,13 @@ package org.apache.lucene.search;
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.util.Comparator;
-
 import org.apache.lucene.index.FilteredTermsEnum;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.util.BytesRef;
 
 /**
  * Subclass of FilteredTermEnum for enumerating all terms that match the
- * specified range parameters.
- * <p>Term enumerations are always ordered by
- * {@link #getComparator}.  Each term in the enumeration is
+ * specified range parameters.  Each term in the enumeration is
  * greater than all that precede it.</p>
  */
 public class TermRangeTermsEnum extends FilteredTermsEnum {
@@ -37,7 +32,6 @@ public class TermRangeTermsEnum extends 
   final private boolean includeUpper;
   final private BytesRef lowerBytesRef;
   final private BytesRef upperBytesRef;
-  private final Comparator<BytesRef> termComp;
 
   /**
    * Enumerates all terms greater/equal than <code>lowerTerm</code>
@@ -82,7 +76,6 @@ public class TermRangeTermsEnum extends 
     }
 
     setInitialSeekTerm(lowerBytesRef);
-    termComp = getComparator();
   }
 
   @Override
@@ -92,7 +85,7 @@ public class TermRangeTermsEnum extends 
     
     // Use this field's default sort ordering
     if (upperBytesRef != null) {
-      final int cmp = termComp.compare(upperBytesRef, term);
+      final int cmp = upperBytesRef.compareTo(term);
       /*
        * if beyond the upper term, or is exclusive and this is equal to
        * the upper term, break out

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TopTermsRewrite.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TopTermsRewrite.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TopTermsRewrite.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/TopTermsRewrite.java Thu Sep 19 20:57:09 2013
@@ -70,20 +70,18 @@ public abstract class TopTermsRewrite<Q 
       private final Map<BytesRef,ScoreTerm> visitedTerms = new HashMap<BytesRef,ScoreTerm>();
       
       private TermsEnum termsEnum;
-      private Comparator<BytesRef> termComp;
       private BoostAttribute boostAtt;        
       private ScoreTerm st;
       
       @Override
       public void setNextEnum(TermsEnum termsEnum) {
         this.termsEnum = termsEnum;
-        this.termComp = termsEnum.getComparator();
         
         assert compareToLastTerm(null);
 
         // lazy init the initial ScoreTerm because comparator is not known on ctor:
         if (st == null)
-          st = new ScoreTerm(this.termComp, new TermContext(topReaderContext));
+          st = new ScoreTerm(new TermContext(topReaderContext));
         boostAtt = termsEnum.attributes().addAttribute(BoostAttribute.class);
       }
     
@@ -95,7 +93,7 @@ public abstract class TopTermsRewrite<Q 
         } else if (t == null) {
           lastTerm = null;
         } else {
-          assert termsEnum.getComparator().compare(lastTerm, t) < 0: "lastTerm=" + lastTerm + " t=" + t;
+          assert lastTerm.compareTo(t) < 0: "lastTerm=" + lastTerm + " t=" + t;
           lastTerm.copyBytes(t);
         }
         return true;
@@ -115,7 +113,7 @@ public abstract class TopTermsRewrite<Q 
           final ScoreTerm t = stQueue.peek();
           if (boost < t.boost)
             return true;
-          if (boost == t.boost && termComp.compare(bytes, t.bytes) > 0)
+          if (boost == t.boost && bytes.compareTo(t.bytes) > 0)
             return true;
         }
         ScoreTerm t = visitedTerms.get(bytes);
@@ -139,7 +137,7 @@ public abstract class TopTermsRewrite<Q 
             visitedTerms.remove(st.bytes);
             st.termState.clear(); // reset the termstate! 
           } else {
-            st = new ScoreTerm(termComp, new TermContext(topReaderContext));
+            st = new ScoreTerm(new TermContext(topReaderContext));
           }
           assert stQueue.size() <= maxSize : "the PQ size must be limited to maxSize";
           // set maxBoostAtt with values to help FuzzyTermsEnum to optimize
@@ -185,26 +183,22 @@ public abstract class TopTermsRewrite<Q 
     new Comparator<ScoreTerm>() {
       @Override
       public int compare(ScoreTerm st1, ScoreTerm st2) {
-        assert st1.termComp == st2.termComp :
-          "term comparator should not change between segments";
-        return st1.termComp.compare(st1.bytes, st2.bytes);
+        return st1.bytes.compareTo(st2.bytes);
       }
     };
 
   static final class ScoreTerm implements Comparable<ScoreTerm> {
-    public final Comparator<BytesRef> termComp;
     public final BytesRef bytes = new BytesRef();
     public float boost;
     public final TermContext termState;
-    public ScoreTerm(Comparator<BytesRef> termComp, TermContext termState) {
-      this.termComp = termComp;
+    public ScoreTerm(TermContext termState) {
       this.termState = termState;
     }
     
     @Override
     public int compareTo(ScoreTerm other) {
       if (this.boost == other.boost)
-        return termComp.compare(other.bytes, this.bytes);
+        return other.bytes.compareTo(this.bytes);
       else
         return Float.compare(this.boost, other.boost);
     }

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/package.html?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/package.html (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/package.html Thu Sep 19 20:57:09 2013
@@ -173,7 +173,7 @@ section for more notes on the process.
     {@link org.apache.lucene.index.Term Term}
     and an upper
     {@link org.apache.lucene.index.Term Term}
-    according to {@link org.apache.lucene.index.TermsEnum#getComparator TermsEnum.getComparator()}. It is not intended
+    according to {@link org.apache.lucene.util.BytesRef#compareTo BytesRef.compareTo()}. It is not intended
     for numerical ranges; use {@link org.apache.lucene.search.NumericRangeQuery NumericRangeQuery} instead.
 
     For example, one could find all documents

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/BytesRefIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/BytesRefIterator.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/BytesRefIterator.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/BytesRefIterator.java Thu Sep 19 20:57:09 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.util;
  */
 
 import java.io.IOException;
-import java.util.Comparator;
 
 /**
  * A simple iterator interface for {@link BytesRef} iteration.
@@ -38,14 +37,6 @@ public interface BytesRefIterator {
    */
   public BytesRef next() throws IOException;
   
-  /**
-   * Return the {@link BytesRef} Comparator used to sort terms provided by the
-   * iterator. This may return null if there are no items or the iterator is not
-   * sorted. Callers may invoke this method many times, so it's best to cache a
-   * single instance & reuse it.
-   */
-  public Comparator<BytesRef> getComparator();
-
   /** Singleton BytesRefIterator that iterates over 0 BytesRefs. */
   public static final BytesRefIterator EMPTY = new BytesRefIterator() {
 
@@ -53,10 +44,5 @@ public interface BytesRefIterator {
     public BytesRef next() {
       return null;
     }
-    
-    @Override
-    public Comparator<BytesRef> getComparator() {
-      return null;
-    }
   };
 }

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat3.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat3.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat3.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat3.java Thu Sep 19 20:57:09 2013
@@ -261,7 +261,6 @@ public class TestBlockPostingsFormat3 ex
    * checks collection-level statistics on Terms 
    */
   public void assertTermsStatistics(Terms leftTerms, Terms rightTerms) throws Exception {
-    assert leftTerms.getComparator() == rightTerms.getComparator();
     if (leftTerms.getDocCount() != -1 && rightTerms.getDocCount() != -1) {
       assertEquals(leftTerms.getDocCount(), rightTerms.getDocCount());
     }

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java Thu Sep 19 20:57:09 2013
@@ -25,11 +25,7 @@ import java.util.Random;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.FieldsConsumer;
 import org.apache.lucene.codecs.FieldsProducer;
-import org.apache.lucene.codecs.PostingsConsumer;
-import org.apache.lucene.codecs.TermStats;
-import org.apache.lucene.codecs.TermsConsumer;
 import org.apache.lucene.codecs.lucene40.Lucene40RWCodec;
 import org.apache.lucene.codecs.lucene41.Lucene41RWCodec;
 import org.apache.lucene.codecs.lucene42.Lucene42RWCodec;
@@ -48,11 +44,11 @@ import org.apache.lucene.search.PhraseQu
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.InfoStream;
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.OpenBitSet;
 import org.apache.lucene.util._TestUtil;
 import org.junit.BeforeClass;
 
@@ -145,22 +141,6 @@ public class TestCodecs extends LuceneTe
     public int compareTo(final FieldData other) {
       return fieldInfo.name.compareTo(other.fieldInfo.name);
     }
-
-    public void write(final FieldsConsumer consumer) throws Throwable {
-      Arrays.sort(terms);
-      final TermsConsumer termsConsumer = consumer.addField(fieldInfo);
-      long sumTotalTermCount = 0;
-      long sumDF = 0;
-      OpenBitSet visitedDocs = new OpenBitSet();
-      for (final TermData term : terms) {
-        for (int i = 0; i < term.docs.length; i++) {
-          visitedDocs.set(term.docs[i]);
-        }
-        sumDF += term.docs.length;
-        sumTotalTermCount += term.write(termsConsumer);
-      }
-      termsConsumer.finish(omitTF ? -1 : sumTotalTermCount, sumDF, (int) visitedDocs.cardinality());
-    }
   }
 
   class PositionData {
@@ -191,30 +171,6 @@ public class TestCodecs extends LuceneTe
     public int compareTo(final TermData o) {
       return text.compareTo(o.text);
     }
-
-    public long write(final TermsConsumer termsConsumer) throws Throwable {
-      final PostingsConsumer postingsConsumer = termsConsumer.startTerm(text);
-      long totTF = 0;
-      for(int i=0;i<docs.length;i++) {
-        final int termDocFreq;
-        if (field.omitTF) {
-          termDocFreq = -1;
-        } else {
-          termDocFreq = positions[i].length;
-        }
-        postingsConsumer.startDoc(docs[i], termDocFreq);
-        if (!field.omitTF) {
-          totTF += positions[i].length;
-          for(int j=0;j<positions[i].length;j++) {
-            final PositionData pos = positions[i][j];
-            postingsConsumer.addPosition(pos.pos, pos.payload, -1, -1);
-          }
-        }
-        postingsConsumer.finishDoc();
-      }
-      termsConsumer.finishTerm(text, new TermStats(docs.length, field.omitTF ? -1 : totTF));
-      return totTF;
-    }
   }
 
   final private static String SEGMENT = "0";
@@ -588,18 +544,16 @@ public class TestCodecs extends LuceneTe
           term = field.terms[upto];
           if (random().nextInt(3) == 1) {
             final DocsEnum docs;
-            final DocsEnum docsAndFreqs;
             final DocsAndPositionsEnum postings;
             if (!field.omitTF) {
               postings = termsEnum.docsAndPositions(null, null);
               if (postings != null) {
-                docs = docsAndFreqs = postings;
+                docs = postings;
               } else {
-                docs = docsAndFreqs = _TestUtil.docs(random(), termsEnum, null, null, DocsEnum.FLAG_FREQS);
+                docs = _TestUtil.docs(random(), termsEnum, null, null, DocsEnum.FLAG_FREQS);
               }
             } else {
               postings = null;
-              docsAndFreqs = null;
               docs = _TestUtil.docs(random(), termsEnum, null, null, DocsEnum.FLAG_NONE);
             }
             assertNotNull(docs);
@@ -657,18 +611,250 @@ public class TestCodecs extends LuceneTe
     }
   }
 
+  private static class DataFields extends Fields {
+    private final FieldData[] fields;
+
+    public DataFields(FieldData[] fields) {
+      // already sorted:
+      this.fields = fields;
+    }
+
+    @Override
+    public Iterator<String> iterator() {
+      return new Iterator<String>() {
+        int upto = -1;
+
+        @Override
+        public boolean hasNext() {
+          return upto+1 < fields.length;
+        }
+
+        @Override
+        public String next() {
+          upto++;
+          return fields[upto].fieldInfo.name;
+        }
+
+        @Override
+        public void remove() {
+          throw new UnsupportedOperationException();
+        }
+      };
+    }
+
+    @Override
+    public Terms terms(String field) {
+      // Slow linear search:
+      for(FieldData fieldData : fields) {
+        if (fieldData.fieldInfo.name.equals(field)) {
+          return new DataTerms(fieldData);
+        }
+      }
+      return null;
+    }
+
+    @Override
+    public int size() {
+      return fields.length;
+    }
+  }
+
+  private static class DataTerms extends Terms {
+    final FieldData fieldData;
+
+    public DataTerms(FieldData fieldData) {
+      this.fieldData = fieldData;
+    }
+
+    @Override
+    public TermsEnum iterator(TermsEnum reuse) {
+      return new DataTermsEnum(fieldData);
+    }
+
+    @Override
+    public long size() {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public long getSumTotalTermFreq() {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public long getSumDocFreq() {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int getDocCount() {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean hasOffsets() {
+      return fieldData.fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+    }
+
+    @Override
+    public boolean hasPositions() {
+      return fieldData.fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+    }
+
+    @Override
+    public boolean hasPayloads() {
+      return fieldData.fieldInfo.hasPayloads();
+    }
+  }
+
+  private static class DataTermsEnum extends TermsEnum {
+    final FieldData fieldData;
+    private int upto = -1;
+
+    public DataTermsEnum(FieldData fieldData) {
+      this.fieldData = fieldData;
+    }
+
+    @Override
+    public BytesRef next() {
+      upto++;
+      if (upto == fieldData.terms.length) {
+        return null;
+      }
+
+      return term();
+    }
+
+    @Override
+    public BytesRef term() {
+      return fieldData.terms[upto].text;
+    }
+
+    @Override
+    public SeekStatus seekCeil(BytesRef text) {
+      // Stupid linear impl:
+      for(int i=0;i<fieldData.terms.length;i++) {
+        int cmp = fieldData.terms[i].text.compareTo(text);
+        if (cmp == 0) {
+          upto = i;
+          return SeekStatus.FOUND;
+        } else if (cmp > 0) {
+          upto = i;
+          return SeekStatus.NOT_FOUND;
+        }
+      }
+
+      return SeekStatus.END;
+    }
+
+    @Override
+    public void seekExact(long ord) {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public long ord() {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int docFreq() {
+      throw new UnsupportedOperationException();
+    }
+  
+    @Override
+    public long totalTermFreq() {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) {
+      assert liveDocs == null;
+      return new DataDocsAndPositionsEnum(fieldData.terms[upto]);
+    }
+
+    @Override
+    public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) {
+      assert liveDocs == null;
+      return new DataDocsAndPositionsEnum(fieldData.terms[upto]);
+    }
+  }
+
+  private static class DataDocsAndPositionsEnum extends DocsAndPositionsEnum {
+    final TermData termData;
+    int docUpto = -1;
+    int posUpto;
+
+    public DataDocsAndPositionsEnum(TermData termData) {
+      this.termData = termData;
+    }
+
+    @Override
+    public long cost() {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int nextDoc() {
+      docUpto++;
+      if (docUpto == termData.docs.length) {
+        return NO_MORE_DOCS;
+      }
+      posUpto = -1;
+      return docID();
+    }
+
+    @Override
+    public int docID() {
+      return termData.docs[docUpto];
+    }
+
+    @Override
+    public int advance(int target) {
+      // Slow linear impl:
+      nextDoc();
+      while (docID() < target) {
+        nextDoc();
+      }
+
+      return docID();
+    }
+
+    @Override
+    public int freq() {
+      return termData.positions[docUpto].length;
+    }
+
+    @Override
+    public int nextPosition() {
+      posUpto++;
+      return termData.positions[docUpto][posUpto].pos;
+    }
+
+    @Override
+    public BytesRef getPayload() {
+      return termData.positions[docUpto][posUpto].payload;
+    }
+    
+    @Override
+    public int startOffset() {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int endOffset() {
+      throw new UnsupportedOperationException();
+    }
+  }
+
   private void write(final FieldInfos fieldInfos, final Directory dir, final FieldData[] fields) throws Throwable {
 
     final Codec codec = Codec.getDefault();
     final SegmentInfo si = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, SEGMENT, 10000, false, codec, null, null);
     final SegmentWriteState state = new SegmentWriteState(InfoStream.getDefault(), dir, si, fieldInfos, null, newIOContext(random()));
 
-    final FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(state);
     Arrays.sort(fields);
-    for (final FieldData field : fields) {
-      field.write(consumer);
-    }
-    consumer.close();
+    codec.postingsFormat().fieldsConsumer(state).write(new DataFields(fields));
   }
   
   public void testDocsOnlyFreq() throws Exception {

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java Thu Sep 19 20:57:09 2013
@@ -215,6 +215,7 @@ public class TestConcurrentMergeSchedule
     IndexWriter writer = new IndexWriter(
         directory,
         newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).
+            // Force excessive merging:
             setMaxBufferedDocs(2).
             setMergePolicy(newLogMergePolicy(100))
     );
@@ -249,7 +250,9 @@ public class TestConcurrentMergeSchedule
           directory,
           newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).
               setOpenMode(OpenMode.APPEND).
-              setMergePolicy(newLogMergePolicy(100))
+              setMergePolicy(newLogMergePolicy(100)).
+              // Force excessive merging:
+              setMaxBufferedDocs(2)
       );
     }
     writer.close();

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java Thu Sep 19 20:57:09 2013
@@ -1094,7 +1094,6 @@ public void testFilesOpenClose() throws 
     File tempDir = _TestUtil.getTempDir("testIndexExistsOnNonExistentDirectory");
     tempDir.delete();
     Directory dir = newFSDirectory(tempDir);
-    System.out.println("dir=" + dir);
     assertFalse(DirectoryReader.indexExists(dir));
     dir.close();
   }

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Thu Sep 19 20:57:09 2013
@@ -29,6 +29,7 @@ import java.util.Random;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.lucene.analysis.*;
+import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
@@ -537,21 +538,15 @@ public class TestIndexWriterExceptions e
     public void eval(MockDirectoryWrapper dir)  throws IOException {
       if (doFail) {
         StackTraceElement[] trace = new Exception().getStackTrace();
-        boolean sawAppend = false;
         boolean sawFlush = false;
         for (int i = 0; i < trace.length; i++) {
-          if (sawAppend && sawFlush) {
-            break;
-          }
-          if (FreqProxTermsWriterPerField.class.getName().equals(trace[i].getClassName()) && "flush".equals(trace[i].getMethodName())) {
-            sawAppend = true;
-          }
           if ("flush".equals(trace[i].getMethodName())) {
             sawFlush = true;
+            break;
           }
         }
 
-        if (sawAppend && sawFlush && count++ >= 30) {
+        if (sawFlush && count++ >= 30) {
           doFail = false;
           throw new IOException("now failing during flush");
         }

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java Thu Sep 19 20:57:09 2013
@@ -199,6 +199,9 @@ public class TestLongPostings extends Lu
           }
 
           if (random().nextInt(6) == 3) {
+            if (VERBOSE) {
+              System.out.println("    check positions");
+            }
             final int freq = postings.freq();
             assertTrue(freq >=1 && freq <= 4);
             for(int pos=0;pos<freq;pos++) {

Modified: lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Thu Sep 19 20:57:09 2013
@@ -804,11 +804,6 @@ public class MemoryIndex {
             }
 
             @Override
-            public Comparator<BytesRef> getComparator() {
-              return BytesRef.getUTF8SortedAsUnicodeComparator();
-            }
-
-            @Override
             public long size() {
               return info.terms.size();
             }
@@ -966,11 +961,6 @@ public class MemoryIndex {
       }
 
       @Override
-      public Comparator<BytesRef> getComparator() {
-        return BytesRef.getUTF8SortedAsUnicodeComparator();
-      }
-
-      @Override
       public void seekExact(BytesRef term, TermState state) throws IOException {
         assert state != null;
         this.seekExact(((OrdTermState)state).ord);

Modified: lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeTermsEnum.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeTermsEnum.java (original)
+++ lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeTermsEnum.java Thu Sep 19 20:57:09 2013
@@ -17,7 +17,6 @@ package org.apache.lucene.sandbox.querie
  * limitations under the License.
  */
 
-import java.io.IOException;
 import java.text.Collator;
 
 import org.apache.lucene.index.TermsEnum;
@@ -28,7 +27,7 @@ import org.apache.lucene.util.BytesRef;
  * Subclass of FilteredTermEnum for enumerating all terms that match the
  * specified range parameters.
  * <p>Term enumerations are always ordered by
- * {@link #getComparator}.  Each term in the enumeration is
+ * {@link BytesRef#compareTo}.  Each term in the enumeration is
  * greater than all that precede it.</p>
  * @deprecated Index collation keys with CollationKeyAnalyzer or ICUCollationKeyAnalyzer instead.
  *  This class will be removed in Lucene 5.0

Modified: lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java (original)
+++ lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java Thu Sep 19 20:57:09 2013
@@ -38,7 +38,7 @@ import org.apache.lucene.util.UnicodeUti
  * fuzzy terms enum method by calling FuzzyTermsEnum's getAutomatonEnum.
  * </p>
  * <p>Term enumerations are always ordered by
- * {@link #getComparator}.  Each term in the enumeration is
+ * {@link BytesRef#compareTo}.  Each term in the enumeration is
  * greater than all that precede it.</p>
  * 
  * @deprecated Use {@link FuzzyTermsEnum} instead.

Modified: lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractVisitingPrefixTreeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractVisitingPrefixTreeFilter.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractVisitingPrefixTreeFilter.java (original)
+++ lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractVisitingPrefixTreeFilter.java Thu Sep 19 20:57:09 2013
@@ -17,7 +17,9 @@ package org.apache.lucene.spatial.prefix
  * limitations under the License.
  */
 
-import com.spatial4j.core.shape.Shape;
+import java.io.IOException;
+import java.util.Iterator;
+
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.DocIdSet;
@@ -26,9 +28,7 @@ import org.apache.lucene.spatial.prefix.
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.StringHelper;
-
-import java.io.IOException;
-import java.util.Iterator;
+import com.spatial4j.core.shape.Shape;
 
 /**
  * Traverses a {@link SpatialPrefixTree} indexed field, using the template &
@@ -176,7 +176,7 @@ public abstract class AbstractVisitingPr
         //Seek to curVNode's cell (or skip if termsEnum has moved beyond)
         curVNodeTerm.bytes = curVNode.cell.getTokenBytes();
         curVNodeTerm.length = curVNodeTerm.bytes.length;
-        int compare = termsEnum.getComparator().compare(thisTerm, curVNodeTerm);
+        int compare = thisTerm.compareTo(curVNodeTerm);
         if (compare > 0) {
           // leap frog (termsEnum is beyond where we would otherwise seek)
           assert ! context.reader().terms(fieldName).iterator(null).seekExact(curVNodeTerm) : "should be absent";

Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java Thu Sep 19 20:57:09 2013
@@ -18,7 +18,6 @@
 package org.apache.lucene.search.spell;
 
 import java.io.IOException;
-import java.util.Comparator;
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.TermsEnum;
@@ -99,14 +98,5 @@ public class HighFrequencyDictionary imp
       }
       return  null;
     }
-
-    @Override
-    public Comparator<BytesRef> getComparator() {
-      if (termsEnum == null) {
-        return null;
-      } else {
-        return termsEnum.getComparator();
-      }
-    }
   }
 }

Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java Thu Sep 19 20:57:09 2013
@@ -17,9 +17,11 @@ package org.apache.lucene.search.spell;
  * limitations under the License.
  */
 
-
-import java.util.Comparator;
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefIterator;
@@ -96,11 +98,5 @@ public class PlainTextDictionary impleme
       }
       return result;
     }
-    
-    @Override
-    public Comparator<BytesRef> getComparator() {
-      return null;
-    }
   }
-
 }

Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqIterator.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqIterator.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqIterator.java Thu Sep 19 20:57:09 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.search.spell;
  */
 
 import java.io.IOException;
-import java.util.Comparator;
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefIterator;
@@ -55,10 +54,5 @@ public interface TermFreqIterator extend
     public BytesRef next() throws IOException {
       return wrapped.next();
     }
-
-    @Override
-    public Comparator<BytesRef> getComparator() {
-      return wrapped.getComparator();
-    }
   }
 }

Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqIteratorWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqIteratorWrapper.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqIteratorWrapper.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqIteratorWrapper.java Thu Sep 19 20:57:09 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.search.suggest
  */
 
 import java.io.IOException;
-import java.util.Comparator;
 import org.apache.lucene.search.spell.TermFreqIterator;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
@@ -37,11 +36,9 @@ public class BufferingTermFreqIteratorWr
   /** buffered weights, parallel with {@link #entries} */
   protected long[] freqs = new long[1];
   private final BytesRef spare = new BytesRef();
-  private final Comparator<BytesRef> comp;
   
   /** Creates a new iterator, buffering entries from the specified iterator */
   public BufferingTermFreqIteratorWrapper(TermFreqIterator source) throws IOException {
-    this.comp = source.getComparator();
     BytesRef spare;
     int freqIndex = 0;
     while((spare = source.next()) != null) {
@@ -67,11 +64,4 @@ public class BufferingTermFreqIteratorWr
     }
     return null;
   }
-
-  @Override
-  public Comparator<BytesRef> getComparator() {
-    return comp;
-  }
-
- 
 }

Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BytesRefArray.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BytesRefArray.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BytesRefArray.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BytesRefArray.java Thu Sep 19 20:57:09 2013
@@ -187,11 +187,6 @@ public final class BytesRefArray {
         }
         return null;
       }
-      
-      @Override
-      public Comparator<BytesRef> getComparator() {
-        return comp;
-      }
     };
   }
 }

Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java Thu Sep 19 20:57:09 2013
@@ -19,7 +19,6 @@ package org.apache.lucene.search.suggest
 
 
 import java.io.*;
-import java.util.Comparator;
 
 import org.apache.lucene.search.spell.Dictionary;
 import org.apache.lucene.search.spell.TermFreqIterator;
@@ -99,11 +98,5 @@ public class FileDictionary implements D
         return null;
       }
     }
-
-    @Override
-    public Comparator<BytesRef> getComparator() {
-      return null;
-    }
   }
-
 }

Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java Thu Sep 19 20:57:09 2013
@@ -65,11 +65,6 @@ public class SortedTermFreqIteratorWrapp
   }
   
   @Override
-  public Comparator<BytesRef> getComparator() {
-    return comparator;
-  }
-  
-  @Override
   public BytesRef next() throws IOException {
     boolean success = false;
     if (done) {

Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java Thu Sep 19 20:57:09 2013
@@ -66,8 +66,7 @@ public class ExternalRefSorter implement
       input = null;
     }
     
-    return new ByteSequenceIterator(new Sort.ByteSequencesReader(sorted),
-        sort.getComparator());
+    return new ByteSequenceIterator(new Sort.ByteSequencesReader(sorted));
   }
   
   private void closeWriter() throws IOException {
@@ -96,12 +95,9 @@ public class ExternalRefSorter implement
   class ByteSequenceIterator implements BytesRefIterator {
     private final ByteSequencesReader reader;
     private BytesRef scratch = new BytesRef();
-    private final Comparator<BytesRef> comparator;
     
-    public ByteSequenceIterator(ByteSequencesReader reader,
-        Comparator<BytesRef> comparator) {
+    public ByteSequenceIterator(ByteSequencesReader reader) {
       this.reader = reader;
-      this.comparator = comparator;
     }
     
     @Override
@@ -128,11 +124,6 @@ public class ExternalRefSorter implement
         }
       }
     }
-    
-    @Override
-    public Comparator<BytesRef> getComparator() {
-      return comparator;
-    }
   }
 
   @Override

Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java Thu Sep 19 20:57:09 2013
@@ -28,7 +28,6 @@ import java.util.List;
 import org.apache.lucene.search.spell.TermFreqIterator;
 import org.apache.lucene.search.spell.TermFreqPayloadIterator;
 import org.apache.lucene.search.suggest.Lookup;
-import org.apache.lucene.search.suggest.UnsortedTermFreqIteratorWrapper;
 import org.apache.lucene.search.suggest.jaspell.JaspellTernarySearchTrie.TSTNode;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CharsRef;
@@ -57,11 +56,6 @@ public class JaspellLookup extends Looku
     if (tfit instanceof TermFreqPayloadIterator) {
       throw new IllegalArgumentException("this suggester doesn't support payloads");
     }
-    if (tfit.getComparator() != null) {
-      // make sure it's unsorted
-      // WTF - this could result in yet another sorted iteration....
-      tfit = new UnsortedTermFreqIteratorWrapper(tfit);
-    }
     trie = new JaspellTernarySearchTrie();
     trie.setMatchAlmostDiff(editDistance);
     BytesRef spare;

Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java Thu Sep 19 20:57:09 2013
@@ -56,11 +56,9 @@ public class TSTLookup extends Lookup {
       throw new IllegalArgumentException("this suggester doesn't support payloads");
     }
     root = new TernaryTreeNode();
-    // buffer first
-    if (tfit.getComparator() != BytesRef.getUTF8SortedAsUTF16Comparator()) {
-      // make sure it's sorted and the comparator uses UTF16 sort order
-      tfit = new SortedTermFreqIteratorWrapper(tfit, BytesRef.getUTF8SortedAsUTF16Comparator());
-    }
+
+    // make sure it's sorted and the comparator uses UTF16 sort order
+    tfit = new SortedTermFreqIteratorWrapper(tfit, BytesRef.getUTF8SortedAsUTF16Comparator());
 
     ArrayList<String> tokens = new ArrayList<String>();
     ArrayList<Number> vals = new ArrayList<Number>();

Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java Thu Sep 19 20:57:09 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.search.suggest
  */
 
 import java.util.Arrays;
-import java.util.Comparator;
 import java.util.Iterator;
 
 import org.apache.lucene.search.spell.TermFreqIterator;
@@ -58,9 +57,4 @@ public final class TermFreqArrayIterator
     }
     return null;
   }
-
-  @Override
-  public Comparator<BytesRef> getComparator() {
-    return null;
-  }
 }
\ No newline at end of file

Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java Thu Sep 19 20:57:09 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.search.suggest
  */
 
 import java.util.Arrays;
-import java.util.Comparator;
 import java.util.Iterator;
 
 import org.apache.lucene.search.spell.TermFreqIterator;
@@ -64,9 +63,4 @@ public final class TermFreqPayloadArrayI
   public BytesRef payload() {
     return current.payload;
   }
-
-  @Override
-  public Comparator<BytesRef> getComparator() {
-    return null;
-  }
 }
\ No newline at end of file

Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestHighFrequencyDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestHighFrequencyDictionary.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestHighFrequencyDictionary.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestHighFrequencyDictionary.java Thu Sep 19 20:57:09 2013
@@ -36,7 +36,6 @@ public class TestHighFrequencyDictionary
     IndexReader ir = DirectoryReader.open(dir);
     Dictionary dictionary = new HighFrequencyDictionary(ir, "bogus", 0.1f);
     BytesRefIterator tf = dictionary.getWordsIterator();
-    assertNull(tf.getComparator());
     assertNull(tf.next());
     dir.close();
   }

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java Thu Sep 19 20:57:09 2013
@@ -25,15 +25,20 @@ import org.apache.lucene.codecs.FieldsCo
 import org.apache.lucene.codecs.FieldsProducer;
 import org.apache.lucene.codecs.PostingsConsumer;
 import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.PushFieldsConsumer;
 import org.apache.lucene.codecs.TermStats;
 import org.apache.lucene.codecs.TermsConsumer;
 import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
 import org.apache.lucene.index.AssertingAtomicReader;
-import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.OpenBitSet;
 
@@ -49,7 +54,12 @@ public final class AssertingPostingsForm
   
   @Override
   public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
-    return new AssertingFieldsConsumer(in.fieldsConsumer(state));
+    FieldsConsumer fieldsConsumer = in.fieldsConsumer(state);
+    if (fieldsConsumer instanceof PushFieldsConsumer) {
+      return new AssertingPushFieldsConsumer(state, (PushFieldsConsumer) fieldsConsumer);
+    } else {
+      return new AssertingFieldsConsumer(state, fieldsConsumer);
+    }
   }
 
   @Override
@@ -92,11 +102,12 @@ public final class AssertingPostingsForm
       return in.ramBytesUsed();
     }
   }
-  
-  static class AssertingFieldsConsumer extends FieldsConsumer {
-    private final FieldsConsumer in;
+
+  static class AssertingPushFieldsConsumer extends PushFieldsConsumer {
+    private final PushFieldsConsumer in;
     
-    AssertingFieldsConsumer(FieldsConsumer in) {
+    AssertingPushFieldsConsumer(SegmentWriteState writeState, PushFieldsConsumer in) {
+      super(writeState);
       this.in = in;
     }
     
@@ -112,6 +123,113 @@ public final class AssertingPostingsForm
       in.close();
     }
   }
+
+  static class AssertingFieldsConsumer extends FieldsConsumer {
+    private final FieldsConsumer in;
+    private final SegmentWriteState writeState;
+
+    AssertingFieldsConsumer(SegmentWriteState writeState, FieldsConsumer in) {
+      this.writeState = writeState;
+      this.in = in;
+    }
+    
+    @Override
+    public void write(Fields fields) throws IOException {
+      in.write(fields);
+
+      // TODO: more asserts?  can we somehow run a
+      // "limited" CheckIndex here???  Or ... can we improve
+      // AssertingFieldsProducer and us it also to wrap the
+      // incoming Fields here?
+ 
+      String lastField = null;
+      TermsEnum termsEnum = null;
+
+      for(String field : fields) {
+
+        FieldInfo fieldInfo = writeState.fieldInfos.fieldInfo(field);
+        assert fieldInfo != null;
+        assert lastField == null || lastField.compareTo(field) < 0;
+        lastField = field;
+
+        Terms terms = fields.terms(field);
+        assert terms != null;
+
+        termsEnum = terms.iterator(termsEnum);
+        BytesRef lastTerm = null;
+        DocsEnum docsEnum = null;
+        DocsAndPositionsEnum posEnum = null;
+
+        boolean hasFreqs = fieldInfo.getIndexOptions().compareTo(FieldInfo.IndexOptions.DOCS_AND_FREQS) >= 0;
+        boolean hasPositions = fieldInfo.getIndexOptions().compareTo(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+        boolean hasOffsets = fieldInfo.getIndexOptions().compareTo(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+
+        assert hasPositions == terms.hasPositions();
+        assert hasOffsets == terms.hasOffsets();
+
+        while(true) {
+          BytesRef term = termsEnum.next();
+          if (term == null) {
+            break;
+          }
+          assert lastTerm == null || lastTerm.compareTo(term) < 0;
+          if (lastTerm == null) {
+            lastTerm = BytesRef.deepCopyOf(term);
+          } else {
+            lastTerm.copyBytes(term);
+          }
+
+          if (hasPositions == false) {
+            int flags = 0;
+            if (hasFreqs) {
+              flags = flags | DocsEnum.FLAG_FREQS;
+            }
+            docsEnum = termsEnum.docs(null, docsEnum, flags);
+          } else {
+            int flags = DocsAndPositionsEnum.FLAG_PAYLOADS;
+            if (hasOffsets) {
+              flags = flags | DocsAndPositionsEnum.FLAG_OFFSETS;
+            }
+            posEnum = termsEnum.docsAndPositions(null, posEnum, flags);
+            docsEnum = posEnum;
+          }
+
+          int lastDocID = -1;
+
+          while(true) {
+            int docID = docsEnum.nextDoc();
+            if (docID == DocsEnum.NO_MORE_DOCS) {
+              break;
+            }
+            assert docID > lastDocID;
+            lastDocID = docID;
+            if (hasFreqs) {
+              int freq = docsEnum.freq();
+              assert freq > 0;
+
+              if (hasPositions) {
+                int lastPos = -1;
+                int lastStartOffset = -1;
+                for(int i=0;i<freq;i++) {
+                  int pos = posEnum.nextPosition();
+                  assert pos > lastPos;
+                  lastPos = pos;
+
+                  if (hasOffsets) {
+                    int startOffset = posEnum.startOffset();
+                    int endOffset = posEnum.endOffset();
+                    assert endOffset > startOffset;
+                    assert startOffset >= lastStartOffset;
+                    lastStartOffset = startOffset;
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
   
   static enum TermsConsumerState { INITIAL, START, FINISHED };
   static class AssertingTermsConsumer extends TermsConsumer {
@@ -123,6 +241,7 @@ public final class AssertingPostingsForm
     private long sumTotalTermFreq = 0;
     private long sumDocFreq = 0;
     private OpenBitSet visitedDocs = new OpenBitSet();
+    private static final Comparator<BytesRef> termComp = BytesRef.getUTF8SortedAsUnicodeComparator();
     
     AssertingTermsConsumer(TermsConsumer in, FieldInfo fieldInfo) {
       this.in = in;
@@ -133,7 +252,7 @@ public final class AssertingPostingsForm
     public PostingsConsumer startTerm(BytesRef text) throws IOException {
       assert state == TermsConsumerState.INITIAL || state == TermsConsumerState.START && lastPostingsConsumer.docFreq == 0;
       state = TermsConsumerState.START;
-      assert lastTerm == null || in.getComparator().compare(text, lastTerm) > 0;
+      assert lastTerm == null || termComp.compare(text, lastTerm) > 0;
       lastTerm = BytesRef.deepCopyOf(text);
       return lastPostingsConsumer = new AssertingPostingsConsumer(in.startTerm(text), fieldInfo, visitedDocs);
     }
@@ -171,11 +290,6 @@ public final class AssertingPostingsForm
       }
       in.finish(sumTotalTermFreq, sumDocFreq, docCount);
     }
-
-    @Override
-    public Comparator<BytesRef> getComparator() throws IOException {
-      return in.getComparator();
-    }
   }
   
   static enum PostingsConsumerState { INITIAL, START };

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java Thu Sep 19 20:57:09 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.codecs.asserti
  */
 
 import java.io.IOException;
-import java.util.Comparator;
 
 import org.apache.lucene.codecs.TermVectorsFormat;
 import org.apache.lucene.codecs.TermVectorsReader;
@@ -181,11 +180,6 @@ public class AssertingTermVectorsFormat 
     }
 
     @Override
-    public Comparator<BytesRef> getComparator() throws IOException {
-      return in.getComparator();
-    }
-
-    @Override
     public void close() throws IOException {
       in.close();
     }

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java?rev=1524840&r1=1524839&r2=1524840&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java Thu Sep 19 20:57:09 2013
@@ -20,7 +20,6 @@ package org.apache.lucene.codecs.ramonly
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -34,6 +33,7 @@ import org.apache.lucene.codecs.FieldsCo
 import org.apache.lucene.codecs.FieldsProducer;
 import org.apache.lucene.codecs.PostingsConsumer;
 import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.PushFieldsConsumer;
 import org.apache.lucene.codecs.TermStats;
 import org.apache.lucene.codecs.TermsConsumer;
 import org.apache.lucene.index.DocsAndPositionsEnum;
@@ -60,41 +60,6 @@ import org.apache.lucene.util.RamUsageEs
 
 public final class RAMOnlyPostingsFormat extends PostingsFormat {
 
-  // For fun, test that we can override how terms are
-  // sorted, and basic things still work -- this comparator
-  // sorts in reversed unicode code point order:
-  private static final Comparator<BytesRef> reverseUnicodeComparator = new Comparator<BytesRef>() {
-      @Override
-      public int compare(BytesRef t1, BytesRef t2) {
-        byte[] b1 = t1.bytes;
-        byte[] b2 = t2.bytes;
-        int b1Stop;
-        int b1Upto = t1.offset;
-        int b2Upto = t2.offset;
-        if (t1.length < t2.length) {
-          b1Stop = t1.offset + t1.length;
-        } else {
-          b1Stop = t1.offset + t2.length;
-        }
-        while(b1Upto < b1Stop) {
-          final int bb1 = b1[b1Upto++] & 0xff;
-          final int bb2 = b2[b2Upto++] & 0xff;
-          if (bb1 != bb2) {
-            //System.out.println("cmp 1=" + t1 + " 2=" + t2 + " return " + (bb2-bb1));
-            return bb2 - bb1;
-          }
-        }
-
-        // One is prefix of another, or they are equal
-        return t2.length-t1.length;
-      }
-
-      @Override
-      public boolean equals(Object other) {
-        return this == other;
-      }
-    };
-
   public RAMOnlyPostingsFormat() {
     super("RAMOnly");
   }
@@ -180,11 +145,6 @@ public final class RAMOnlyPostingsFormat
     }
 
     @Override
-    public Comparator<BytesRef> getComparator() {
-      return reverseUnicodeComparator;
-    }
-
-    @Override
     public boolean hasOffsets() {
       return info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
     }
@@ -243,12 +203,13 @@ public final class RAMOnlyPostingsFormat
   }
 
   // Classes for writing to the postings state
-  private static class RAMFieldsConsumer extends FieldsConsumer {
+  private static class RAMFieldsConsumer extends PushFieldsConsumer {
 
     private final RAMPostings postings;
     private final RAMTermsConsumer termsConsumer = new RAMTermsConsumer();
 
-    public RAMFieldsConsumer(RAMPostings postings) {
+    public RAMFieldsConsumer(SegmentWriteState writeState, RAMPostings postings) {
+      super(writeState);
       this.postings = postings;
     }
 
@@ -286,12 +247,6 @@ public final class RAMOnlyPostingsFormat
       return postingsWriter;
     }
 
-      
-    @Override
-    public Comparator<BytesRef> getComparator() {
-      return BytesRef.getUTF8SortedAsUnicodeComparator();
-    }
-
     @Override
     public void finishTerm(BytesRef text, TermStats stats) {
       assert stats.docFreq > 0;
@@ -355,11 +310,6 @@ public final class RAMOnlyPostingsFormat
     }
       
     @Override
-    public Comparator<BytesRef> getComparator() {
-      return BytesRef.getUTF8SortedAsUnicodeComparator();
-    }
-
-    @Override
     public BytesRef next() {
       if (it == null) {
         if (current == null) {
@@ -586,7 +536,7 @@ public final class RAMOnlyPostingsFormat
     }
     
     final RAMPostings postings = new RAMPostings();
-    final RAMFieldsConsumer consumer = new RAMFieldsConsumer(postings);
+    final RAMFieldsConsumer consumer = new RAMFieldsConsumer(writeState, postings);
 
     synchronized(state) {
       state.put(id, postings);



Mime
View raw message