lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r920378 [2/3] - in /lucene/java/branches/flex_1458: contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/ contrib/misc/src/java/org/apache/lucene/index/ contrib/misc/src/test/org/apache/lucene/index/ contrib/queries/src/java/org/ap...
Date Mon, 08 Mar 2010 16:10:33 GMT
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHash.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHash.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHash.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHash.java Mon Mar  8 16:10:31 2010
@@ -44,9 +44,6 @@
   final int postingsFreeChunk;
   final DocumentsWriter docWriter;
   
-  //nocommit: did I lose code for this below in merge?
-  private TermsHash primaryTermsHash;
-
   private RawPostingList[] postingsFreeList = new RawPostingList[1];
   private int postingsFreeCount;
   private int postingsAllocCount;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerField.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerField.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerField.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerField.java Mon Mar  8 16:10:31 2010
@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Comparator;
 
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.document.Fieldable;
@@ -52,12 +53,7 @@
   private RawPostingList[] postingsHash = new RawPostingList[postingsHashSize];
   private RawPostingList p;
   private final BytesRef utf8;
-  private BytesRef.Comparator termComp;
-
-  // nocommit -- move to thread level
-  // Used when comparing postings via termRefComp
-  private final BytesRef tr1 = new BytesRef();
-  private final BytesRef tr2 = new BytesRef();
+  private Comparator<BytesRef> termComp;
 
   public TermsHashPerField(DocInverterPerField docInverterPerField, final TermsHashPerThread perThread, final TermsHashPerThread nextPerThread, final FieldInfo fieldInfo) {
     this.perThread = perThread;
@@ -68,9 +64,6 @@
     fieldState = docInverterPerField.fieldState;
     this.consumer = perThread.consumer.addField(this, fieldInfo);
 
-    tr1.length = 3*((int) (Short.MAX_VALUE));
-    tr2.length = 3*((int) (Short.MAX_VALUE));
-
     streamCount = consumer.getStreamCount();
     numPostingInt = 2*streamCount;
     utf8 = perThread.utf8;
@@ -146,7 +139,7 @@
   }
 
   /** Collapse the hash table & sort in-place. */
-  public RawPostingList[] sortPostings(BytesRef.Comparator termComp) {
+  public RawPostingList[] sortPostings(Comparator<BytesRef> termComp) {
     this.termComp = termComp;
     compactPostings();
     quickSort(postingsHash, 0, numPostings-1);
@@ -219,14 +212,14 @@
   int comparePostings(RawPostingList p1, RawPostingList p2) {
 
     if (p1 == p2) {
-      // nocommit -- why does this happen again?
+      // Our quicksort does this, eg during partition
       return 0;
     }
 
-    termBytePool.setBytesRef(tr1, p1.textStart);
-    termBytePool.setBytesRef(tr2, p2.textStart);
+    termBytePool.setBytesRef(perThread.tr1, p1.textStart);
+    termBytePool.setBytesRef(perThread.tr2, p2.textStart);
 
-    return termComp.compare(tr1, tr2);
+    return termComp.compare(perThread.tr1, perThread.tr2);
   }
 
   /** Test whether the text for current RawPostingList p equals

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerThread.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerThread.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerThread.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsHashPerThread.java Mon Mar  8 16:10:31 2010
@@ -36,6 +36,10 @@
   final RawPostingList freePostings[] = new RawPostingList[256];
   int freePostingsCount;
 
+  // Used when comparing postings via termRefComp, in TermsHashPerField
+  final BytesRef tr1 = new BytesRef();
+  final BytesRef tr2 = new BytesRef();
+
   // Used by perField:
   final BytesRef utf8 = new BytesRef(10);
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codec.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codec.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codec.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codec.java Mon Mar  8 16:10:31 2010
@@ -18,7 +18,7 @@
  */
 
 import java.io.IOException;
-import java.util.Collection;
+import java.util.Set;
 
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.FieldInfos;
@@ -56,16 +56,14 @@
   /** Reads a segment.  NOTE: by the time this call
    *  returns, it must hold open any files it will need to
    *  use; else, those files may be deleted. */
-  // nocommit -- add a "required capabilities" here; this
-  // way merging could say only "TERMS_LINEAR_SCAN" but
-  // searching would say "TERMS_RANDOM_ACCESS"?
+  // nocommit -- make a SegmentReadState (symmetric)?
   public abstract FieldsProducer fieldsProducer(Directory dir, FieldInfos fieldInfos, SegmentInfo si, int readBufferSize, int indexDivisor) throws IOException;
 
   /** Gathers files associated with this segment */
-  public abstract void files(Directory dir, SegmentInfo segmentInfo, Collection<String> files) throws IOException;
+  public abstract void files(Directory dir, SegmentInfo segmentInfo, Set<String> files) throws IOException;
 
   /** Records all file extensions this codec uses */
-  public abstract void getExtensions(Collection<String> extensions);
+  public abstract void getExtensions(Set<String> extensions);
 
   /** @return Actual version of the file */
   public static int checkHeader(IndexInput in, String codec, int version) throws IOException {

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codecs.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codecs.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codecs.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codecs.java Mon Mar  8 16:10:31 2010
@@ -20,6 +20,7 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Set;
 
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.codecs.intblock.IntBlockCodec;
@@ -39,7 +40,7 @@
 
   private final HashMap<String, Codec> codecs = new HashMap<String, Codec>();
 
-  private final Collection<String> knownExtensions = new HashSet<String>();
+  private final Set<String> knownExtensions = new HashSet<String>();
 
   public void register(Codec codec) {
     if (codec.name == null) {

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java Mon Mar  8 16:10:31 2010
@@ -20,7 +20,6 @@
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.FieldsEnum;
-import org.apache.lucene.index.TermsEnum;
 
 import java.io.IOException;
 
@@ -41,14 +40,12 @@
 
   public void merge(MergeState mergeState, Fields fields) throws IOException {
     FieldsEnum fieldsEnum = fields.iterator();
+    assert fieldsEnum != null;
     String field;
     while((field = fieldsEnum.next()) != null) {
       mergeState.fieldInfo = mergeState.fieldInfos.fieldInfo(field);
       final TermsConsumer termsConsumer = addField(mergeState.fieldInfo);
-      final TermsEnum termsEnum = fieldsEnum.terms();
-      if (termsEnum != null) {
-        termsConsumer.merge(mergeState, termsEnum);
-      }
+      termsConsumer.merge(mergeState, fieldsEnum.terms());
     }
   }
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java Mon Mar  8 16:10:31 2010
@@ -29,14 +29,13 @@
 
 public abstract class PostingsConsumer {
 
-  // nocommit
+  // nocommit -- debugging
   public String desc;
 
-  // nocommit -- rename to startDoc?
   /** Adds a new doc in this term.  Return null if this
    *  consumer doesn't need to see the positions for this
    *  doc. */
-  public abstract void addDoc(int docID, int termDocFreq) throws IOException;
+  public abstract void startDoc(int docID, int termDocFreq) throws IOException;
 
   public static class PostingsMergeState {
     DocsEnum docsEnum;
@@ -67,7 +66,7 @@
         if (doc == DocsAndPositionsEnum.NO_MORE_DOCS) {
           break;
         }
-        addDoc(doc, postings.freq());
+        startDoc(doc, postings.freq());
         df++;
       }
     } else {
@@ -78,7 +77,7 @@
           break;
         }
         final int freq = postingsEnum.freq();
-        addDoc(doc, freq);
+        startDoc(doc, freq);
         for(int i=0;i<freq;i++) {
           final int position = postingsEnum.nextPosition();
           final int payloadLength = postingsEnum.getPayloadLength();

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/TermsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/TermsConsumer.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/TermsConsumer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/TermsConsumer.java Mon Mar  8 16:10:31 2010
@@ -18,6 +18,7 @@
  */
 
 import java.io.IOException;
+import java.util.Comparator;
 
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.MultiDocsEnum;
@@ -31,10 +32,12 @@
 
 public abstract class TermsConsumer {
 
-  /** Starts a new term in this field. */
+  /** Starts a new term in this field; this may be called
+   *  with no corresponding call to finish if the term had
+   *  no docs. */
   public abstract PostingsConsumer startTerm(BytesRef text) throws IOException;
 
-  /** Finishes the current term */
+  /** Finishes the current term; numDocs must be > 0. */
   public abstract void finishTerm(BytesRef text, int numDocs) throws IOException;
 
   /** Called when we are done adding terms to this field */
@@ -42,7 +45,7 @@
 
   /** Return the BytesRef Comparator used to sort terms
    *  before feeding to this API. */
-  public abstract BytesRef.Comparator getComparator() throws IOException;
+  public abstract Comparator<BytesRef> getComparator() throws IOException;
 
   /** Default merge impl */
   private MappingMultiDocsEnum docsEnum = null;
@@ -51,6 +54,7 @@
   public void merge(MergeState mergeState, TermsEnum termsEnum) throws IOException {
 
     BytesRef term;
+    assert termsEnum != null;
 
     if (mergeState.fieldInfo.omitTermFreqAndPositions) {
       if (docsEnum == null) {
@@ -61,13 +65,14 @@
       MultiDocsEnum docsEnumIn = null;
 
       while((term = termsEnum.next()) != null) {
-        MultiDocsEnum docsEnumIn2 = (MultiDocsEnum) termsEnum.docs(mergeState.multiDeletedDocs, docsEnumIn);
-        if (docsEnumIn2 != null) {
-          docsEnumIn = docsEnumIn2;
+        docsEnumIn = (MultiDocsEnum) termsEnum.docs(mergeState.multiDeletedDocs, docsEnumIn);
+        if (docsEnumIn != null) {
           docsEnum.reset(docsEnumIn);
           final PostingsConsumer postingsConsumer = startTerm(term);
           final int numDocs = postingsConsumer.merge(mergeState, docsEnum);
-          finishTerm(term, numDocs);
+          if (numDocs > 0) {
+            finishTerm(term, numDocs);
+          }
         }
       }
     } else {
@@ -77,13 +82,14 @@
       postingsEnum.setMergeState(mergeState);
       MultiDocsAndPositionsEnum postingsEnumIn = null;
       while((term = termsEnum.next()) != null) {
-        MultiDocsAndPositionsEnum postingsEnumIn2 = (MultiDocsAndPositionsEnum) termsEnum.docsAndPositions(mergeState.multiDeletedDocs, postingsEnumIn);
-        if (postingsEnumIn2 != null) {
-          postingsEnumIn = postingsEnumIn2;
+        postingsEnumIn = (MultiDocsAndPositionsEnum) termsEnum.docsAndPositions(mergeState.multiDeletedDocs, postingsEnumIn);
+        if (postingsEnumIn != null) {
           postingsEnum.reset(postingsEnumIn);
           final PostingsConsumer postingsConsumer = startTerm(term);
           final int numDocs = postingsConsumer.merge(mergeState, postingsEnum);
-          finishTerm(term, numDocs);
+          if (numDocs > 0) {
+            finishTerm(term, numDocs);
+          }
         }
       }
     }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java Mon Mar  8 16:10:31 2010
@@ -18,7 +18,7 @@
  */
 
 import java.io.IOException;
-import java.util.Collection;
+import java.util.Set;
 
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.SegmentInfo;
@@ -122,14 +122,14 @@
   }
 
   @Override
-  public void files(Directory dir, SegmentInfo segmentInfo, Collection<String> files) {
+  public void files(Directory dir, SegmentInfo segmentInfo, Set<String> files) {
     SepPostingsReaderImpl.files(segmentInfo, files);
     StandardTermsDictReader.files(dir, segmentInfo, files);
     SimpleStandardTermsIndexReader.files(dir, segmentInfo, files);
   }
 
   @Override
-  public void getExtensions(Collection<String> extensions) {
+  public void getExtensions(Set<String> extensions) {
     SepCodec.getSepExtensions(extensions);
   }
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java Mon Mar  8 16:10:31 2010
@@ -17,7 +17,7 @@
  * limitations under the License.
  */
 
-import java.util.Collection;
+import java.util.Set;
 import java.io.IOException;
 
 import org.apache.lucene.store.Directory;
@@ -66,12 +66,12 @@
   }
 
   @Override
-  public void files(Directory dir, SegmentInfo info, Collection<String> files) throws IOException {
+  public void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException {
     PreFlexFields.files(dir, info, files);
   }
 
   @Override
-  public void getExtensions(Collection<String> extensions) {
+  public void getExtensions(Set<String> extensions) {
     extensions.add(FREQ_EXTENSION);
     extensions.add(PROX_EXTENSION);
     extensions.add(TERMS_EXTENSION);

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java Mon Mar  8 16:10:31 2010
@@ -21,6 +21,7 @@
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.TreeMap;
+import java.util.Comparator;
 
 import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.DocsAndPositionsEnum;
@@ -218,7 +219,7 @@
     }
 
     @Override
-    public BytesRef.Comparator getComparator() {
+    public Comparator<BytesRef> getComparator() {
       // Pre-flex indexes always sorted in UTF16 order
       return BytesRef.getUTF8SortedAsUTF16Comparator();
     }
@@ -256,7 +257,7 @@
     }
 
     @Override
-    public BytesRef.Comparator getComparator() {
+    public Comparator<BytesRef> getComparator() {
       // Pre-flex indexes always sorted in UTF16 order
       return BytesRef.getUTF8SortedAsUTF16Comparator();
     }
@@ -348,8 +349,6 @@
 
     @Override
     public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException {
-      // nocommit -- must assert that skipDocs "matches" the
-      // underlying deletedDocs?
       if (reuse != null) {
         return ((PreDocsEnum) reuse).reset(termEnum, skipDocs);        
       } else {
@@ -359,8 +358,6 @@
 
     @Override
     public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException {
-      // nocommit -- must assert that skipDocs "matches" the
-      // underlying deletedDocs?
       if (reuse != null) {
         return ((PreDocsAndPositionsEnum) reuse).reset(termEnum, skipDocs);        
       } else {

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java Mon Mar  8 16:10:31 2010
@@ -144,8 +144,10 @@
         doc += docCode >>> 1;       // shift off low bit
         if ((docCode & 1) != 0)       // if low bit is set
           freq = 1;         // freq is one
-        else
+        else {
           freq = freqStream.readVInt();     // else read freq
+          assert freq != 1;
+        }
       }
       
       count++;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java Mon Mar  8 16:10:31 2010
@@ -206,9 +206,6 @@
       }
     }
 
-    // nocommit -- make sure these optimizations survive
-    // into flex 
-
     // optimize sequential access: first try scanning cached enum w/o seeking
     SegmentTermEnum enumerator = resources.termEnum;
     if (enumerator.term() != null                 // term is at or past current

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java Mon Mar  8 16:10:31 2010
@@ -18,7 +18,7 @@
  */
 
 import java.io.IOException;
-import java.util.Collection;
+import java.util.Set;
 
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.SegmentInfo;
@@ -142,14 +142,14 @@
   }
 
   @Override
-  public void files(Directory dir, SegmentInfo segmentInfo, Collection<String> files) throws IOException {
+  public void files(Directory dir, SegmentInfo segmentInfo, Set<String> files) throws IOException {
     StandardPostingsReaderImpl.files(dir, segmentInfo, files);
     StandardTermsDictReader.files(dir, segmentInfo, files);
     SimpleStandardTermsIndexReader.files(dir, segmentInfo, files);
   }
 
   @Override
-  public void getExtensions(Collection<String> extensions) {
+  public void getExtensions(Set<String> extensions) {
     StandardCodec.getStandardExtensions(extensions);
   }
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsWriterImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsWriterImpl.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsWriterImpl.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsWriterImpl.java Mon Mar  8 16:10:31 2010
@@ -20,7 +20,6 @@
 import java.io.IOException;
 
 import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.index.codecs.standard.StandardPostingsWriter;
 import org.apache.lucene.store.IndexOutput;
@@ -53,7 +52,7 @@
   // Starts a new term
   FieldInfo fieldInfo;
 
-  // nocommit
+  // nocommit -- debugging
   String desc;
 
   /** @lucene.experimental */
@@ -159,7 +158,7 @@
   }
 
   @Override
-  public void addDoc(int docID, int termDocFreq) throws IOException {
+  public void startDoc(int docID, int termDocFreq) throws IOException {
 
     assert docID >= 0: "got docID=" + docID;
         
@@ -183,7 +182,7 @@
         if (Codec.DEBUG)
           System.out.println("  docID=" + doc.docID);
 
-        wrappedPostingsWriter.addDoc(doc.docID, doc.termDocFreq);
+        wrappedPostingsWriter.startDoc(doc.docID, doc.termDocFreq);
 
         if (!omitTF) {
           assert doc.termDocFreq == doc.numPositions;
@@ -208,7 +207,7 @@
     if (pulsed) {
       // We've already seen too many docs for this term --
       // just forward to our fallback writer
-      wrappedPostingsWriter.addDoc(docID, termDocFreq);
+      wrappedPostingsWriter.startDoc(docID, termDocFreq);
     } else {
       currentDoc = pendingDocs[pendingDocCount++];
       currentDoc.docID = docID;
@@ -255,6 +254,8 @@
   @Override
   public void finishTerm(int docCount, boolean isIndexTerm) throws IOException {
 
+    assert docCount > 0;
+
     if (Codec.DEBUG) {
       System.out.println("PW: finishTerm pendingDocCount=" + pendingDocCount);
     }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/IntIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/IntIndexInput.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/IntIndexInput.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/IntIndexInput.java Mon Mar  8 16:10:31 2010
@@ -39,7 +39,7 @@
   // TODO: -- can we simplify this?
   public abstract static class Index {
 
-    // nocommit
+    // nocommit -- debugging
     public String desc;
 
     public abstract void read(IndexInput indexIn, boolean absolute) throws IOException;
@@ -75,7 +75,7 @@
       return bulkResult;
     }
 
-    // nocommit
+    // nocommit -- debugging
     public abstract String descFilePointer() throws IOException;
   }
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/IntIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/IntIndexOutput.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/IntIndexOutput.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/IntIndexOutput.java Mon Mar  8 16:10:31 2010
@@ -39,7 +39,7 @@
 
   public abstract static class Index {
 
-    // nocommit
+    // nocommit -- debugging
     public String desc;
 
     /** Internally records the current location */
@@ -59,6 +59,6 @@
 
   public abstract void close() throws IOException;
 
-  // nocommit
+  // nocommit -- debugging
   public abstract String descFilePointer() throws IOException;
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepCodec.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepCodec.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepCodec.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepCodec.java Mon Mar  8 16:10:31 2010
@@ -18,7 +18,7 @@
  */
 
 import java.io.IOException;
-import java.util.Collection;
+import java.util.Set;
 
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.SegmentInfo;
@@ -125,18 +125,18 @@
   }
 
   @Override
-  public void files(Directory dir, SegmentInfo segmentInfo, Collection<String> files) {
+  public void files(Directory dir, SegmentInfo segmentInfo, Set<String> files) {
     SepPostingsReaderImpl.files(segmentInfo, files);
     StandardTermsDictReader.files(dir, segmentInfo, files);
     SimpleStandardTermsIndexReader.files(dir, segmentInfo, files);
   }
 
   @Override
-  public void getExtensions(Collection<String> extensions) {
+  public void getExtensions(Set<String> extensions) {
     getSepExtensions(extensions);
   }
 
-  public static void getSepExtensions(Collection<String> extensions) {
+  public static void getSepExtensions(Set<String> extensions) {
     extensions.add(DOC_EXTENSION);
     extensions.add(FREQ_EXTENSION);
     extensions.add(SKIP_EXTENSION);

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java Mon Mar  8 16:10:31 2010
@@ -352,7 +352,7 @@
         }
       }
 
-      // nocommit
+      // nocommit -- debugging
       if (Codec.DEBUG) {
         System.out.println("  return doc=" + doc);
       }
@@ -594,7 +594,7 @@
         }
       }
 
-      // nocommit
+      // nocommit -- debugging
       if (Codec.DEBUG) {
         System.out.println("  return doc=" + doc);
       }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriterImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriterImpl.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriterImpl.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriterImpl.java Mon Mar  8 16:10:31 2010
@@ -164,7 +164,7 @@
   /** Adds a new doc in this term.  If this returns null
    *  then we just skip consuming positions/payloads. */
   @Override
-  public void addDoc(int docID, int termDocFreq) throws IOException {
+  public void startDoc(int docID, int termDocFreq) throws IOException {
 
     final int delta = docID - lastDocID;
 
@@ -260,6 +260,7 @@
     long skipPos = skipOut.getFilePointer();
 
     // TODO: -- wasteful we are counting this in two places?
+    assert docCount > 0;
     assert docCount == df;
     if (Codec.DEBUG) {
       System.out.println("dw.finishTerm termsFP=" + termsOut.getFilePointer() + " df=" + df + " skipPos=" + skipPos);
@@ -302,7 +303,7 @@
     lastDocID = 0;
     df = 0;
 
-    // nocommit
+    // nocommit -- debugging
     count = 0;
   }
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListWriter.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListWriter.java Mon Mar  8 16:10:31 2010
@@ -37,8 +37,7 @@
   private long[] lastSkipProxPointer;
   
   private IndexOutput freqOutput;
-  // nocommit -- private again
-  public IndexOutput proxOutput;
+  private IndexOutput proxOutput;
 
   private int curDoc;
   private boolean curStorePayloads;
@@ -57,16 +56,6 @@
     lastSkipProxPointer = new long[numberOfSkipLevels];
   }
 
-  // nocommit -- made public
-  //public void setFreqOutput(IndexOutput freqOutput) {
-  //this.freqOutput = freqOutput;
-  //}
-
-  // nocommit -- made public
-  //public void setProxOutput(IndexOutput proxOutput) {
-  //this.proxOutput = proxOutput;
-  //}
-
   /**
    * Sets the values for the current skip data. 
    */

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java Mon Mar  8 16:10:31 2010
@@ -31,6 +31,7 @@
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Collection;
+import java.util.Comparator;
 import java.io.IOException;
 
 /**
@@ -69,11 +70,11 @@
   final private IndexInput in;
   private volatile boolean indexLoaded;
 
-  private final BytesRef.Comparator termComp;
+  private final Comparator<BytesRef> termComp;
 
   final HashMap<FieldInfo,FieldIndexReader> fields = new HashMap<FieldInfo,FieldIndexReader>();
 
-  public SimpleStandardTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, BytesRef.Comparator termComp)
+  public SimpleStandardTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator<BytesRef> termComp)
     throws IOException {
 
     this.termComp = termComp;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java Mon Mar  8 16:10:31 2010
@@ -18,7 +18,7 @@
  */
 
 import java.io.IOException;
-import java.util.Collection;
+import java.util.Set;
 
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.SegmentInfo;
@@ -125,18 +125,18 @@
   static final String TERMS_INDEX_EXTENSION = "tii";
 
   @Override
-  public void files(Directory dir, SegmentInfo segmentInfo, Collection<String> files) throws IOException {
+  public void files(Directory dir, SegmentInfo segmentInfo, Set<String> files) throws IOException {
     StandardPostingsReaderImpl.files(dir, segmentInfo, files);
     StandardTermsDictReader.files(dir, segmentInfo, files);
     SimpleStandardTermsIndexReader.files(dir, segmentInfo, files);
   }
 
   @Override
-  public void getExtensions(Collection<String> extensions) {
+  public void getExtensions(Set<String> extensions) {
     getStandardExtensions(extensions);
   }
 
-  public static void getStandardExtensions(Collection<String> extensions) {
+  public static void getStandardExtensions(Set<String> extensions) {
     extensions.add(FREQ_EXTENSION);
     extensions.add(PROX_EXTENSION);
     StandardTermsDictReader.getExtensions(extensions);

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReaderImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReaderImpl.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReaderImpl.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReaderImpl.java Mon Mar  8 16:10:31 2010
@@ -35,10 +35,6 @@
  *  postings format. 
  *  @lucene.experimental */
 
-// nocommit -- should we switch "hasProx" higher up?  and
-// create two separate docs readers, one that also reads
-// prox and one that doesn't?
-
 public class StandardPostingsReaderImpl extends StandardPostingsReader {
 
   private final IndexInput freqIn;
@@ -227,10 +223,9 @@
       freqOffset = termState.freqOffset;
       skipOffset = termState.skipOffset;
 
-      // nocommit this seek frequently isn't needed, when
-      // we enum terms and all docs for each term (MTQ,
-      // or, merging).  is this seek costing us anything?
-      // we should avoid it so...
+      // TODO: for full enum case (eg segment merging) this
+      // seek is unnecessary; maybe we can avoid in such
+      // cases
       freqIn.seek(termState.freqOffset);
       limit = termState.docFreq;
       ord = 0;
@@ -447,10 +442,9 @@
 
       this.skipDocs = skipDocs;
 
-      // nocommit this seek frequently isn't needed, when
-      // we enum terms and all docs for each term (MTQ,
-      // or, merging).  is this seek costing us anything?
-      // we should avoid it so...
+      // TODO: for full enum case (eg segment merging) this
+      // seek is unnecessary; maybe we can avoid in such
+      // cases
       freqIn.seek(termState.freqOffset);
       lazyProxPointer = termState.proxOffset;
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriterImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriterImpl.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriterImpl.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriterImpl.java Mon Mar  8 16:10:31 2010
@@ -105,9 +105,6 @@
     skipListWriter.resetSkip();
   }
 
-  // nocommit -- should we NOT reuse across fields?  would
-  // be cleaner
-
   // Currently, this instance is re-used across fields, so
   // our parent calls setField whenever the field changes
   @Override
@@ -125,7 +122,7 @@
   /** Adds a new doc in this term.  If this returns null
    *  then we just skip consuming positions/payloads. */
   @Override
-  public void addDoc(int docID, int termDocFreq) throws IOException {
+  public void startDoc(int docID, int termDocFreq) throws IOException {
 
     final int delta = docID - lastDocID;
     
@@ -140,9 +137,6 @@
     if ((++df % skipInterval) == 0) {
       skipListWriter.setSkipData(lastDocID, storePayloads, lastPayloadLength);
       skipListWriter.bufferSkip(df);
-      if (Codec.DEBUG) {
-        System.out.println("    bufferSkip lastDocID=" + lastDocID + " df=" + df + " freqFP=" + freqOut.getFilePointer() + " proxFP=" + skipListWriter.proxOutput.getFilePointer());
-      }
     }
 
     assert docID < totalNumDocs: "docID=" + docID + " totalNumDocs=" + totalNumDocs;
@@ -212,8 +206,12 @@
   /** Called when we are done adding docs to this term */
   @Override
   public void finishTerm(int docCount, boolean isIndexTerm) throws IOException {
-    // nocommit -- wasteful we are counting this in two places?
+    assert docCount > 0;
+
+    // TODO: wasteful we are counting this (counting # docs
+    // for this term) in two places?
     assert docCount == df;
+
     // mxx
     if (Codec.DEBUG) {
       Codec.debug("dw.finishTerm termsOut.fp=" + termsOut.getFilePointer() + " freqStart=" + freqStart + " df=" + df + " isIndex?=" + isIndexTerm);
@@ -251,7 +249,7 @@
     lastDocID = 0;
     df = 0;
     
-    // nocommit
+    // nocommit -- debugging
     count = 0;
   }
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java Mon Mar  8 16:10:31 2010
@@ -22,6 +22,7 @@
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.TreeMap;
+import java.util.Comparator;
 
 import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.DocsAndPositionsEnum;
@@ -66,7 +67,7 @@
   private final String segment;
 
   // Comparator that orders our terms
-  private final BytesRef.Comparator termComp;
+  private final Comparator<BytesRef> termComp;
 
   // Caches the most recently looked-up field + terms:
   private final Cache<FieldAndTerm,TermState> termsCache;
@@ -98,7 +99,7 @@
   }
   
   public StandardTermsDictReader(StandardTermsIndexReader indexReader, Directory dir, FieldInfos fieldInfos, String segment, StandardPostingsReader postingsReader, int readBufferSize,
-                                 BytesRef.Comparator termComp, int termsCacheSize)
+                                 Comparator<BytesRef> termComp, int termsCacheSize)
     throws IOException {
     
     this.segment = segment;
@@ -256,7 +257,7 @@
     }
 
     @Override
-    public BytesRef.Comparator getComparator() {
+    public Comparator<BytesRef> getComparator() {
       return termComp;
     }
 
@@ -296,7 +297,7 @@
       }
 
       @Override
-      public BytesRef.Comparator getComparator() {
+      public Comparator<BytesRef> getComparator() {
         return termComp;
       }
 
@@ -522,12 +523,6 @@
         // a "how many terms until next index entry" in each
         // index entry, but that'd require some tricky
         // lookahead work when writing the index
-
-        // nocommit -- this call to isIndexTerm is not
-        // right, when indexDivisor > 1?  ie, this will
-        // return false for entries that actually are index
-        // terms, and then the postings impl will read the
-        // wrong offset.  make a test...
         postingsReader.readTerm(in,
                                 fieldInfo, state,
                                 indexReader.isIndexTerm(1+state.ord, state.docFreq, false));
@@ -552,6 +547,7 @@
           System.out.println("stdr.docs");
         }
         DocsEnum docsEnum = postingsReader.docs(fieldInfo, state, skipDocs, reuse);
+        assert docsEnum != null;
         if (Codec.DEBUG) {
           docsEnum.desc = fieldInfo.name + ":" + bytesReader.term.utf8ToString();
         }
@@ -567,14 +563,12 @@
           return null;
         } else {
           DocsAndPositionsEnum postingsEnum = postingsReader.docsAndPositions(fieldInfo, state, skipDocs, reuse);
-          if (Codec.DEBUG) {
-            if (postingsEnum != null) {
+          if (postingsEnum != null) {
+            if (Codec.DEBUG) {
               postingsEnum.desc = fieldInfo.name + ":" + bytesReader.term.utf8ToString();
+              Codec.debug("  return enum=" + postingsEnum);
             }
           }
-          if (Codec.DEBUG) {
-            Codec.debug("  return enum=" + postingsEnum);
-          }
           return postingsEnum;
         }
       }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictWriter.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictWriter.java Mon Mar  8 16:10:31 2010
@@ -20,6 +20,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Comparator;
 
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
@@ -60,11 +61,11 @@
   FieldInfo currentField;
   private final StandardTermsIndexWriter indexWriter;
   private final List<TermsConsumer> fields = new ArrayList<TermsConsumer>();
-  private final BytesRef.Comparator termComp;
+  private final Comparator<BytesRef> termComp;
 
   private String segment;
 
-  public StandardTermsDictWriter(StandardTermsIndexWriter indexWriter, SegmentWriteState state, StandardPostingsWriter postingsWriter, BytesRef.Comparator termComp) throws IOException {
+  public StandardTermsDictWriter(StandardTermsIndexWriter indexWriter, SegmentWriteState state, StandardPostingsWriter postingsWriter, Comparator<BytesRef> termComp) throws IOException {
     final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, StandardCodec.TERMS_EXTENSION);
     this.indexWriter = indexWriter;
     this.termComp = termComp;
@@ -165,7 +166,7 @@
     }
     
     @Override
-    public BytesRef.Comparator getComparator() {
+    public Comparator<BytesRef> getComparator() {
       return termComp;
     }
 
@@ -182,23 +183,23 @@
     @Override
     public void finishTerm(BytesRef text, int numDocs) throws IOException {
 
+      assert numDocs > 0;
+
       if (Codec.DEBUG) {
         Codec.debug("finishTerm seg=" + segment + " text=" + fieldInfo.name + ":" + text.utf8ToString() + " numDocs=" + numDocs + " numTerms=" + numTerms);
       }
 
-      if (numDocs > 0) {
-        final boolean isIndexTerm = fieldIndexWriter.checkIndexTerm(text, numDocs);
-
-        if (Codec.DEBUG) {
-          Codec.debug("  tis.fp=" + out.getFilePointer() + " isIndexTerm?=" + isIndexTerm);
-          System.out.println("  term bytes=" + text.utf8ToString());
-        }
-        termWriter.write(text);
-        out.writeVInt(numDocs);
+      final boolean isIndexTerm = fieldIndexWriter.checkIndexTerm(text, numDocs);
 
-        postingsWriter.finishTerm(numDocs, isIndexTerm);
-        numTerms++;
+      if (Codec.DEBUG) {
+        Codec.debug("  tis.fp=" + out.getFilePointer() + " isIndexTerm?=" + isIndexTerm);
+        System.out.println("  term bytes=" + text.utf8ToString());
       }
+      termWriter.write(text);
+      out.writeVInt(numDocs);
+
+      postingsWriter.finishTerm(numDocs, isIndexTerm);
+      numTerms++;
     }
 
     // Finishes all terms in this field

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/AutomatonQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/AutomatonQuery.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/AutomatonQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/AutomatonQuery.java Mon Mar  8 16:10:31 2010
@@ -21,7 +21,6 @@
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.util.ToStringUtils;
@@ -79,9 +78,7 @@
     
     // matches all possible strings
     if (BasicOperations.isTotal(automaton)) {
-      final Terms terms = MultiFields.getTerms(reader, getField());
-      // nocommit -- should we just return null?  singleton?
-      return (terms != null) ? terms.iterator() : TermsEnum.EMPTY;
+      return MultiFields.getTerms(reader, getField()).iterator();
     }
     
     // matches a fixed string in singleton representation

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCache.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCache.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCache.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCache.java Mon Mar  8 16:10:31 2010
@@ -150,12 +150,11 @@
   /** The default parser for byte values, which are encoded by {@link Byte#toString(byte)} */
   public static final ByteParser DEFAULT_BYTE_PARSER = new ByteParser() {
     public byte parseByte(BytesRef term) {
-      final long num = FieldCacheImpl.parseLong(term);
-      if (num >= Byte.MIN_VALUE && num <= Byte.MAX_VALUE) {
-        return (byte) num;
-      } else {
-        throw new IllegalArgumentException("value \"" + term + "\" is out of bounds for Byte");
-      }
+      // TODO: would be far better to directly parse from
+      // UTF8 bytes... but really users should use
+      // NumericField, instead, which already decodes
+      // directly from byte[]
+      return Byte.parseByte(term.utf8ToString());
     }
     protected Object readResolve() {
       return DEFAULT_BYTE_PARSER;
@@ -169,12 +168,11 @@
   /** The default parser for short values, which are encoded by {@link Short#toString(short)} */
   public static final ShortParser DEFAULT_SHORT_PARSER = new ShortParser() {
     public short parseShort(BytesRef term) {
-      final long num = FieldCacheImpl.parseLong(term);
-      if (num >= Short.MIN_VALUE && num <= Short.MAX_VALUE) {
-        return (short) num;
-      } else {
-        throw new IllegalArgumentException("value \"" + term + "\" is out of bounds for Short");
-      }
+      // TODO: would be far better to directly parse from
+      // UTF8 bytes... but really users should use
+      // NumericField, instead, which already decodes
+      // directly from byte[]
+      return Short.parseShort(term.utf8ToString());
     }
     protected Object readResolve() {
       return DEFAULT_SHORT_PARSER;
@@ -188,12 +186,11 @@
   /** The default parser for int values, which are encoded by {@link Integer#toString(int)} */
   public static final IntParser DEFAULT_INT_PARSER = new IntParser() {
     public int parseInt(BytesRef term) {
-      final long num = FieldCacheImpl.parseLong(term);
-      if (num >= Integer.MIN_VALUE && num <= Integer.MAX_VALUE) {
-        return (int) num;
-      } else {
-        throw new IllegalArgumentException("value \"" + term + "\" is out of bounds for Int");
-      }
+      // TODO: would be far better to directly parse from
+      // UTF8 bytes... but really users should use
+      // NumericField, instead, which already decodes
+      // directly from byte[]
+      return Integer.parseInt(term.utf8ToString());
     }
     protected Object readResolve() {
       return DEFAULT_INT_PARSER;
@@ -207,8 +204,10 @@
   /** The default parser for float values, which are encoded by {@link Float#toString(float)} */
   public static final FloatParser DEFAULT_FLOAT_PARSER = new FloatParser() {
     public float parseFloat(BytesRef term) {
-      // TODO: would be far better to directly parse
-      // the UTF-8 bytes into float, but that's tricky?
+      // TODO: would be far better to directly parse from
+      // UTF8 bytes... but really users should use
+      // NumericField, instead, which already decodes
+      // directly from byte[]
       return Float.parseFloat(term.utf8ToString());
     }
     protected Object readResolve() {
@@ -223,7 +222,11 @@
   /** The default parser for long values, which are encoded by {@link Long#toString(long)} */
   public static final LongParser DEFAULT_LONG_PARSER = new LongParser() {
     public long parseLong(BytesRef term) {
-      return FieldCacheImpl.parseLong(term);
+      // TODO: would be far better to directly parse from
+      // UTF8 bytes... but really users should use
+      // NumericField, instead, which already decodes
+      // directly from byte[]
+      return Long.parseLong(term.utf8ToString());
     }
     protected Object readResolve() {
       return DEFAULT_LONG_PARSER;
@@ -237,8 +240,10 @@
   /** The default parser for double values, which are encoded by {@link Double#toString(double)} */
   public static final DoubleParser DEFAULT_DOUBLE_PARSER = new DoubleParser() {
     public double parseDouble(BytesRef term) {
-      // TODO: would be far better to directly parse
-      // the UTF-8 bytes into float, but that's tricky?
+      // TODO: would be far better to directly parse from
+      // UTF8 bytes... but really users should use
+      // NumericField, instead, which already decodes
+      // directly from byte[]
       return Double.parseDouble(term.utf8ToString());
     }
     protected Object readResolve() {

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java Mon Mar  8 16:10:31 2010
@@ -283,7 +283,7 @@
       Terms terms = MultiFields.getTerms(reader, field);
       if (terms != null) {
         final TermsEnum termsEnum = terms.iterator();
-        final Bits delDocs = reader.getDeletedDocs();
+        final Bits delDocs = MultiFields.getDeletedDocs(reader);
         DocsEnum docs = null;
         try {
           while(true) {
@@ -337,7 +337,7 @@
       Terms terms = MultiFields.getTerms(reader, field);
       if (terms != null) {
         final TermsEnum termsEnum = terms.iterator();
-        final Bits delDocs = reader.getDeletedDocs();
+        final Bits delDocs = MultiFields.getDeletedDocs(reader);
         DocsEnum docs = null;
         try {
           while(true) {
@@ -396,7 +396,7 @@
       Terms terms = MultiFields.getTerms(reader, field);
       if (terms != null) {
         final TermsEnum termsEnum = terms.iterator();
-        final Bits delDocs = reader.getDeletedDocs();
+        final Bits delDocs = MultiFields.getDeletedDocs(reader);
         DocsEnum docs = null;
         try {
           while(true) {
@@ -468,7 +468,7 @@
       Terms terms = MultiFields.getTerms(reader, field);
       if (terms != null) {
         final TermsEnum termsEnum = terms.iterator();
-        final Bits delDocs = reader.getDeletedDocs();
+        final Bits delDocs = MultiFields.getDeletedDocs(reader);
         DocsEnum docs = null;
         try {
           while(true) {
@@ -536,7 +536,7 @@
       Terms terms = MultiFields.getTerms(reader, field);
       if (terms != null) {
         final TermsEnum termsEnum = terms.iterator();
-        final Bits delDocs = reader.getDeletedDocs();
+        final Bits delDocs = MultiFields.getDeletedDocs(reader);
         DocsEnum docs = null;
         try {
           while(true) {
@@ -606,7 +606,7 @@
       Terms terms = MultiFields.getTerms(reader, field);
       if (terms != null) {
         final TermsEnum termsEnum = terms.iterator();
-        final Bits delDocs = reader.getDeletedDocs();
+        final Bits delDocs = MultiFields.getDeletedDocs(reader);
         DocsEnum docs = null;
         try {
           while(true) {
@@ -658,7 +658,7 @@
       Terms terms = MultiFields.getTerms(reader, field);
       if (terms != null) {
         final TermsEnum termsEnum = terms.iterator();
-        final Bits delDocs = reader.getDeletedDocs();
+        final Bits delDocs = MultiFields.getDeletedDocs(reader);
         DocsEnum docs = null;
         while(true) {
           final BytesRef term = termsEnum.next();
@@ -711,7 +711,7 @@
 
       if (terms != null) {
         final TermsEnum termsEnum = terms.iterator();
-        final Bits delDocs = reader.getDeletedDocs();
+        final Bits delDocs = MultiFields.getDeletedDocs(reader);
         DocsEnum docs = null;
         while(true) {
           final BytesRef term = termsEnum.next();
@@ -763,29 +763,5 @@
   public PrintStream getInfoStream() {
     return infoStream;
   }
-  
-  // Directly parses a numeric value from UTF8 bytes
-  // nocommit -- whitespace?  +e syntax?
-  final static long parseLong(BytesRef term) {
-    int upto = term.offset;
-    final int negMul;
-    if (term.bytes[upto] == '-') {
-      negMul = -1;
-      upto++;
-    } else {
-      negMul = 1;
-    }
-    final int end = term.offset + term.length;
-    long number = 0;
-    while(upto < end) {
-      final int b = term.bytes[upto++];
-      if (b >= '0' && b <= '9') {
-        number = 10*number + (int) (b-'0');
-      } else {
-        throw new NumberFormatException("could not parse \"" + term.utf8ToString() + "\" to a number");
-      }
-    }
-    return negMul * number;
-  }
 }
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java Mon Mar  8 16:10:31 2010
@@ -19,6 +19,7 @@
 import java.io.IOException;
 
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.util.NumericUtils;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.document.NumericField; // for javadocs
@@ -537,7 +538,7 @@
         if (isCacheable()) {
           skipDocs = null;
         } else {
-          skipDocs = reader.getDeletedDocs();
+          skipDocs = MultiFields.getDeletedDocs(reader);
         }
       }
       final int maxDoc = reader.maxDoc();

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FilteredTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FilteredTermsEnum.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FilteredTermsEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FilteredTermsEnum.java Mon Mar  8 16:10:31 2010
@@ -18,10 +18,12 @@
  */
 
 import java.io.IOException;
+import java.util.Comparator;
+
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Terms;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.DocsAndPositionsEnum;
@@ -127,17 +129,19 @@
   
   @Override
   public BytesRef term() throws IOException {
-    return (tenum == null) ? null : tenum.term();
+    assert tenum != null;
+    return tenum.term();
   }
 
   @Override
-  public BytesRef.Comparator getComparator() throws IOException {
+  public Comparator<BytesRef> getComparator() throws IOException {
     return (tenum == null) ? null : tenum.getComparator();
   }
     
   @Override
   public int docFreq() {
-    return (tenum == null) ? -1 : tenum.docFreq();
+    assert tenum != null;
+    return tenum.docFreq();
   }
 
   /** This enum does not support seeking!
@@ -158,17 +162,20 @@
 
   @Override
   public long ord() throws IOException {
-    return (tenum == null) ? -1 : tenum.ord();
+    assert tenum != null;
+    return tenum.ord();
   }
 
   @Override
   public DocsEnum docs(Bits bits, DocsEnum reuse) throws IOException {
-    return (tenum == null) ? null : tenum.docs(bits, reuse);
+    assert tenum != null;
+    return tenum.docs(bits, reuse);
   }
     
   @Override
   public DocsAndPositionsEnum docsAndPositions(Bits bits, DocsAndPositionsEnum reuse) throws IOException {
-    return (tenum == null) ? null : tenum.docsAndPositions(bits, reuse);
+    assert tenum != null;
+    return tenum.docsAndPositions(bits, reuse);
   }
     
   @Override

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MatchAllDocsQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MatchAllDocsQuery.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MatchAllDocsQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MatchAllDocsQuery.java Mon Mar  8 16:10:31 2010
@@ -19,6 +19,7 @@
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.util.ToStringUtils;
 import org.apache.lucene.util.Bits;
 
@@ -54,7 +55,7 @@
     MatchAllScorer(IndexReader reader, Similarity similarity, Weight w,
         byte[] norms) throws IOException {
       super(similarity);
-      delDocs = reader.getDeletedDocs();
+      delDocs = MultiFields.getDeletedDocs(reader);
       score = w.getValue();
       maxDoc = reader.maxDoc();
       this.norms = norms;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiPhraseQuery.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiPhraseQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiPhraseQuery.java Mon Mar  8 16:10:31 2010
@@ -23,6 +23,7 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.index.DocsAndPositionsEnum;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.ToStringUtils;
@@ -178,7 +179,7 @@
         if (terms.length > 1) {
           postingsEnum = new UnionDocsAndPositionsEnum(reader, terms);
         } else {
-          postingsEnum = reader.termPositionsEnum(reader.getDeletedDocs(),
+          postingsEnum = reader.termPositionsEnum(MultiFields.getDeletedDocs(reader),
                                                   terms[0].field(),
                                                   new BytesRef(terms[0].text()));
         }
@@ -382,8 +383,7 @@
  * Takes the logical union of multiple DocsEnum iterators.
  */
 
-// nocommit -- this must carefully take union of attr source
-// as well -- this is tricky
+// TODO: if ever we allow subclassing of the *PhraseScorer
 class UnionDocsAndPositionsEnum extends DocsAndPositionsEnum {
 
   private static final class DocsQueue extends PriorityQueue<DocsAndPositionsEnum> {
@@ -454,7 +454,7 @@
 
   public UnionDocsAndPositionsEnum(IndexReader indexReader, Term[] terms) throws IOException {
     List<DocsAndPositionsEnum> docsEnums = new LinkedList<DocsAndPositionsEnum>();
-    final Bits delDocs = indexReader.getDeletedDocs();
+    final Bits delDocs = MultiFields.getDeletedDocs(indexReader);
     for (int i = 0; i < terms.length; i++) {
       DocsAndPositionsEnum postings = indexReader.termPositionsEnum(delDocs,
                                                                     terms[i].field(),

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQuery.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQuery.java Mon Mar  8 16:10:31 2010
@@ -26,6 +26,9 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.Terms;
 import org.apache.lucene.queryParser.QueryParser; // for javadoc
 import org.apache.lucene.util.Attribute;
 import org.apache.lucene.util.AttributeImpl;
@@ -186,13 +189,28 @@
   private abstract static class BooleanQueryRewrite extends RewriteMethod {
   
     protected final int collectTerms(IndexReader reader, MultiTermQuery query, TermCollector collector) throws IOException {
+
       if (query.hasNewAPI) {
-        final TermsEnum termsEnum = query.getTermsEnum(reader);
-        if (termsEnum == null) {
-          return 0; // nocommit, subclass shouldn't deal with this case of nonexistent field
-        }
-        if (query.field == null)
+
+        if (query.field == null) {
           throw new NullPointerException("If you implement getTermsEnum(), you must specify a non-null field in the constructor of MultiTermQuery.");
+        }
+
+        final Fields fields = MultiFields.getFields(reader);
+        if (fields == null) {
+          // reader has no fields
+          return 0;
+        }
+
+        final Terms terms = fields.terms(query.field);
+        if (terms == null) {
+          // field does not exist
+          return 0;
+        }
+
+        final TermsEnum termsEnum = query.getTermsEnum(reader);
+        assert termsEnum != null;
+
         if (termsEnum == TermsEnum.EMPTY)
           return 0;
         final BoostAttribute boostAtt =
@@ -384,7 +402,7 @@
     public Query rewrite(IndexReader reader, MultiTermQuery query) throws IOException {
       Query result = super.rewrite(reader, query);
       assert result instanceof BooleanQuery;
-      // nocommit: if empty boolean query return NullQuery
+      // TODO: if empty boolean query return NullQuery?
       if (!((BooleanQuery) result).clauses().isEmpty()) {
         // strip the scores off
         result = new ConstantScoreQuery(new QueryWrapperFilter(result));
@@ -614,12 +632,13 @@
   }
 
   /** Construct the enumeration to be used, expanding the
-   *  pattern term.  This method must return null if no
-   *  terms fall in the range; else, it must return a
-   *  TermsEnum already positioned to the first matching
-   *  term.
-   *
-   *  nocommit in 3.x this will become abstract  */
+   *  pattern term.  This method should only be called if
+   *  the field exists (ie, implementations can assume the
+   *  field does exist).  This method should not return null
+   *  (should instead return {@link TermsEnum#EMPTY} if no
+   *  terms match).  The TermsEnum must already be
+   *  positioned to the first matching term. */
+  // TODO 4.0: make this method abstract
   protected TermsEnum getTermsEnum(IndexReader reader) throws IOException {
     throw new UnsupportedOperationException();
   }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java Mon Mar  8 16:10:31 2010
@@ -21,10 +21,13 @@
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermDocs;
 import org.apache.lucene.index.TermEnum;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.util.OpenBitSet;
 import org.apache.lucene.util.Bits;
 
@@ -107,17 +110,31 @@
   @Override
   public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
     if (query.hasNewAPI) {
-      final TermsEnum termsEnum = query.getTermsEnum(reader);
-      if (termsEnum == null) {
-        return DocIdSet.EMPTY_DOCIDSET;// nocommit;
+      if (query.field == null) {
+        throw new NullPointerException("If you implement getTermsEnum(), you must specify a non-null field in the constructor of MultiTermQuery.");
       }
+
+      final Fields fields = MultiFields.getFields(reader);
+      if (fields == null) {
+        // reader has no fields
+        return DocIdSet.EMPTY_DOCIDSET;
+      }
+
+      final Terms terms = fields.terms(query.field);
+      if (terms == null) {
+        // field does not exist
+        return DocIdSet.EMPTY_DOCIDSET;
+      }
+
+      final TermsEnum termsEnum = query.getTermsEnum(reader);
+      assert termsEnum != null;
       if (termsEnum.next() != null) {
         // fill into a OpenBitSet
         final OpenBitSet bitSet = new OpenBitSet(reader.maxDoc());
         final int[] docs = new int[32];
         final int[] freqs = new int[32];
         int termCount = 0;
-        final Bits delDocs = reader.getDeletedDocs();
+        final Bits delDocs = MultiFields.getDeletedDocs(reader);
         DocsEnum docsEnum = null;
         do {
           termCount++;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/NumericRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/NumericRangeQuery.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/NumericRangeQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/NumericRangeQuery.java Mon Mar  8 16:10:31 2010
@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 import java.util.LinkedList;
+import java.util.Comparator;
 
 import org.apache.lucene.analysis.NumericTokenStream; // for javadocs
 import org.apache.lucene.document.NumericField; // for javadocs
@@ -381,7 +382,7 @@
     private final BytesRef currentLowerBound = new BytesRef(), currentUpperBound = new BytesRef();
 
     private final LinkedList<String> rangeBounds = new LinkedList<String>();
-    private final BytesRef.Comparator termComp;
+    private final Comparator<BytesRef> termComp;
 
     NumericRangeTermsEnum(final IndexReader reader) throws IOException {
       super(reader, getField());

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseQuery.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseQuery.java Mon Mar  8 16:10:31 2010
@@ -154,21 +154,18 @@
         return null;
 
       DocsAndPositionsEnum[] postings = new DocsAndPositionsEnum[terms.size()];
-      final Bits delDocs = reader.getDeletedDocs();
+      final Bits delDocs = MultiFields.getDeletedDocs(reader);
       for (int i = 0; i < terms.size(); i++) {
         final Term t = terms.get(i);
         final BytesRef text = new BytesRef(t.text());
-        // NOTE: debateably, the caller should never pass in a
-        // multi reader...
         DocsAndPositionsEnum postingsEnum = MultiFields.getTermPositionsEnum(reader,
                                                                              delDocs,
                                                                              t.field(),
                                                                              text);
+        // PhraseQuery on a field that did not index
+        // positions.
         if (postingsEnum == null) {
-          if (MultiFields.getTermDocsEnum(reader,
-                                          delDocs,
-                                          t.field(),
-                                          text) != null) {
+          if (MultiFields.getTermDocsEnum(reader, delDocs, t.field(), text) != null) {
             // term does exist, but has no positions
             throw new IllegalStateException("field \"" + t.field() + "\" was indexed with Field.omitTermFreqAndPositions=true; cannot run PhraseQuery (term=" + t.text() + ")");
           } else {

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PrefixQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PrefixQuery.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PrefixQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PrefixQuery.java Mon Mar  8 16:10:31 2010
@@ -20,9 +20,9 @@
 import java.io.IOException;
 
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.util.ToStringUtils;
 
@@ -52,6 +52,7 @@
   @Override  
   protected TermsEnum getTermsEnum(IndexReader reader) throws IOException {
     if (prefix.text().length() == 0) {
+      // no prefix -- match all terms for this field:
       final Terms terms = MultiFields.getTerms(reader, getField());
       return (terms != null) ? terms.iterator() : TermsEnum.EMPTY;
     }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/Similarity.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/Similarity.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/Similarity.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/Similarity.java Mon Mar  8 16:10:31 2010
@@ -857,7 +857,7 @@
    * @return An implementation dependent float to be used as a scoring factor
    *
    */
-  // nocommit -- swtich to BytesRef
+  // TODO: maybe switch this API to BytesRef?
   public float scorePayload(int docId, String fieldName, int start, int end, byte [] payload, int offset, int length)
   {
     return 1;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermQuery.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermQuery.java Mon Mar  8 16:10:31 2010
@@ -75,7 +75,7 @@
     public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException {
       // NOTE: debateably, the caller should never pass in a
       // multi reader...
-      DocsEnum docs = MultiFields.getTermDocsEnum(reader, reader.getDeletedDocs(), term.field(), new BytesRef(term.text()));
+      DocsEnum docs = MultiFields.getTermDocsEnum(reader, MultiFields.getDeletedDocs(reader), term.field(), new BytesRef(term.text()));
       if (docs == null) {
         return null;
       }
@@ -118,7 +118,7 @@
 
       Explanation tfExplanation = new Explanation();
       int tf = 0;
-      DocsEnum docs = reader.termDocsEnum(reader.getDeletedDocs(), term.field(), new BytesRef(term.text()));
+      DocsEnum docs = reader.termDocsEnum(MultiFields.getDeletedDocs(reader), term.field(), new BytesRef(term.text()));
       if (docs != null) {
           int newDoc = docs.advance(doc);
           if (newDoc == doc) {

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeQuery.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeQuery.java Mon Mar  8 16:10:31 2010
@@ -144,7 +144,7 @@
       // NOTE: debateably, the caller should never pass in a
       // multi reader...
       final Terms terms = MultiFields.getTerms(reader, field);
-      return (terms != null) ? terms.iterator() : TermsEnum.EMPTY;
+      return (terms != null) ? terms.iterator() : null;
     }
     return new TermRangeTermsEnum(reader, field,
         lowerTerm, upperTerm, includeLower, includeUpper, collator);

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeTermsEnum.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeTermsEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeTermsEnum.java Mon Mar  8 16:10:31 2010
@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 import java.text.Collator;
+import java.util.Comparator;
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.util.BytesRef;
@@ -39,7 +40,7 @@
   private boolean includeUpper;
   final private BytesRef lowerBytesRef;
   final private BytesRef upperBytesRef;
-  private final BytesRef.Comparator termComp;
+  private final Comparator<BytesRef> termComp;
 
   /**
    * Enumerates all terms greater/equal than <code>lowerTerm</code>

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/function/ValueSourceQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/function/ValueSourceQuery.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/function/ValueSourceQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/function/ValueSourceQuery.java Mon Mar  8 16:10:31 2010
@@ -20,6 +20,7 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.*;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.util.ToStringUtils;
 import org.apache.lucene.util.Bits;
 
@@ -137,7 +138,7 @@
       qWeight = w.getValue();
       // this is when/where the values are first created.
       vals = valSrc.getValues(reader);
-      delDocs = reader.getDeletedDocs();
+      delDocs = MultiFields.getDeletedDocs(reader);
       maxDoc = reader.maxDoc();
     }
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/SpanTermQuery.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/SpanTermQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/SpanTermQuery.java Mon Mar  8 16:10:31 2010
@@ -86,7 +86,7 @@
     // NOTE: debateably, the caller should never pass in a
     // multi reader...
     final DocsAndPositionsEnum postings = MultiFields.getTermPositionsEnum(reader,
-                                                                           reader.getDeletedDocs(),
+                                                                           MultiFields.getDeletedDocs(reader),
                                                                            term.field(),
                                                                            new BytesRef(term.text()));
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/Bits.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/Bits.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/Bits.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/Bits.java Mon Mar  8 16:10:31 2010
@@ -17,7 +17,13 @@
  * limitations under the License.
  */
 
+/**
+ * @lucene.experimental
+ */
+
 public interface Bits {
   public boolean get(int index);
   public int length();
+
+  public static final Bits[] EMPTY_ARRAY = new Bits[0];
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/BytesRef.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/BytesRef.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/BytesRef.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/BytesRef.java Mon Mar  8 16:10:31 2010
@@ -17,6 +17,7 @@
  * limitations under the License.
  */
 
+import java.util.Comparator;
 import java.io.UnsupportedEncodingException;
 
 /** Represents byte[], as a slice (offset + length) into an
@@ -65,16 +66,15 @@
    * Copies the UTF8 bytes for this string.
    * 
    * @param text Must be well-formed unicode text, with no
-   * unpaired surrogates or U+FFFF.
+   * unpaired surrogates or invalid UTF16 code units.
    */
   public void copy(CharSequence text) {
-    // nocommit -- remove this paranoia
-    assert UnicodeUtil.validUTF16String(text);
     // nocommit -- new byte[10] is waste of resources,
     // it should simply allocate text.length()*4 like UnicodeUtil.
     // Ideally, I would remove this here and add a null-check in UnicodeUtil. (Uwe)
-    if (bytes == null)
+    if (bytes == null) {
       bytes = new byte[10];
+    }
     UnicodeUtil.UTF16toUTF8(text, 0, text.length(), this);
   }
 
@@ -99,8 +99,7 @@
     return new BytesRef(this);
   }
 
-  public boolean startsWith(BytesRef other, int pos) {
-    // nocommit: maybe this one shouldn't be public...
+  private boolean sliceEquals(BytesRef other, int pos) {
     if (pos < 0 || length - pos < other.length) {
       return false;
     }
@@ -108,19 +107,21 @@
     int j = other.offset;
     final int k = other.offset + other.length;
     
-    while (j < k)
-      if (bytes[i++] != other.bytes[j++])
+    while (j < k) {
+      if (bytes[i++] != other.bytes[j++]) {
         return false;
+      }
+    }
     
     return true;
   }
   
   public boolean startsWith(BytesRef other) {
-    return startsWith(other, 0);
+    return sliceEquals(other, 0);
   }
 
   public boolean endsWith(BytesRef other) {
-    return startsWith(other, length - other.length);
+    return sliceEquals(other, length - other.length);
   }
   
   @Override
@@ -139,15 +140,6 @@
     return this.bytesEquals((BytesRef) other);
   }
 
-  // nocommit -- catch statically all places where this is
-  // being incorrectly called & switch to utf8ToString
-  public String toString() {
-    // nocommit -- do this, to fix all places using
-    // toString, to use utf8ToString instead:
-    throw new RuntimeException();
-    //return utf8ToString();
-  }
-
   /** Interprets stored bytes as UTF8 bytes, returning the
    *  resulting string */
   public String utf8ToString() {
@@ -160,7 +152,8 @@
     }
   }
 
-  public String toBytesString() {
+  /** Returns hex encoded bytes, eg [0x6c 0x75 0x63 0x65 0x6e 0x65] */
+  public String toString() {
     StringBuilder sb = new StringBuilder();
     sb.append('[');
     final int end = offset + length;
@@ -174,25 +167,6 @@
     return sb.toString();
   }
 
-  private final String asUnicodeChar(char c) {
-    return "U+" + Integer.toHexString(c);
-  }
-
-  // for debugging only -- this is slow
-  public String toUnicodeString() {
-    StringBuilder sb = new StringBuilder();
-    sb.append('[');
-    final String s = toString();
-    for(int i=0;i<s.length();i++) {
-      if (i > 0) {
-        sb.append(' ');
-      }
-      sb.append(asUnicodeChar(s.charAt(i)));
-    }
-    sb.append(']');
-    return sb.toString();
-  }
-
   public void copy(BytesRef other) {
     if (bytes == null) {
       bytes = new byte[other.length];
@@ -208,17 +182,16 @@
     bytes = ArrayUtil.grow(bytes, newLength);
   }
 
-  public abstract static class Comparator {
-    abstract public int compare(BytesRef a, BytesRef b);
-  }
-
-  private final static Comparator utf8SortedAsUTF16SortOrder = new UTF8SortedAsUTF16Comparator();
+  private final static Comparator<BytesRef> utf8SortedAsUTF16SortOrder = new UTF8SortedAsUTF16Comparator();
 
-  public static Comparator getUTF8SortedAsUTF16Comparator() {
+  public static Comparator<BytesRef> getUTF8SortedAsUTF16Comparator() {
     return utf8SortedAsUTF16SortOrder;
   }
 
-  public static class UTF8SortedAsUTF16Comparator extends Comparator {
+  public static class UTF8SortedAsUTF16Comparator implements Comparator<BytesRef> {
+    // Only singleton
+    private UTF8SortedAsUTF16Comparator() {};
+
     public int compare(BytesRef a, BytesRef b) {
 
       final byte[] aBytes = a.bytes;
@@ -259,5 +232,9 @@
       // One is a prefix of the other, or, they are equal:
       return a.length - b.length;
     }
+
+    public boolean equals(Object other) {
+      return this == other;
+    }
   }
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/MultiBits.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/MultiBits.java?rev=920378&r1=920377&r2=920378&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/MultiBits.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/MultiBits.java Mon Mar  8 16:10:31 2010
@@ -17,21 +17,30 @@
  * limitations under the License.
  */
 
+import java.util.List;
+
 /**
  * Concatenates multiple Bits together, on every lookup.
  *
  * <p><b>NOTE</b>: This is very costly, as every lookup must
  * do a binary search to locate the right sub-reader.
+ *
+ * @lucene.experimental
  */
+
 public final class MultiBits implements Bits {
   private final Bits[] subs;
 
   // length is 1+subs.length (the last entry has the maxDoc):
   private final int[] starts;
 
-  public MultiBits(Bits[] subs, int[] starts) {
-    this.subs = subs;
-    this.starts = starts;
+  public MultiBits(List<Bits> bits, List<Integer> starts) {
+    assert starts.size() == 1+bits.size();
+    this.subs = bits.toArray(Bits.EMPTY_ARRAY);
+    this.starts = new int[starts.size()];
+    for(int i=0;i<this.starts.length;i++) {
+      this.starts[i] = starts.get(i);
+    }
   }
 
   private boolean checkLength(int reader, int doc) {
@@ -42,6 +51,7 @@
 
   public boolean get(int doc) {
     final int reader = ReaderUtil.subIndex(doc, starts);
+    assert reader != -1;
     final Bits bits = subs[reader];
     if (bits == null) {
       return false;
@@ -51,13 +61,24 @@
     }
   }
 
-  public Bits getMatchingSub(ReaderUtil.Slice slice) {
+  public final static class SubResult {
+    public boolean matches;
+    public Bits result;
+  }
+
+  private final SubResult subResult = new SubResult();
+
+  public SubResult getMatchingSub(ReaderUtil.Slice slice) {
     int reader = ReaderUtil.subIndex(slice.start, starts);
+    assert reader != -1;
+    assert reader < subs.length: "slice=" + slice + " starts[-1]=" + starts[starts.length-1];
     if (starts[reader] == slice.start && starts[1+reader] == slice.start+slice.length) {
-      return subs[reader];
+      subResult.matches = true;
+      subResult.result = subs[reader];
     } else {
-      return null;
+      subResult.matches = false;
     }
+    return subResult;
   }
 
   public int length() {



Mime
View raw message