lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r824918 [9/11] - in /lucene/java/branches/flex_1458: contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/ contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/ contrib/benchmark/src/test/org/apache/lucene/benc...
Date Tue, 13 Oct 2009 20:44:59 GMT
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java Tue Oct 13 20:44:51 2009
@@ -29,9 +29,13 @@
 
 import org.apache.lucene.document.NumericField; // javadoc
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.index.TermDocs;          // deprecated
+import org.apache.lucene.index.TermRef;
+import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.StringHelper;
 import org.apache.lucene.util.FieldCacheSanityChecker;
 
@@ -264,22 +268,28 @@
         return wrapper.getBytes(reader, field, FieldCache.DEFAULT_BYTE_PARSER);
       }
       final byte[] retArray = new byte[reader.maxDoc()];
-      TermDocs termDocs = reader.termDocs();
-      TermEnum termEnum = reader.terms (new Term (field));
-      try {
-        do {
-          Term term = termEnum.term();
-          if (term==null || term.field() != field) break;
-          byte termval = parser.parseByte(term.text());
-          termDocs.seek (termEnum);
-          while (termDocs.next()) {
-            retArray[termDocs.doc()] = termval;
-          }
-        } while (termEnum.next());
-      } catch (StopFillCacheException stop) {
-      } finally {
-        termDocs.close();
-        termEnum.close();
+      Terms terms = reader.fields().terms(field);
+      if (terms != null) {
+        final TermsEnum termsEnum = terms.iterator();
+        final Bits delDocs = reader.getDeletedDocs();
+        try {
+          while(true) {
+            final TermRef term = termsEnum.next();
+            if (term == null) {
+              break;
+            }
+            final byte termval = parser.parseByte(term);
+            final DocsEnum docs = termsEnum.docs(delDocs);
+            while (true) {
+              final int docID = docs.next();
+              if (docID == DocsEnum.NO_MORE_DOCS) {
+                break;
+              }
+              retArray[docID] = termval;
+            }
+          }
+        } catch (StopFillCacheException stop) {
+        }
       }
       return retArray;
     }
@@ -310,22 +320,28 @@
         return wrapper.getShorts(reader, field, FieldCache.DEFAULT_SHORT_PARSER);
       }
       final short[] retArray = new short[reader.maxDoc()];
-      TermDocs termDocs = reader.termDocs();
-      TermEnum termEnum = reader.terms (new Term (field));
-      try {
-        do {
-          Term term = termEnum.term();
-          if (term==null || term.field() != field) break;
-          short termval = parser.parseShort(term.text());
-          termDocs.seek (termEnum);
-          while (termDocs.next()) {
-            retArray[termDocs.doc()] = termval;
-          }
-        } while (termEnum.next());
-      } catch (StopFillCacheException stop) {
-      } finally {
-        termDocs.close();
-        termEnum.close();
+      Terms terms = reader.fields().terms(field);
+      if (terms != null) {
+        final TermsEnum termsEnum = terms.iterator();
+        final Bits delDocs = reader.getDeletedDocs();
+        try {
+          while(true) {
+            final TermRef term = termsEnum.next();
+            if (term == null) {
+              break;
+            }
+            final short termval = parser.parseShort(term);
+            final DocsEnum docs = termsEnum.docs(delDocs);
+            while (true) {
+              final int docID = docs.next();
+              if (docID == DocsEnum.NO_MORE_DOCS) {
+                break;
+              }
+              retArray[docID] = termval;
+            }
+          }
+        } catch (StopFillCacheException stop) {
+        }
       }
       return retArray;
     }
@@ -360,27 +376,40 @@
         }
       }
       int[] retArray = null;
-      TermDocs termDocs = reader.termDocs();
-      TermEnum termEnum = reader.terms (new Term (field));
-      try {
-        do {
-          Term term = termEnum.term();
-          if (term==null || term.field() != field) break;
-          int termval = parser.parseInt(term.text());
-          if (retArray == null) // late init
-            retArray = new int[reader.maxDoc()];
-          termDocs.seek (termEnum);
-          while (termDocs.next()) {
-            retArray[termDocs.doc()] = termval;
-          }
-        } while (termEnum.next());
-      } catch (StopFillCacheException stop) {
-      } finally {
-        termDocs.close();
-        termEnum.close();
+
+      Terms terms = reader.fields().terms(field);
+      if (terms != null) {
+        final TermsEnum termsEnum = terms.iterator();
+        final Bits delDocs = reader.getDeletedDocs();
+        try {
+          while(true) {
+            final TermRef term = termsEnum.next();
+            if (term == null) {
+              break;
+            }
+            final int termval = parser.parseInt(term);
+            if (retArray == null) {
+              // late init so numeric fields don't double allocate
+              retArray = new int[reader.maxDoc()];
+            }
+
+            final DocsEnum docs = termsEnum.docs(delDocs);
+            while (true) {
+              final int docID = docs.next();
+              if (docID == DocsEnum.NO_MORE_DOCS) {
+                break;
+              }
+              retArray[docID] = termval;
+            }
+          }
+        } catch (StopFillCacheException stop) {
+        }
       }
-      if (retArray == null) // no values
+
+      if (retArray == null) {
+        // no values
         retArray = new int[reader.maxDoc()];
+      }
       return retArray;
     }
   };
@@ -415,29 +444,42 @@
         } catch (NumberFormatException ne) {
           return wrapper.getFloats(reader, field, NUMERIC_UTILS_FLOAT_PARSER);      
         }
-    }
+      }
       float[] retArray = null;
-      TermDocs termDocs = reader.termDocs();
-      TermEnum termEnum = reader.terms (new Term (field));
-      try {
-        do {
-          Term term = termEnum.term();
-          if (term==null || term.field() != field) break;
-          float termval = parser.parseFloat(term.text());
-          if (retArray == null) // late init
-            retArray = new float[reader.maxDoc()];
-          termDocs.seek (termEnum);
-          while (termDocs.next()) {
-            retArray[termDocs.doc()] = termval;
-          }
-        } while (termEnum.next());
-      } catch (StopFillCacheException stop) {
-      } finally {
-        termDocs.close();
-        termEnum.close();
+
+      Terms terms = reader.fields().terms(field);
+      if (terms != null) {
+        final TermsEnum termsEnum = terms.iterator();
+        final Bits delDocs = reader.getDeletedDocs();
+        try {
+          while(true) {
+            final TermRef term = termsEnum.next();
+            if (term == null) {
+              break;
+            }
+            final float termval = parser.parseFloat(term);
+            if (retArray == null) {
+              // late init so numeric fields don't double allocate
+              retArray = new float[reader.maxDoc()];
+            }
+
+            final DocsEnum docs = termsEnum.docs(delDocs);
+            while (true) {
+              final int docID = docs.next();
+              if (docID == DocsEnum.NO_MORE_DOCS) {
+                break;
+              }
+              retArray[docID] = termval;
+            }
+          }
+        } catch (StopFillCacheException stop) {
+        }
       }
-      if (retArray == null) // no values
+
+      if (retArray == null) {
+        // no values
         retArray = new float[reader.maxDoc()];
+      }
       return retArray;
     }
   };
@@ -471,27 +513,39 @@
         }
       }
       long[] retArray = null;
-      TermDocs termDocs = reader.termDocs();
-      TermEnum termEnum = reader.terms (new Term(field));
-      try {
-        do {
-          Term term = termEnum.term();
-          if (term==null || term.field() != field) break;
-          long termval = parser.parseLong(term.text());
-          if (retArray == null) // late init
-            retArray = new long[reader.maxDoc()];
-          termDocs.seek (termEnum);
-          while (termDocs.next()) {
-            retArray[termDocs.doc()] = termval;
-          }
-        } while (termEnum.next());
-      } catch (StopFillCacheException stop) {
-      } finally {
-        termDocs.close();
-        termEnum.close();
+      Terms terms = reader.fields().terms(field);
+      if (terms != null) {
+        final TermsEnum termsEnum = terms.iterator();
+        final Bits delDocs = reader.getDeletedDocs();
+        try {
+          while(true) {
+            final TermRef term = termsEnum.next();
+            if (term == null) {
+              break;
+            }
+            final long termval = parser.parseLong(term);
+            if (retArray == null) {
+              // late init so numeric fields don't double allocate
+              retArray = new long[reader.maxDoc()];
+            }
+
+            final DocsEnum docs = termsEnum.docs(delDocs);
+            while (true) {
+              final int docID = docs.next();
+              if (docID == DocsEnum.NO_MORE_DOCS) {
+                break;
+              }
+              retArray[docID] = termval;
+            }
+          }
+        } catch (StopFillCacheException stop) {
+        }
       }
-      if (retArray == null) // no values
+
+      if (retArray == null) {
+        // no values
         retArray = new long[reader.maxDoc()];
+      }
       return retArray;
     }
   };
@@ -526,24 +580,33 @@
         }
       }
       double[] retArray = null;
-      TermDocs termDocs = reader.termDocs();
-      TermEnum termEnum = reader.terms (new Term (field));
-      try {
-        do {
-          Term term = termEnum.term();
-          if (term==null || term.field() != field) break;
-          double termval = parser.parseDouble(term.text());
-          if (retArray == null) // late init
-            retArray = new double[reader.maxDoc()];
-          termDocs.seek (termEnum);
-          while (termDocs.next()) {
-            retArray[termDocs.doc()] = termval;
-          }
-        } while (termEnum.next());
-      } catch (StopFillCacheException stop) {
-      } finally {
-        termDocs.close();
-        termEnum.close();
+      Terms terms = reader.fields().terms(field);
+      if (terms != null) {
+        final TermsEnum termsEnum = terms.iterator();
+        final Bits delDocs = reader.getDeletedDocs();
+        try {
+          while(true) {
+            final TermRef term = termsEnum.next();
+            if (term == null) {
+              break;
+            }
+            final double termval = parser.parseDouble(term);
+            if (retArray == null) {
+              // late init so numeric fields don't double allocate
+              retArray = new double[reader.maxDoc()];
+            }
+
+            final DocsEnum docs = termsEnum.docs(delDocs);
+            while (true) {
+              final int docID = docs.next();
+              if (docID == DocsEnum.NO_MORE_DOCS) {
+                break;
+              }
+              retArray[docID] = termval;
+            }
+          }
+        } catch (StopFillCacheException stop) {
+        }
       }
       if (retArray == null) // no values
         retArray = new double[reader.maxDoc()];
@@ -566,21 +629,26 @@
         throws IOException {
       String field = StringHelper.intern((String) entryKey.field);
       final String[] retArray = new String[reader.maxDoc()];
-      TermDocs termDocs = reader.termDocs();
-      TermEnum termEnum = reader.terms (new Term (field));
-      try {
-        do {
-          Term term = termEnum.term();
-          if (term==null || term.field() != field) break;
-          String termval = term.text();
-          termDocs.seek (termEnum);
-          while (termDocs.next()) {
-            retArray[termDocs.doc()] = termval;
-          }
-        } while (termEnum.next());
-      } finally {
-        termDocs.close();
-        termEnum.close();
+
+      Terms terms = reader.fields().terms(field);
+      if (terms != null) {
+        final TermsEnum termsEnum = terms.iterator();
+        final Bits delDocs = reader.getDeletedDocs();
+        while(true) {
+          final TermRef term = termsEnum.next();
+          if (term == null) {
+            break;
+          }
+          final DocsEnum docs = termsEnum.docs(delDocs);
+          final String termval = term.toString();
+          while (true) {
+            final int docID = docs.next();
+            if (docID == DocsEnum.NO_MORE_DOCS) {
+              break;
+            }
+            retArray[docID] = termval;
+          }
+        }
       }
       return retArray;
     }
@@ -602,8 +670,9 @@
       String field = StringHelper.intern((String) entryKey.field);
       final int[] retArray = new int[reader.maxDoc()];
       String[] mterms = new String[reader.maxDoc()+1];
-      TermDocs termDocs = reader.termDocs();
-      TermEnum termEnum = reader.terms (new Term (field));
+
+      Terms terms = reader.fields().terms(field);
+
       int t = 0;  // current term number
 
       // an entry for documents that have no terms in this field
@@ -612,28 +681,34 @@
       // needs to change as well.
       mterms[t++] = null;
 
-      try {
-        do {
-          Term term = termEnum.term();
-          if (term==null || term.field() != field) break;
+      if (terms != null) {
+        final TermsEnum termsEnum = terms.iterator();
+        final Bits delDocs = reader.getDeletedDocs();
+        while(true) {
+          final TermRef term = termsEnum.next();
+          if (term == null) {
+            break;
+          }
 
           // store term text
           // we expect that there is at most one term per document
-          if (t >= mterms.length) throw new RuntimeException ("there are more terms than " +
-                  "documents in field \"" + field + "\", but it's impossible to sort on " +
-                  "tokenized fields");
-          mterms[t] = term.text();
-
-          termDocs.seek (termEnum);
-          while (termDocs.next()) {
-            retArray[termDocs.doc()] = t;
+          if (t >= mterms.length) {
+            throw new RuntimeException ("there are more terms than " +
+                                        "documents in field \"" + field + "\", but it's impossible to sort on " +
+                                        "tokenized fields");
+          }
+          mterms[t] = term.toString();
+
+          final DocsEnum docs = termsEnum.docs(delDocs);
+          while (true) {
+            final int docID = docs.next();
+            if (docID == DocsEnum.NO_MORE_DOCS) {
+              break;
+            }
+            retArray[docID] = t;
           }
-
           t++;
-        } while (termEnum.next());
-      } finally {
-        termDocs.close();
-        termEnum.close();
+        }
       }
 
       if (t == 0) {
@@ -643,16 +718,16 @@
       } else if (t < mterms.length) {
         // if there are less terms than documents,
         // trim off the dead array space
-        String[] terms = new String[t];
-        System.arraycopy (mterms, 0, terms, 0, t);
-        mterms = terms;
+        String[] newTerms = new String[t];
+        System.arraycopy (mterms, 0, newTerms, 0, t);
+        mterms = newTerms;
       }
 
       StringIndex value = new StringIndex (retArray, mterms);
       return value;
     }
   };
-
+  
   private volatile PrintStream infoStream;
 
   public void setInfoStream(PrintStream stream) {
@@ -662,5 +737,29 @@
   public PrintStream getInfoStream() {
     return infoStream;
   }
+  
+  // Directly parses a numeric value from UTF8 bytes
+  // nocommit -- whitespace?  +e syntax?
+  final static long parseLong(TermRef term) {
+    int upto = term.offset;
+    final int negMul;
+    if (term.bytes[upto] == '-') {
+      negMul = -1;
+      upto++;
+    } else {
+      negMul = 1;
+    }
+    final int end = term.offset + term.length;
+    long number = 0;
+    while(upto < end) {
+      final int b = term.bytes[upto++];
+      if (b >= '0' && b <= '9') {
+        number = 10*number + (int) (b-'0');
+      } else {
+        throw new NumberFormatException("could not parse \"" + term + "\" to a number");
+      }
+    }
+    return negMul * number;
+  }
 }
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java Tue Oct 13 20:44:51 2009
@@ -19,8 +19,8 @@
 import java.io.IOException;
 
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.TermDocs;
 import org.apache.lucene.util.NumericUtils;
+import org.apache.lucene.util.Bits;
 import org.apache.lucene.document.NumericField; // for javadocs
 
 /**
@@ -117,9 +117,9 @@
         
         assert inclusiveLowerPoint > 0 && inclusiveUpperPoint > 0;
         
-        // for this DocIdSet, we never need to use TermDocs,
+        // for this DocIdSet, we can ignore deleted docs
         // because deleted docs have an order of 0 (null entry in StringIndex)
-        return new FieldCacheDocIdSet(reader, false) {
+        return new FieldCacheDocIdSet(reader, true) {
           final boolean matchDoc(int doc) {
             return fcsi.order[doc] >= inclusiveLowerPoint && fcsi.order[doc] <= inclusiveUpperPoint;
           }
@@ -167,8 +167,8 @@
           return DocIdSet.EMPTY_DOCIDSET;
         
         final byte[] values = FieldCache.DEFAULT.getBytes(reader, field, (FieldCache.ByteParser) parser);
-        // we only request the usage of termDocs, if the range contains 0
-        return new FieldCacheDocIdSet(reader, (inclusiveLowerPoint <= 0 && inclusiveUpperPoint >= 0)) {
+        // we only respect deleted docs if the range contains 0
+        return new FieldCacheDocIdSet(reader, !(inclusiveLowerPoint <= 0 && inclusiveUpperPoint >= 0)) {
           boolean matchDoc(int doc) {
             return values[doc] >= inclusiveLowerPoint && values[doc] <= inclusiveUpperPoint;
           }
@@ -216,8 +216,8 @@
           return DocIdSet.EMPTY_DOCIDSET;
         
         final short[] values = FieldCache.DEFAULT.getShorts(reader, field, (FieldCache.ShortParser) parser);
-        // we only request the usage of termDocs, if the range contains 0
-        return new FieldCacheDocIdSet(reader, (inclusiveLowerPoint <= 0 && inclusiveUpperPoint >= 0)) {
+        // ignore deleted docs if range doesn't contain 0
+        return new FieldCacheDocIdSet(reader, !(inclusiveLowerPoint <= 0 && inclusiveUpperPoint >= 0)) {
           boolean matchDoc(int doc) {
             return values[doc] >= inclusiveLowerPoint && values[doc] <= inclusiveUpperPoint;
           }
@@ -265,8 +265,8 @@
           return DocIdSet.EMPTY_DOCIDSET;
         
         final int[] values = FieldCache.DEFAULT.getInts(reader, field, (FieldCache.IntParser) parser);
-        // we only request the usage of termDocs, if the range contains 0
-        return new FieldCacheDocIdSet(reader, (inclusiveLowerPoint <= 0 && inclusiveUpperPoint >= 0)) {
+        // ignore deleted docs if range doesn't contain 0
+        return new FieldCacheDocIdSet(reader, !(inclusiveLowerPoint <= 0 && inclusiveUpperPoint >= 0)) {
           boolean matchDoc(int doc) {
             return values[doc] >= inclusiveLowerPoint && values[doc] <= inclusiveUpperPoint;
           }
@@ -314,8 +314,8 @@
           return DocIdSet.EMPTY_DOCIDSET;
         
         final long[] values = FieldCache.DEFAULT.getLongs(reader, field, (FieldCache.LongParser) parser);
-        // we only request the usage of termDocs, if the range contains 0
-        return new FieldCacheDocIdSet(reader, (inclusiveLowerPoint <= 0L && inclusiveUpperPoint >= 0L)) {
+        // ignore deleted docs if range doesn't contain 0
+        return new FieldCacheDocIdSet(reader, !(inclusiveLowerPoint <= 0L && inclusiveUpperPoint >= 0L)) {
           boolean matchDoc(int doc) {
             return values[doc] >= inclusiveLowerPoint && values[doc] <= inclusiveUpperPoint;
           }
@@ -367,8 +367,8 @@
           return DocIdSet.EMPTY_DOCIDSET;
         
         final float[] values = FieldCache.DEFAULT.getFloats(reader, field, (FieldCache.FloatParser) parser);
-        // we only request the usage of termDocs, if the range contains 0
-        return new FieldCacheDocIdSet(reader, (inclusiveLowerPoint <= 0.0f && inclusiveUpperPoint >= 0.0f)) {
+        // ignore deleted docs if range doesn't contain 0
+        return new FieldCacheDocIdSet(reader, !(inclusiveLowerPoint <= 0.0f && inclusiveUpperPoint >= 0.0f)) {
           boolean matchDoc(int doc) {
             return values[doc] >= inclusiveLowerPoint && values[doc] <= inclusiveUpperPoint;
           }
@@ -420,8 +420,8 @@
           return DocIdSet.EMPTY_DOCIDSET;
         
         final double[] values = FieldCache.DEFAULT.getDoubles(reader, field, (FieldCache.DoubleParser) parser);
-        // we only request the usage of termDocs, if the range contains 0
-        return new FieldCacheDocIdSet(reader, (inclusiveLowerPoint <= 0.0 && inclusiveUpperPoint >= 0.0)) {
+        // ignore deleted docs if range doesn't contain 0
+        return new FieldCacheDocIdSet(reader, !(inclusiveLowerPoint <= 0.0 && inclusiveUpperPoint >= 0.0)) {
           boolean matchDoc(int doc) {
             return values[doc] >= inclusiveLowerPoint && values[doc] <= inclusiveUpperPoint;
           }
@@ -467,91 +467,77 @@
   
   static abstract class FieldCacheDocIdSet extends DocIdSet {
     private final IndexReader reader;
-    private boolean mayUseTermDocs;
-  
-    FieldCacheDocIdSet(IndexReader reader, boolean mayUseTermDocs) {
+    private boolean canIgnoreDeletedDocs;
+
+    FieldCacheDocIdSet(IndexReader reader, boolean canIgnoreDeletedDocs) {
       this.reader = reader;
-      this.mayUseTermDocs = mayUseTermDocs;
+      this.canIgnoreDeletedDocs = canIgnoreDeletedDocs;
     }
-  
-    /** this method checks, if a doc is a hit, should throw AIOBE, when position invalid */
+
+    /**
+     * this method checks, if a doc is a hit, should throw AIOBE, when position
+     * invalid
+     */
     abstract boolean matchDoc(int doc) throws ArrayIndexOutOfBoundsException;
-    
-    /** this DocIdSet is cacheable, if it works solely with FieldCache and no TermDocs */
+
+    /**
+     * this DocIdSet is cacheable, if it can ignore deletions
+     */
     public boolean isCacheable() {
-      return !(mayUseTermDocs && reader.hasDeletions());
+      return canIgnoreDeletedDocs || !reader.hasDeletions();
     }
 
     public DocIdSetIterator iterator() throws IOException {
       // Synchronization needed because deleted docs BitVector
       // can change after call to hasDeletions until TermDocs creation.
-      // We only use an iterator with termDocs, when this was requested (e.g. range contains 0)
+      // We only use an iterator with termDocs, when this was requested (e.g.
+      // range contains 0)
       // and the index has deletions
-      final TermDocs termDocs;
-      synchronized(reader) {
-        termDocs = isCacheable() ? null : reader.termDocs(null);
+
+      final Bits skipDocs;
+      synchronized (reader) {
+        if (isCacheable()) {
+          skipDocs = null;
+        } else {
+          skipDocs = reader.getDeletedDocs();
+        }
       }
-      if (termDocs != null) {
-        // a DocIdSetIterator using TermDocs to iterate valid docIds
-        return new DocIdSetIterator() {
-          private int doc = -1;
-          
-          public int docID() {
-            return doc;
-          }
-          
-          public int nextDoc() throws IOException {
+      final int maxDoc = reader.maxDoc();
+
+      // a DocIdSetIterator generating docIds by
+      // incrementing a variable & checking skipDocs -
+      return new DocIdSetIterator() {
+        private int doc = -1;
+
+        public int docID() {
+          return doc;
+        }
+
+        public int nextDoc() {
+          try {
             do {
-              if (!termDocs.next())
-                return doc = NO_MORE_DOCS;
-            } while (!matchDoc(doc = termDocs.doc()));
+              doc++;
+            } while ((skipDocs != null && doc < maxDoc && skipDocs.get(doc))
+                || !matchDoc(doc));
             return doc;
+          } catch (ArrayIndexOutOfBoundsException e) {
+            return doc = NO_MORE_DOCS;
           }
-          
-          public int advance(int target) throws IOException {
-            if (!termDocs.skipTo(target))
-              return doc = NO_MORE_DOCS;
-            while (!matchDoc(doc = termDocs.doc())) { 
-              if (!termDocs.next())
-                return doc = NO_MORE_DOCS;
+        }
+
+        public int advance(int target) {
+          try {
+            doc = target;
+            while (!matchDoc(doc)) {
+              doc++;
             }
             return doc;
+          } catch (ArrayIndexOutOfBoundsException e) {
+            return doc = NO_MORE_DOCS;
           }
-        };
-      } else {
-        // a DocIdSetIterator generating docIds by incrementing a variable -
-        // this one can be used if there are no deletions are on the index
-        return new DocIdSetIterator() {
-          private int doc = -1;
-          
-          public int docID() {
-            return doc;
-          }
-          
-          public int nextDoc() {
-            try {
-              do {
-                doc++;
-              } while (!matchDoc(doc));
-              return doc;
-            } catch (ArrayIndexOutOfBoundsException e) {
-              return doc = NO_MORE_DOCS;
-            }
-          }
-          
-          public int advance(int target) {
-            try {
-              doc = target;
-              while (!matchDoc(doc)) { 
-                doc++;
-              }
-              return doc;
-            } catch (ArrayIndexOutOfBoundsException e) {
-              return doc = NO_MORE_DOCS;
-            }
-          }
-        };
-      }
+
+        }
+      };
     }
   }
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FilteredTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FilteredTermEnum.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FilteredTermEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FilteredTermEnum.java Tue Oct 13 20:44:51 2009
@@ -24,7 +24,10 @@
 /** Abstract class for enumerating a subset of all terms. 
 
   <p>Term enumerations are always ordered by Term.compareTo().  Each term in
-  the enumeration is greater than all that precede it.  */
+  the enumeration is greater than all that precede it.
+
+  @deprecated Switch to {@link FilteredTermsEnum} instead.
+*/
 public abstract class FilteredTermEnum extends TermEnum {
     /** the current term */
     protected Term currentTerm = null;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FuzzyQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FuzzyQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FuzzyQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FuzzyQuery.java Tue Oct 13 20:44:51 2009
@@ -109,10 +109,15 @@
     return prefixLength;
   }
 
+  // @deprecated see #getTermsEnum
   protected FilteredTermEnum getEnum(IndexReader reader) throws IOException {
     return new FuzzyTermEnum(reader, getTerm(), minimumSimilarity, prefixLength);
   }
   
+  protected FilteredTermsEnum getTermsEnum(IndexReader reader) throws IOException {
+    return new FuzzyTermsEnum(reader, getTerm(), minimumSimilarity, prefixLength);
+  }
+  
   /**
    * Returns the pattern term.
    */

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FuzzyTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FuzzyTermEnum.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FuzzyTermEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FuzzyTermEnum.java Tue Oct 13 20:44:51 2009
@@ -27,6 +27,8 @@
  *
  * <p>Term enumerations are always ordered by Term.compareTo().  Each term in
  * the enumeration is greater than all that precede it.
+ *
+ * @deprecated Please use {@link FuzzyTermsEnum} instead.
  */
 public final class FuzzyTermEnum extends FilteredTermEnum {
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MatchAllDocsQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MatchAllDocsQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MatchAllDocsQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MatchAllDocsQuery.java Tue Oct 13 20:44:51 2009
@@ -19,8 +19,8 @@
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
 import org.apache.lucene.util.ToStringUtils;
+import org.apache.lucene.util.Bits;
 
 import java.util.Set;
 import java.io.IOException;
@@ -45,16 +45,18 @@
   }
 
   private class MatchAllScorer extends Scorer {
-    final TermDocs termDocs;
     final float score;
     final byte[] norms;
     private int doc = -1;
+    private final int maxDoc;
+    private final Bits delDocs;
     
     MatchAllScorer(IndexReader reader, Similarity similarity, Weight w,
         byte[] norms) throws IOException {
       super(similarity);
-      this.termDocs = reader.termDocs(null);
+      delDocs = reader.getDeletedDocs();
       score = w.getValue();
+      maxDoc = reader.maxDoc();
       this.norms = norms;
     }
 
@@ -67,7 +69,14 @@
     }
 
     public int nextDoc() throws IOException {
-      return doc = termDocs.next() ? termDocs.doc() : NO_MORE_DOCS;
+      doc++;
+      while(delDocs != null && doc < maxDoc && delDocs.get(doc)) {
+        doc++;
+      }
+      if (doc == maxDoc) {
+        doc = NO_MORE_DOCS;
+      }
+      return doc;
     }
     
     public float score() {
@@ -75,7 +84,8 @@
     }
 
     public int advance(int target) throws IOException {
-      return doc = termDocs.skipTo(target) ? termDocs.doc() : NO_MORE_DOCS;
+      doc = target-1;
+      return nextDoc();
     }
   }
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiPhraseQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiPhraseQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiPhraseQuery.java Tue Oct 13 20:44:51 2009
@@ -21,10 +21,13 @@
 import java.util.*;
 
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.MultipleTermPositions;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.PositionsEnum;
+import org.apache.lucene.index.TermRef;
 import org.apache.lucene.util.ToStringUtils;
+import org.apache.lucene.util.PriorityQueue;
+import org.apache.lucene.util.Bits;
 
 /**
  * MultiPhraseQuery is a generalized version of PhraseQuery, with an added
@@ -113,7 +116,7 @@
   }
 
   // inherit javadoc
-  public void extractTerms(Set<Term> terms) {
+  public void extractTerms(Set terms) {
     for (Iterator iter = termArrays.iterator(); iter.hasNext();) {
       Term[] arr = (Term[])iter.next();
       for (int i=0; i<arr.length; i++) {
@@ -162,27 +165,31 @@
       if (termArrays.size() == 0)                  // optimize zero-term case
         return null;
 
-      TermPositions[] tps = new TermPositions[termArrays.size()];
-      for (int i=0; i<tps.length; i++) {
+      DocsEnum[] docs = new DocsEnum[termArrays.size()];
+      for (int i=0; i<docs.length; i++) {
         Term[] terms = (Term[])termArrays.get(i);
 
-        TermPositions p;
-        if (terms.length > 1)
-          p = new MultipleTermPositions(reader, terms);
-        else
-          p = reader.termPositions(terms[0]);
+        final DocsEnum docsEnum;
+        if (terms.length > 1) {
+          docsEnum = new UnionDocsEnum(reader, terms);
+        } else {
+          docsEnum = reader.termDocsEnum(reader.getDeletedDocs(),
+                                         terms[0].field(),
+                                         new TermRef(terms[0].text()));
+        }
 
-        if (p == null)
+        if (docsEnum == null) {
           return null;
+        }
 
-        tps[i] = p;
+        docs[i] = docsEnum;
       }
 
       if (slop == 0)
-        return new ExactPhraseScorer(this, tps, getPositions(), similarity,
+        return new ExactPhraseScorer(this, docs, getPositions(), similarity,
                                      reader.norms(field));
       else
-        return new SloppyPhraseScorer(this, tps, getPositions(), similarity,
+        return new SloppyPhraseScorer(this, docs, getPositions(), similarity,
                                       slop, reader.norms(field));
     }
 
@@ -371,3 +378,187 @@
     return true;
   }
 }
+
+/**
+ * Takes the logical union of multiple DocsEnum iterators.
+ */
+
+class UnionDocsEnum extends DocsEnum {
+
+  private final static class DocsEnumWrapper {
+    int doc;
+    final DocsEnum docsEnum;
+    public DocsEnumWrapper(DocsEnum docsEnum) {
+      this.docsEnum = docsEnum;
+    }
+  }
+
+  private static final class DocsQueue extends PriorityQueue {
+    DocsQueue(List docsEnums) throws IOException {
+      initialize(docsEnums.size());
+
+      Iterator i = docsEnums.iterator();
+      while (i.hasNext()) {
+        DocsEnumWrapper docs = (DocsEnumWrapper) i.next();
+        docs.doc = docs.docsEnum.next();
+        if (docs.doc != DocsEnum.NO_MORE_DOCS) {
+          add(docs);
+        }
+      }
+    }
+
+    final public DocsEnumWrapper peek() {
+      return (DocsEnumWrapper) top();
+    }
+
+    public final boolean lessThan(Object a, Object b) {
+      return ((DocsEnumWrapper) a).doc < ((DocsEnumWrapper) b).doc;
+    }
+  }
+
+  private static final class IntQueue {
+    private int _arraySize = 16;
+    private int _index = 0;
+    private int _lastIndex = 0;
+    private int[] _array = new int[_arraySize];
+
+    final void add(int i) {
+      if (_lastIndex == _arraySize)
+        growArray();
+
+      _array[_lastIndex++] = i;
+    }
+
+    final int next() {
+      return _array[_index++];
+    }
+
+    final void sort() {
+      Arrays.sort(_array, _index, _lastIndex);
+    }
+
+    final void clear() {
+      _index = 0;
+      _lastIndex = 0;
+    }
+
+    final int size() {
+      return (_lastIndex - _index);
+    }
+
+    private void growArray() {
+      int[] newArray = new int[_arraySize * 2];
+      System.arraycopy(_array, 0, newArray, 0, _arraySize);
+      _array = newArray;
+      _arraySize *= 2;
+    }
+  }
+
+  private int _doc;
+  private int _freq;
+  private DocsQueue _queue;
+  private IntQueue _posList;
+
+  private final UnionPositionsEnum unionPositionsEnum;
+
+  public UnionDocsEnum(IndexReader indexReader, Term[] terms) throws IOException {
+    List docsEnums = new LinkedList();
+    final Bits delDocs = indexReader.getDeletedDocs();
+
+    for (int i = 0; i < terms.length; i++) {
+      DocsEnum docs = indexReader.termDocsEnum(delDocs,
+                                               terms[i].field(),
+                                               new TermRef(terms[i].text()));
+      if (docs != null) {
+        docsEnums.add(new DocsEnumWrapper(docs));
+      }
+    }
+
+    _queue = new DocsQueue(docsEnums);
+    _posList = new IntQueue();
+    unionPositionsEnum = new UnionPositionsEnum();
+  }
+
+  public PositionsEnum positions() {
+    return unionPositionsEnum;
+  }
+
+  public final int next() throws IOException {
+    if (_queue.size() == 0) {
+      return NO_MORE_DOCS;
+    }
+
+    // TODO: move this init into positions(): if the search
+    // doesn't need the positions for this doc then don't
+    // waste CPU merging them:
+    _posList.clear();
+    _doc = _queue.peek().doc;
+
+    // merge sort all positions together
+    DocsEnumWrapper docs;
+    do {
+      docs = _queue.peek();
+      final PositionsEnum positions = docs.docsEnum.positions();
+
+      final int freq = docs.docsEnum.freq();
+      for (int i = 0; i < freq; i++) {
+        _posList.add(positions.next());
+      }
+
+      docs.doc = docs.docsEnum.next();
+
+      if (docs.doc != NO_MORE_DOCS) {
+        _queue.updateTop();
+      } else {
+        _queue.pop();
+      }
+    } while (_queue.size() > 0 && _queue.peek().doc == _doc);
+
+    _posList.sort();
+    _freq = _posList.size();
+
+    return _doc;
+  }
+
+  private class UnionPositionsEnum extends PositionsEnum {
+
+    public int next() {
+      return _posList.next();
+    }
+
+    public int getPayloadLength() {
+      throw new UnsupportedOperationException();
+    }
+
+    public byte[] getPayload(byte[] data, int offset) {
+      throw new UnsupportedOperationException();
+    }
+
+    public boolean hasPayload() {
+      throw new UnsupportedOperationException();
+    }
+  }
+
+  public final int advance(int target) throws IOException {
+    while (_queue.peek() != null && target > _queue.peek().doc) {
+      DocsEnumWrapper docs = (DocsEnumWrapper) _queue.pop();
+      docs.doc = docs.docsEnum.advance(target);
+      if (docs.doc != NO_MORE_DOCS) {
+        _queue.add(docs);
+      }
+    }
+    return next();
+  }
+
+  public final int freq() {
+    return _freq;
+  }
+
+  /**
+   * Not implemented.
+   * @throws UnsupportedOperationException
+   */
+  public int read(int[] arg0, int[] arg1) throws IOException {
+    throw new UnsupportedOperationException();
+  }
+}

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQuery.java Tue Oct 13 20:44:51 2009
@@ -25,6 +25,8 @@
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermRef;
+import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.util.ToStringUtils;
 import org.apache.lucene.queryParser.QueryParser; // for javadoc
 
@@ -96,24 +98,49 @@
   private static class ScoringBooleanQueryRewrite extends RewriteMethod implements Serializable {
     public Query rewrite(IndexReader reader, MultiTermQuery query) throws IOException {
 
-      FilteredTermEnum enumerator = query.getEnum(reader);
-      BooleanQuery result = new BooleanQuery(true);
-      int count = 0;
-      try {
-        do {
-          Term t = enumerator.term();
-          if (t != null) {
-            TermQuery tq = new TermQuery(t); // found a match
-            tq.setBoost(query.getBoost() * enumerator.difference()); // set the boost
+      FilteredTermsEnum termsEnum = query.getTermsEnum(reader);
+      if (termsEnum != null) {
+
+        // nocommit -- if no terms we'd want to return NullQuery
+        BooleanQuery result = new BooleanQuery(true);
+        if (!termsEnum.empty()) {
+          final String field = termsEnum.field();
+          assert field != null;
+          int count = 0;
+          TermRef term = termsEnum.term();
+          // first term must exist since termsEnum wasn't null
+          assert term != null;
+          do {
+            TermQuery tq = new TermQuery(new Term(field, term.toString())); // found a match
+            tq.setBoost(query.getBoost() * termsEnum.difference()); // set the boost
             result.add(tq, BooleanClause.Occur.SHOULD); // add to query
             count++;
-          }
-        } while (enumerator.next());    
-      } finally {
-        enumerator.close();
+            term = termsEnum.next();
+          } while(term != null);
+          query.incTotalNumberOfTerms(count);
+        }
+        return result;
+      } else {
+        // deprecated case
+        FilteredTermEnum enumerator = query.getEnum(reader);
+        BooleanQuery result = new BooleanQuery(true);
+        int count = 0;
+        try {
+          do {
+            Term t = enumerator.term();
+            if (t != null) {
+              TermQuery tq = new TermQuery(t); // found a match
+              tq.setBoost(query.getBoost() * enumerator.difference()); // set the boost
+              result.add(tq, BooleanClause.Occur.SHOULD); // add to query
+              count++;
+            }
+          } while (enumerator.next());    
+        } finally {
+          enumerator.close();
+        }
+        query.incTotalNumberOfTerms(count);
+        return result;
       }
-      query.incTotalNumberOfTerms(count);
-      return result;
     }
 
     // Make sure we are still a singleton even after deserializing
@@ -213,6 +240,7 @@
     }
 
     public Query rewrite(IndexReader reader, MultiTermQuery query) throws IOException {
+
       // Get the enum and start visiting terms.  If we
       // exhaust the enum before hitting either of the
       // cutoffs, we use ConstantBooleanQueryRewrite; else,
@@ -222,53 +250,97 @@
       final int termCountLimit = Math.min(BooleanQuery.getMaxClauseCount(), termCountCutoff);
       int docVisitCount = 0;
 
-      FilteredTermEnum enumerator = query.getEnum(reader);
-      try {
-        while(true) {
-          Term t = enumerator.term();
-          if (t != null) {
-            pendingTerms.add(t);
+      FilteredTermsEnum termsEnum = query.getTermsEnum(reader);
+      if (termsEnum != null) {
+        if (!termsEnum.empty()) {
+          final String field = termsEnum.field();
+          assert field != null;
+          TermRef term = termsEnum.term();
+          // first term must exist since termsEnum wasn't null
+          assert term != null;
+          do {
+            pendingTerms.add(term.clone());
+            if (pendingTerms.size() >= termCountLimit || docVisitCount >= docCountCutoff) {
+              // Too many terms -- cut our losses now and make a filter.
+              Query result = new ConstantScoreQuery(new MultiTermQueryWrapperFilter(query));
+              result.setBoost(query.getBoost());
+              return result;
+            }
             // Loading the TermInfo from the terms dict here
             // should not be costly, because 1) the
             // query/filter will load the TermInfo when it
             // runs, and 2) the terms dict has a cache:
-            docVisitCount += reader.docFreq(t);
+            docVisitCount += reader.docFreq(field, term);
+            term = termsEnum.next();
+          } while(term != null);
+        
+          // Enumeration is done, and we hit a small
+          // enough number of terms & docs -- just make a
+          // BooleanQuery, now
+          Iterator it = pendingTerms.iterator();
+          BooleanQuery bq = new BooleanQuery(true);
+          while(it.hasNext()) {
+            TermQuery tq = new TermQuery(new Term(field, ((TermRef) it.next()).toString()));
+            bq.add(tq, BooleanClause.Occur.SHOULD);
           }
+          // Strip scores
+          Query result = new ConstantScoreQuery(new QueryWrapperFilter(bq));
+          result.setBoost(query.getBoost());
+          query.incTotalNumberOfTerms(pendingTerms.size());
+          return result;
+        } else {
+          // nocommit -- need NullQuery here
+          return new BooleanQuery();
+        }
+      } else {
 
-          if (pendingTerms.size() >= termCountLimit || docVisitCount >= docCountCutoff) {
-            // Too many terms -- make a filter.
-            Query result = new ConstantScoreQuery(new MultiTermQueryWrapperFilter(query));
-            result.setBoost(query.getBoost());
-            return result;
-          } else  if (!enumerator.next()) {
-            // Enumeration is done, and we hit a small
-            // enough number of terms & docs -- just make a
-            // BooleanQuery, now
-            Iterator it = pendingTerms.iterator();
-            BooleanQuery bq = new BooleanQuery(true);
-            while(it.hasNext()) {
-              TermQuery tq = new TermQuery((Term) it.next());
-              bq.add(tq, BooleanClause.Occur.SHOULD);
+        // deprecated case
+        FilteredTermEnum enumerator = query.getEnum(reader);
+        try {
+          while(true) {
+            Term t = enumerator.term();
+            if (t != null) {
+              pendingTerms.add(t);
+              // Loading the TermInfo from the terms dict here
+              // should not be costly, because 1) the
+              // query/filter will load the TermInfo when it
+              // runs, and 2) the terms dict has a cache:
+              docVisitCount += reader.docFreq(t);
+            }
+
+            if (pendingTerms.size() >= termCountLimit || docVisitCount >= docCountCutoff) {
+              // Too many terms -- make a filter.
+              Query result = new ConstantScoreQuery(new MultiTermQueryWrapperFilter(query));
+              result.setBoost(query.getBoost());
+              return result;
+            } else  if (!enumerator.next()) {
+              // Enumeration is done, and we hit a small
+              // enough number of terms & docs -- just make a
+              // BooleanQuery, now
+              Iterator it = pendingTerms.iterator();
+              BooleanQuery bq = new BooleanQuery(true);
+              while(it.hasNext()) {
+                TermQuery tq = new TermQuery((Term) it.next());
+                bq.add(tq, BooleanClause.Occur.SHOULD);
+              }
+              // Strip scores
+              Query result = new ConstantScoreQuery(new QueryWrapperFilter(bq));
+              result.setBoost(query.getBoost());
+              query.incTotalNumberOfTerms(pendingTerms.size());
+              return result;
             }
-            // Strip scores
-            Query result = new ConstantScoreQuery(new QueryWrapperFilter(bq));
-            result.setBoost(query.getBoost());
-            query.incTotalNumberOfTerms(pendingTerms.size());
-            return result;
           }
+        } finally {
+          enumerator.close();
         }
-      } finally {
-        enumerator.close();
       }
     }
     
-    @Override
     public int hashCode() {
       final int prime = 1279;
       return (int) (prime * termCountCutoff + Double.doubleToLongBits(docCountPercent));
     }
 
-    @Override
     public boolean equals(Object obj) {
       if (this == obj)
         return true;
@@ -326,9 +398,25 @@
   public MultiTermQuery() {
   }
 
-  /** Construct the enumeration to be used, expanding the pattern term. */
-  protected abstract FilteredTermEnum getEnum(IndexReader reader)
-      throws IOException;
+  /** Construct the enumeration to be used, expanding the
+   * pattern term.
+   * @deprecated Please override {@link #getTermsEnum} instead */
+  protected FilteredTermEnum getEnum(IndexReader reader)
+    throws IOException {
+    return null;
+  }
+
+  /** Construct the enumeration to be used, expanding the
+   *  pattern term.  This method must return null if no
+   *  terms fall in the range; else, it must return a
+   *  TermsEnum already positioned to the first matching
+   *  term.
+   *
+   *  nocommit in 3.x this will become abstract  */
+  protected FilteredTermsEnum getTermsEnum(IndexReader reader)
+    throws IOException {
+    return null;
+  }
 
   /**
    * Expert: Return the number of unique terms visited during execution of the query.

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java Tue Oct 13 20:44:51 2009
@@ -23,7 +23,11 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermDocs;
 import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.index.TermRef;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.Bits;
 
 /**
  * A wrapper for {@link MultiTermQuery}, that exposes its
@@ -95,6 +99,7 @@
   }
 
   abstract class TermGenerator {
+    // @deprecated
     public void generate(IndexReader reader, TermEnum enumerator) throws IOException {
       final int[] docs = new int[32];
       final int[] freqs = new int[32];
@@ -125,6 +130,38 @@
         termDocs.close();
       }
     }
+
+    public void generate(IndexReader reader, TermsEnum enumerator) throws IOException {
+      //System.out.println("mtq.filter generate");
+      final int[] docs = new int[32];
+      final int[] freqs = new int[32];
+      int termCount = 0;
+      final Bits delDocs = reader.getDeletedDocs();
+      while(true) {
+        termCount++;
+        //System.out.println("  iter termCount=" + termCount + " term=" + enumerator.term().toBytesString());
+        DocsEnum docsEnum = enumerator.docs(delDocs);
+        while (true) {
+          final int count = docsEnum.read(docs, freqs);
+          if (count != 0) {
+            for(int i=0;i<count;i++) {
+              handleDoc(docs[i]);
+            }
+          } else {
+            break;
+          }
+        }
+        TermRef term = enumerator.next();
+        if (term == null) {
+          break;
+        }
+        //System.out.println("  enum next term=" + term.toBytesString());
+        assert term.termEquals(enumerator.term());
+      }
+      //System.out.println("  done termCount=" + termCount);
+
+      query.incTotalNumberOfTerms(termCount);
+    }
     abstract public void handleDoc(int doc);
   }
   
@@ -134,21 +171,37 @@
    */
   @Override
   public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
-    final TermEnum enumerator = query.getEnum(reader);
-    try {
-      // if current term in enum is null, the enum is empty -> shortcut
-      if (enumerator.term() == null)
+    final FilteredTermsEnum termsEnum = query.getTermsEnum(reader);
+    if (termsEnum != null) {
+      if (!termsEnum.empty()) {
+        // fill into a OpenBitSet
+        final OpenBitSet bitSet = new OpenBitSet(reader.maxDoc());
+        new TermGenerator() {
+          public void handleDoc(int doc) {
+            bitSet.set(doc);
+          }
+        }.generate(reader, termsEnum);
+        return bitSet;
+      } else {
         return DocIdSet.EMPTY_DOCIDSET;
-      // else fill into a OpenBitSet
-      final OpenBitSet bitSet = new OpenBitSet(reader.maxDoc());
-      new TermGenerator() {
-        public void handleDoc(int doc) {
-          bitSet.set(doc);
-        }
-      }.generate(reader, enumerator);
-      return bitSet;
-    } finally {
-      enumerator.close();
+      }
+    } else {
+      final TermEnum enumerator = query.getEnum(reader);
+      try {
+        // if current term in enum is null, the enum is empty -> shortcut
+        if (enumerator.term() == null)
+          return DocIdSet.EMPTY_DOCIDSET;
+        // else fill into a OpenBitSet
+        final OpenBitSet bitSet = new OpenBitSet(reader.maxDoc());
+        new TermGenerator() {
+          public void handleDoc(int doc) {
+            bitSet.set(doc);
+          }
+        }.generate(reader, enumerator);
+        return bitSet;
+      } finally {
+        enumerator.close();
+      }
     }
   }
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/NumericRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/NumericRangeQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/NumericRangeQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/NumericRangeQuery.java Tue Oct 13 20:44:51 2009
@@ -27,6 +27,8 @@
 import org.apache.lucene.util.StringHelper;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermRef;
+import org.apache.lucene.index.Terms;
 
 /**
  * <p>A {@link Query} that matches numeric values within a
@@ -305,6 +307,10 @@
     return new NumericRangeTermEnum(reader);
   }
 
+  protected FilteredTermsEnum getTermsEnum(final IndexReader reader) throws IOException {
+    return new NumericRangeTermsEnum(reader);
+  }
+
   /** Returns the field name for this query */
   public String getField() { return field; }
 
@@ -378,7 +384,11 @@
    * The ordering depends on how {@link NumericUtils#splitLongRange} and
    * {@link NumericUtils#splitIntRange} generates the sub-ranges. For
    * {@link MultiTermQuery} ordering is not relevant.
+   *
+   * @deprecated use NumericRangeTermsEnum instead
    */
+  // nocommit -- can we remove this?  only back compat
+  // concern would be subclasses of NRQ that invoke getEnum
   private final class NumericRangeTermEnum extends FilteredTermEnum {
 
     private final IndexReader reader;
@@ -532,5 +542,174 @@
     }
 
   }
+
+
+  /**
+   * Subclass of FilteredTermsEnum for enumerating all terms that match the
+   * sub-ranges for trie range queries, using flex API.
+   * <p>
+   * WARNING: This term enumeration is not guaranteed to be always ordered by
+   * {@link Term#compareTo}.
+   * The ordering depends on how {@link NumericUtils#splitLongRange} and
+   * {@link NumericUtils#splitIntRange} generates the sub-ranges. For
+   * {@link MultiTermQuery} ordering is not relevant.
+   */
+  private final class NumericRangeTermsEnum extends FilteredTermsEnum {
+
+    private final IndexReader reader;
+    private final LinkedList<String> rangeBounds = new LinkedList<String>();
+    private TermRef currentUpperBound = null;
+    private final boolean empty;
+
+    NumericRangeTermsEnum(final IndexReader reader) throws IOException {
+      this.reader = reader;
+      
+      switch (valSize) {
+        case 64: {
+          // lower
+          long minBound = Long.MIN_VALUE;
+          if (min instanceof Long) {
+            minBound = min.longValue();
+          } else if (min instanceof Double) {
+            minBound = NumericUtils.doubleToSortableLong(min.doubleValue());
+          }
+          if (!minInclusive && min != null) {
+            if (minBound == Long.MAX_VALUE) break;
+            minBound++;
+          }
+          
+          // upper
+          long maxBound = Long.MAX_VALUE;
+          if (max instanceof Long) {
+            maxBound = max.longValue();
+          } else if (max instanceof Double) {
+            maxBound = NumericUtils.doubleToSortableLong(max.doubleValue());
+          }
+          if (!maxInclusive && max != null) {
+            if (maxBound == Long.MIN_VALUE) break;
+            maxBound--;
+          }
+          
+          NumericUtils.splitLongRange(new NumericUtils.LongRangeBuilder() {
+            //@Override
+            public final void addRange(String minPrefixCoded, String maxPrefixCoded) {
+              rangeBounds.add(minPrefixCoded);
+              rangeBounds.add(maxPrefixCoded);
+            }
+          }, precisionStep, minBound, maxBound);
+          break;
+        }
+          
+        case 32: {
+          // lower
+          int minBound = Integer.MIN_VALUE;
+          if (min instanceof Integer) {
+            minBound = min.intValue();
+          } else if (min instanceof Float) {
+            minBound = NumericUtils.floatToSortableInt(min.floatValue());
+          }
+          if (!minInclusive && min != null) {
+            if (minBound == Integer.MAX_VALUE) break;
+            minBound++;
+          }
+          
+          // upper
+          int maxBound = Integer.MAX_VALUE;
+          if (max instanceof Integer) {
+            maxBound = max.intValue();
+          } else if (max instanceof Float) {
+            maxBound = NumericUtils.floatToSortableInt(max.floatValue());
+          }
+          if (!maxInclusive && max != null) {
+            if (maxBound == Integer.MIN_VALUE) break;
+            maxBound--;
+          }
+          
+          NumericUtils.splitIntRange(new NumericUtils.IntRangeBuilder() {
+            //@Override
+            public final void addRange(String minPrefixCoded, String maxPrefixCoded) {
+              rangeBounds.add(minPrefixCoded);
+              rangeBounds.add(maxPrefixCoded);
+            }
+          }, precisionStep, minBound, maxBound);
+          break;
+        }
+          
+        default:
+          // should never happen
+          throw new IllegalArgumentException("valSize must be 32 or 64");
+      }
+      
+      // seek to first term
+      empty = next() == null;
+    }
+
+    @Override
+    public float difference() {
+      return 1.0f;
+    }
+    
+    /** this is a dummy, it is not used by this class. */
+    @Override
+    public boolean empty() {
+      return empty;
+    }
+
+    public String field() {
+      return field;
+    }
+
+    /**
+     * Compares if current upper bound is reached,
+     * this also updates the term count for statistics.
+     * In contrast to {@link FilteredTermEnum}, a return value
+     * of <code>false</code> ends iterating the current enum
+     * and forwards to the next sub-range.
+     */
+    @Override
+    protected boolean accept(TermRef term) {
+      return (term.compareTerm(currentUpperBound) <= 0);
+    }
+
+    /** Increments the enumeration to the next element.  True if one exists. */
+    @Override
+    public TermRef next() throws IOException {
+      //System.out.println("nrq.next");
+      // if the actual enum is initialized, try change to
+      // next term, if no such term exists, fall-through
+      if (actualEnum != null) {
+        TermRef term = actualEnum.next();
+        if (term != null && accept(term)) {
+          //System.out.println("  return term=" + term.toBytesString());
+          return term;
+        }
+      }
+
+      //System.out.println("  ranges = " + rangeBounds.size());
+
+      // if all above fails, we go forward to the next enum,
+      // if one is available
+      if (rangeBounds.size() < 2) {
+        assert rangeBounds.size() == 0;
+        //System.out.println("  return null0");
+        return null;
+      }
+
+      final TermRef lowerBound = new TermRef(rangeBounds.removeFirst());
+      this.currentUpperBound = new TermRef(rangeBounds.removeFirst());
+
+      // this call recursively uses next(), if no valid term in
+      // next enum found.
+      // if this behavior is changed/modified in the superclass,
+      // this enum will not work anymore!
+      Terms terms = reader.fields().terms(field);
+      if (terms != null) {
+        return setEnum(terms.iterator(), lowerBound);
+      } else {
+        //System.out.println("  return null");
+        return null;
+      }
+    }
+  }
   
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhrasePositions.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhrasePositions.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhrasePositions.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhrasePositions.java Tue Oct 13 20:44:51 2009
@@ -28,40 +28,43 @@
   int position;					  // position in doc
   int count;					  // remaining pos in this doc
   int offset;					  // position in phrase
-  TermPositions tp;				  // stream of positions
-  PhrasePositions next;				  // used to make lists
+  final DocsEnum docs;				  // stream of docs
+  PositionsEnum positions;                        // positions in current doc
+  PhrasePositions next;	                          // used to make lists
   boolean repeats;       // there's other pp for same term (e.g. query="1st word 2nd word"~1) 
 
-  PhrasePositions(TermPositions t, int o) {
-    tp = t;
+  PhrasePositions(DocsEnum docs, int o) {
+    this.docs = docs;
     offset = o;
   }
 
   final boolean next() throws IOException {	  // increments to next doc
-    if (!tp.next()) {
-      tp.close();				  // close stream
-      doc = Integer.MAX_VALUE;			  // sentinel value
+    doc = docs.next();
+    if (doc == docs.NO_MORE_DOCS) {
       return false;
     }
-    doc = tp.doc();
-    position = 0;
+    positions = docs.positions();
+
+    // nocommit -- really needed?
+    //position = 0;
+
     return true;
   }
 
   final boolean skipTo(int target) throws IOException {
-    if (!tp.skipTo(target)) {
-      tp.close();				  // close stream
-      doc = Integer.MAX_VALUE;			  // sentinel value
+    doc = docs.advance(target);
+    if (doc == docs.NO_MORE_DOCS) {
       return false;
     }
-    doc = tp.doc();
-    position = 0;
+    // nocommit -- really needed?
+    // position = 0;
     return true;
   }
 
 
   final void firstPosition() throws IOException {
-    count = tp.freq();				  // read first pos
+    count = docs.freq();				  // read first pos
+    positions = docs.positions();
     nextPosition();
   }
 
@@ -73,7 +76,7 @@
    */
   final boolean nextPosition() throws IOException {
     if (count-- > 0) {				  // read subsequent pos's
-      position = tp.nextPosition() - offset;
+      position = positions.next() - offset;
       return true;
     } else
       return false;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseQuery.java Tue Oct 13 20:44:51 2009
@@ -22,10 +22,12 @@
 import java.util.ArrayList;
 
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.index.TermRef;
+import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.Explanation.IDFExplanation;
 import org.apache.lucene.util.ToStringUtils;
+import org.apache.lucene.util.Bits;
 
 /** A Query that matches documents containing a particular sequence of terms.
  * A PhraseQuery is built by QueryParser for input like <code>"new york"</code>.
@@ -143,20 +145,25 @@
       if (terms.size() == 0)			  // optimize zero-term case
         return null;
 
-      TermPositions[] tps = new TermPositions[terms.size()];
+      DocsEnum[] docs = new DocsEnum[terms.size()];
+      final Bits delDocs = reader.getDeletedDocs();
       for (int i = 0; i < terms.size(); i++) {
-        TermPositions p = reader.termPositions((Term)terms.get(i));
-        if (p == null)
+        final Term t = (Term) terms.get(i);
+        DocsEnum docsEnum = reader.termDocsEnum(delDocs,
+                                                t.field(),
+                                                new TermRef(t.text()));
+        if (docsEnum == null) {
           return null;
-        tps[i] = p;
+        }
+        docs[i] = docsEnum;
       }
 
       if (slop == 0)				  // optimize exact case
-        return new ExactPhraseScorer(this, tps, getPositions(), similarity,
+        return new ExactPhraseScorer(this, docs, getPositions(), similarity,
                                      reader.norms(field));
       else
         return
-          new SloppyPhraseScorer(this, tps, getPositions(), similarity, slop,
+          new SloppyPhraseScorer(this, docs, getPositions(), similarity, slop,
                                  reader.norms(field));
 
     }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseScorer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseScorer.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseScorer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseScorer.java Tue Oct 13 20:44:51 2009
@@ -19,7 +19,7 @@
 
 import java.io.IOException;
 
-import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.index.DocsEnum;
 
 /** Expert: Scoring functionality for phrase queries.
  * <br>A document is considered matching if it contains the phrase-query terms  
@@ -43,7 +43,7 @@
 
   private float freq; //phrase frequency in current doc as computed by phraseFreq().
 
-  PhraseScorer(Weight weight, TermPositions[] tps, int[] offsets,
+  PhraseScorer(Weight weight, DocsEnum[] docs, int[] offsets,
       Similarity similarity, byte[] norms) {
     super(similarity);
     this.norms = norms;
@@ -55,8 +55,8 @@
     // reflects the phrase offset: pp.pos = tp.pos - offset.
     // this allows to easily identify a matching (exact) phrase 
     // when all PhrasePositions have exactly the same position.
-    for (int i = 0; i < tps.length; i++) {
-      PhrasePositions pp = new PhrasePositions(tps[i], offsets[i]);
+    for (int i = 0; i < docs.length; i++) {
+      PhrasePositions pp = new PhrasePositions(docs[i], offsets[i]);
       if (last != null) {			  // add next to end of list
         last.next = pp;
       } else {
@@ -65,7 +65,7 @@
       last = pp;
     }
 
-    pq = new PhraseQueue(tps.length);             // construct empty pq
+    pq = new PhraseQueue(docs.length);             // construct empty pq
     first.doc = -1;
   }
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PrefixQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PrefixQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PrefixQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PrefixQuery.java Tue Oct 13 20:44:51 2009
@@ -40,8 +40,8 @@
   /** Returns the prefix of this query. */
   public Term getPrefix() { return prefix; }
   
-  protected FilteredTermEnum getEnum(IndexReader reader) throws IOException {
-    return new PrefixTermEnum(reader, prefix);
+  protected FilteredTermsEnum getTermsEnum(IndexReader reader) throws IOException {
+    return new PrefixTermsEnum(reader, prefix);
   }
 
   /** Prints a user-readable version of this query. */

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PrefixTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PrefixTermEnum.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PrefixTermEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PrefixTermEnum.java Tue Oct 13 20:44:51 2009
@@ -29,6 +29,7 @@
  * Term enumerations are always ordered by Term.compareTo().  Each term in
  * the enumeration is greater than all that precede it.
  *
+ * @deprecated Use {@link PrefixTermsEnum} instead.
  */
 public class PrefixTermEnum extends FilteredTermEnum {
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/SloppyPhraseScorer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/SloppyPhraseScorer.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/SloppyPhraseScorer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/SloppyPhraseScorer.java Tue Oct 13 20:44:51 2009
@@ -17,7 +17,7 @@
  * limitations under the License.
  */
 
-import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.index.DocsEnum;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -28,9 +28,9 @@
     private PhrasePositions tmpPos[]; // for flipping repeating pps.
     private boolean checkedRepeats;
 
-    SloppyPhraseScorer(Weight weight, TermPositions[] tps, int[] offsets, Similarity similarity,
+    SloppyPhraseScorer(Weight weight, DocsEnum[] docs, int[] offsets, Similarity similarity,
                        int slop, byte[] norms) {
-        super(weight, tps, offsets, similarity, norms);
+        super(weight, docs, offsets, similarity, norms);
         this.slop = slop;
     }
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermQuery.java Tue Oct 13 20:44:51 2009
@@ -20,8 +20,9 @@
 import java.io.IOException;
 import java.util.Set;
 
+import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.TermRef;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.Explanation.IDFExplanation;
 import org.apache.lucene.util.ToStringUtils;
@@ -64,12 +65,12 @@
     }
 
     public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException {
-      TermDocs termDocs = reader.termDocs(term);
-
-      if (termDocs == null)
+      DocsEnum docs = reader.termDocsEnum(reader.getDeletedDocs(), term.field(), new TermRef(term.text()));
+      if (docs == null) {
         return null;
+      }
 
-      return new TermScorer(this, termDocs, similarity, reader.norms(term.field()));
+      return new TermScorer(this, docs, similarity, reader.norms(term.field()));
     }
 
     public Explanation explain(IndexReader reader, int doc)

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeQuery.java Tue Oct 13 20:44:51 2009
@@ -135,6 +135,17 @@
         upperTerm, includeLower, includeUpper, collator);
   }
 
+  public String field() {
+    return field;
+  }
+
+  protected FilteredTermsEnum getTermsEnum(IndexReader reader) throws IOException {
+    return new TermRangeTermsEnum(reader, field,
+                                  lowerTerm, upperTerm,
+                                  includeLower, includeUpper,
+                                  collator);
+  }
+
   /** Prints a user-readable version of this query. */
   public String toString(String field) {
       StringBuilder buffer = new StringBuilder();

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeTermEnum.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeTermEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeTermEnum.java Tue Oct 13 20:44:51 2009
@@ -31,6 +31,7 @@
  * Term enumerations are always ordered by Term.compareTo().  Each term in
  * the enumeration is greater than all that precede it.
  * @since 2.9
+ * @deprecated Please switch to {@link TermRangeTermsEnum}
  */
 public class TermRangeTermEnum extends FilteredTermEnum {
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermScorer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermScorer.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermScorer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermScorer.java Tue Oct 13 20:44:51 2009
@@ -19,7 +19,7 @@
 
 import java.io.IOException;
 
-import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.DocsEnum;
 
 /** Expert: A <code>Scorer</code> for documents matching a <code>Term</code>.
  */
@@ -28,7 +28,7 @@
   private static final float[] SIM_NORM_DECODER = Similarity.getNormDecoder();
   
   private Weight weight;
-  private TermDocs termDocs;
+  private DocsEnum docsEnum;
   private byte[] norms;
   private float weightValue;
   private int doc = -1;
@@ -54,10 +54,10 @@
    * @param norms
    *          The field norms of the document fields for the <code>Term</code>.
    */
-  TermScorer(Weight weight, TermDocs td, Similarity similarity, byte[] norms) {
+  TermScorer(Weight weight, DocsEnum td, Similarity similarity, byte[] norms) {
     super(similarity);
     this.weight = weight;
-    this.termDocs = td;
+    this.docsEnum = td;
     this.norms = norms;
     this.weightValue = weight.getValue();
 
@@ -71,17 +71,17 @@
 
   // firstDocID is ignored since nextDoc() sets 'doc'
   protected boolean score(Collector c, int end, int firstDocID) throws IOException {
+    //System.out.println("top score " + firstDocID + " max=" + pointerMax);
     c.setScorer(this);
     while (doc < end) {                           // for docs in window
       c.collect(doc);                      // collect score
-        
+      //System.out.println("done collect");
       if (++pointer >= pointerMax) {
-        pointerMax = termDocs.read(docs, freqs);  // refill buffers
+        pointerMax = docsEnum.read(docs, freqs);  // refill buffers
         if (pointerMax != 0) {
           pointer = 0;
         } else {
-          termDocs.close();                       // close stream
-          doc = Integer.MAX_VALUE;                // set to sentinel value
+          doc = NO_MORE_DOCS;                // set to sentinel value
           return false;
         }
       } 
@@ -97,25 +97,28 @@
    * The iterator over the matching documents is buffered using
    * {@link TermDocs#read(int[],int[])}.
    * 
-   * @return the document matching the query or -1 if there are no more documents.
+   * @return the document matching the query or NO_MORE_DOCS if there are no more documents.
    */
   public int nextDoc() throws IOException {
+    //System.out.println("ts.nextDoc pointer=" + pointer + " max=" + pointerMax + " this=" + this + " docsEnum=" + docsEnum);
     pointer++;
     if (pointer >= pointerMax) {
-      pointerMax = termDocs.read(docs, freqs);    // refill buffer
+      pointerMax = docsEnum.read(docs, freqs);    // refill buffer
+      //System.out.println("ts set max=" + pointerMax);
       if (pointerMax != 0) {
         pointer = 0;
       } else {
-        termDocs.close();                         // close stream
+        //System.out.println("ts no more docs");
         return doc = NO_MORE_DOCS;
       }
     } 
     doc = docs[pointer];
+    assert doc != NO_MORE_DOCS;
     return doc;
   }
   
   public float score() {
-    assert doc != -1;
+    assert doc != NO_MORE_DOCS;
     int f = freqs[pointer];
     float raw =                                   // compute tf(f)*weight
       f < SCORE_CACHE_SIZE                        // check cache
@@ -128,11 +131,11 @@
   /**
    * Advances to the first match beyond the current whose document number is
    * greater than or equal to a given target. <br>
-   * The implementation uses {@link TermDocs#skipTo(int)}.
+   * The implementation uses {@link DocsEnum#adnvace(int)}.
    * 
    * @param target
    *          The target document number.
-   * @return the matching document or -1 if none exist.
+   * @return the matching document or NO_MORE_DOCS if none exist.
    */
   public int advance(int target) throws IOException {
     // first scan in cache
@@ -142,13 +145,14 @@
       }
     }
 
-    // not found in cache, seek underlying stream
-    boolean result = termDocs.skipTo(target);
-    if (result) {
+    // not found in readahead cache, seek underlying stream
+    int newDoc = docsEnum.advance(target);
+    //System.out.println("ts.advance docsEnum=" + docsEnum);
+    if (newDoc != DocsEnum.NO_MORE_DOCS) {
       pointerMax = 1;
       pointer = 0;
-      docs[pointer] = doc = termDocs.doc();
-      freqs[pointer] = termDocs.freq();
+      docs[pointer] = doc = newDoc;
+      freqs[pointer] = docsEnum.freq();
     } else {
       doc = NO_MORE_DOCS;
     }
@@ -168,15 +172,11 @@
       pointer++;
     }
     if (tf == 0) {
-        if (termDocs.skipTo(doc))
-        {
-            if (termDocs.doc() == doc)
-            {
-                tf = termDocs.freq();
-            }
-        }
+      int newDoc = docsEnum.advance(doc);
+      if (newDoc == doc) {
+        tf = docsEnum.freq();
+      }
     }
-    termDocs.close();
     tfExplanation.setValue(getSimilarity().tf(tf));
     tfExplanation.setDescription("tf(termFreq("+query.getTerm()+")="+tf+")");
     
@@ -184,5 +184,6 @@
   }
 
   /** Returns a string representation of this <code>TermScorer</code>. */
-  public String toString() { return "scorer(" + weight + ")"; }
+  // nocommit
+  //public String toString() { return "scorer(" + weight + ")"; }
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/WildcardQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/WildcardQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/WildcardQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/WildcardQuery.java Tue Oct 13 20:44:51 2009
@@ -34,7 +34,7 @@
  * MultiTermQuery#CONSTANT_SCORE_AUTO_REWRITE_DEFAULT}
  * rewrite method.
  *
- * @see WildcardTermEnum */
+ * @see WildcardTermEnums */
 public class WildcardQuery extends MultiTermQuery {
   private boolean termContainsWildcard;
   private boolean termIsPrefix;
@@ -49,7 +49,13 @@
         && (text.indexOf('?') == -1) 
         && (text.indexOf('*') == text.length() - 1);
   }
-
+  
+  // nocommit: needs singletermenum stuff
+  protected FilteredTermsEnum getTermsEnum(IndexReader reader) throws IOException {
+    return new WildcardTermsEnum(reader, getTerm());
+  }
+  
+  // @deprecated see getTermsEnum
   protected FilteredTermEnum getEnum(IndexReader reader) throws IOException {
     if (termContainsWildcard)
       return new WildcardTermEnum(reader, getTerm());

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/WildcardTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/WildcardTermEnum.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/WildcardTermEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/WildcardTermEnum.java Tue Oct 13 20:44:51 2009
@@ -28,6 +28,7 @@
  * <p>
  * Term enumerations are always ordered by Term.compareTo().  Each term in
  * the enumeration is greater than all that precede it.
+ * @deprecated Please use {@link WildcardTermsEnum} instead.
  */
 public class WildcardTermEnum extends FilteredTermEnum {
   final Term searchTerm;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/function/ValueSourceQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/function/ValueSourceQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/function/ValueSourceQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/function/ValueSourceQuery.java Tue Oct 13 20:44:51 2009
@@ -18,10 +18,10 @@
  */
 
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
 import org.apache.lucene.search.*;
+import org.apache.lucene.index.Term;
 import org.apache.lucene.util.ToStringUtils;
+import org.apache.lucene.util.Bits;
 
 import java.io.IOException;
 import java.util.Set;
@@ -58,7 +58,7 @@
     return this;
   }
 
-  /*(non-Javadoc) @see org.apache.lucene.search.Query#extractTerms(java.util.Set) */
+  /*(non-Javadoc) @see org.apache.lucene.search.Query#extractTerms(Set) */
   public void extractTerms(Set<Term> terms) {
     // no terms involved here
   }
@@ -114,7 +114,8 @@
     private final ValueSourceWeight weight;
     private final float qWeight;
     private final DocValues vals;
-    private final TermDocs termDocs;
+    private final Bits delDocs;
+    private final int maxDoc;
     private int doc = -1;
 
     // constructor
@@ -124,24 +125,33 @@
       this.qWeight = w.getValue();
       // this is when/where the values are first created.
       vals = valSrc.getValues(reader);
-      termDocs = reader.termDocs(null);
+      delDocs = reader.getDeletedDocs();
+      maxDoc = reader.maxDoc();
     }
 
     public int nextDoc() throws IOException {
-      return doc = termDocs.next() ? termDocs.doc() : NO_MORE_DOCS;
+      doc++;
+      while (delDocs != null && doc < maxDoc && delDocs.get(doc)) {
+        doc++;
+      }
+      if (doc == maxDoc) {
+        doc = NO_MORE_DOCS;
+      }
+      return doc;
     }
-    
+
     public int docID() {
       return doc;
     }
-    
+
     public int advance(int target) throws IOException {
-      return doc = termDocs.skipTo(target) ? termDocs.doc() : NO_MORE_DOCS;
+      doc = target - 1;
+      return nextDoc();
     }
     
     /*(non-Javadoc) @see org.apache.lucene.search.Scorer#score() */
     public float score() throws IOException {
-      return qWeight * vals.floatVal(termDocs.doc());
+      return qWeight * vals.floatVal(doc);
     }
 
     /*(non-Javadoc) @see org.apache.lucene.search.Scorer#explain(int) */

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java Tue Oct 13 20:44:51 2009
@@ -19,7 +19,7 @@
 
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.index.PositionsEnum;
 import org.apache.lucene.search.Searcher;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.Weight;
@@ -80,14 +80,14 @@
     protected class PayloadTermSpanScorer extends SpanScorer {
       // TODO: is this the best way to allocate this?
       protected byte[] payload = new byte[256];
-      protected TermPositions positions;
       protected float payloadScore;
       protected int payloadsSeen;
+      private final TermSpans termSpans;
 
       public PayloadTermSpanScorer(TermSpans spans, Weight weight,
           Similarity similarity, byte[] norms) throws IOException {
         super(spans, weight, similarity, norms);
-        positions = spans.getPositions();
+        termSpans = spans;
       }
 
       protected boolean setFreqCurrentDoc() throws IOException {
@@ -112,7 +112,8 @@
       }
 
       protected void processPayload(Similarity similarity) throws IOException {
-        if (positions.isPayloadAvailable()) {
+        final PositionsEnum positions = termSpans.getPositions();
+        if (positions.hasPayload()) {
           payload = positions.getPayload(payload, 0);
           payloadScore = function.currentScore(doc, term.field(),
               spans.start(), spans.end(), payloadsSeen, payloadScore,

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/SpanTermQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/SpanTermQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/SpanTermQuery.java Tue Oct 13 20:44:51 2009
@@ -19,6 +19,7 @@
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermRef;
 import org.apache.lucene.util.ToStringUtils;
 
 import java.io.IOException;
@@ -76,7 +77,9 @@
   }
 
   public Spans getSpans(final IndexReader reader) throws IOException {
-    return new TermSpans(reader.termPositions(term), term);
+    return new TermSpans(reader.termDocsEnum(reader.getDeletedDocs(),
+                                             term.field(),
+                                             new TermRef(term.text())), term);
   }
 
 }



Mime
View raw message