lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From uschind...@apache.org
Subject svn commit: r791173 [3/4] - in /lucene/java/branches/lucene_2_4_back_compat_tests: ./ contrib/highlighter/src/test/org/apache/lucene/search/highlight/ src/java/org/apache/lucene/analysis/ src/java/org/apache/lucene/analysis/standard/ src/java/org/apach...
Date Sat, 04 Jul 2009 20:08:56 GMT
Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/RangeFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/RangeFilter.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/RangeFilter.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/RangeFilter.java Sat Jul  4 20:08:54 2009
@@ -18,6 +18,10 @@
  */
 
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.util.OpenBitSet;
 
 import java.io.IOException;
 import java.util.BitSet;
@@ -31,9 +35,6 @@
  * This code borrows heavily from {@link RangeQuery}, but is implemented as a Filter
  * 
  * </p>
- * 
- * If you construct a large number of range filters with different ranges but on the 
- * same field, {@link FieldCacheRangeFilter} may have significantly better performance. 
  */
 public class RangeFilter extends Filter {
     
@@ -43,7 +44,6 @@
     private boolean includeLower;
     private boolean includeUpper;
     private Collator collator;
-    private RangeQuery rangeQuery;
 
   /**
      * @param fieldName The field this range applies to
@@ -75,7 +75,6 @@
             throw new IllegalArgumentException
                 ("The upper bound must be non-null to be inclusive");
         }
-        initRangeQuery();
     }
 
     /**
@@ -100,11 +99,6 @@
                        Collator collator) {
         this(fieldName, lowerTerm, upperTerm, includeLower, includeUpper);
         this.collator = collator;
-        initRangeQuery();
-    }
-
-    private void initRangeQuery() {
-      rangeQuery = new RangeQuery(fieldName, lowerTerm, upperTerm, includeLower, includeUpper, collator);
     }
 
     /**
@@ -130,7 +124,81 @@
      * @deprecated Use {@link #getDocIdSet(IndexReader)} instead.
      */
     public BitSet bits(IndexReader reader) throws IOException {
-      return rangeQuery.getFilter().bits(reader);
+        BitSet bits = new BitSet(reader.maxDoc());
+        TermEnum enumerator =
+            (null != lowerTerm && collator == null
+             ? reader.terms(new Term(fieldName, lowerTerm))
+             : reader.terms(new Term(fieldName)));
+        
+        try {
+            
+            if (enumerator.term() == null) {
+                return bits;
+            }
+            
+            TermDocs termDocs = reader.termDocs();
+            try {
+                if (collator != null) {
+                    do {
+                        Term term = enumerator.term();
+                        if (term != null && term.field().equals(fieldName)) {
+                            if ((lowerTerm == null
+                                 || (includeLower
+                                     ? collator.compare(term.text(), lowerTerm) >= 0
+                                     : collator.compare(term.text(), lowerTerm) > 0))
+                                && (upperTerm == null
+                                    || (includeUpper
+                                        ? collator.compare(term.text(), upperTerm) <= 0
+                                        : collator.compare(term.text(), upperTerm) < 0))) {
+                              /* we have a good term, find the docs */
+                                termDocs.seek(enumerator.term());
+                                while (termDocs.next()) {
+                                    bits.set(termDocs.doc());
+                                }
+                            }
+                        }
+                    }
+                    while (enumerator.next());
+                } else { // collator is null - use Unicode code point ordering
+                    boolean checkLower = false;
+                    if (!includeLower) // make adjustments to set to exclusive
+                        checkLower = true;
+       
+                    do {
+                        Term term = enumerator.term();
+                        if (term != null && term.field().equals(fieldName)) {
+                            if (!checkLower || null==lowerTerm || term.text().compareTo(lowerTerm) > 0) {
+                                checkLower = false;
+                                if (upperTerm != null) {
+                                    int compare = upperTerm.compareTo(term.text());
+                                    /* if beyond the upper term, or is exclusive and
+                                     * this is equal to the upper term, break out */
+                                    if ((compare < 0) ||
+                                        (!includeUpper && compare==0)) {
+                                        break;
+                                    }
+                                }
+                                /* we have a good term, find the docs */
+                            
+                                termDocs.seek(enumerator.term());
+                                while (termDocs.next()) {
+                                    bits.set(termDocs.doc());
+                                }
+                            }
+                        } else {
+                            break;
+                        }
+                    }
+                    while (enumerator.next());
+                }
+            } finally {
+                termDocs.close();
+            }
+        } finally {
+            enumerator.close();
+        }
+
+        return bits;
     }
     
     /**
@@ -138,7 +206,84 @@
      * permitted in search results.
      */
     public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
-      return rangeQuery.getFilter().getDocIdSet(reader);
+        OpenBitSet bits = new OpenBitSet(reader.maxDoc());
+        
+        TermEnum enumerator =
+            (null != lowerTerm && collator == null
+             ? reader.terms(new Term(fieldName, lowerTerm))
+             : reader.terms(new Term(fieldName)));
+        
+        try {
+            
+            if (enumerator.term() == null) {
+                return bits;
+            }
+
+            TermDocs termDocs = reader.termDocs();
+
+            try {
+                if (collator != null) {
+                    do {
+                        Term term = enumerator.term();
+                        if (term != null && term.field().equals(fieldName)) {
+                            if ((lowerTerm == null
+                                 || (includeLower
+                                     ? collator.compare(term.text(), lowerTerm) >= 0
+                                     : collator.compare(term.text(), lowerTerm) > 0))
+                                && (upperTerm == null
+                                    || (includeUpper
+                                        ? collator.compare(term.text(), upperTerm) <= 0
+                                        : collator.compare(term.text(), upperTerm) < 0))) {
+                                /* we have a good term, find the docs */
+                                termDocs.seek(enumerator.term());
+                                while (termDocs.next()) {
+                                    bits.set(termDocs.doc());
+                                }
+                            }
+                        }
+                    }
+                    while (enumerator.next());
+                } else { // collator is null - use Unicode code point ordering
+                    boolean checkLower = false;
+                    if (!includeLower) // make adjustments to set to exclusive
+                        checkLower = true;
+        
+                    do {
+                        Term term = enumerator.term();
+                        if (term != null && term.field().equals(fieldName)) {
+                            if (!checkLower || null==lowerTerm || term.text().compareTo(lowerTerm) > 0) {
+                                checkLower = false;
+                                if (upperTerm != null) {
+                                    int compare = upperTerm.compareTo(term.text());
+                                    /* if beyond the upper term, or is exclusive and
+                                     * this is equal to the upper term, break out */
+                                    if ((compare < 0) ||
+                                        (!includeUpper && compare==0)) {
+                                        break;
+                                    }
+                                }
+                                /* we have a good term, find the docs */
+                            
+                                termDocs.seek(enumerator.term());
+                                while (termDocs.next()) {
+                                    bits.set(termDocs.doc());
+                                }
+                            }
+                        } else {
+                            break;
+                        }
+                    }
+                    while (enumerator.next());
+                }
+                
+            } finally {
+                termDocs.close();
+            }
+        } finally {
+            enumerator.close();
+        }
+
+        return bits;
     }
     
     public String toString() {

Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/RangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/RangeQuery.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/RangeQuery.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/RangeQuery.java Sat Jul  4 20:08:54 2009
@@ -21,214 +21,234 @@
 import java.text.Collator;
 
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermEnum;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.ToStringUtils;
 
 /**
  * A Query that matches documents within an exclusive range. A RangeQuery
  * is built by QueryParser for input like <code>[010 TO 120]</code> but only if the QueryParser has 
  * the useOldRangeQuery property set to true. The QueryParser default behaviour is to use
- * the newer ConstantScore mode. This is generally preferable because:
+ * the newer ConstantScoreRangeQuery class. This is generally preferable because:
  * <ul>
- *  <li>It is faster than the standard RangeQuery mode</li>
- *  <li>Unlike the RangeQuery mode, it does not cause a BooleanQuery.TooManyClauses exception if the range of values is large</li>
- *  <li>Unlike the RangeQuery mode, it does not influence scoring based on the scarcity of individual terms that may match</li>
+ * 	<li>It is faster than RangeQuery</li>
+ * 	<li>Unlike RangeQuery, it does not cause a BooleanQuery.TooManyClauses exception if the range of values is large</li>
+ * 	<li>Unlike RangeQuery it does not influence scoring based on the scarcity of individual terms that may match</li>
  * </ul>
  * 
+ * 
+ * @see ConstantScoreRangeQuery
+ * 
  *
  * @version $Id$
  */
-public class RangeQuery extends MultiTermQuery {
-  private Term lowerTerm;
-  private Term upperTerm;
-  private Collator collator;
-  private String field;
-  private boolean includeLower;
-  private boolean includeUpper;
-
-
-  /**
-   * Constructs a query selecting all terms greater/equal than <code>lowerTerm</code>
-   * but less/equal than <code>upperTerm</code>. 
-   * 
-   * <p>
-   * If an endpoint is null, it is said 
-   * to be "open". Either or both endpoints may be open.  Open endpoints may not 
-   * be exclusive (you can't select all but the first or last term without 
-   * explicitly specifying the term to exclude.)
-   * 
-   * @param field The field that holds both lower and upper terms.
-   * @param lowerTerm
-   *          The term text at the lower end of the range
-   * @param upperTerm
-   *          The term text at the upper end of the range
-   * @param includeLower
-   *          If true, the <code>lowerTerm</code> is
-   *          included in the range.
-   * @param includeUpper
-   *          If true, the <code>upperTerm</code> is
-   *          included in the range.
-   */
-  public RangeQuery(String field, String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) {
-    init(new Term(field, lowerTerm), new Term(field, upperTerm), includeLower, includeUpper, null);
-  }
-
-  /** Constructs a query selecting all terms greater/equal than
-   * <code>lowerTerm</code> but less/equal than <code>upperTerm</code>.
-   * <p>
-   * If an endpoint is null, it is said 
-   * to be "open". Either or both endpoints may be open.  Open endpoints may not 
-   * be exclusive (you can't select all but the first or last term without 
-   * explicitly specifying the term to exclude.)
-   * <p>
-   * If <code>collator</code> is not null, it will be used to decide whether
-   * index terms are within the given range, rather than using the Unicode code
-   * point order in which index terms are stored.
-   * <p>
-   * <strong>WARNING:</strong> Using this constructor and supplying a non-null
-   * value in the <code>collator</code> parameter will cause every single 
-   * index Term in the Field referenced by lowerTerm and/or upperTerm to be
-   * examined.  Depending on the number of index Terms in this Field, the 
-   * operation could be very slow.
-   *
-   * @param lowerTerm The Term text at the lower end of the range
-   * @param upperTerm The Term text at the upper end of the range
-   * @param includeLower
-   *          If true, the <code>lowerTerm</code> is
-   *          included in the range.
-   * @param includeUpper
-   *          If true, the <code>upperTerm</code> is
-   *          included in the range.
-   * @param collator The collator to use to collate index Terms, to determine
-   *  their membership in the range bounded by <code>lowerTerm</code> and
-   *  <code>upperTerm</code>.
-   */
-  public RangeQuery(String field, String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper,
-                    Collator collator) {
-    init(new Term(field, lowerTerm), new Term(field,upperTerm), includeLower, includeUpper, collator);
-  }
-
-  /** @deprecated Please use {@link #RangeQuery(String,
-   *  String, String, boolean, boolean, Collator)} instead */
-  public RangeQuery(Term lowerTerm, Term upperTerm, boolean inclusive,
-                    Collator collator) {
-    init(lowerTerm, upperTerm, inclusive, inclusive, collator);
-  }
-  
-  /** @deprecated Please use {@link #RangeQuery(String,
-   *  String, String, boolean, boolean)} instead */
-  public RangeQuery(Term lowerTerm, Term upperTerm, boolean inclusive) {
-    init(lowerTerm, upperTerm, inclusive, inclusive, null);
-  }
-
-  private void init(Term lowerTerm, Term upperTerm, boolean includeLower, boolean includeUpper, Collator collator) {
-    if (lowerTerm == null && upperTerm == null)
-      throw new IllegalArgumentException("At least one term must be non-null");
-    if (lowerTerm != null && upperTerm != null && lowerTerm.field() != upperTerm.field())
-      throw new IllegalArgumentException("Both terms must be for the same field");
-
-    if (lowerTerm == null)
-      this.field = upperTerm.field();
-    else
-      this.field = lowerTerm.field();
-    this.lowerTerm = lowerTerm;
-    this.upperTerm = upperTerm;
-    this.includeLower = includeLower;
-    this.includeUpper = includeUpper;
-    this.collator = collator;
-  }
-  
-  /** Returns the field name for this query */
-  public String getField() {
-    return field;
-  }
-
-  /** Returns the lower term of this range query.
-   *  @deprecated Use {@link #getLowerTermText} instead. */
-  public Term getLowerTerm() { return lowerTerm; }
-
-  /** Returns the upper term of this range query.
-   *  @deprecated Use {@link #getUpperTermText} instead. */
-  public Term getUpperTerm() { return upperTerm; }
-  
-  /** Returns the lower value of this range query */
-  public String getLowerTermText() { return lowerTerm == null ? null : lowerTerm.text(); }
-
-  /** Returns the upper value of this range query */
-  public String getUpperTermText() { return upperTerm == null ? null : upperTerm.text(); }
-  
-  /** Returns <code>true</code> if the lower endpoint is inclusive */
-  public boolean includesLower() { return includeLower; }
-  
-  /** Returns <code>true</code> if the upper endpoint is inclusive */
-  public boolean includesUpper() { return includeUpper; }
-
-  /** Returns <code>true</code> if the range query is inclusive 
-   *  @deprecated Use {@link #includesLower}, {@link #includesUpper}  instead. 
-   */
-  public boolean isInclusive() { return includeUpper && includeLower; }
-
-  /** Returns the collator used to determine range inclusion, if any. */
-  public Collator getCollator() { return collator; }
-  
-  protected FilteredTermEnum getEnum(IndexReader reader) throws IOException {
-    //TODO: when the deprecated 'Term' constructors are removed we can remove these null checks
-    return new RangeTermEnum(reader, collator, getField(), lowerTerm == null ? null : lowerTerm.text(),
-        upperTerm == null ? null : upperTerm.text(), includeLower, includeUpper);
-  }
-
-  /** Prints a user-readable version of this query. */
-  public String toString(String field) {
-      StringBuffer buffer = new StringBuffer();
-      if (!getField().equals(field)) {
-          buffer.append(getField());
-          buffer.append(":");
-      }
-      buffer.append(includeLower ? '[' : '{');
-      buffer.append(lowerTerm != null ? lowerTerm.text() : "*");
-      buffer.append(" TO ");
-      buffer.append(upperTerm != null ? upperTerm.text() : "*");
-      buffer.append(includeUpper ? ']' : '}');
-      if (getBoost() != 1.0f) {
-          buffer.append("^");
-          buffer.append(Float.toString(getBoost()));
-      }
-      return buffer.toString();
-  }
-
-  /** Returns true iff <code>o</code> is equal to this. */
-  public boolean equals(Object o) {
-    if (this == o) return true;
-    if (!(o instanceof RangeQuery)) return false;
-    RangeQuery other = (RangeQuery) o;
-
-    if (this.field != other.field  // interned comparison
-        || this.includeLower != other.includeLower
-        || this.includeUpper != other.includeUpper
-        || (this.collator != null && ! this.collator.equals(other.collator))
-       ) { return false; }
-    String lowerVal = this.lowerTerm == null ? null : lowerTerm.text();
-    String upperVal = this.upperTerm == null ? null : upperTerm.text();
-    String olowerText = other.lowerTerm == null ? null : other.lowerTerm.text();
-    String oupperText = other.upperTerm == null ? null : other.upperTerm.text();
-    if (lowerVal != null ? !lowerVal.equals(olowerText) : olowerText != null) return false;
-    if (upperVal != null ? !upperVal.equals(oupperText) : oupperText != null) return false;
-    return this.getBoost() == other.getBoost();
-  }
-
-  /** Returns a hash code value for this object.*/
-  public int hashCode() {
-    int h = Float.floatToIntBits(getBoost()) ^ field.hashCode();
-    String lowerVal = this.lowerTerm == null ? null : lowerTerm.text();
-    String upperVal = this.upperTerm == null ? null : upperTerm.text();
-    // hashCode of "" is 0, so don't use that for null...
-    h ^= lowerVal != null ? lowerVal.hashCode() : 0x965a965a;
-    // don't just XOR upperVal with out mixing either it or h, as it will cancel
-    // out lowerVal if they are equal.
-    h ^= (h << 17) | (h >>> 16);  // a reversible (one to one) 32 bit mapping mix
-    h ^= (upperVal != null ? (upperVal.hashCode()) : 0x5a695a69);
-    h ^= (includeLower ? 0x665599aa : 0)
-       ^ (includeUpper ? 0x99aa5566 : 0);
-    h ^= collator != null ? collator.hashCode() : 0;
-    return h;
-  }
+public class RangeQuery extends Query
+{
+    private Term lowerTerm;
+    private Term upperTerm;
+    private boolean inclusive;
+    private Collator collator;
+
+    /** Constructs a query selecting all terms greater than
+     * <code>lowerTerm</code> but less than <code>upperTerm</code>.
+     * There must be at least one term and either term may be null,
+     * in which case there is no bound on that side, but if there are
+     * two terms, both terms <b>must</b> be for the same field.
+     *
+     * @param lowerTerm The Term at the lower end of the range
+     * @param upperTerm The Term at the upper end of the range
+     * @param inclusive If true, both <code>lowerTerm</code> and
+     *  <code>upperTerm</code> will themselves be included in the range.
+     */
+    public RangeQuery(Term lowerTerm, Term upperTerm, boolean inclusive)
+    {
+        if (lowerTerm == null && upperTerm == null)
+        {
+            throw new IllegalArgumentException("At least one term must be non-null");
+        }
+        if (lowerTerm != null && upperTerm != null && lowerTerm.field() != upperTerm.field())
+        {
+            throw new IllegalArgumentException("Both terms must be for the same field");
+        }
+
+        // if we have a lowerTerm, start there. otherwise, start at beginning
+        if (lowerTerm != null) {
+            this.lowerTerm = lowerTerm;
+        }
+        else {
+            this.lowerTerm = new Term(upperTerm.field());
+        }
+
+        this.upperTerm = upperTerm;
+        this.inclusive = inclusive;
+    }
+
+    /** Constructs a query selecting all terms greater than
+     * <code>lowerTerm</code> but less than <code>upperTerm</code>.
+     * There must be at least one term and either term may be null,
+     * in which case there is no bound on that side, but if there are
+     * two terms, both terms <b>must</b> be for the same field.
+     * <p>
+     * If <code>collator</code> is not null, it will be used to decide whether
+     * index terms are within the given range, rather than using the Unicode code
+     * point order in which index terms are stored.
+     * <p>
+     * <strong>WARNING:</strong> Using this constructor and supplying a non-null
+     * value in the <code>collator</code> parameter will cause every single 
+     * index Term in the Field referenced by lowerTerm and/or upperTerm to be
+     * examined.  Depending on the number of index Terms in this Field, the 
+     * operation could be very slow.
+     *
+     * @param lowerTerm The Term at the lower end of the range
+     * @param upperTerm The Term at the upper end of the range
+     * @param inclusive If true, both <code>lowerTerm</code> and
+     *  <code>upperTerm</code> will themselves be included in the range.
+     * @param collator The collator to use to collate index Terms, to determine
+     *  their membership in the range bounded by <code>lowerTerm</code> and
+     *  <code>upperTerm</code>.
+     */
+    public RangeQuery(Term lowerTerm, Term upperTerm, boolean inclusive,
+                      Collator collator)
+    {
+        this(lowerTerm, upperTerm, inclusive);
+        this.collator = collator;
+    }
+
+    public Query rewrite(IndexReader reader) throws IOException {
+
+        BooleanQuery query = new BooleanQuery(true);
+        String testField = getField();
+        if (collator != null) {
+            TermEnum enumerator = reader.terms(new Term(testField, ""));
+            String lowerTermText = lowerTerm != null ? lowerTerm.text() : null;
+            String upperTermText = upperTerm != null ? upperTerm.text() : null;
+
+            try {
+                do {
+                    Term term = enumerator.term();
+                    if (term != null && term.field() == testField) { // interned comparison
+                        if ((lowerTermText == null
+                             || (inclusive ? collator.compare(term.text(), lowerTermText) >= 0
+                                           : collator.compare(term.text(), lowerTermText) > 0))
+                            && (upperTermText == null
+                                || (inclusive ? collator.compare(term.text(), upperTermText) <= 0
+                                              : collator.compare(term.text(), upperTermText) < 0))) {
+                            addTermToQuery(term, query);
+                        }
+                    }
+                }
+                while (enumerator.next());
+            }
+            finally {
+                enumerator.close();
+            }
+        }
+        else { // collator is null
+            TermEnum enumerator = reader.terms(lowerTerm);
+
+            try {
+
+                boolean checkLower = false;
+                if (!inclusive) // make adjustments to set to exclusive
+                    checkLower = true;
+
+                do {
+                    Term term = enumerator.term();
+                    if (term != null && term.field() == testField) { // interned comparison
+                        if (!checkLower || term.text().compareTo(lowerTerm.text()) > 0) {
+                            checkLower = false;
+                            if (upperTerm != null) {
+                                int compare = upperTerm.text().compareTo(term.text());
+                                /* if beyond the upper term, or is exclusive and
+                                 * this is equal to the upper term, break out */
+                                if ((compare < 0) || (!inclusive && compare == 0))
+                                    break;
+                            }
+                            addTermToQuery(term, query); // Found a match
+                        }
+                    }
+                    else {
+                        break;
+                    }
+                }
+                while (enumerator.next());
+            }
+            finally {
+                enumerator.close();
+            }
+        }
+        return query;
+    }
+
+    private void addTermToQuery(Term term, BooleanQuery query) {
+        TermQuery tq = new TermQuery(term);
+        tq.setBoost(getBoost()); // set the boost
+        query.add(tq, BooleanClause.Occur.SHOULD); // add to query
+    }
+
+    /** Returns the field name for this query */
+    public String getField() {
+      return (lowerTerm != null ? lowerTerm.field() : upperTerm.field());
+    }
+
+    /** Returns the lower term of this range query */
+    public Term getLowerTerm() { return lowerTerm; }
+
+    /** Returns the upper term of this range query */
+    public Term getUpperTerm() { return upperTerm; }
+
+    /** Returns <code>true</code> if the range query is inclusive */
+    public boolean isInclusive() { return inclusive; }
+
+    /** Returns the collator used to determine range inclusion, if any. */
+    public Collator getCollator() { return collator; }
+
+
+    /** Prints a user-readable version of this query. */
+    public String toString(String field)
+    {
+        StringBuffer buffer = new StringBuffer();
+        if (!getField().equals(field))
+        {
+            buffer.append(getField());
+            buffer.append(":");
+        }
+        buffer.append(inclusive ? "[" : "{");
+        buffer.append(lowerTerm != null ? lowerTerm.text() : "null");
+        buffer.append(" TO ");
+        buffer.append(upperTerm != null ? upperTerm.text() : "null");
+        buffer.append(inclusive ? "]" : "}");
+        buffer.append(ToStringUtils.boost(getBoost()));
+        return buffer.toString();
+    }
+
+    /** Returns true iff <code>o</code> is equal to this. */
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof RangeQuery)) return false;
+
+        final RangeQuery other = (RangeQuery) o;
+        if (this.getBoost() != other.getBoost()) return false;
+        if (this.inclusive != other.inclusive) return false;
+        if (this.collator != null && ! this.collator.equals(other.collator)) 
+            return false;
+
+        // one of lowerTerm and upperTerm can be null
+        if (this.lowerTerm != null ? !this.lowerTerm.equals(other.lowerTerm) : other.lowerTerm != null) return false;
+        if (this.upperTerm != null ? !this.upperTerm.equals(other.upperTerm) : other.upperTerm != null) return false;
+        return true;
+    }
+
+    /** Returns a hash code value for this object.*/
+    public int hashCode() {
+      int h = Float.floatToIntBits(getBoost());
+      h ^= lowerTerm != null ? lowerTerm.hashCode() : 0;
+      // reversible mix to make lower and upper position dependent and
+      // to prevent them from cancelling out.
+      h ^= (h << 25) | (h >>> 8);
+      h ^= upperTerm != null ? upperTerm.hashCode() : 0;
+      h ^= this.inclusive ? 0x2742E74A : 0;
+      h ^= collator != null ? collator.hashCode() : 0; 
+      return h;
+    }
 }

Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/WildcardQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/WildcardQuery.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/WildcardQuery.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/WildcardQuery.java Sat Jul  4 20:08:54 2009
@@ -48,11 +48,12 @@
 
     return false;
   }
-
+  
   public Query rewrite(IndexReader reader) throws IOException {
-    if (!termContainsWildcard)
+      if (this.termContainsWildcard) {
+          return super.rewrite(reader);
+      }
+      
       return new TermQuery(getTerm());
-    else
-      return super.rewrite(reader);
   }
 }

Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/WildcardTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/WildcardTermEnum.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/WildcardTermEnum.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/WildcardTermEnum.java Sat Jul  4 20:08:54 2009
@@ -40,7 +40,9 @@
   boolean endEnum = false;
 
   /**
-   * Creates a new <code>WildcardTermEnum</code>.
+   * Creates a new <code>WildcardTermEnum</code>.  Passing in a
+   * {@link org.apache.lucene.index.Term Term} that does not contain a
+   * <code>WILDCARD_CHAR</code> will cause an exception to be thrown.
    * <p>
    * After calling the constructor the enumeration is already pointing to the first 
    * valid term if such a term exists.
@@ -60,12 +62,8 @@
     else if (cidx >= 0) {
       idx = Math.min(idx, cidx);
     }
-    if (idx != -1) {
-      pre = searchTerm.text().substring(0,idx);
-    } else {
-      pre = "";
-    }
 
+    pre = searchTerm.text().substring(0,idx);
     preLen = pre.length();
     text = text.substring(preLen);
     setEnum(reader.terms(new Term(searchTerm.field(), pre)));

Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java Sat Jul  4 20:08:54 2009
@@ -18,9 +18,6 @@
 
 import org.apache.lucene.analysis.standard.StandardFilter;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.English;
 import org.apache.lucene.util.LuceneTestCase;
 
@@ -43,8 +40,7 @@
     super(s);
   }
 
-  protected void setUp() throws Exception {
-    super.setUp();
+  protected void setUp() {
     tokens1 = new String[]{"The", "quick", "Burgundy", "Fox", "jumped", "over", "the", "lazy", "Red", "Dogs"};
     tokens2 = new String[]{"The", "Lazy", "Dogs", "should", "stay", "on", "the", "porch"};
     buffer1 = new StringBuffer();
@@ -66,29 +62,24 @@
   public void test() throws IOException {
 
     SinkTokenizer sink1 = new SinkTokenizer(null) {
-      public void add(AttributeSource a) throws IOException {
-        TermAttribute termAtt = null;
-        if (a.hasAttribute(TermAttribute.class)) {
-          termAtt = (TermAttribute) a.getAttribute(TermAttribute.class);
-        }
-        if (termAtt != null && termAtt.term().equalsIgnoreCase("The")) {
-          super.add(a);
+      public void add(Token t) {
+        if (t != null && t.term().equalsIgnoreCase("The")) {
+          super.add(t);
         }
       }
     };
     TokenStream source = new TeeTokenFilter(new WhitespaceTokenizer(new StringReader(buffer1.toString())), sink1);
     int i = 0;
-    TermAttribute termAtt = (TermAttribute) source.getAttribute(TermAttribute.class);
-    while (source.incrementToken()) {
-      assertTrue(termAtt.term() + " is not equal to " + tokens1[i], termAtt.term().equals(tokens1[i]) == true);
+    final Token reusableToken = new Token();
+    for (Token nextToken = source.next(reusableToken); nextToken != null; nextToken = source.next(reusableToken)) {
+      assertTrue(nextToken.term() + " is not equal to " + tokens1[i], nextToken.term().equals(tokens1[i]) == true);
       i++;
     }
     assertTrue(i + " does not equal: " + tokens1.length, i == tokens1.length);
     assertTrue("sink1 Size: " + sink1.getTokens().size() + " is not: " + 2, sink1.getTokens().size() == 2);
     i = 0;
-    termAtt = (TermAttribute) sink1.getAttribute(TermAttribute.class);
-    while (sink1.incrementToken()) {
-      assertTrue(termAtt.term() + " is not equal to " + "The", termAtt.term().equalsIgnoreCase("The") == true);
+    for (Token token = sink1.next(reusableToken); token != null; token = sink1.next(reusableToken)) {
+      assertTrue(token.term() + " is not equal to " + "The", token.term().equalsIgnoreCase("The") == true);
       i++;
     }
     assertTrue(i + " does not equal: " + sink1.getTokens().size(), i == sink1.getTokens().size());
@@ -96,67 +87,55 @@
 
   public void testMultipleSources() throws Exception {
     SinkTokenizer theDetector = new SinkTokenizer(null) {
-      public void add(AttributeSource a) throws IOException {
-        TermAttribute termAtt = null;
-        if (a.hasAttribute(TermAttribute.class)) {
-          termAtt = (TermAttribute) a.getAttribute(TermAttribute.class);
-        }
-        if (termAtt != null && termAtt.term().equalsIgnoreCase("The")) {
-          super.add(a);
+      public void add(Token t) {
+        if (t != null && t.term().equalsIgnoreCase("The")) {
+          super.add(t);
         }
       }
     };
-    SinkTokenizer dogDetector = new SinkTokenizer(null) {      
-      public void add(AttributeSource a) throws IOException {
-        TermAttribute termAtt = null;
-        if (a.hasAttribute(TermAttribute.class)) {
-          termAtt = (TermAttribute) a.getAttribute(TermAttribute.class);
-        }
-        if (termAtt != null && termAtt.term().equalsIgnoreCase("Dogs")) {
-          super.add(a);
+    SinkTokenizer dogDetector = new SinkTokenizer(null) {
+      public void add(Token t) {
+        if (t != null && t.term().equalsIgnoreCase("Dogs")) {
+          super.add(t);
         }
       }
     };
     TokenStream source1 = new CachingTokenFilter(new TeeTokenFilter(new TeeTokenFilter(new WhitespaceTokenizer(new StringReader(buffer1.toString())), theDetector), dogDetector));
     TokenStream source2 = new TeeTokenFilter(new TeeTokenFilter(new WhitespaceTokenizer(new StringReader(buffer2.toString())), theDetector), dogDetector);
     int i = 0;
-    TermAttribute termAtt = (TermAttribute) source1.getAttribute(TermAttribute.class);
-    while (source1.incrementToken()) {
-      assertTrue(termAtt.term() + " is not equal to " + tokens1[i], termAtt.term().equals(tokens1[i]) == true);
+    final Token reusableToken = new Token();
+    for (Token nextToken = source1.next(reusableToken); nextToken != null; nextToken = source1.next(reusableToken)) {
+      assertTrue(nextToken.term() + " is not equal to " + tokens1[i], nextToken.term().equals(tokens1[i]) == true);
       i++;
     }
     assertTrue(i + " does not equal: " + tokens1.length, i == tokens1.length);
     assertTrue("theDetector Size: " + theDetector.getTokens().size() + " is not: " + 2, theDetector.getTokens().size() == 2);
     assertTrue("dogDetector Size: " + dogDetector.getTokens().size() + " is not: " + 1, dogDetector.getTokens().size() == 1);
     i = 0;
-    termAtt = (TermAttribute) source2.getAttribute(TermAttribute.class);
-    while (source2.incrementToken()) {
-      assertTrue(termAtt.term() + " is not equal to " + tokens2[i], termAtt.term().equals(tokens2[i]) == true);
+    for (Token nextToken = source2.next(reusableToken); nextToken != null; nextToken = source2.next(reusableToken)) {
+      assertTrue(nextToken.term() + " is not equal to " + tokens2[i], nextToken.term().equals(tokens2[i]) == true);
       i++;
     }
     assertTrue(i + " does not equal: " + tokens2.length, i == tokens2.length);
     assertTrue("theDetector Size: " + theDetector.getTokens().size() + " is not: " + 4, theDetector.getTokens().size() == 4);
     assertTrue("dogDetector Size: " + dogDetector.getTokens().size() + " is not: " + 2, dogDetector.getTokens().size() == 2);
     i = 0;
-    termAtt = (TermAttribute) theDetector.getAttribute(TermAttribute.class);
-    while (theDetector.incrementToken()) {
-      assertTrue(termAtt.term() + " is not equal to " + "The", termAtt.term().equalsIgnoreCase("The") == true);
+    for (Token nextToken = theDetector.next(reusableToken); nextToken != null; nextToken = theDetector.next(reusableToken)) {
+      assertTrue(nextToken.term() + " is not equal to " + "The", nextToken.term().equalsIgnoreCase("The") == true);
       i++;
     }
     assertTrue(i + " does not equal: " + theDetector.getTokens().size(), i == theDetector.getTokens().size());
     i = 0;
-    termAtt = (TermAttribute) dogDetector.getAttribute(TermAttribute.class);
-    while (dogDetector.incrementToken()) {
-      assertTrue(termAtt.term() + " is not equal to " + "Dogs", termAtt.term().equalsIgnoreCase("Dogs") == true);
+    for (Token nextToken = dogDetector.next(reusableToken); nextToken != null; nextToken = dogDetector.next(reusableToken)) {
+      assertTrue(nextToken.term() + " is not equal to " + "Dogs", nextToken.term().equalsIgnoreCase("Dogs") == true);
       i++;
     }
     assertTrue(i + " does not equal: " + dogDetector.getTokens().size(), i == dogDetector.getTokens().size());
     source1.reset();
     TokenStream lowerCasing = new LowerCaseFilter(source1);
     i = 0;
-    termAtt = (TermAttribute) lowerCasing.getAttribute(TermAttribute.class);
-    while (lowerCasing.incrementToken()) {
-      assertTrue(termAtt.term() + " is not equal to " + tokens1[i].toLowerCase(), termAtt.term().equals(tokens1[i].toLowerCase()) == true);
+    for (Token nextToken = lowerCasing.next(reusableToken); nextToken != null; nextToken = lowerCasing.next(reusableToken)) {
+      assertTrue(nextToken.term() + " is not equal to " + tokens1[i].toLowerCase(), nextToken.term().equals(tokens1[i].toLowerCase()) == true);
       i++;
     }
     assertTrue(i + " does not equal: " + tokens1.length, i == tokens1.length);
@@ -178,20 +157,21 @@
       }
       //make sure we produce the same tokens
       ModuloSinkTokenizer sink = new ModuloSinkTokenizer(tokCount[k], 100);
+      final Token reusableToken = new Token();
       TokenStream stream = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), sink);
-      while (stream.incrementToken()) {
+      while (stream.next(reusableToken) != null) {
       }
       stream = new ModuloTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), 100);
       List tmp = new ArrayList();
-      while (stream.incrementToken()) {
-        tmp.add(stream.captureState());
+      for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+        tmp.add(nextToken.clone());
       }
       List sinkList = sink.getTokens();
       assertTrue("tmp Size: " + tmp.size() + " is not: " + sinkList.size(), tmp.size() == sinkList.size());
       for (int i = 0; i < tmp.size(); i++) {
-        AttributeSource tfTok = (AttributeSource) tmp.get(i);
-        AttributeSource sinkTok = (AttributeSource) sinkList.get(i);
-        assertTrue(tfTok + " is not equal to " + sinkTok + " at token: " + i, tfTok.equals(sinkTok) == true);
+        Token tfTok = (Token) tmp.get(i);
+        Token sinkTok = (Token) sinkList.get(i);
+        assertTrue(tfTok.term() + " is not equal to " + sinkTok.term() + " at token: " + i, tfTok.term().equals(sinkTok.term()) == true);
       }
       //simulate two fields, each being analyzed once, for 20 documents
 
@@ -200,14 +180,12 @@
         long start = System.currentTimeMillis();
         for (int i = 0; i < 20; i++) {
           stream = new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString())));
-          PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) stream.getAttribute(PositionIncrementAttribute.class);
-          while (stream.incrementToken()) {
-            tfPos += posIncrAtt.getPositionIncrement();
+          for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+            tfPos += nextToken.getPositionIncrement();
           }
           stream = new ModuloTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), modCounts[j]);
-          posIncrAtt = (PositionIncrementAttribute) stream.getAttribute(PositionIncrementAttribute.class);
-          while (stream.incrementToken()) {
-            tfPos += posIncrAtt.getPositionIncrement();
+          for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+            tfPos += nextToken.getPositionIncrement();
           }
         }
         long finish = System.currentTimeMillis();
@@ -218,15 +196,13 @@
         for (int i = 0; i < 20; i++) {
           sink = new ModuloSinkTokenizer(tokCount[k], modCounts[j]);
           stream = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), sink);
-          PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) stream.getAttribute(PositionIncrementAttribute.class);
-          while (stream.incrementToken()) {
-            sinkPos += posIncrAtt.getPositionIncrement();
+          for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+            sinkPos += nextToken.getPositionIncrement();
           }
           //System.out.println("Modulo--------");
           stream = sink;
-          posIncrAtt = (PositionIncrementAttribute) stream.getAttribute(PositionIncrementAttribute.class);
-          while (stream.incrementToken()) {
-            sinkPos += posIncrAtt.getPositionIncrement();
+          for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+            sinkPos += nextToken.getPositionIncrement();
           }
         }
         finish = System.currentTimeMillis();
@@ -252,15 +228,15 @@
     int count = 0;
 
     //return every 100 tokens
-    public boolean incrementToken() throws IOException {
-      boolean hasNext;
-      for (hasNext = input.incrementToken();
-           hasNext && count % modCount != 0;
-           hasNext = input.incrementToken()) {
+    public Token next(final Token reusableToken) throws IOException {
+      Token nextToken = null;
+      for (nextToken = input.next(reusableToken);
+           nextToken != null && count % modCount != 0;
+           nextToken = input.next(reusableToken)) {
         count++;
       }
       count++;
-      return hasNext;
+      return nextToken;
     }
   }
 
@@ -274,9 +250,9 @@
       lst = new ArrayList(numToks % mc);
     }
 
-    public void add(AttributeSource a) throws IOException {
-      if (a != null && count % modCount == 0) {
-        super.add(a);
+    public void add(Token t) {
+      if (t != null && count % modCount == 0) {
+        super.add(t);
       }
       count++;
     }

Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestAnalyzers.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestAnalyzers.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestAnalyzers.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestAnalyzers.java Sat Jul  4 20:08:54 2009
@@ -19,10 +19,10 @@
 
 import java.io.IOException;
 import java.io.StringReader;
+import java.util.LinkedList;
+import java.util.List;
 
 import org.apache.lucene.analysis.standard.StandardTokenizer;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.index.Payload;
 import org.apache.lucene.util.LuceneTestCase;
 
@@ -36,12 +36,13 @@
                                String input, 
                                String[] output) throws Exception {
     TokenStream ts = a.tokenStream("dummy", new StringReader(input));
-    TermAttribute termAtt = (TermAttribute) ts.getAttribute(TermAttribute.class);
+    final Token reusableToken  = new Token();
     for (int i=0; i<output.length; i++) {
-      assertTrue(ts.incrementToken());
-      assertEquals(termAtt.term(), output[i]);
+      Token nextToken = ts.next(reusableToken);
+      assertNotNull(nextToken);
+      assertEquals(nextToken.term(), output[i]);
     }
-    assertFalse(ts.incrementToken());
+    assertNull(ts.next(reusableToken));
     ts.close();
   }
 
@@ -94,13 +95,14 @@
   }
 
   void verifyPayload(TokenStream ts) throws IOException {
-    PayloadAttribute payloadAtt = (PayloadAttribute) ts.getAttribute(PayloadAttribute.class);
+    final Token reusableToken = new Token();
     for(byte b=1;;b++) {
-      boolean hasNext = ts.incrementToken();
-      if (!hasNext) break;
+      reusableToken.clear();
+      Token nextToken = ts.next(reusableToken);
+      if (nextToken==null) break;
       // System.out.println("id="+System.identityHashCode(nextToken) + " " + t);
       // System.out.println("payload=" + (int)nextToken.getPayload().toByteArray()[0]);
-      assertEquals(b, payloadAtt.getPayload().toByteArray()[0]);
+      assertEquals(b, nextToken.getPayload().toByteArray()[0]);
     }
   }
 
@@ -109,11 +111,13 @@
     String s = "how now brown cow";
     TokenStream ts;
     ts = new WhitespaceTokenizer(new StringReader(s));
+    ts = new BuffTokenFilter(ts);
     ts = new PayloadSetter(ts);
     verifyPayload(ts);
 
     ts = new WhitespaceTokenizer(new StringReader(s));
     ts = new PayloadSetter(ts);
+    ts = new BuffTokenFilter(ts);
     verifyPayload(ts);
   }
 
@@ -132,21 +136,38 @@
   }
 }
 
+class BuffTokenFilter extends TokenFilter {
+  List lst;
+
+  public BuffTokenFilter(TokenStream input) {
+    super(input);
+  }
+
+  public Token next(final Token reusableToken) throws IOException {
+    if (lst == null) {
+      lst = new LinkedList();
+      for(Token nextToken = input.next(reusableToken); nextToken != null; nextToken = input.next(reusableToken)) {
+        lst.add(nextToken.clone());
+      }
+    }
+    return lst.size()==0 ? null : (Token)lst.remove(0);
+  }
+}
+
 class PayloadSetter extends TokenFilter {
-  PayloadAttribute payloadAtt;
   public  PayloadSetter(TokenStream input) {
     super(input);
-    payloadAtt = (PayloadAttribute) addAttribute(PayloadAttribute.class);
   }
 
   byte[] data = new byte[1];
   Payload p = new Payload(data,0,1);
 
-  public boolean incrementToken() throws IOException {
-    boolean hasNext = input.incrementToken();
-    if (!hasNext) return false;
-    payloadAtt.setPayload(p);  // reuse the payload / byte[]
+  public Token next(final Token reusableToken) throws IOException {
+    assert reusableToken != null;
+    Token nextToken = input.next(reusableToken);
+    if (nextToken==null) return null;
+    nextToken.setPayload(p);  // reuse the payload / byte[]
     data[0]++;
-    return true;
+    return nextToken;
   }
 }
\ No newline at end of file

Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java Sat Jul  4 20:08:54 2009
@@ -22,8 +22,6 @@
 
 import org.apache.lucene.util.LuceneTestCase;
 
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Field.TermVector;
@@ -43,16 +41,13 @@
     Document doc = new Document();
     TokenStream stream = new TokenStream() {
       private int index = 0;
-      private TermAttribute termAtt = (TermAttribute) addAttribute(TermAttribute.class);
-      private OffsetAttribute offsetAtt = (OffsetAttribute) addAttribute(OffsetAttribute.class);
       
-      public boolean incrementToken() throws IOException {
+      public Token next(final Token reusableToken) throws IOException {
+        assert reusableToken != null;
         if (index == tokens.length) {
-          return false;
+          return null;
         } else {
-          termAtt.setTermBuffer(tokens[index++]);
-          offsetAtt.setOffset(0, 0);
-          return true;
+          return reusableToken.reinit(tokens[index++], 0, 0);
         }        
       }
       
@@ -97,12 +92,10 @@
   
   private void checkTokens(TokenStream stream) throws IOException {
     int count = 0;
-    
-    TermAttribute termAtt = (TermAttribute) stream.getAttribute(TermAttribute.class);
-    assertNotNull(termAtt);
-    while (stream.incrementToken()) {
+    final Token reusableToken = new Token();
+    for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
       assertTrue(count < tokens.length);
-      assertEquals(tokens[count], termAtt.term());
+      assertEquals(tokens[count], nextToken.term());
       count++;
     }
     

Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java Sat Jul  4 20:08:54 2009
@@ -17,7 +17,6 @@
  * limitations under the License.
  */
 
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.util.LuceneTestCase;
 
 import java.io.StringReader;
@@ -26,87 +25,82 @@
   public void testU() throws Exception {
     TokenStream stream = new WhitespaceTokenizer(new StringReader("Des mot clés À LA CHAÎNE À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï IJ Ð Ñ Ò Ó Ô Õ Ö Ø Œ Þ Ù Ú Û Ü Ý Ÿ à á â ã ä å æ ç è é ê ë ì í î ï ij ð ñ ò ó ô õ ö ø œ ß þ ù ú û ü ý ÿ fi fl"));
     ISOLatin1AccentFilter filter = new ISOLatin1AccentFilter(stream);
-    TermAttribute termAtt = (TermAttribute) filter.getAttribute(TermAttribute.class);
-    assertTermEquals("Des", filter, termAtt);
-    assertTermEquals("mot", filter, termAtt);
-    assertTermEquals("cles", filter, termAtt);
-    assertTermEquals("A", filter, termAtt);
-    assertTermEquals("LA", filter, termAtt);
-    assertTermEquals("CHAINE", filter, termAtt);
-    assertTermEquals("A", filter, termAtt);
-    assertTermEquals("A", filter, termAtt);
-    assertTermEquals("A", filter, termAtt);
-    assertTermEquals("A", filter, termAtt);
-    assertTermEquals("A", filter, termAtt);
-    assertTermEquals("A", filter, termAtt);
-    assertTermEquals("AE", filter, termAtt);
-    assertTermEquals("C", filter, termAtt);
-    assertTermEquals("E", filter, termAtt);
-    assertTermEquals("E", filter, termAtt);
-    assertTermEquals("E", filter, termAtt);
-    assertTermEquals("E", filter, termAtt);
-    assertTermEquals("I", filter, termAtt);
-    assertTermEquals("I", filter, termAtt);
-    assertTermEquals("I", filter, termAtt);
-    assertTermEquals("I", filter, termAtt);
-    assertTermEquals("IJ", filter, termAtt);
-    assertTermEquals("D", filter, termAtt);
-    assertTermEquals("N", filter, termAtt);
-    assertTermEquals("O", filter, termAtt);
-    assertTermEquals("O", filter, termAtt);
-    assertTermEquals("O", filter, termAtt);
-    assertTermEquals("O", filter, termAtt);
-    assertTermEquals("O", filter, termAtt);
-    assertTermEquals("O", filter, termAtt);
-    assertTermEquals("OE", filter, termAtt);
-    assertTermEquals("TH", filter, termAtt);
-    assertTermEquals("U", filter, termAtt);
-    assertTermEquals("U", filter, termAtt);
-    assertTermEquals("U", filter, termAtt);
-    assertTermEquals("U", filter, termAtt);
-    assertTermEquals("Y", filter, termAtt);
-    assertTermEquals("Y", filter, termAtt);
-    assertTermEquals("a", filter, termAtt);
-    assertTermEquals("a", filter, termAtt);
-    assertTermEquals("a", filter, termAtt);
-    assertTermEquals("a", filter, termAtt);
-    assertTermEquals("a", filter, termAtt);
-    assertTermEquals("a", filter, termAtt);
-    assertTermEquals("ae", filter, termAtt);
-    assertTermEquals("c", filter, termAtt);
-    assertTermEquals("e", filter, termAtt);
-    assertTermEquals("e", filter, termAtt);
-    assertTermEquals("e", filter, termAtt);
-    assertTermEquals("e", filter, termAtt);
-    assertTermEquals("i", filter, termAtt);
-    assertTermEquals("i", filter, termAtt);
-    assertTermEquals("i", filter, termAtt);
-    assertTermEquals("i", filter, termAtt);
-    assertTermEquals("ij", filter, termAtt);
-    assertTermEquals("d", filter, termAtt);
-    assertTermEquals("n", filter, termAtt);
-    assertTermEquals("o", filter, termAtt);
-    assertTermEquals("o", filter, termAtt);
-    assertTermEquals("o", filter, termAtt);
-    assertTermEquals("o", filter, termAtt);
-    assertTermEquals("o", filter, termAtt);
-    assertTermEquals("o", filter, termAtt);
-    assertTermEquals("oe", filter, termAtt);
-    assertTermEquals("ss", filter, termAtt);
-    assertTermEquals("th", filter, termAtt);
-    assertTermEquals("u", filter, termAtt);
-    assertTermEquals("u", filter, termAtt);
-    assertTermEquals("u", filter, termAtt);
-    assertTermEquals("u", filter, termAtt);
-    assertTermEquals("y", filter, termAtt);
-    assertTermEquals("y", filter, termAtt);
-    assertTermEquals("fi", filter, termAtt);
-    assertTermEquals("fl", filter, termAtt);
-    assertFalse(filter.incrementToken());
-  }
-  
-  void assertTermEquals(String expected, TokenStream stream, TermAttribute termAtt) throws Exception {
-    assertTrue(stream.incrementToken());
-    assertEquals(expected, termAtt.term());
+    final Token reusableToken = new Token();
+    assertEquals("Des", filter.next(reusableToken).term());
+    assertEquals("mot", filter.next(reusableToken).term());
+    assertEquals("cles", filter.next(reusableToken).term());
+    assertEquals("A", filter.next(reusableToken).term());
+    assertEquals("LA", filter.next(reusableToken).term());
+    assertEquals("CHAINE", filter.next(reusableToken).term());
+    assertEquals("A", filter.next(reusableToken).term());
+    assertEquals("A", filter.next(reusableToken).term());
+    assertEquals("A", filter.next(reusableToken).term());
+    assertEquals("A", filter.next(reusableToken).term());
+    assertEquals("A", filter.next(reusableToken).term());
+    assertEquals("A", filter.next(reusableToken).term());
+    assertEquals("AE", filter.next(reusableToken).term());
+    assertEquals("C", filter.next(reusableToken).term());
+    assertEquals("E", filter.next(reusableToken).term());
+    assertEquals("E", filter.next(reusableToken).term());
+    assertEquals("E", filter.next(reusableToken).term());
+    assertEquals("E", filter.next(reusableToken).term());
+    assertEquals("I", filter.next(reusableToken).term());
+    assertEquals("I", filter.next(reusableToken).term());
+    assertEquals("I", filter.next(reusableToken).term());
+    assertEquals("I", filter.next(reusableToken).term());
+    assertEquals("IJ", filter.next(reusableToken).term());
+    assertEquals("D", filter.next(reusableToken).term());
+    assertEquals("N", filter.next(reusableToken).term());
+    assertEquals("O", filter.next(reusableToken).term());
+    assertEquals("O", filter.next(reusableToken).term());
+    assertEquals("O", filter.next(reusableToken).term());
+    assertEquals("O", filter.next(reusableToken).term());
+    assertEquals("O", filter.next(reusableToken).term());
+    assertEquals("O", filter.next(reusableToken).term());
+    assertEquals("OE", filter.next(reusableToken).term());
+    assertEquals("TH", filter.next(reusableToken).term());
+    assertEquals("U", filter.next(reusableToken).term());
+    assertEquals("U", filter.next(reusableToken).term());
+    assertEquals("U", filter.next(reusableToken).term());
+    assertEquals("U", filter.next(reusableToken).term());
+    assertEquals("Y", filter.next(reusableToken).term());
+    assertEquals("Y", filter.next(reusableToken).term());
+    assertEquals("a", filter.next(reusableToken).term());
+    assertEquals("a", filter.next(reusableToken).term());
+    assertEquals("a", filter.next(reusableToken).term());
+    assertEquals("a", filter.next(reusableToken).term());
+    assertEquals("a", filter.next(reusableToken).term());
+    assertEquals("a", filter.next(reusableToken).term());
+    assertEquals("ae", filter.next(reusableToken).term());
+    assertEquals("c", filter.next(reusableToken).term());
+    assertEquals("e", filter.next(reusableToken).term());
+    assertEquals("e", filter.next(reusableToken).term());
+    assertEquals("e", filter.next(reusableToken).term());
+    assertEquals("e", filter.next(reusableToken).term());
+    assertEquals("i", filter.next(reusableToken).term());
+    assertEquals("i", filter.next(reusableToken).term());
+    assertEquals("i", filter.next(reusableToken).term());
+    assertEquals("i", filter.next(reusableToken).term());
+    assertEquals("ij", filter.next(reusableToken).term());
+    assertEquals("d", filter.next(reusableToken).term());
+    assertEquals("n", filter.next(reusableToken).term());
+    assertEquals("o", filter.next(reusableToken).term());
+    assertEquals("o", filter.next(reusableToken).term());
+    assertEquals("o", filter.next(reusableToken).term());
+    assertEquals("o", filter.next(reusableToken).term());
+    assertEquals("o", filter.next(reusableToken).term());
+    assertEquals("o", filter.next(reusableToken).term());
+    assertEquals("oe", filter.next(reusableToken).term());
+    assertEquals("ss", filter.next(reusableToken).term());
+    assertEquals("th", filter.next(reusableToken).term());
+    assertEquals("u", filter.next(reusableToken).term());
+    assertEquals("u", filter.next(reusableToken).term());
+    assertEquals("u", filter.next(reusableToken).term());
+    assertEquals("u", filter.next(reusableToken).term());
+    assertEquals("y", filter.next(reusableToken).term());
+    assertEquals("y", filter.next(reusableToken).term());
+    assertEquals("fi", filter.next(reusableToken).term());
+    assertEquals("fl", filter.next(reusableToken).term());
+    assertNull(filter.next(reusableToken));
   }
 }

Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java Sat Jul  4 20:08:54 2009
@@ -19,7 +19,6 @@
 
 import java.io.StringReader;
 
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexReader;
@@ -89,9 +88,9 @@
   // LUCENE-1441
   public void testOffsets() throws Exception {
     TokenStream stream = new KeywordAnalyzer().tokenStream("field", new StringReader("abcd"));
-    OffsetAttribute offsetAtt = (OffsetAttribute) stream.addAttribute(OffsetAttribute.class);
-    assertTrue(stream.incrementToken());
-    assertEquals(0, offsetAtt.startOffset());
-    assertEquals(4, offsetAtt.endOffset());
+    Token token = new Token();
+    assertTrue(stream.next(token) != null);
+    assertEquals(0, token.startOffset);
+    assertEquals(4, token.endOffset);
   }
 }

Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestLengthFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestLengthFilter.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestLengthFilter.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestLengthFilter.java Sat Jul  4 20:08:54 2009
@@ -17,7 +17,6 @@
  * limitations under the License.
  */
 
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.util.LuceneTestCase;
 
 import java.io.StringReader;
@@ -28,15 +27,11 @@
     TokenStream stream = new WhitespaceTokenizer(
         new StringReader("short toolong evenmuchlongertext a ab toolong foo"));
     LengthFilter filter = new LengthFilter(stream, 2, 6);
-    TermAttribute termAtt = (TermAttribute) filter.getAttribute(TermAttribute.class);
-
-    assertTrue(filter.incrementToken());
-    assertEquals("short", termAtt.term());
-    assertTrue(filter.incrementToken());
-    assertEquals("ab", termAtt.term());
-    assertTrue(filter.incrementToken());
-    assertEquals("foo", termAtt.term());
-    assertFalse(filter.incrementToken());
+    final Token reusableToken = new Token();
+    assertEquals("short", filter.next(reusableToken).term());
+    assertEquals("ab", filter.next(reusableToken).term());
+    assertEquals("foo", filter.next(reusableToken).term());
+    assertNull(filter.next(reusableToken));
   }
 
 }

Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java Sat Jul  4 20:08:54 2009
@@ -1,9 +1,7 @@
 package org.apache.lucene.analysis;
 
-import java.io.StringReader;
-
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.util.LuceneTestCase;
+import java.io.StringReader;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -31,19 +29,17 @@
 
     TokenStream tokenStream = analyzer.tokenStream("field",
                                             new StringReader(text));
-    TermAttribute termAtt = (TermAttribute) tokenStream.getAttribute(TermAttribute.class);
-
-    assertTrue(tokenStream.incrementToken());
+    final Token reusableToken = new Token();
+    Token nextToken = tokenStream.next(reusableToken);
     assertEquals("WhitespaceAnalyzer does not lowercase",
                  "Qwerty",
-                 termAtt.term());
+                 nextToken.term());
 
     tokenStream = analyzer.tokenStream("special",
                                             new StringReader(text));
-    termAtt = (TermAttribute) tokenStream.getAttribute(TermAttribute.class);
-    assertTrue(tokenStream.incrementToken());
+    nextToken = tokenStream.next(reusableToken);
     assertEquals("SimpleAnalyzer lowercases",
                  "qwerty",
-                 termAtt.term());
+                 nextToken.term());
   }
 }

Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java Sat Jul  4 20:08:54 2009
@@ -1,10 +1,6 @@
 package org.apache.lucene.analysis;
 
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.util.LuceneTestCase;
 
 import java.io.StringReader;
@@ -39,25 +35,19 @@
 
   public void assertAnalyzesTo(Analyzer a, String input, String[] expectedImages, String[] expectedTypes, int[] expectedPosIncrs) throws Exception {
     TokenStream ts = a.tokenStream("dummy", new StringReader(input));
-    // TODO Java 1.5
-    //final TypeAttribute typeAtt = reusableToken.getAttribute(TypeAttribute.class);
-    //final PositionIncrementAttribute posIncrAtt = reusableToken.getAttribute(PositionIncrementAttribute.class);
-
-    final TermAttribute termAtt = (TermAttribute) ts.getAttribute(TermAttribute.class);
-    final TypeAttribute typeAtt = (TypeAttribute) ts.getAttribute(TypeAttribute.class);
-    final PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) ts.getAttribute(PositionIncrementAttribute.class);
-    
+    final Token reusableToken = new Token();
     for (int i = 0; i < expectedImages.length; i++) {
-      assertTrue(ts.incrementToken());
-      assertEquals(expectedImages[i], new String(termAtt.termBuffer(), 0, termAtt.termLength()));
+      Token nextToken = ts.next(reusableToken);
+      assertNotNull(nextToken);
+      assertEquals(expectedImages[i], nextToken.term());
       if (expectedTypes != null) {
-        assertEquals(expectedTypes[i], typeAtt.type());
+        assertEquals(expectedTypes[i], nextToken.type());
       }
       if (expectedPosIncrs != null) {
-        assertEquals(expectedPosIncrs[i], posIncrAtt.getPositionIncrement());
+        assertEquals(expectedPosIncrs[i], nextToken.getPositionIncrement());
       }
     }
-    assertFalse(ts.incrementToken());
+    assertNull(ts.next(reusableToken));
     ts.close();
   }
 

Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java Sat Jul  4 20:08:54 2009
@@ -17,8 +17,6 @@
  * limitations under the License.
  */
 
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.util.LuceneTestCase;
 
 import java.io.StringReader;
@@ -47,10 +45,9 @@
     StringReader reader = new StringReader("This is a test of the english stop analyzer");
     TokenStream stream = stop.tokenStream("test", reader);
     assertTrue(stream != null);
-    TermAttribute termAtt = (TermAttribute) stream.getAttribute(TermAttribute.class);
-    
-    while (stream.incrementToken()) {
-      assertFalse(inValidTokens.contains(termAtt.term()));
+    final Token reusableToken = new Token();
+    for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+      assertFalse(inValidTokens.contains(nextToken.term()));
     }
   }
 
@@ -63,13 +60,11 @@
     StringReader reader = new StringReader("This is a good test of the english stop analyzer");
     TokenStream stream = newStop.tokenStream("test", reader);
     assertNotNull(stream);
-    TermAttribute termAtt = (TermAttribute) stream.getAttribute(TermAttribute.class);
-    PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) stream.addAttribute(PositionIncrementAttribute.class);
-    
-    while (stream.incrementToken()) {
-      String text = termAtt.term();
+    final Token reusableToken = new Token();
+    for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+      String text = nextToken.term();
       assertFalse(stopWordsSet.contains(text));
-      assertEquals(1,posIncrAtt.getPositionIncrement()); // by default stop tokenizer does not apply increments.
+      assertEquals(1,nextToken.getPositionIncrement()); // by default stop tokenizer does not apply increments.
     }
   }
 
@@ -87,13 +82,11 @@
       TokenStream stream = newStop.tokenStream("test", reader);
       assertNotNull(stream);
       int i = 0;
-      TermAttribute termAtt = (TermAttribute) stream.getAttribute(TermAttribute.class);
-      PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) stream.addAttribute(PositionIncrementAttribute.class);
-
-      while (stream.incrementToken()) {
-        String text = termAtt.term();
+      final Token reusableToken = new Token();
+      for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+        String text = nextToken.term();
         assertFalse(stopWordsSet.contains(text));
-        assertEquals(expectedIncr[i++],posIncrAtt.getPositionIncrement());
+        assertEquals(expectedIncr[i++],nextToken.getPositionIncrement());
       }
     } finally {
       StopFilter.setEnablePositionIncrementsDefault(defaultEnable);

Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStopFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStopFilter.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStopFilter.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStopFilter.java Sat Jul  4 20:08:54 2009
@@ -16,8 +16,6 @@
  * limitations under the License.
  */
 
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.util.English;
 import org.apache.lucene.util.LuceneTestCase;
 
@@ -37,22 +35,19 @@
     StringReader reader = new StringReader("Now is The Time");
     String[] stopWords = new String[] { "is", "the", "Time" };
     TokenStream stream = new StopFilter(new WhitespaceTokenizer(reader), stopWords);
-    final TermAttribute termAtt = (TermAttribute) stream.getAttribute(TermAttribute.class);
-    assertTrue(stream.incrementToken());
-    assertEquals("Now", termAtt.term());
-    assertTrue(stream.incrementToken());
-    assertEquals("The", termAtt.term());
-    assertFalse(stream.incrementToken());
+    final Token reusableToken = new Token();
+    assertEquals("Now", stream.next(reusableToken).term());
+    assertEquals("The", stream.next(reusableToken).term());
+    assertEquals(null, stream.next(reusableToken));
   }
 
   public void testIgnoreCase() throws IOException {
     StringReader reader = new StringReader("Now is The Time");
     String[] stopWords = new String[] { "is", "the", "Time" };
     TokenStream stream = new StopFilter(new WhitespaceTokenizer(reader), stopWords, true);
-    final TermAttribute termAtt = (TermAttribute) stream.getAttribute(TermAttribute.class);
-    assertTrue(stream.incrementToken());
-    assertEquals("Now", termAtt.term());
-    assertFalse(stream.incrementToken());
+    final Token reusableToken = new Token();
+    assertEquals("Now", stream.next(reusableToken).term());
+    assertEquals(null,stream.next(reusableToken));
   }
 
   public void testStopFilt() throws IOException {
@@ -60,12 +55,10 @@
     String[] stopWords = new String[] { "is", "the", "Time" };
     Set stopSet = StopFilter.makeStopSet(stopWords);
     TokenStream stream = new StopFilter(new WhitespaceTokenizer(reader), stopSet);
-    final TermAttribute termAtt = (TermAttribute) stream.getAttribute(TermAttribute.class);
-    assertTrue(stream.incrementToken());
-    assertEquals("Now", termAtt.term());
-    assertTrue(stream.incrementToken());
-    assertEquals("The", termAtt.term());
-    assertFalse(stream.incrementToken());
+    final Token reusableToken = new Token();
+    assertEquals("Now", stream.next(reusableToken).term());
+    assertEquals("The", stream.next(reusableToken).term());
+    assertEquals(null, stream.next(reusableToken));
   }
 
   /**
@@ -117,16 +110,15 @@
   private void doTestStopPositons(StopFilter stpf, boolean enableIcrements) throws IOException {
     log("---> test with enable-increments-"+(enableIcrements?"enabled":"disabled"));
     stpf.setEnablePositionIncrements(enableIcrements);
-    TermAttribute termAtt = (TermAttribute) stpf.getAttribute(TermAttribute.class);
-    PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) stpf.getAttribute(PositionIncrementAttribute.class);
+    final Token reusableToken = new Token();
     for (int i=0; i<20; i+=3) {
-      assertTrue(stpf.incrementToken());
-      log("Token "+i+": "+stpf);
+      Token nextToken = stpf.next(reusableToken);
+      log("Token "+i+": "+nextToken);
       String w = English.intToEnglish(i).trim();
-      assertEquals("expecting token "+i+" to be "+w,w,termAtt.term());
-      assertEquals("all but first token must have position increment of 3",enableIcrements?(i==0?1:3):1,posIncrAtt.getPositionIncrement());
+      assertEquals("expecting token "+i+" to be "+w,w,nextToken.term());
+      assertEquals("all but first token must have position increment of 3",enableIcrements?(i==0?1:3):1,nextToken.getPositionIncrement());
     }
-    assertFalse(stpf.incrementToken());
+    assertNull(stpf.next(reusableToken));
   }
   
   // print debug info depending on VERBOSE

Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestToken.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestToken.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestToken.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestToken.java Sat Jul  4 20:08:54 2009
@@ -19,7 +19,6 @@
 
 import org.apache.lucene.util.LuceneTestCase;
 
-/** @deprecated */
 public class TestToken extends LuceneTestCase {
 
   public TestToken(String name) {

Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/index/TestDocumentWriter.java Sat Jul  4 20:08:54 2009
@@ -22,14 +22,12 @@
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.SimpleAnalyzer;
+import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.WhitespaceAnalyzer;
 import org.apache.lucene.analysis.WhitespaceTokenizer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
@@ -37,7 +35,6 @@
 import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.Field.TermVector;
 import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
 
@@ -141,38 +138,33 @@
       public TokenStream tokenStream(String fieldName, Reader reader) {
         return new TokenFilter(new WhitespaceTokenizer(reader)) {
           boolean first=true;
-          AttributeSource state;
+          Token buffered;
 
-          public boolean incrementToken() throws IOException {
-            if (state != null) {
-              state.restoreState(this);
-              payloadAtt.setPayload(null);
-              posIncrAtt.setPositionIncrement(0);
-              termAtt.setTermBuffer(new char[]{'b'}, 0, 1);
-              state = null;
-              return true;
+          public Token next(final Token reusableToken) throws IOException {
+            if (buffered != null) {
+              Token nextToken = buffered;
+              buffered=null;
+              return nextToken;
             }
-
-            boolean hasNext = input.incrementToken();
-            if (!hasNext) return false;
-            if (Character.isDigit(termAtt.termBuffer()[0])) {
-              posIncrAtt.setPositionIncrement(termAtt.termBuffer()[0] - '0');
+            Token nextToken = input.next(reusableToken);
+            if (nextToken==null) return null;
+            if (Character.isDigit(nextToken.termBuffer()[0])) {
+              nextToken.setPositionIncrement(nextToken.termBuffer()[0] - '0');
             }
             if (first) {
               // set payload on first position only
-              payloadAtt.setPayload(new Payload(new byte[]{100}));
+              nextToken.setPayload(new Payload(new byte[]{100}));
               first = false;
             }
 
             // index a "synonym" for every token
-            state = captureState();
-            return true;
+            buffered = (Token)nextToken.clone();
+            buffered.setPayload(null);
+            buffered.setPositionIncrement(0);
+            buffered.setTermBuffer(new char[]{'b'}, 0, 1);
 
+            return nextToken;
           }
-
-          TermAttribute termAtt = (TermAttribute) addAttribute(TermAttribute.class);
-          PayloadAttribute payloadAtt = (PayloadAttribute) addAttribute(PayloadAttribute.class);
-          PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class);          
         };
       }
     };
@@ -209,14 +201,12 @@
       private String[] tokens = new String[] {"term1", "term2", "term3", "term2"};
       private int index = 0;
       
-      private TermAttribute termAtt = (TermAttribute) addAttribute(TermAttribute.class);
-      
-      public boolean incrementToken() throws IOException {
+      public Token next(final Token reusableToken) throws IOException {
+        assert reusableToken != null;
         if (index == tokens.length) {
-          return false;
+          return null;
         } else {
-          termAtt.setTermBuffer(tokens[index++]);
-          return true;
+          return reusableToken.reinit(tokens[index++], 0, 0);
         }        
       }
       

Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/index/TestIndexWriter.java Sat Jul  4 20:08:54 2009
@@ -17,48 +17,48 @@
  * limitations under the License.
  */
 
-import java.io.ByteArrayOutputStream;
-import java.io.File;
 import java.io.IOException;
-import java.io.PrintStream;
 import java.io.Reader;
-import java.util.ArrayList;
+import java.io.File;
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
 import java.util.Arrays;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Random;
 
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.UnicodeUtil;
+
+import org.apache.lucene.analysis.WhitespaceAnalyzer;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.SinkTokenizer;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceAnalyzer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.Token;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.PhraseQuery;
-import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.search.spans.SpanTermQuery;
-import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.Lock;
-import org.apache.lucene.store.LockFactory;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.util._TestUtil;
+
 import org.apache.lucene.store.MockRAMDirectory;
-import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.store.LockFactory;
+import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.SingleInstanceLockFactory;
-import org.apache.lucene.util.AttributeSource;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.UnicodeUtil;
-import org.apache.lucene.util._TestUtil;
 
 /**
  *
@@ -1793,11 +1793,11 @@
         return new TokenFilter(new StandardTokenizer(reader)) {
           private int count = 0;
 
-          public boolean incrementToken() throws IOException {
+          public Token next(final Token reusableToken) throws IOException {
             if (count++ == 5) {
               throw new IOException();
             }
-            return input.incrementToken();
+            return input.next(reusableToken);
           }
         };
       }
@@ -1916,10 +1916,10 @@
       this.fieldName = fieldName;
     }
 
-    public boolean incrementToken() throws IOException {
+    public Token next(final Token reusableToken) throws IOException {
       if (this.fieldName.equals("crash") && count++ >= 4)
         throw new IOException("I'm experiencing problems");
-      return input.incrementToken();
+      return input.next(reusableToken);
     }
 
     public void reset() throws IOException {
@@ -3577,47 +3577,21 @@
     }
   }
 
-  private static class MyAnalyzer extends Analyzer {
-
-    public TokenStream tokenStream(String fieldName, Reader reader) {
-      TokenStream s = new WhitespaceTokenizer(reader);
-      s.addAttribute(PositionIncrementAttribute.class);
-      return s;
-    }
-    
-  }
-  
   // LUCENE-1255
   public void testNegativePositions() throws Throwable {
     SinkTokenizer tokens = new SinkTokenizer();
-    tokens.addAttribute(TermAttribute.class);
-    tokens.addAttribute(PositionIncrementAttribute.class);
-
-    AttributeSource state = new AttributeSource();
-    TermAttribute termAtt = (TermAttribute) state.addAttribute(TermAttribute.class);
-    PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) state.addAttribute(PositionIncrementAttribute.class);
-    termAtt.setTermBuffer("a");
-    posIncrAtt.setPositionIncrement(0);
-    tokens.add(state);
-
-    state = new AttributeSource();
-    termAtt = (TermAttribute) state.addAttribute(TermAttribute.class);
-    posIncrAtt = (PositionIncrementAttribute) state.addAttribute(PositionIncrementAttribute.class);
-
-    termAtt.setTermBuffer("b");
-    posIncrAtt.setPositionIncrement(1);
-    tokens.add(state);
-    
-    state = new AttributeSource();
-    termAtt = (TermAttribute) state.addAttribute(TermAttribute.class);
-    posIncrAtt = (PositionIncrementAttribute) state.addAttribute(PositionIncrementAttribute.class);
-
-    termAtt.setTermBuffer("c");
-    posIncrAtt.setPositionIncrement(1);
-    tokens.add(state);
+    Token t = new Token();
+    t.setTermBuffer("a");
+    t.setPositionIncrement(0);
+    tokens.add(t);
+    t.setTermBuffer("b");
+    t.setPositionIncrement(1);
+    tokens.add(t);
+    t.setTermBuffer("c");
+    tokens.add(t);
 
     MockRAMDirectory dir = new MockRAMDirectory();
-    IndexWriter w = new IndexWriter(dir, new MyAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);
+    IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);
     w.setAllowMinus1Position();
     Document doc = new Document();
     doc.add(new Field("field", tokens));



Mime
View raw message