lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From uschind...@apache.org
Subject svn commit: r762710 - in /lucene/java/trunk/contrib/queries/src: java/org/apache/lucene/search/trie/ test/org/apache/lucene/search/trie/
Date Tue, 07 Apr 2009 11:47:35 GMT
Author: uschindler
Date: Tue Apr  7 11:47:35 2009
New Revision: 762710

URL: http://svn.apache.org/viewvc?rev=762710&view=rev
Log:
LUCENE-1582: Make TrieRange completely independent from Document/Field with TokenStream of prefix encoded values

Added:
    lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/IntTrieTokenStream.java
    lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/LongTrieTokenStream.java
    lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/ShiftAttribute.java
    lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestIntTrieTokenStream.java
    lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestLongTrieTokenStream.java
Modified:
    lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/AbstractTrieRangeFilter.java
    lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/IntTrieRangeFilter.java
    lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/LongTrieRangeFilter.java
    lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/TrieUtils.java
    lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/package.html
    lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestIntTrieRangeFilter.java
    lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestLongTrieRangeFilter.java
    lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestTrieUtils.java

Modified: lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/AbstractTrieRangeFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/AbstractTrieRangeFilter.java?rev=762710&r1=762709&r2=762710&view=diff
==============================================================================
--- lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/AbstractTrieRangeFilter.java (original)
+++ lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/AbstractTrieRangeFilter.java Tue Apr  7 11:47:35 2009
@@ -18,7 +18,6 @@
  */
 
 import java.io.IOException;
-import java.util.Arrays;
 
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.search.Query;
@@ -33,10 +32,10 @@
 
 abstract class AbstractTrieRangeFilter extends Filter {
 
-  AbstractTrieRangeFilter(final String[] fields, final int precisionStep,
+  AbstractTrieRangeFilter(final String field, final int precisionStep,
     Number min, Number max, final boolean minInclusive, final boolean maxInclusive
   ) {
-    this.fields=(String[])fields.clone();
+    this.field=field.intern();
     this.precisionStep=precisionStep;
     this.min=min;
     this.max=max;
@@ -51,7 +50,7 @@
 
   public String toString(final String field) {
     final StringBuffer sb=new StringBuffer();
-    if (!this.fields[0].equals(field)) sb.append(this.fields[0]).append(':');
+    if (!this.field.equals(field)) sb.append(this.field).append(':');
     return sb.append(minInclusive ? '[' : '{')
       .append((min==null) ? "*" : min.toString())
       .append(" TO ")
@@ -66,7 +65,7 @@
     if (this.getClass().equals(o.getClass())) {
       AbstractTrieRangeFilter q=(AbstractTrieRangeFilter)o;
       return (
-        Arrays.equals(fields,q.fields) &&
+        field==q.field &&
         (q.min == null ? min == null : q.min.equals(min)) &&
         (q.max == null ? max == null : q.max.equals(max)) &&
         minInclusive==q.minInclusive &&
@@ -79,7 +78,7 @@
 
   //@Override
   public final int hashCode() {
-    int hash=Arrays.asList(fields).hashCode()+(precisionStep^0x64365465);
+    int hash = field.hashCode() + (precisionStep^0x64365465);
     if (min!=null) hash += min.hashCode()^0x14fa55fb;
     if (max!=null) hash += max.hashCode()^0x733fa5fe;
     return hash+
@@ -123,12 +122,10 @@
   void fillBits(
     final IndexReader reader,
     final OpenBitSet bits, final TermDocs termDocs,
-    String field,
     final String lowerTerm, final String upperTerm
   ) throws IOException {
     final int len=lowerTerm.length();
     assert upperTerm.length()==len;
-    field=field.intern();
     
     // find the docs
     final TermEnum enumerator = reader.terms(new Term(field, lowerTerm));
@@ -151,7 +148,7 @@
   }
   
   // members
-  final String[] fields;
+  final String field;
   final int precisionStep;
   final Number min,max;
   final boolean minInclusive,maxInclusive;

Modified: lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/IntTrieRangeFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/IntTrieRangeFilter.java?rev=762710&r1=762709&r2=762710&view=diff
==============================================================================
--- lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/IntTrieRangeFilter.java (original)
+++ lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/IntTrieRangeFilter.java Tue Apr  7 11:47:35 2009
@@ -30,7 +30,7 @@
 /**
  * Implementation of a Lucene {@link Filter} that implements trie-based range filtering for ints/floats.
  * This filter depends on a specific structure of terms in the index that can only be created
- * by {@link TrieUtils} methods.
+ * by indexing via {@link IntTrieTokenStream} methods.
  * For more information, how the algorithm works, see the {@linkplain org.apache.lucene.search.trie package description}.
  */
 public class IntTrieRangeFilter extends AbstractTrieRangeFilter {
@@ -43,49 +43,11 @@
    * You can leave the bounds open, by supplying <code>null</code> for <code>min</code> and/or
    * <code>max</code>. Inclusive/exclusive bounds can also be supplied.
    * To query float values use the converter {@link TrieUtils#floatToSortableInt}.
-   * <p>This is the counterpart to {@link TrieUtils#addIndexedFields(Document,String,String[])}.
-   * <p><b>This is the recommended usage of TrieUtils/IntTrieRangeFilter.</b>
    */
   public IntTrieRangeFilter(final String field, final int precisionStep,
     final Integer min, final Integer max, final boolean minInclusive, final boolean maxInclusive
   ) {
-    this(
-      new String[]{field, field+TrieUtils.LOWER_PRECISION_FIELD_NAME_SUFFIX},
-      precisionStep,min,max,minInclusive,maxInclusive
-    );
-  }
-  
-  /**
-   * Expert: A trie filter for matching trie coded values using the given field names.
-   * You can specify the main and helper field name, that was used to idex the values.
-   * <code>precisionStep</code> must me equal or a multiple of the <code>precisionStep</code>
-   * used for indexing the values.
-   * You can leave the bounds open, by supplying <code>null</code> for <code>min</code> and/or
-   * <code>max</code>. Inclusive/exclusive bounds can also be supplied.
-   * To query float values use the converter {@link TrieUtils#floatToSortableInt}.
-   * <p>This is the counterpart to {@link TrieUtils#addIndexedFields(Document,String,String,String[])}.
-   */
-  public IntTrieRangeFilter(final String field, final String lowerPrecisionField, final int precisionStep,
-    final Integer min, final Integer max, final boolean minInclusive, final boolean maxInclusive
-  ) {
-    this(new String[]{field, lowerPrecisionField},precisionStep,min,max,minInclusive,maxInclusive);
-  }
-
-  /**
-   * Expert: A trie filter for matching trie coded values
-   * using the given field names. If the array of field names is shorter than the
-   * trieCoded one, all trieCoded values with higher index get the last field name.
-   * <code>precisionStep</code> must me equal or a multiple of the <code>precisionStep</code>
-   * used for indexing the values.
-   * You can leave the bounds open, by supplying <code>null</code> for <code>min</code> and/or
-   * <code>max</code>. Inclusive/exclusive bounds can also be supplied.
-   * To query float values use the converter {@link TrieUtils#floatToSortableInt}.
-   * <p>This is the counterpart to {@link TrieUtils#addIndexedFields(Document,String[],String[])}.
-   */
-  public IntTrieRangeFilter(final String[] fields, final int precisionStep,
-    Integer min, Integer max, final boolean minInclusive, final boolean maxInclusive
-  ) {
-    super(fields, precisionStep, min, max, minInclusive, maxInclusive);
+    super(field,precisionStep,min,max,minInclusive,maxInclusive);
   }
 
   /**
@@ -112,11 +74,10 @@
         TrieUtils.splitIntRange(new TrieUtils.IntRangeBuilder() {
         
           //@Override
-          public final void addRange(String minPrefixCoded, String maxPrefixCoded, int level) {
+          public final void addRange(String minPrefixCoded, String maxPrefixCoded) {
             try {
               fillBits(
                 reader, bits, termDocs,
-                fields[Math.min(fields.length-1, level)],
                 minPrefixCoded, maxPrefixCoded
               );
             } catch (IOException ioe) {

Added: lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/IntTrieTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/IntTrieTokenStream.java?rev=762710&view=auto
==============================================================================
--- lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/IntTrieTokenStream.java (added)
+++ lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/IntTrieTokenStream.java Tue Apr  7 11:47:35 2009
@@ -0,0 +1,172 @@
+package org.apache.lucene.search.trie;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+
+/**
+ * This class provides a {@link TokenStream} for indexing <code>int</code> values
+ * that can be queried by {@link IntTrieRangeFilter}. This stream is not intended
+ * to be used in analyzers, its more for iterating the different precisions during
+ * indexing a specific numeric value.
+ * <p>A <code>int</code> value is indexed as multiple string encoded terms, each reduced
+ * by zeroing bits from the right. Each value is also prefixed (in the first char) by the
+ * <code>shift</code> value (number of bits removed) used during encoding.
+ * <p>The number of bits removed from the right for each trie entry is called
+ * <code>precisionStep</code> in this API. For comparing the different step values, see the
+ * {@linkplain org.apache.lucene.search.trie package description}.
+ * <p>The usage pattern is (it is recommened to switch off norms and term frequencies
+ * for numeric fields; it does not make sense to have them):
+ * <pre>
+ *  Field field = new Field(name, new IntTrieTokenStream(value, precisionStep));
+ *  field.setOmitNorms(true);
+ *  field.setOmitTermFreqAndPositions(true);
+ *  document.add(field);
+ * </pre>
+ * <p>For optimal performance, re-use the TokenStream and Field instance
+ * for more than one document:
+ * <pre>
+ *  <em>// init</em>
+ *  TokenStream stream = new IntTrieTokenStream(precisionStep);
+ *  Field field = new Field(name, stream);
+ *  field.setOmitNorms(true);
+ *  field.setOmitTermFreqAndPositions(true);
+ *  <em>// use this code to index many documents:</em>
+ *  stream.setValue(value1)
+ *  document.add(field);
+ *  writer.addDocument(document);
+ *  stream.setValue(value2)
+ *  document.add(field);
+ *  writer.addDocument(document);
+ *  ...
+ * </pre>
+ * <p><em>Please note:</em> Token streams are read, when the document is added to index.
+ * If you index more than one numeric field, use a separate instance for each.
+ * <p>For more information, how trie fields work, see the
+ * {@linkplain org.apache.lucene.search.trie package description}.
+ */
+public class IntTrieTokenStream extends TokenStream {
+
+  /** The full precision token gets this token type assigned. */
+  public static final String TOKEN_TYPE_FULL_PREC  = "fullPrecTrieInt";
+
+  /** The lower precision tokens gets this token type assigned. */
+  public static final String TOKEN_TYPE_LOWER_PREC = "lowerPrecTrieInt";
+
+  /**
+   * Creates a token stream for indexing <code>value</code> with the given
+   * <code>precisionStep</code>. As instance creating is a major cost,
+   * consider using a {@link #IntTrieTokenStream(int)} instance once for
+   * indexing a large number of documents and assign a value with
+   * {@link #setValue} for each document.
+   * To index float values use the converter {@link TrieUtils#doubleToSortableLong}.
+   */
+  public IntTrieTokenStream(final int value, final int precisionStep) {
+    if (precisionStep<1 || precisionStep>32)
+      throw new IllegalArgumentException("precisionStep may only be 1..32");
+    this.value = value;
+    this.precisionStep = precisionStep;
+    termAtt = (TermAttribute) addAttribute(TermAttribute.class);
+    typeAtt = (TypeAttribute) addAttribute(TypeAttribute.class);
+    posIncrAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class);
+    shiftAtt = (ShiftAttribute) addAttribute(ShiftAttribute.class);
+  }
+  
+  /**
+   * Creates a token stream for indexing values with the given
+   * <code>precisionStep</code>. This stream is initially &quot;empty&quot;
+   * (using a numeric value of 0), assign a value before indexing
+   * each document using {@link #setValue}.
+   */
+  public IntTrieTokenStream(final int precisionStep) {
+    this(0, precisionStep);
+  }
+  
+  /**
+   * Resets the token stream to deliver prefix encoded values
+   * for <code>value</code>. Use this method to index the same
+   * numeric field for a large number of documents and reuse the
+   * current stream instance.
+   * To index float values use the converter {@link TrieUtils#doubleToSortableLong}.
+   */
+  public void setValue(final int value) {
+    this.value = value;
+    reset();
+  }
+  
+  // @Override
+  public void reset() {
+    shift = 0;
+  }
+
+  // @Override
+  public boolean incrementToken() {
+    if (shift>=32) return false;
+    final char[] buffer = termAtt.resizeTermBuffer(TrieUtils.INT_BUF_SIZE);
+    termAtt.setTermLength(TrieUtils.intToPrefixCoded(value, shift, buffer));
+    shiftAtt.setShift(shift);
+    if (shift==0) {
+      typeAtt.setType(TOKEN_TYPE_FULL_PREC);
+      posIncrAtt.setPositionIncrement(1);
+    } else {
+      typeAtt.setType(TOKEN_TYPE_LOWER_PREC);
+      posIncrAtt.setPositionIncrement(0);
+    }
+    shift += precisionStep;
+    return true;
+  }
+
+  // @Override
+  /** @deprecated */
+  public Token next(final Token reusableToken) {
+    if (shift>=32) return null;
+    reusableToken.clear();
+    final char[] buffer = reusableToken.resizeTermBuffer(TrieUtils.INT_BUF_SIZE);
+    reusableToken.setTermLength(TrieUtils.intToPrefixCoded(value, shift, buffer));
+    if (shift==0) {
+      reusableToken.setType(TOKEN_TYPE_FULL_PREC);
+      reusableToken.setPositionIncrement(1);
+    } else {
+      reusableToken.setType(TOKEN_TYPE_LOWER_PREC);
+      reusableToken.setPositionIncrement(0);
+    }
+    shift += precisionStep;
+    return reusableToken;
+  }
+  
+  // @Override
+  public String toString() {
+    final StringBuffer sb = new StringBuffer("(trie-int,value=").append(value);
+    sb.append(",precisionStep=").append(precisionStep).append(')');
+    return sb.toString();
+  }
+
+  // members
+  private final TermAttribute termAtt;
+  private final TypeAttribute typeAtt;
+  private final PositionIncrementAttribute posIncrAtt;
+  private final ShiftAttribute shiftAtt;
+  
+  private int shift = 0;
+  private int value;
+  private final int precisionStep;
+}

Modified: lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/LongTrieRangeFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/LongTrieRangeFilter.java?rev=762710&r1=762709&r2=762710&view=diff
==============================================================================
--- lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/LongTrieRangeFilter.java (original)
+++ lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/LongTrieRangeFilter.java Tue Apr  7 11:47:35 2009
@@ -30,7 +30,7 @@
 /**
  * Implementation of a Lucene {@link Filter} that implements trie-based range filtering for longs/doubles.
  * This filter depends on a specific structure of terms in the index that can only be created
- * by {@link TrieUtils} methods.
+ * by indexing via {@link LongTrieTokenStream} methods.
  * For more information, how the algorithm works, see the {@linkplain org.apache.lucene.search.trie package description}.
  */
 public class LongTrieRangeFilter extends AbstractTrieRangeFilter {
@@ -43,49 +43,11 @@
    * You can leave the bounds open, by supplying <code>null</code> for <code>min</code> and/or
    * <code>max</code>. Inclusive/exclusive bounds can also be supplied.
    * To query double values use the converter {@link TrieUtils#doubleToSortableLong}.
-   * <p>This is the counterpart to {@link TrieUtils#addIndexedFields(Document,String,String[])}.
-   * <p><b>This is the recommended usage of TrieUtils/LongTrieRangeFilter.</b>
    */
   public LongTrieRangeFilter(final String field, final int precisionStep,
     final Long min, final Long max, final boolean minInclusive, final boolean maxInclusive
   ) {
-    this(
-      new String[]{field, field+TrieUtils.LOWER_PRECISION_FIELD_NAME_SUFFIX},
-      precisionStep,min,max,minInclusive,maxInclusive
-    );
-  }
-  
-  /**
-   * Expert: A trie filter for matching trie coded values using the given field names.
-   * You can specify the main and helper field name, that was used to idex the values.
-   * <code>precisionStep</code> must me equal or a multiple of the <code>precisionStep</code>
-   * used for indexing the values.
-   * You can leave the bounds open, by supplying <code>null</code> for <code>min</code> and/or
-   * <code>max</code>. Inclusive/exclusive bounds can also be supplied.
-   * To query double values use the converter {@link TrieUtils#doubleToSortableLong}.
-   * <p>This is the counterpart to {@link TrieUtils#addIndexedFields(Document,String,String,String[])}.
-   */
-  public LongTrieRangeFilter(final String field, final String lowerPrecisionField, final int precisionStep,
-    final Long min, final Long max, final boolean minInclusive, final boolean maxInclusive
-  ) {
-    this(new String[]{field, lowerPrecisionField},precisionStep,min,max,minInclusive,maxInclusive);
-  }
-
-  /**
-   * Expert: A trie filter for matching trie coded values
-   * using the given field names. If the array of field names is shorter than the
-   * trieCoded one, all trieCoded values with higher index get the last field name.
-   * <code>precisionStep</code> must me equal or a multiple of the <code>precisionStep</code>
-   * used for indexing the values.
-   * You can leave the bounds open, by supplying <code>null</code> for <code>min</code> and/or
-   * <code>max</code>. Inclusive/exclusive bounds can also be supplied.
-   * To query double values use the converter {@link TrieUtils#doubleToSortableLong}.
-   * <p>This is the counterpart to {@link TrieUtils#addIndexedFields(Document,String[],String[])}.
-   */
-  public LongTrieRangeFilter(final String[] fields, final int precisionStep,
-    Long min, Long max, final boolean minInclusive, final boolean maxInclusive
-  ) {
-    super(fields, precisionStep, min, max, minInclusive, maxInclusive);
+    super(field,precisionStep,min,max,minInclusive,maxInclusive);
   }
 
   /**
@@ -112,11 +74,10 @@
         TrieUtils.splitLongRange(new TrieUtils.LongRangeBuilder() {
         
           //@Override
-          public final void addRange(String minPrefixCoded, String maxPrefixCoded, int level) {
+          public final void addRange(String minPrefixCoded, String maxPrefixCoded) {
             try {
               fillBits(
                 reader, bits, termDocs,
-                fields[Math.min(fields.length-1, level)],
                 minPrefixCoded, maxPrefixCoded
               );
             } catch (IOException ioe) {

Added: lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/LongTrieTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/LongTrieTokenStream.java?rev=762710&view=auto
==============================================================================
--- lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/LongTrieTokenStream.java (added)
+++ lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/LongTrieTokenStream.java Tue Apr  7 11:47:35 2009
@@ -0,0 +1,172 @@
+package org.apache.lucene.search.trie;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+
+/**
+ * This class provides a {@link TokenStream} for indexing <code>long</code> values
+ * that can be queried by {@link LongTrieRangeFilter}. This stream is not intended
+ * to be used in analyzers, its more for iterating the different precisions during
+ * indexing a specific numeric value.
+ * <p>A <code>long</code> value is indexed as multiple string encoded terms, each reduced
+ * by zeroing bits from the right. Each value is also prefixed (in the first char) by the
+ * <code>shift</code> value (number of bits removed) used during encoding.
+ * <p>The number of bits removed from the right for each trie entry is called
+ * <code>precisionStep</code> in this API. For comparing the different step values, see the
+ * {@linkplain org.apache.lucene.search.trie package description}.
+ * <p>The usage pattern is (it is recommened to switch off norms and term frequencies
+ * for numeric fields; it does not make sense to have them):
+ * <pre>
+ *  Field field = new Field(name, new LongTrieTokenStream(value, precisionStep));
+ *  field.setOmitNorms(true);
+ *  field.setOmitTermFreqAndPositions(true);
+ *  document.add(field);
+ * </pre>
+ * <p>For optimal performance, re-use the TokenStream and Field instance
+ * for more than one document:
+ * <pre>
+ *  <em>// init</em>
+ *  TokenStream stream = new LongTrieTokenStream(precisionStep);
+ *  Field field = new Field(name, stream);
+ *  field.setOmitNorms(true);
+ *  field.setOmitTermFreqAndPositions(true);
+ *  <em>// use this code to index many documents:</em>
+ *  stream.setValue(value1)
+ *  document.add(field);
+ *  writer.addDocument(document);
+ *  stream.setValue(value2)
+ *  document.add(field);
+ *  writer.addDocument(document);
+ *  ...
+ * </pre>
+ * <p><em>Please note:</em> Token streams are read, when the document is added to index.
+ * If you index more than one numeric field, use a separate instance for each.
+ * <p>For more information, how trie fields work, see the
+ * {@linkplain org.apache.lucene.search.trie package description}.
+ */
+public class LongTrieTokenStream extends TokenStream {
+
+  /** The full precision token gets this token type assigned. */
+  public static final String TOKEN_TYPE_FULL_PREC  = "fullPrecTrieLong";
+
+  /** The lower precision tokens gets this token type assigned. */
+  public static final String TOKEN_TYPE_LOWER_PREC = "lowerPrecTrieLong";
+
+  /**
+   * Creates a token stream for indexing <code>value</code> with the given
+   * <code>precisionStep</code>. As instance creating is a major cost,
+   * consider using a {@link #LongTrieTokenStream(int)} instance once for
+   * indexing a large number of documents and assign a value with
+   * {@link #setValue} for each document.
+   * To index double values use the converter {@link TrieUtils#doubleToSortableLong}.
+   */
+  public LongTrieTokenStream(final long value, final int precisionStep) {
+    if (precisionStep<1 || precisionStep>64)
+      throw new IllegalArgumentException("precisionStep may only be 1..64");
+    this.value = value;
+    this.precisionStep = precisionStep;
+    termAtt = (TermAttribute) addAttribute(TermAttribute.class);
+    typeAtt = (TypeAttribute) addAttribute(TypeAttribute.class);
+    posIncrAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class);
+    shiftAtt = (ShiftAttribute) addAttribute(ShiftAttribute.class);
+  }
+  
+  /**
+   * Creates a token stream for indexing values with the given
+   * <code>precisionStep</code>. This stream is initially &quot;empty&quot;
+   * (using a numeric value of 0), assign a value before indexing
+   * each document using {@link #setValue}.
+   */
+  public LongTrieTokenStream(final int precisionStep) {
+    this(0L, precisionStep);
+  }
+
+  /**
+   * Resets the token stream to deliver prefix encoded values
+   * for <code>value</code>. Use this method to index the same
+   * numeric field for a large number of documents and reuse the
+   * current stream instance.
+   * To index double values use the converter {@link TrieUtils#doubleToSortableLong}.
+   */
+  public void setValue(final long value) {
+    this.value = value;
+    reset();
+  }
+  
+  // @Override
+  public void reset() {
+    shift = 0;
+  }
+
+  // @Override
+  public boolean incrementToken() {
+    if (shift>=64) return false;
+    final char[] buffer = termAtt.resizeTermBuffer(TrieUtils.LONG_BUF_SIZE);
+    termAtt.setTermLength(TrieUtils.longToPrefixCoded(value, shift, buffer));
+    shiftAtt.setShift(shift);
+    if (shift==0) {
+      typeAtt.setType(TOKEN_TYPE_FULL_PREC);
+      posIncrAtt.setPositionIncrement(1);
+    } else {
+      typeAtt.setType(TOKEN_TYPE_LOWER_PREC);
+      posIncrAtt.setPositionIncrement(0);
+    }
+    shift += precisionStep;
+    return true;
+  }
+
+  // @Override
+  /** @deprecated */
+  public Token next(final Token reusableToken) {
+    if (shift>=64) return null;
+    reusableToken.clear();
+    final char[] buffer = reusableToken.resizeTermBuffer(TrieUtils.LONG_BUF_SIZE);
+    reusableToken.setTermLength(TrieUtils.longToPrefixCoded(value, shift, buffer));
+    if (shift==0) {
+      reusableToken.setType(TOKEN_TYPE_FULL_PREC);
+      reusableToken.setPositionIncrement(1);
+    } else {
+      reusableToken.setType(TOKEN_TYPE_LOWER_PREC);
+      reusableToken.setPositionIncrement(0);
+    }
+    shift += precisionStep;
+    return reusableToken;
+  }
+  
+  // @Override
+  public String toString() {
+    final StringBuffer sb = new StringBuffer("(trie-long,value=").append(value);
+    sb.append(",precisionStep=").append(precisionStep).append(')');
+    return sb.toString();
+  }
+
+  // members
+  private final TermAttribute termAtt;
+  private final TypeAttribute typeAtt;
+  private final PositionIncrementAttribute posIncrAtt;
+  private final ShiftAttribute shiftAtt;
+  
+  private int shift = 0;
+  private long value;
+  private final int precisionStep;
+}

Added: lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/ShiftAttribute.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/ShiftAttribute.java?rev=762710&view=auto
==============================================================================
--- lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/ShiftAttribute.java (added)
+++ lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/ShiftAttribute.java Tue Apr  7 11:47:35 2009
@@ -0,0 +1,67 @@
+package org.apache.lucene.search.trie;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.Attribute;
+
+import java.io.Serializable;
+
+/**
+ * This attribute is updated by {@link IntTrieTokenStream} and {@link LongTrieTokenStream}
+ * to the shift value of the current prefix-encoded token.
+ * It may be used by filters or consumers to e.g. distribute the values to various fields.
+ */
+public final class ShiftAttribute extends Attribute implements Cloneable, Serializable {
+  private int shift = 0;
+  
+  /**
+   * Returns the shift value of the current prefix encoded token.
+   */
+  public int getShift() {
+    return shift;
+  }
+
+  void setShift(final int shift) {
+    this.shift = shift;
+  }
+  
+  public void clear() {
+    shift = 0;
+  }
+
+  public String toString() {
+    return "shift=" + shift;
+  }
+
+  public boolean equals(Object other) {
+    if (this == other) return true;
+    if (other instanceof ShiftAttribute) {
+      return ((ShiftAttribute) other).shift == shift;
+    }
+    return false;
+  }
+
+  public int hashCode() {
+    return shift;
+  }
+  
+  public void copyTo(Attribute target) {
+    final ShiftAttribute t = (ShiftAttribute) target;
+    t.setShift(shift);
+  }
+}

Modified: lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/TrieUtils.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/TrieUtils.java?rev=762710&r1=762709&r2=762710&view=diff
==============================================================================
--- lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/TrieUtils.java (original)
+++ lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/TrieUtils.java Tue Apr  7 11:47:35 2009
@@ -17,17 +17,13 @@
  * limitations under the License.
  */
 
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.FieldCache;
 import org.apache.lucene.search.ExtendedFieldCache;
 
 /**
- * This is a helper class to construct the trie-based index entries for numerical values.
- * For more information on how the algorithm works, see the
- * {@linkplain org.apache.lucene.search.trie package description}.
- * <h3>The trie format using prefix encoded numerical values</h3>
+ * This is a helper class to generate prefix-encoded representations for numerical values
+ * and supplies converters to represent float/double values as sortable integers/longs.
  * <p>To quickly execute range queries in Apache Lucene, a range is divided recursively
  * into multiple intervals for searching: The center of the range is searched only with
  * the lowest possible precision in the trie, while the boundaries are matched
@@ -35,54 +31,48 @@
  * <p>This class generates terms to achive this: First the numerical integer values need to
  * be converted to strings. For that integer values (32 bit or 64 bit) are made unsigned
  * and the bits are converted to ASCII chars with each 7 bit. The resulting string is
- * sortable like the original integer value.
+ * sortable like the original integer value. Each value is also prefixed
+ * (in the first char) by the <code>shift</code> value (number of bits removed) used
+ * during encoding.
  * <p>To also index floating point numbers, this class supplies two methods to convert them
  * to integer values by changing their bit layout: {@link #doubleToSortableLong},
  * {@link #floatToSortableInt}. You will have no precision loss by
  * converting floating point numbers to integers and back (only that the integer form
  * is not usable). Other data types like dates can easily converted to longs or ints (e.g.
  * date to long: {@link java.util.Date#getTime}).
- * <p>To index the different precisions of the long values each encoded value is also reduced
- * by zeroing bits from the right. Each value is also prefixed (in the first char) by the
- * <code>shift</code> value (number of bits removed) used during encoding. This series of
- * different precision values can be indexed into a Lucene {@link Document} using
- * {@link #addIndexedFields(Document,String,String[])}. The default is to index the original
- * precision in the supplied field name and the lower precisions in an additional helper field.
- * Because of this, the full-precision field can also be sorted (using {@link #getLongSortField}
- * or {@link #getIntSortField}).
- * <p>The number of bits removed from the right for each trie entry is called
- * <code>precisionStep</code> in this API. For comparing the different step values, see the
- * {@linkplain org.apache.lucene.search.trie package description}.
+ * <p>Prefix encoded fields can also be sorted using the {@link SortField} factories
+ * {@link #getLongSortField} or {@link #getIntSortField}.
  */
 public final class TrieUtils {
 
   private TrieUtils() {} // no instance!
 
   /**
-   * The default &quot;helper&quot; field containing the lower precision terms is the original
-   * fieldname with this appended. This suffix is used in
-   * {@link #addIndexedFields(Document,String,String[])} and the corresponding c'tor
-   * of <code>(Long|Int)TrieRangeFilter</code>.
-   */
-  public static final String LOWER_PRECISION_FIELD_NAME_SUFFIX="#trie";
-
-  /**
    * Longs are stored at lower precision by shifting off lower bits. The shift count is
    * stored as <code>SHIFT_START_LONG+shift</code> in the first character
    */
   public static final char SHIFT_START_LONG = (char)0x20;
 
+  /** internal: maximum needed <code>char[]</code> buffer size for encoding */
+  static final int LONG_BUF_SIZE = 63/7 + 2;
+
   /**
    * Integers are stored at lower precision by shifting off lower bits. The shift count is
    * stored as <code>SHIFT_START_INT+shift</code> in the first character
    */
   public static final char SHIFT_START_INT  = (char)0x60;
 
+  /** internal: maximum needed <code>char[]</code> buffer size for encoding */
+  static final int INT_BUF_SIZE = 31/7 + 2;
+
   /**
    * A parser instance for filling a {@link ExtendedFieldCache}, that parses prefix encoded fields as longs.
    */
   public static final ExtendedFieldCache.LongParser FIELD_CACHE_LONG_PARSER=new ExtendedFieldCache.LongParser(){
     public final long parseLong(final String val) {
+      final int shift = val.charAt(0)-SHIFT_START_LONG;
+      if (shift>0 && shift<=63)
+        throw new FieldCache.StopFillCacheException();
       return prefixCodedToLong(val);
     }
   };
@@ -92,6 +82,9 @@
    */
   public static final FieldCache.IntParser FIELD_CACHE_INT_PARSER=new FieldCache.IntParser(){
     public final int parseInt(final String val) {
+      final int shift = val.charAt(0)-SHIFT_START_INT;
+      if (shift>0 && shift<=31)
+        throw new FieldCache.StopFillCacheException();
       return prefixCodedToInt(val);
     }
   };
@@ -102,6 +95,9 @@
    */
   public static final ExtendedFieldCache.DoubleParser FIELD_CACHE_DOUBLE_PARSER=new ExtendedFieldCache.DoubleParser(){
     public final double parseDouble(final String val) {
+      final int shift = val.charAt(0)-SHIFT_START_LONG;
+      if (shift>0 && shift<=63)
+        throw new FieldCache.StopFillCacheException();
       return sortableLongToDouble(prefixCodedToLong(val));
     }
   };
@@ -112,9 +108,28 @@
    */
   public static final FieldCache.FloatParser FIELD_CACHE_FLOAT_PARSER=new FieldCache.FloatParser(){
     public final float parseFloat(final String val) {
+      final int shift = val.charAt(0)-SHIFT_START_INT;
+      if (shift>0 && shift<=31)
+        throw new FieldCache.StopFillCacheException();
       return sortableIntToFloat(prefixCodedToInt(val));
     }
   };
+  
+  /** internal */
+  static int longToPrefixCoded(final long val, final int shift, final char[] buffer) {
+    int nChars = (63-shift)/7 + 1, len = nChars+1;
+    buffer[0] = (char)(SHIFT_START_LONG + shift);
+    long sortableBits = val ^ 0x8000000000000000L;
+    sortableBits >>>= shift;
+    while (nChars>=1) {
+      // Store 7 bits per character for good efficiency when UTF-8 encoding.
+      // The whole number is right-justified so that lucene can prefix-encode
+      // the terms more efficiently.
+      buffer[nChars--] = (char)(sortableBits & 0x7f);
+      sortableBits >>>= 7;
+    }
+    return len;
+  }
 
   /**
    * This is a convenience method, that returns prefix coded bits of a long without
@@ -125,27 +140,33 @@
   public static String longToPrefixCoded(final long val) {
     return longToPrefixCoded(val, 0);
   }
-
+  
   /**
    * Expert: Returns prefix coded bits after reducing the precision by <code>shift</code> bits.
-   * This is method is used by {@link #trieCodeLong}.
+   * This is method is used by {@link LongRangeBuilder}.
    */
   public static String longToPrefixCoded(final long val, final int shift) {
     if (shift>63 || shift<0)
       throw new IllegalArgumentException("Illegal shift value, must be 0..63");
-    int nChars = (63-shift)/7 + 1;
-    final char[] arr = new char[nChars+1];
-    arr[0] = (char)(SHIFT_START_LONG + shift);
-    long sortableBits = val ^ 0x8000000000000000L;
+    final char[] buffer = new char[LONG_BUF_SIZE];
+    final int len = longToPrefixCoded(val, shift, buffer);
+    return new String(buffer, 0, len);
+  }
+
+  /** internal */
+  static int intToPrefixCoded(final int val, final int shift, final char[] buffer) {
+    int nChars = (31-shift)/7 + 1, len = nChars+1;
+    buffer[0] = (char)(SHIFT_START_INT + shift);
+    int sortableBits = val ^ 0x80000000;
     sortableBits >>>= shift;
     while (nChars>=1) {
       // Store 7 bits per character for good efficiency when UTF-8 encoding.
       // The whole number is right-justified so that lucene can prefix-encode
       // the terms more efficiently.
-      arr[nChars--] = (char)(sortableBits & 0x7f);
+      buffer[nChars--] = (char)(sortableBits & 0x7f);
       sortableBits >>>= 7;
     }
-    return new String(arr);
+    return len;
   }
 
   /**
@@ -160,39 +181,30 @@
 
   /**
    * Expert: Returns prefix coded bits after reducing the precision by <code>shift</code> bits.
-   * This is method is used by {@link #trieCodeInt}.
+   * This is method is used by {@link IntRangeBuilder}.
    */
   public static String intToPrefixCoded(final int val, final int shift) {
     if (shift>31 || shift<0)
       throw new IllegalArgumentException("Illegal shift value, must be 0..31");
-    int nChars = (31-shift)/7 + 1;
-    final char[] arr = new char[nChars+1];
-    arr[0] = (char)(SHIFT_START_INT + shift);
-    int sortableBits = val ^ 0x80000000;
-    sortableBits >>>= shift;
-    while (nChars>=1) {
-      // Store 7 bits per character for good efficiency when UTF-8 encoding.
-      // The whole number is right-justified so that lucene can prefix-encode
-      // the terms more efficiently.
-      arr[nChars--] = (char)(sortableBits & 0x7f);
-      sortableBits >>>= 7;
-    }
-    return new String(arr);
+    final char[] buffer = new char[INT_BUF_SIZE];
+    final int len = intToPrefixCoded(val, shift, buffer);
+    return new String(buffer, 0, len);
   }
 
   /**
    * Returns a long from prefixCoded characters.
    * Rightmost bits will be zero for lower precision codes.
    * This method can be used to decode e.g. a stored field.
+   * @throws NumberFormatException if the supplied string is
+   * not correctly prefix encoded.
    * @see #longToPrefixCoded(long)
    */
   public static long prefixCodedToLong(final String prefixCoded) {
-    final int len = prefixCoded.length();
     final int shift = prefixCoded.charAt(0)-SHIFT_START_LONG;
     if (shift>63 || shift<0)
       throw new NumberFormatException("Invalid shift value in prefixCoded string (is encoded value really a LONG?)");
     long sortableBits = 0L;
-    for (int i=1; i<len; i++) {
+    for (int i=1, len=prefixCoded.length(); i<len; i++) {
       sortableBits <<= 7;
       final char ch = prefixCoded.charAt(i);
       if (ch>0x7f) {
@@ -201,7 +213,7 @@
           Integer.toHexString((int)ch)+" at position "+i+" is invalid)"
         );
       }
-      sortableBits |= (long)(ch & 0x7f);
+      sortableBits |= (long)ch;
     }
     return (sortableBits << shift) ^ 0x8000000000000000L;
   }
@@ -210,15 +222,16 @@
    * Returns an int from prefixCoded characters.
    * Rightmost bits will be zero for lower precision codes.
    * This method can be used to decode e.g. a stored field.
+   * @throws NumberFormatException if the supplied string is
+   * not correctly prefix encoded.
    * @see #intToPrefixCoded(int)
    */
   public static int prefixCodedToInt(final String prefixCoded) {
-    final int len = prefixCoded.length();
     final int shift = prefixCoded.charAt(0)-SHIFT_START_INT;
     if (shift>31 || shift<0)
       throw new NumberFormatException("Invalid shift value in prefixCoded string (is encoded value really an INT?)");
     int sortableBits = 0;
-    for (int i=1; i<len; i++) {
+    for (int i=1, len=prefixCoded.length(); i<len; i++) {
       sortableBits <<= 7;
       final char ch = prefixCoded.charAt(i);
       if (ch>0x7f) {
@@ -227,7 +240,7 @@
           Integer.toHexString((int)ch)+" at position "+i+" is invalid)"
         );
       }
-      sortableBits |= (int)(ch & 0x7f);
+      sortableBits |= (int)ch;
     }
     return (sortableBits << shift) ^ 0x80000000;
   }
@@ -277,116 +290,20 @@
   }
 
   /** A factory method, that generates a {@link SortField} instance for sorting prefix encoded long values. */
-  public static SortField getLongSortField(final String field, boolean reverse) {
+  public static SortField getLongSortField(final String field, final boolean reverse) {
     return new SortField(field, FIELD_CACHE_LONG_PARSER, reverse);
   }
   
   /** A factory method, that generates a {@link SortField} instance for sorting prefix encoded int values. */
-  public static SortField getIntSortField(final String field, boolean reverse) {
+  public static SortField getIntSortField(final String field, final boolean reverse) {
     return new SortField(field, FIELD_CACHE_INT_PARSER, reverse);
   }
 
   /**
-   * Returns a sequence of trie coded numbers suitable for {@link LongTrieRangeFilter}.
-   * Each successive string in the list has had it's precision reduced by <code>precisionStep</code>.
-   * For sorting, index the first full-precision value into a separate field and the
-   * remaining values into another field.
-   * <p>To achieve this, use {@link #addIndexedFields(Document,String,String[])}.
-   */
-  public static String[] trieCodeLong(long val, int precisionStep) {
-    if (precisionStep<1 || precisionStep>64)
-      throw new IllegalArgumentException("precisionStep may only be 1..64");
-    String[] arr = new String[63/precisionStep+1];
-    int idx = 0;
-    for (int shift=0; shift<64; shift+=precisionStep) {
-      arr[idx++] = longToPrefixCoded(val, shift);
-    }
-    return arr;
-  }
-
-  /**
-   * Returns a sequence of trie coded numbers suitable for {@link IntTrieRangeFilter}.
-   * Each successive string in the list has had it's precision reduced by <code>precisionStep</code>.
-   * For sorting, index the first full-precision value into a separate field and the
-   * remaining values into another field.
-   * <p>To achieve this, use {@link #addIndexedFields(Document,String,String[])}.
-   */
-  public static String[] trieCodeInt(int val, int precisionStep) {
-    if (precisionStep<1 || precisionStep>32)
-      throw new IllegalArgumentException("precisionStep may only be 1..32");
-    String[] arr = new String[31/precisionStep+1];
-    int idx = 0;
-    for (int shift=0; shift<32; shift+=precisionStep) {
-      arr[idx++] = intToPrefixCoded(val, shift);
-    }
-    return arr;
-  }
-
-  /**
-   * Indexes the full precision value only in the main field (for sorting), and indexes all other
-   * lower precision values in <code>field+LOWER_PRECISION_FIELD_NAME_SUFFIX</code>.
-   * <p><b>This is the recommended variant to add trie fields to the index.</b>
-   * By this it is possible to sort the field using a <code>SortField</code> instance
-   * returned by {@link #getLongSortField} or {@link #getIntSortField}.
-   * <p>This method does not store the fields and saves no term frequency or norms
-   * (which are normally not needed for trie fields). If you want to additionally store
-   * the value, you can use the normal methods of {@link Document} to achive this, just specify
-   * <code>Field.Store.YES</code>, <code>Field.Index.NO</code> and the same field name.
-   * <p>Examples:
-   * <pre>
-   *  addIndexedFields(doc, "mydouble", trieCodeLong(doubleToSortableLong(1.414d), 4));
-   *  addIndexedFields(doc, "mylong", trieCodeLong(123456L, 4));
-   * </pre>
-   **/
-  public static void addIndexedFields(Document doc, String field, String[] trieCoded) {
-    addIndexedFields(doc, new String[]{field, field+LOWER_PRECISION_FIELD_NAME_SUFFIX}, trieCoded);
-  }
-
-  /**
-   * Expert: Indexes the full precision value only in the main field (for sorting), and indexes all other
-   * lower precision values in the <code>lowerPrecision</code> field.
-   * If you do not specify the same field name for the main and lower precision one,
-   * it is possible to sort the field using a <code>SortField</code> instance
-   * returned by {@link #getLongSortField} or {@link #getIntSortField}.
-   * <p>This method does not store the fields and saves no term frequency or norms
-   * (which are normally not needed for trie fields). If you want to additionally store
-   * the value, you can use the normal methods of {@link Document} to achive this, just specify
-   * <code>Field.Store.YES</code>, <code>Field.Index.NO</code> and the same main field name.
-   * <p>Examples:
-   * <pre>
-   *  addIndexedFields(doc, "mydouble", "mydoubletrie", trieCodeLong(doubleToSortableLong(1.414d), 4));
-   *  addIndexedFields(doc, "mylong", "mylongtrie", trieCodeLong(123456L, 4));
-   * </pre>
-   * @see #addIndexedFields(Document,String,String[])
-   **/
-  public static void addIndexedFields(Document doc, String field, String lowerPrecisionField, String[] trieCoded) {
-    addIndexedFields(doc, new String[]{field, lowerPrecisionField}, trieCoded);
-  }
-
-  /**
-   * Expert: Indexes a series of trie coded values into a lucene {@link Document}
-   * using the given field names.
-   * If the array of field names is shorter than the trie coded one, all trie coded
-   * values with higher index get the last field name.
-   * <p>This method does not store the fields and saves no term frequency or norms
-   * (which are normally not needed for trie fields). If you want to additionally store
-   * the value, you can use the normal methods of {@link Document} to achive this, just specify
-   * <code>Field.Store.YES</code>, <code>Field.Index.NO</code> and the same main field name.
-   **/
-  public static void addIndexedFields(Document doc, String[] fields, String[] trieCoded) {
-    for (int i=0; i<trieCoded.length; i++) {
-      final int fnum = Math.min(fields.length-1, i);
-      final Field f = new Field(fields[fnum], trieCoded[i], Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS);
-      f.setOmitTermFreqAndPositions(true);
-      doc.add(f);
-    }
-  }
-
-  /**
    * Expert: Splits a long range recursively.
    * You may implement a builder that adds clauses to a
    * {@link org.apache.lucene.search.BooleanQuery} for each call to its
-   * {@link LongRangeBuilder#addRange(String,String,int)}
+   * {@link LongRangeBuilder#addRange(String,String)}
    * method.
    * <p>This method is used by {@link LongTrieRangeFilter}.
    */
@@ -402,7 +319,7 @@
    * Expert: Splits an int range recursively.
    * You may implement a builder that adds clauses to a
    * {@link org.apache.lucene.search.BooleanQuery} for each call to its
-   * {@link IntRangeBuilder#addRange(String,String,int)}
+   * {@link IntRangeBuilder#addRange(String,String)}
    * method.
    * <p>This method is used by {@link IntTrieRangeFilter}.
    */
@@ -419,7 +336,7 @@
     final Object builder, final int valSize,
     final int precisionStep, long minBound, long maxBound
   ) {
-    for (int level=0,shift=0;; level++) {
+    for (int shift=0; ; shift += precisionStep) {
       // calculate new bounds for inner precision
       final long diff = 1L << (shift+precisionStep),
         mask = ((1L<<precisionStep) - 1L) << shift;
@@ -432,20 +349,19 @@
 
       if (shift+precisionStep>=valSize || nextMinBound>nextMaxBound) {
         // We are in the lowest precision or the next precision is not available.
-        addRange(builder, valSize, minBound, maxBound, shift, level);
+        addRange(builder, valSize, minBound, maxBound, shift);
         // exit the split recursion loop
         break;
       }
       
       if (hasLower)
-        addRange(builder, valSize, minBound, minBound | mask, shift, level);
+        addRange(builder, valSize, minBound, minBound | mask, shift);
       if (hasUpper)
-        addRange(builder, valSize, maxBound & ~mask, maxBound, shift, level);
+        addRange(builder, valSize, maxBound & ~mask, maxBound, shift);
       
       // recurse to next precision
       minBound = nextMinBound;
       maxBound = nextMaxBound;
-      shift += precisionStep;
     }
   }
   
@@ -453,7 +369,7 @@
   private static void addRange(
     final Object builder, final int valSize,
     long minBound, long maxBound,
-    final int shift, final int level
+    final int shift
   ) {
     // for the max bound set all lower bits (that were shifted away):
     // this is important for testing or other usages of the splitted range
@@ -463,10 +379,10 @@
     // delegate to correct range builder
     switch(valSize) {
       case 64:
-        ((LongRangeBuilder)builder).addRange(minBound, maxBound, shift, level);
+        ((LongRangeBuilder)builder).addRange(minBound, maxBound, shift);
         break;
       case 32:
-        ((IntRangeBuilder)builder).addRange((int)minBound, (int)maxBound, shift, level);
+        ((IntRangeBuilder)builder).addRange((int)minBound, (int)maxBound, shift);
         break;
       default:
         // Should not happen!
@@ -484,16 +400,9 @@
     
     /**
      * Overwrite this method, if you like to receive the already prefix encoded range bounds.
-     * You can directly build classical range queries from them.
-     * The level gives the precision level (0 = highest precision) of the encoded values.
-     * This parameter could be used as an index to an array of fieldnames like the
-     * parameters to {@link #addIndexedFields(Document,String[],String[])} for specifying
-     * the field names for each precision:
-     * <pre>
-     *  String field = fields[Math.min(fields.length-1, level)];
-     * </pre>
+     * You can directly build classical (inclusive) range queries from them.
      */
-    public void addRange(String minPrefixCoded, String maxPrefixCoded, int level) {
+    public void addRange(String minPrefixCoded, String maxPrefixCoded) {
       throw new UnsupportedOperationException();
     }
     
@@ -501,10 +410,8 @@
      * Overwrite this method, if you like to receive the raw long range bounds.
      * You can use this for e.g. debugging purposes (print out range bounds).
      */
-    public void addRange(final long min, final long max, final int shift, final int level) {
-      /*System.out.println(Long.toHexString((min^0x8000000000000000L) >>> shift)+".."+
-        Long.toHexString((max^0x8000000000000000L) >>> shift));*/
-      addRange(longToPrefixCoded(min, shift), longToPrefixCoded(max, shift), level);
+    public void addRange(final long min, final long max, final int shift) {
+      addRange(longToPrefixCoded(min, shift), longToPrefixCoded(max, shift));
     }
   
   }
@@ -519,16 +426,9 @@
     
     /**
      * Overwrite this method, if you like to receive the already prefix encoded range bounds.
-     * You can directly build classical range queries from them.
-     * The level gives the precision level (0 = highest precision) of the encoded values.
-     * This parameter could be used as an index to an array of fieldnames like the
-     * parameters to {@link #addIndexedFields(Document,String[],String[])} for specifying
-     * the field names for each precision:
-     * <pre>
-     *  String field = fields[Math.min(fields.length-1, level)];
-     * </pre>
+     * You can directly build classical range (inclusive) queries from them.
      */
-    public void addRange(String minPrefixCoded, String maxPrefixCoded, int level) {
+    public void addRange(String minPrefixCoded, String maxPrefixCoded) {
       throw new UnsupportedOperationException();
     }
     
@@ -536,10 +436,8 @@
      * Overwrite this method, if you like to receive the raw int range bounds.
      * You can use this for e.g. debugging purposes (print out range bounds).
      */
-    public void addRange(final int min, final int max, final int shift, final int level) {
-      /*System.out.println(Integer.toHexString((min^0x80000000) >>> shift)+".."+
-        Integer.toHexString((max^0x80000000) >>> shift));*/
-      addRange(intToPrefixCoded(min, shift), intToPrefixCoded(max, shift), level);
+    public void addRange(final int min, final int max, final int shift) {
+      addRange(intToPrefixCoded(min, shift), intToPrefixCoded(max, shift));
     }
   
   }

Modified: lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/package.html
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/package.html?rev=762710&r1=762709&r2=762710&view=diff
==============================================================================
--- lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/package.html (original)
+++ lucene/java/trunk/contrib/queries/src/java/org/apache/lucene/search/trie/package.html Tue Apr  7 11:47:35 2009
@@ -50,10 +50,14 @@
 are no longer dependent on the index size and the number of distinct values because there is
 an upper limit unrelated to either of these properties.</p>
 
-<h3>Usage</h3>
+<h3>Indexing Usage</h3>
 <p>To use the new query types the numerical values, which may be<code>long</code>, <code>double</code>, <code>int</code>,
 <code>float</code>, or <code>Date</code>, the values must be indexed in a special prefix encoded format
-(using {@link org.apache.lucene.search.trie.TrieUtils}). This can be done like this:</p>
+using {@link org.apache.lucene.search.trie.LongTrieTokenStream} or
+{@link org.apache.lucene.search.trie.IntTrieTokenStream}, which generate the necessary tokens.
+Use {@link org.apache.lucene.search.trie.TrieUtils} to convert floating point values to integers.
+Example code for indexing (it is recommened to disable norms and term frequencies during indexing
+trie encoded fields):</p>
 
 <pre>
   <em>// chose a step value, 8 is a general good value for large indexes:</em>
@@ -67,15 +71,25 @@
   
   <em>// add some numerical fields:</em>
   long lvalue = 121345L;
-  TrieUtils.addIndexedFields(doc, "exampleLong", TrieUtils.trieCodeLong(lvalue, precisionStep));
+  Field f = new Field("exampleLong", new LongTrieTokenStream(lvalue, precisionStep));
+  f.setOmitNorms(true); f.setOmitTermFreqAndPositions(true);
+  doc.add(f);
   double dvalue = 1.057E17;
-  TrieUtils.addIndexedFields(doc, "exampleDouble", TrieUtils.trieCodeLong(TrieUtils.doubleToSortableLong(dvalue), precisionStep));
+  f = new Field("exampleDouble", new LongTrieTokenStream(TrieUtils.doubleToSortableLong(dvalue), precisionStep));
+  f.setOmitNorms(true); f.setOmitTermFreqAndPositions(true);
+  doc.add(f);
   int ivalue = 121345;
-  TrieUtils.addIndexedFields(doc, "exampleInt", TrieUtils.trieCodeInt(ivalue, precisionStep));
+  f = new Field("exampleInt", new IntTrieTokenStream(ivalue, precisionStep));
+  f.setOmitNorms(true); f.setOmitTermFreqAndPositions(true);
+  doc.add(f);
   float fvalue = 1.057E17f;
-  TrieUtils.addIndexedFields(doc, "exampleFloat", TrieUtils.trieCodeInt(TrieUtils.floatToSortableInt(fvalue), precisionStep));
+  f = new Field("exampleFloat", new IntTrieTokenStream(TrieUtils.floatToSortableInt(fvalue), precisionStep));
+  f.setOmitNorms(true); f.setOmitTermFreqAndPositions(true);
+  doc.add(f);
   Date datevalue = new Date(); <em>// actual time</em>
-  TrieUtils.addIndexedFields(doc, "exampleDate", TrieUtils.trieCodeLong(datevalue.getTime(), precisionStep));
+  f = new Field("exampleDate", new LongTrieTokenStream(datevalue.getTime(), precisionStep));
+  f.setOmitNorms(true); f.setOmitTermFreqAndPositions(true);
+  doc.add(f);
   
   <em>// if you want to also store one of the values:</em>
   doc.add(new Field("exampleLong", Long.toString(lvalue), Field.Store.YES, Field.Index.NO));
@@ -86,6 +100,11 @@
   <em>// now add document to IndexWriter, as usual</em>
 </pre>
 
+<p><em>(for higher indexing performance, you can reuse the TokenStreams &ndash;
+more info about this in the stream documentation)</em></p>
+
+<h3>Searching</h3>
+
 <p>The numeric index fields you prepared in this way can be searched by
 {@link org.apache.lucene.search.trie.LongTrieRangeFilter} or {@link org.apache.lucene.search.trie.IntTrieRangeFilter}:</p>
 

Modified: lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestIntTrieRangeFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestIntTrieRangeFilter.java?rev=762710&r1=762709&r2=762710&view=diff
==============================================================================
--- lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestIntTrieRangeFilter.java (original)
+++ lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestIntTrieRangeFilter.java Tue Apr  7 11:47:35 2009
@@ -33,8 +33,7 @@
 import org.apache.lucene.search.RangeQuery;
 import org.apache.lucene.util.LuceneTestCase;
 
-public class TestIntTrieRangeFilter extends LuceneTestCase
-{
+public class TestIntTrieRangeFilter extends LuceneTestCase {
   // distance of entries
   private static final int distance = 6666;
   // shift the starting of the values to the left, to also have negative values:
@@ -42,6 +41,15 @@
   // number of docs to generate for testing
   private static final int noDocs = 10000;
   
+  private static Field newField(String name, int precisionStep) {
+    IntTrieTokenStream stream = new IntTrieTokenStream(precisionStep);
+    stream.setUseNewAPI(true);
+    Field f=new Field(name, stream);
+    f.setOmitTermFreqAndPositions(true);
+    f.setOmitNorms(true);
+    return f;
+  }
+  
   private static final RAMDirectory directory;
   private static final IndexSearcher searcher;
   static {
@@ -50,21 +58,34 @@
       IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(),
       true, MaxFieldLength.UNLIMITED);
       
+      Field
+        field8 = newField("field8", 8),
+        field4 = newField("field4", 4),
+        field2 = newField("field2", 2),
+        ascfield8 = newField("ascfield8", 8),
+        ascfield4 = newField("ascfield4", 4),
+        ascfield2 = newField("ascfield2", 2);
+      
       // Add a series of noDocs docs with increasing int values
       for (int l=0; l<noDocs; l++) {
         Document doc=new Document();
         // add fields, that have a distance to test general functionality
-        final int val=distance*l+startOffset;
-        TrieUtils.addIndexedFields(doc,"field8", TrieUtils.trieCodeInt(val, 8));
-        doc.add(new Field("field8", TrieUtils.intToPrefixCoded(val), Field.Store.YES, Field.Index.NO));
-        TrieUtils.addIndexedFields(doc,"field4", TrieUtils.trieCodeInt(val, 4));
-        doc.add(new Field("field4", TrieUtils.intToPrefixCoded(val), Field.Store.YES, Field.Index.NO));
-        TrieUtils.addIndexedFields(doc,"field2", TrieUtils.trieCodeInt(val, 2));
-        doc.add(new Field("field2", TrieUtils.intToPrefixCoded(val), Field.Store.YES, Field.Index.NO));
+        int val=distance*l+startOffset;
+        doc.add(new Field("value", TrieUtils.intToPrefixCoded(val), Field.Store.YES, Field.Index.NO));
+        ((IntTrieTokenStream)field8.tokenStreamValue()).setValue(val);
+        doc.add(field8);
+        ((IntTrieTokenStream)field4.tokenStreamValue()).setValue(val);
+        doc.add(field4);
+        ((IntTrieTokenStream)field2.tokenStreamValue()).setValue(val);
+        doc.add(field2);
         // add ascending fields with a distance of 1, beginning at -noDocs/2 to test the correct splitting of range and inclusive/exclusive
-        TrieUtils.addIndexedFields(doc,"ascfield8", TrieUtils.trieCodeInt(l-(noDocs/2), 8));
-        TrieUtils.addIndexedFields(doc,"ascfield4", TrieUtils.trieCodeInt(l-(noDocs/2), 4));
-        TrieUtils.addIndexedFields(doc,"ascfield2", TrieUtils.trieCodeInt(l-(noDocs/2), 2));
+        val=l-(noDocs/2);
+        ((IntTrieTokenStream)ascfield8.tokenStreamValue()).setValue(val);
+        doc.add(ascfield8);
+        ((IntTrieTokenStream)ascfield4.tokenStreamValue()).setValue(val);
+        doc.add(ascfield4);
+        ((IntTrieTokenStream)ascfield2.tokenStreamValue()).setValue(val);
+        doc.add(ascfield2);
         writer.addDocument(doc);
       }
     
@@ -87,9 +108,9 @@
     assertNotNull(sd);
     assertEquals("Score doc count", count, sd.length );
     Document doc=searcher.doc(sd[0].doc);
-    assertEquals("First doc", 2*distance+startOffset, TrieUtils.prefixCodedToInt(doc.get(field)) );
+    assertEquals("First doc", 2*distance+startOffset, TrieUtils.prefixCodedToInt(doc.get("value")) );
     doc=searcher.doc(sd[sd.length-1].doc);
-    assertEquals("Last doc", (1+count)*distance+startOffset, TrieUtils.prefixCodedToInt(doc.get(field)) );
+    assertEquals("Last doc", (1+count)*distance+startOffset, TrieUtils.prefixCodedToInt(doc.get("value")) );
   }
 
   public void testRange_8bit() throws Exception {
@@ -115,9 +136,9 @@
     assertNotNull(sd);
     assertEquals("Score doc count", count, sd.length );
     Document doc=searcher.doc(sd[0].doc);
-    assertEquals("First doc", startOffset, TrieUtils.prefixCodedToInt(doc.get(field)) );
+    assertEquals("First doc", startOffset, TrieUtils.prefixCodedToInt(doc.get("value")) );
     doc=searcher.doc(sd[sd.length-1].doc);
-    assertEquals("Last doc", (count-1)*distance+startOffset, TrieUtils.prefixCodedToInt(doc.get(field)) );
+    assertEquals("Last doc", (count-1)*distance+startOffset, TrieUtils.prefixCodedToInt(doc.get("value")) );
   }
   
   public void testLeftOpenRange_8bit() throws Exception {
@@ -143,9 +164,9 @@
     assertNotNull(sd);
     assertEquals("Score doc count", noDocs-count, sd.length );
     Document doc=searcher.doc(sd[0].doc);
-    assertEquals("First doc", count*distance+startOffset, TrieUtils.prefixCodedToInt(doc.get(field)) );
+    assertEquals("First doc", count*distance+startOffset, TrieUtils.prefixCodedToInt(doc.get("value")) );
     doc=searcher.doc(sd[sd.length-1].doc);
-    assertEquals("Last doc", (noDocs-1)*distance+startOffset, TrieUtils.prefixCodedToInt(doc.get(field)) );
+    assertEquals("Last doc", (noDocs-1)*distance+startOffset, TrieUtils.prefixCodedToInt(doc.get("value")) );
   }
   
   public void testRightOpenRange_8bit() throws Exception {
@@ -163,39 +184,47 @@
   private void testRandomTrieAndClassicRangeQuery(int precisionStep) throws Exception {
     final Random rnd=newRandom();
     String field="field"+precisionStep;
-    // 50 random tests, the tests may also return 0 results, if min>max, but this is ok
+    int termCount=0;
     for (int i=0; i<50; i++) {
       int lower=(int)(rnd.nextDouble()*noDocs*distance)+startOffset;
       int upper=(int)(rnd.nextDouble()*noDocs*distance)+startOffset;
+      if (lower>upper) {
+        int a=lower; lower=upper; upper=a;
+      }
       // test inclusive range
-      Query tq=new IntTrieRangeFilter(field, precisionStep, new Integer(lower), new Integer(upper), true, true).asQuery();
+      IntTrieRangeFilter tf=new IntTrieRangeFilter(field, precisionStep, new Integer(lower), new Integer(upper), true, true);
       RangeQuery cq=new RangeQuery(field, TrieUtils.intToPrefixCoded(lower), TrieUtils.intToPrefixCoded(upper), true, true);
       cq.setConstantScoreRewrite(true);
-      TopDocs tTopDocs = searcher.search(tq, 1);
+      TopDocs tTopDocs = searcher.search(tf.asQuery(), 1);
       TopDocs cTopDocs = searcher.search(cq, 1);
       assertEquals("Returned count for IntTrieRangeFilter and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+      termCount += tf.getLastNumberOfTerms();
       // test exclusive range
-      tq=new IntTrieRangeFilter(field, precisionStep, new Integer(lower), new Integer(upper), false, false).asQuery();
+      tf=new IntTrieRangeFilter(field, precisionStep, new Integer(lower), new Integer(upper), false, false);
       cq=new RangeQuery(field, TrieUtils.intToPrefixCoded(lower), TrieUtils.intToPrefixCoded(upper), false, false);
       cq.setConstantScoreRewrite(true);
-      tTopDocs = searcher.search(tq, 1);
+      tTopDocs = searcher.search(tf.asQuery(), 1);
       cTopDocs = searcher.search(cq, 1);
       assertEquals("Returned count for IntTrieRangeFilter and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+      termCount += tf.getLastNumberOfTerms();
       // test left exclusive range
-      tq=new IntTrieRangeFilter(field, precisionStep, new Integer(lower), new Integer(upper), false, true).asQuery();
+      tf=new IntTrieRangeFilter(field, precisionStep, new Integer(lower), new Integer(upper), false, true);
       cq=new RangeQuery(field, TrieUtils.intToPrefixCoded(lower), TrieUtils.intToPrefixCoded(upper), false, true);
       cq.setConstantScoreRewrite(true);
-      tTopDocs = searcher.search(tq, 1);
+      tTopDocs = searcher.search(tf.asQuery(), 1);
       cTopDocs = searcher.search(cq, 1);
       assertEquals("Returned count for IntTrieRangeFilter and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+      termCount += tf.getLastNumberOfTerms();
       // test right exclusive range
-      tq=new IntTrieRangeFilter(field, precisionStep, new Integer(lower), new Integer(upper), true, false).asQuery();
+      tf=new IntTrieRangeFilter(field, precisionStep, new Integer(lower), new Integer(upper), true, false);
       cq=new RangeQuery(field, TrieUtils.intToPrefixCoded(lower), TrieUtils.intToPrefixCoded(upper), true, false);
       cq.setConstantScoreRewrite(true);
-      tTopDocs = searcher.search(tq, 1);
+      tTopDocs = searcher.search(tf.asQuery(), 1);
       cTopDocs = searcher.search(cq, 1);
       assertEquals("Returned count for IntTrieRangeFilter and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+      termCount += tf.getLastNumberOfTerms();
     }
+    System.out.println("Average number of terms during random search: " + (((double)termCount)/(50*4)));
   }
   
   public void testRandomTrieAndClassicRangeQuery_8bit() throws Exception {
@@ -267,9 +296,9 @@
       if (topDocs.totalHits==0) continue;
       ScoreDoc[] sd = topDocs.scoreDocs;
       assertNotNull(sd);
-      int last=TrieUtils.prefixCodedToInt(searcher.doc(sd[0].doc).get(field));
+      int last=TrieUtils.prefixCodedToInt(searcher.doc(sd[0].doc).get("value"));
       for (int j=1; j<sd.length; j++) {
-        int act=TrieUtils.prefixCodedToInt(searcher.doc(sd[j].doc).get(field));
+        int act=TrieUtils.prefixCodedToInt(searcher.doc(sd[j].doc).get("value"));
         assertTrue("Docs should be sorted backwards", last>act );
         last=act;
       }

Added: lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestIntTrieTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestIntTrieTokenStream.java?rev=762710&view=auto
==============================================================================
--- lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestIntTrieTokenStream.java (added)
+++ lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestIntTrieTokenStream.java Tue Apr  7 11:47:35 2009
@@ -0,0 +1,53 @@
+package org.apache.lucene.search.trie;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+
+public class TestIntTrieTokenStream extends LuceneTestCase {
+
+  static final int precisionStep = 8;
+  static final int value = 123456;
+
+  public void testStreamNewAPI() throws Exception {
+    final IntTrieTokenStream stream=new IntTrieTokenStream(value, precisionStep);
+    stream.setUseNewAPI(true);
+    final ShiftAttribute shiftAtt = (ShiftAttribute) stream.addAttribute(ShiftAttribute.class);
+    final TermAttribute termAtt = (TermAttribute) stream.addAttribute(TermAttribute.class);
+    for (int shift=0; shift<32; shift+=precisionStep) {
+      assertTrue("New token is available", stream.incrementToken());
+      assertEquals("Shift value", shift, shiftAtt.getShift());
+      assertEquals("Term is correctly encoded", TrieUtils.intToPrefixCoded(value, shift), termAtt.term());
+    }
+    assertFalse("No more tokens available", stream.incrementToken());
+  }
+  
+  public void testStreamOldAPI() throws Exception {
+    final IntTrieTokenStream stream=new IntTrieTokenStream(value, precisionStep);
+    stream.setUseNewAPI(false);
+    Token tok=new Token();
+    for (int shift=0; shift<32; shift+=precisionStep) {
+      assertNotNull("New token is available", tok=stream.next(tok));
+      assertEquals("Term is correctly encoded", TrieUtils.intToPrefixCoded(value, shift), tok.term());
+    }
+    assertNull("No more tokens available", stream.next(tok));
+  }
+  
+}

Modified: lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestLongTrieRangeFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestLongTrieRangeFilter.java?rev=762710&r1=762709&r2=762710&view=diff
==============================================================================
--- lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestLongTrieRangeFilter.java (original)
+++ lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestLongTrieRangeFilter.java Tue Apr  7 11:47:35 2009
@@ -33,8 +33,7 @@
 import org.apache.lucene.search.RangeQuery;
 import org.apache.lucene.util.LuceneTestCase;
 
-public class TestLongTrieRangeFilter extends LuceneTestCase
-{
+public class TestLongTrieRangeFilter extends LuceneTestCase {
   // distance of entries
   private static final long distance = 66666L;
   // shift the starting of the values to the left, to also have negative values:
@@ -42,6 +41,15 @@
   // number of docs to generate for testing
   private static final int noDocs = 10000;
   
+  private static Field newField(String name, int precisionStep) {
+    LongTrieTokenStream stream = new LongTrieTokenStream(precisionStep);
+    stream.setUseNewAPI(true);
+    Field f=new Field(name, stream);
+    f.setOmitTermFreqAndPositions(true);
+    f.setOmitNorms(true);
+    return f;
+  }
+  
   private static final RAMDirectory directory;
   private static final IndexSearcher searcher;
   static {
@@ -50,21 +58,34 @@
       IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(),
       true, MaxFieldLength.UNLIMITED);
       
+      Field
+        field8 = newField("field8", 8),
+        field4 = newField("field4", 4),
+        field2 = newField("field2", 2),
+        ascfield8 = newField("ascfield8", 8),
+        ascfield4 = newField("ascfield4", 4),
+        ascfield2 = newField("ascfield2", 2);
+      
       // Add a series of noDocs docs with increasing long values
       for (int l=0; l<noDocs; l++) {
         Document doc=new Document();
         // add fields, that have a distance to test general functionality
-        final long val=distance*l+startOffset;
-        TrieUtils.addIndexedFields(doc,"field8", TrieUtils.trieCodeLong(val, 8));
-        doc.add(new Field("field8", TrieUtils.longToPrefixCoded(val), Field.Store.YES, Field.Index.NO));
-        TrieUtils.addIndexedFields(doc,"field4", TrieUtils.trieCodeLong(val, 4));
-        doc.add(new Field("field4", TrieUtils.longToPrefixCoded(val), Field.Store.YES, Field.Index.NO));
-        TrieUtils.addIndexedFields(doc,"field2", TrieUtils.trieCodeLong(val, 2));
-        doc.add(new Field("field2", TrieUtils.longToPrefixCoded(val), Field.Store.YES, Field.Index.NO));
+        long val=distance*l+startOffset;
+        doc.add(new Field("value", TrieUtils.longToPrefixCoded(val), Field.Store.YES, Field.Index.NO));
+        ((LongTrieTokenStream)field8.tokenStreamValue()).setValue(val);
+        doc.add(field8);
+        ((LongTrieTokenStream)field4.tokenStreamValue()).setValue(val);
+        doc.add(field4);
+        ((LongTrieTokenStream)field2.tokenStreamValue()).setValue(val);
+        doc.add(field2);
         // add ascending fields with a distance of 1, beginning at -noDocs/2 to test the correct splitting of range and inclusive/exclusive
-        TrieUtils.addIndexedFields(doc,"ascfield8", TrieUtils.trieCodeLong(l-(noDocs/2), 8));
-        TrieUtils.addIndexedFields(doc,"ascfield4", TrieUtils.trieCodeLong(l-(noDocs/2), 4));
-        TrieUtils.addIndexedFields(doc,"ascfield2", TrieUtils.trieCodeLong(l-(noDocs/2), 2));
+        val=l-(noDocs/2);
+        ((LongTrieTokenStream)ascfield8.tokenStreamValue()).setValue(val);
+        doc.add(ascfield8);
+        ((LongTrieTokenStream)ascfield4.tokenStreamValue()).setValue(val);
+        doc.add(ascfield4);
+        ((LongTrieTokenStream)ascfield2.tokenStreamValue()).setValue(val);
+        doc.add(ascfield2);
         writer.addDocument(doc);
       }
     
@@ -87,9 +108,9 @@
     assertNotNull(sd);
     assertEquals("Score doc count", count, sd.length );
     Document doc=searcher.doc(sd[0].doc);
-    assertEquals("First doc", 2*distance+startOffset, TrieUtils.prefixCodedToLong(doc.get(field)) );
+    assertEquals("First doc", 2*distance+startOffset, TrieUtils.prefixCodedToLong(doc.get("value")) );
     doc=searcher.doc(sd[sd.length-1].doc);
-    assertEquals("Last doc", (1+count)*distance+startOffset, TrieUtils.prefixCodedToLong(doc.get(field)) );
+    assertEquals("Last doc", (1+count)*distance+startOffset, TrieUtils.prefixCodedToLong(doc.get("value")) );
   }
 
   public void testRange_8bit() throws Exception {
@@ -115,9 +136,9 @@
     assertNotNull(sd);
     assertEquals("Score doc count", count, sd.length );
     Document doc=searcher.doc(sd[0].doc);
-    assertEquals("First doc", startOffset, TrieUtils.prefixCodedToLong(doc.get(field)) );
+    assertEquals("First doc", startOffset, TrieUtils.prefixCodedToLong(doc.get("value")) );
     doc=searcher.doc(sd[sd.length-1].doc);
-    assertEquals("Last doc", (count-1)*distance+startOffset, TrieUtils.prefixCodedToLong(doc.get(field)) );
+    assertEquals("Last doc", (count-1)*distance+startOffset, TrieUtils.prefixCodedToLong(doc.get("value")) );
   }
   
   public void testLeftOpenRange_8bit() throws Exception {
@@ -143,9 +164,9 @@
     assertNotNull(sd);
     assertEquals("Score doc count", noDocs-count, sd.length );
     Document doc=searcher.doc(sd[0].doc);
-    assertEquals("First doc", count*distance+startOffset, TrieUtils.prefixCodedToLong(doc.get(field)) );
+    assertEquals("First doc", count*distance+startOffset, TrieUtils.prefixCodedToLong(doc.get("value")) );
     doc=searcher.doc(sd[sd.length-1].doc);
-    assertEquals("Last doc", (noDocs-1)*distance+startOffset, TrieUtils.prefixCodedToLong(doc.get(field)) );
+    assertEquals("Last doc", (noDocs-1)*distance+startOffset, TrieUtils.prefixCodedToLong(doc.get("value")) );
   }
   
   public void testRightOpenRange_8bit() throws Exception {
@@ -163,39 +184,47 @@
   private void testRandomTrieAndClassicRangeQuery(int precisionStep) throws Exception {
     final Random rnd=newRandom();
     String field="field"+precisionStep;
-    // 50 random tests, the tests may also return 0 results, if min>max, but this is ok
+    int termCount=0;
     for (int i=0; i<50; i++) {
       long lower=(long)(rnd.nextDouble()*noDocs*distance)+startOffset;
       long upper=(long)(rnd.nextDouble()*noDocs*distance)+startOffset;
+      if (lower>upper) {
+        long a=lower; lower=upper; upper=a;
+      }
       // test inclusive range
-      Query tq=new LongTrieRangeFilter(field, precisionStep, new Long(lower), new Long(upper), true, true).asQuery();
+      LongTrieRangeFilter tf=new LongTrieRangeFilter(field, precisionStep, new Long(lower), new Long(upper), true, true);
       RangeQuery cq=new RangeQuery(field, TrieUtils.longToPrefixCoded(lower), TrieUtils.longToPrefixCoded(upper), true, true);
       cq.setConstantScoreRewrite(true);
-      TopDocs tTopDocs = searcher.search(tq, 1);
+      TopDocs tTopDocs = searcher.search(tf.asQuery(), 1);
       TopDocs cTopDocs = searcher.search(cq, 1);
       assertEquals("Returned count for LongTrieRangeFilter and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+      termCount += tf.getLastNumberOfTerms();
       // test exclusive range
-      tq=new LongTrieRangeFilter(field, precisionStep, new Long(lower), new Long(upper), false, false).asQuery();
+      tf=new LongTrieRangeFilter(field, precisionStep, new Long(lower), new Long(upper), false, false);
       cq=new RangeQuery(field, TrieUtils.longToPrefixCoded(lower), TrieUtils.longToPrefixCoded(upper), false, false);
       cq.setConstantScoreRewrite(true);
-      tTopDocs = searcher.search(tq, 1);
+      tTopDocs = searcher.search(tf.asQuery(), 1);
       cTopDocs = searcher.search(cq, 1);
       assertEquals("Returned count for LongTrieRangeFilter and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+      termCount += tf.getLastNumberOfTerms();
       // test left exclusive range
-      tq=new LongTrieRangeFilter(field, precisionStep, new Long(lower), new Long(upper), false, true).asQuery();
+      tf=new LongTrieRangeFilter(field, precisionStep, new Long(lower), new Long(upper), false, true);
       cq=new RangeQuery(field, TrieUtils.longToPrefixCoded(lower), TrieUtils.longToPrefixCoded(upper), false, true);
       cq.setConstantScoreRewrite(true);
-      tTopDocs = searcher.search(tq, 1);
+      tTopDocs = searcher.search(tf.asQuery(), 1);
       cTopDocs = searcher.search(cq, 1);
       assertEquals("Returned count for LongTrieRangeFilter and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+      termCount += tf.getLastNumberOfTerms();
       // test right exclusive range
-      tq=new LongTrieRangeFilter(field, precisionStep, new Long(lower), new Long(upper), true, false).asQuery();
+      tf=new LongTrieRangeFilter(field, precisionStep, new Long(lower), new Long(upper), true, false);
       cq=new RangeQuery(field, TrieUtils.longToPrefixCoded(lower), TrieUtils.longToPrefixCoded(upper), true, false);
       cq.setConstantScoreRewrite(true);
-      tTopDocs = searcher.search(tq, 1);
+      tTopDocs = searcher.search(tf.asQuery(), 1);
       cTopDocs = searcher.search(cq, 1);
       assertEquals("Returned count for LongTrieRangeFilter and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+      termCount += tf.getLastNumberOfTerms();
     }
+    System.out.println("Average number of terms during random search: " + (((double)termCount)/(50*4)));
   }
   
   public void testRandomTrieAndClassicRangeQuery_8bit() throws Exception {
@@ -267,9 +296,9 @@
       if (topDocs.totalHits==0) continue;
       ScoreDoc[] sd = topDocs.scoreDocs;
       assertNotNull(sd);
-      long last=TrieUtils.prefixCodedToLong(searcher.doc(sd[0].doc).get(field));
+      long last=TrieUtils.prefixCodedToLong(searcher.doc(sd[0].doc).get("value"));
       for (int j=1; j<sd.length; j++) {
-        long act=TrieUtils.prefixCodedToLong(searcher.doc(sd[j].doc).get(field));
+        long act=TrieUtils.prefixCodedToLong(searcher.doc(sd[j].doc).get("value"));
         assertTrue("Docs should be sorted backwards", last>act );
         last=act;
       }

Added: lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestLongTrieTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestLongTrieTokenStream.java?rev=762710&view=auto
==============================================================================
--- lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestLongTrieTokenStream.java (added)
+++ lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestLongTrieTokenStream.java Tue Apr  7 11:47:35 2009
@@ -0,0 +1,53 @@
+package org.apache.lucene.search.trie;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+
+public class TestLongTrieTokenStream extends LuceneTestCase {
+
+  static final int precisionStep = 8;
+  static final long value = 4573245871874382L;
+
+  public void testStreamNewAPI() throws Exception {
+    final LongTrieTokenStream stream=new LongTrieTokenStream(value, precisionStep);
+    stream.setUseNewAPI(true);
+    final ShiftAttribute shiftAtt = (ShiftAttribute) stream.addAttribute(ShiftAttribute.class);
+    final TermAttribute termAtt = (TermAttribute) stream.addAttribute(TermAttribute.class);
+    for (int shift=0; shift<64; shift+=precisionStep) {
+      assertTrue("New token is available", stream.incrementToken());
+      assertEquals("Shift value", shift, shiftAtt.getShift());
+      assertEquals("Term is correctly encoded", TrieUtils.longToPrefixCoded(value, shift), termAtt.term());
+    }
+    assertFalse("No more tokens available", stream.incrementToken());
+  }
+  
+  public void testStreamOldAPI() throws Exception {
+    final LongTrieTokenStream stream=new LongTrieTokenStream(value, precisionStep);
+    stream.setUseNewAPI(false);
+    Token tok=new Token();
+    for (int shift=0; shift<64; shift+=precisionStep) {
+      assertNotNull("New token is available", tok=stream.next(tok));
+      assertEquals("Term is correctly encoded", TrieUtils.longToPrefixCoded(value, shift), tok.term());
+    }
+    assertNull("No more tokens available", stream.next(tok));
+  }
+  
+}

Modified: lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestTrieUtils.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestTrieUtils.java?rev=762710&r1=762709&r2=762710&view=diff
==============================================================================
--- lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestTrieUtils.java (original)
+++ lucene/java/trunk/contrib/queries/src/test/org/apache/lucene/search/trie/TestTrieUtils.java Tue Apr  7 11:47:35 2009
@@ -179,7 +179,7 @@
     
     TrieUtils.splitLongRange(new TrieUtils.LongRangeBuilder() {
       //@Override
-      public void addRange(long min, long max, int shift, int level) {
+      public void addRange(long min, long max, int shift) {
         assertTrue("min, max should be inside bounds", min>=lower && min<=upper && max>=lower && max<=upper);
         if (useBitSet) for (long l=min; l<=max; l++) {
           assertFalse("ranges should not overlap", bits.getAndSet(l-lower) );
@@ -253,7 +253,7 @@
     
     TrieUtils.splitIntRange(new TrieUtils.IntRangeBuilder() {
       //@Override
-      public void addRange(int min, int max, int shift, int level) {
+      public void addRange(int min, int max, int shift) {
         assertTrue("min, max should be inside bounds", min>=lower && min<=upper && max>=lower && max<=upper);
         if (useBitSet) for (int i=min; i<=max; i++) {
           assertFalse("ranges should not overlap", bits.getAndSet(i-lower) );



Mime
View raw message