lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From uschind...@apache.org
Subject svn commit: r930821 - in /lucene/java/branches/flex_1458: ./ backwards/src/test/org/apache/lucene/analysis/ backwards/src/test/org/apache/lucene/search/ backwards/src/test/org/apache/lucene/util/ contrib/spatial/src/java/org/apache/lucene/spatial/tier/...
Date Mon, 05 Apr 2010 08:07:19 GMT
Author: uschindler
Date: Mon Apr  5 08:07:18 2010
New Revision: 930821

URL: http://svn.apache.org/viewvc?rev=930821&view=rev
Log:
LUCENE-2354: NumericUtils and NumericTokenStream now work directly on ByteRefs, update CartesianShapeFilter to flex API

Removed:
    lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java
Modified:
    lucene/java/branches/flex_1458/CHANGES.txt
    lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java
    lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java
    lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/util/TestNumericUtils.java
    lucene/java/branches/flex_1458/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java
    lucene/java/branches/flex_1458/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesian.java
    lucene/java/branches/flex_1458/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestDistance.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/NumericTokenStream.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCache.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/NumericRangeQuery.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/NumericUtils.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/util/TestNumericUtils.java

Modified: lucene/java/branches/flex_1458/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/CHANGES.txt?rev=930821&r1=930820&r2=930821&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/CHANGES.txt (original)
+++ lucene/java/branches/flex_1458/CHANGES.txt Mon Apr  5 08:07:18 2010
@@ -4,7 +4,7 @@ Lucene Change Log
 
 Changes in backwards compatibility policy
 
-* LUCENE-1458, LUCENE-2111: Changes from flexible indexing:
+* LUCENE-1458, LUCENE-2111, LUCENE-2354: Changes from flexible indexing:
 
   - MultiReader ctor now throws IOException
 
@@ -28,6 +28,23 @@ Changes in backwards compatibility polic
     if you could fix your parser to instead operate directly on the
     byte[] in the BytesRef.
 
+  - The internal (experimental) API of NumericUtils changed completely
+    from String to BytesRef. Client code should never use this class,
+    so the change would normally not affect you. If you used some of
+    the methods to inspect terms or create TermQueries out of
+    prefix encoded terms, change to use BytesRef. Please note:
+    Do not use TermQueries to search for single numeric terms.
+    The recommended way is to create a corresponding NumericRangeQuery
+    with upper and lower bound equal and included. TermQueries do not
+    score correct, so the constant score mode of NRQ is the only
+    correct way to handle single value queries.
+  
+  - NumericTokenStream now works directly on byte[] terms. If you
+    plug a TokenFilter on top of this stream, you will likely get
+    an IllegalArgumentException, because the NTS does not support
+    TermAttribute/CharTermAttribute. If you want to further filter
+    or attach Payloads to NTS, use the new NumericTermAttribute.
+
 Bug Fixes
 
 * LUCENE-2222: FixedIntBlockIndexInput incorrectly read one block of

Modified: lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java?rev=930821&r1=930820&r2=930821&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java (original)
+++ lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java Mon Apr  5 08:07:18 2010
@@ -230,6 +230,8 @@ public class TestNumericRangeQuery32 ext
     testRightOpenRange(2);
   }
   
+  /* TESTs disabled, because incompatible API change in 3.1/flex:
+  
   private void testRandomTrieAndClassicRangeQuery(int precisionStep) throws Exception {
     final Random rnd=newRandom();
     String field="field"+precisionStep;
@@ -298,6 +300,8 @@ public class TestNumericRangeQuery32 ext
     testRandomTrieAndClassicRangeQuery(Integer.MAX_VALUE);
   }
   
+  */
+  
   private void testRangeSplit(int precisionStep) throws Exception {
     final Random rnd=newRandom();
     String field="ascfield"+precisionStep;

Modified: lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java?rev=930821&r1=930820&r2=930821&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java (original)
+++ lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java Mon Apr  5 08:07:18 2010
@@ -245,6 +245,8 @@ public class TestNumericRangeQuery64 ext
     testRightOpenRange(2);
   }
   
+  /* TESTs disabled, because incompatible API change in 3.1/flex:
+  
   private void testRandomTrieAndClassicRangeQuery(int precisionStep) throws Exception {
     final Random rnd=newRandom();
     String field="field"+precisionStep;
@@ -317,6 +319,8 @@ public class TestNumericRangeQuery64 ext
     testRandomTrieAndClassicRangeQuery(Integer.MAX_VALUE);
   }
   
+  */
+  
   private void testRangeSplit(int precisionStep) throws Exception {
     final Random rnd=newRandom();
     String field="ascfield"+precisionStep;

Modified: lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/util/TestNumericUtils.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/util/TestNumericUtils.java?rev=930821&r1=930820&r2=930821&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/util/TestNumericUtils.java (original)
+++ lucene/java/branches/flex_1458/backwards/src/test/org/apache/lucene/util/TestNumericUtils.java Mon Apr  5 08:07:18 2010
@@ -26,6 +26,8 @@ import java.util.Iterator;
 
 public class TestNumericUtils extends LuceneTestCase {
 
+  /* TESTs disabled, because incompatible API change in 3.1/flex:
+  
   public void testLongConversionAndOrdering() throws Exception {
     // generate a series of encoded longs, each numerical one bigger than the one before
     String last=null;
@@ -131,6 +133,8 @@ public class TestNumericUtils extends Lu
       }
     }
   }
+  
+  */
 
   public void testDoubles() throws Exception {
     double[] vals=new double[]{

Modified: lucene/java/branches/flex_1458/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java?rev=930821&r1=930820&r2=930821&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java Mon Apr  5 08:07:18 2010
@@ -19,12 +19,15 @@ package org.apache.lucene.spatial.tier;
 import java.io.IOException;
 import java.util.List;
 
+import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.search.DocIdSet;
+import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.util.NumericUtils;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.OpenBitSet;
 
 /**
@@ -44,22 +47,41 @@ public class CartesianShapeFilter extend
   
   @Override
   public DocIdSet getDocIdSet(final IndexReader reader) throws IOException {
-    final OpenBitSet bits = new OpenBitSet(reader.maxDoc());
-    final TermDocs termDocs = reader.termDocs();
+    final Bits delDocs = MultiFields.getDeletedDocs(reader);
     final List<Double> area = shape.getArea();
-    int sz = area.size();
+    final int sz = area.size();
     
-    final Term term = new Term(fieldName);
     // iterate through each boxid
-    for (int i =0; i< sz; i++) {
-      double boxId = area.get(i).doubleValue();
-      termDocs.seek(term.createTerm(NumericUtils.doubleToPrefixCoded(boxId)));
-      // iterate through all documents
-      // which have this boxId
-      while (termDocs.next()) {
-        bits.fastSet(termDocs.doc());
+    final BytesRef bytesRef = new BytesRef(NumericUtils.BUF_SIZE_LONG);
+    if (sz == 1) {
+      double boxId = area.get(0).doubleValue();
+      NumericUtils.longToPrefixCoded(NumericUtils.doubleToSortableLong(boxId), 0, bytesRef);
+      return new DocIdSet() {
+        @Override
+        public DocIdSetIterator iterator() throws IOException {
+          return MultiFields.getTermDocsEnum(reader, delDocs, fieldName, bytesRef);
+        }
+        
+        @Override
+        public boolean isCacheable() {
+          return false;
+        }
+      };
+    } else {
+      final OpenBitSet bits = new OpenBitSet(reader.maxDoc());
+      for (int i =0; i< sz; i++) {
+        double boxId = area.get(i).doubleValue();
+        NumericUtils.longToPrefixCoded(NumericUtils.doubleToSortableLong(boxId), 0, bytesRef);
+        final DocsEnum docsEnum = MultiFields.getTermDocsEnum(reader, delDocs, fieldName, bytesRef);
+        if (docsEnum == null) continue;
+        // iterate through all documents
+        // which have this boxId
+        int doc;
+        while ((doc = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
+          bits.fastSet(doc);
+        }
       }
+      return bits;
     }
-    return bits;
   }
 }

Modified: lucene/java/branches/flex_1458/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesian.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesian.java?rev=930821&r1=930820&r2=930821&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesian.java (original)
+++ lucene/java/branches/flex_1458/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestCartesian.java Mon Apr  5 08:07:18 2010
@@ -24,6 +24,7 @@ import java.util.Map;
 import org.apache.lucene.analysis.WhitespaceAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericField;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriterConfig;
@@ -49,7 +50,6 @@ import org.apache.lucene.spatial.tier.pr
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.NumericUtils;
 
 public class TestCartesian extends LuceneTestCase {
 
@@ -96,8 +96,8 @@ public class TestCartesian extends Lucen
     doc.add(new Field("name", name,Field.Store.YES, Field.Index.ANALYZED));
     
     // convert the lat / long to lucene fields
-    doc.add(new Field(latField, NumericUtils.doubleToPrefixCoded(lat),Field.Store.YES, Field.Index.NOT_ANALYZED));
-    doc.add(new Field(lngField, NumericUtils.doubleToPrefixCoded(lng),Field.Store.YES, Field.Index.NOT_ANALYZED));
+    doc.add(new NumericField(latField, Integer.MAX_VALUE, Field.Store.YES, true).setDoubleValue(lat));
+    doc.add(new NumericField(lngField, Integer.MAX_VALUE, Field.Store.YES, true).setDoubleValue(lng));
     
     // add a default meta field to make searching all documents easy 
     doc.add(new Field("metafile", "doc",Field.Store.YES, Field.Index.ANALYZED));
@@ -105,10 +105,9 @@ public class TestCartesian extends Lucen
     int ctpsize = ctps.size();
     for (int i =0; i < ctpsize; i++){
       CartesianTierPlotter ctp = ctps.get(i);
-      doc.add(new Field(ctp.getTierFieldName(), 
-          NumericUtils.doubleToPrefixCoded(ctp.getTierBoxId(lat,lng)),
+      doc.add(new NumericField(ctp.getTierFieldName(), Integer.MAX_VALUE, 
           Field.Store.YES, 
-          Field.Index.NOT_ANALYZED_NO_NORMS));
+          true).setDoubleValue(ctp.getTierBoxId(lat,lng)));
       
       doc.add(new Field(geoHashPrefix, GeoHashUtils.encode(lat,lng), 
     		  Field.Store.YES, 
@@ -245,8 +244,8 @@ public class TestCartesian extends Lucen
       Document d = searcher.doc(scoreDocs[i].doc);
 
       String name = d.get("name");
-      double rsLat = NumericUtils.prefixCodedToDouble(d.get(latField));
-      double rsLng = NumericUtils.prefixCodedToDouble(d.get(lngField));
+      double rsLat = Double.parseDouble(d.get(latField));
+      double rsLng = Double.parseDouble(d.get(lngField));
       Double geo_distance = distances.get(scoreDocs[i].doc);
 
       double distance = DistanceUtils.getInstance().getDistanceMi(lat, lng, rsLat, rsLng);
@@ -339,8 +338,8 @@ public class TestCartesian extends Lucen
     for(int i =0 ; i < results; i++){
       Document d = searcher.doc(scoreDocs[i].doc);
       String name = d.get("name");
-      double rsLat = NumericUtils.prefixCodedToDouble(d.get(latField));
-      double rsLng = NumericUtils.prefixCodedToDouble(d.get(lngField));
+      double rsLat = Double.parseDouble(d.get(latField));
+      double rsLng = Double.parseDouble(d.get(lngField));
       Double geo_distance = distances.get(scoreDocs[i].doc);
 
       double distance = DistanceUtils.getInstance().getDistanceMi(lat, lng, rsLat, rsLng);
@@ -434,8 +433,8 @@ public class TestCartesian extends Lucen
         Document d = searcher.doc(scoreDocs[i].doc);
       
         String name = d.get("name");
-        double rsLat = NumericUtils.prefixCodedToDouble(d.get(latField));
-        double rsLng = NumericUtils.prefixCodedToDouble(d.get(lngField)); 
+        double rsLat = Double.parseDouble(d.get(latField));
+        double rsLng = Double.parseDouble(d.get(lngField)); 
         Double geo_distance = distances.get(scoreDocs[i].doc);
       
         double distance = DistanceUtils.getInstance().getDistanceMi(lat, lng, rsLat, rsLng);
@@ -528,8 +527,8 @@ public class TestCartesian extends Lucen
         Document d = searcher.doc(scoreDocs[i].doc);
 	      
         String name = d.get("name");
-        double rsLat = NumericUtils.prefixCodedToDouble(d.get(latField));
-        double rsLng = NumericUtils.prefixCodedToDouble(d.get(lngField)); 
+        double rsLat = Double.parseDouble(d.get(latField));
+        double rsLng = Double.parseDouble(d.get(lngField)); 
         Double geo_distance = distances.get(scoreDocs[i].doc);
 	      
         double distance = DistanceUtils.getInstance().getDistanceMi(lat, lng, rsLat, rsLng);

Modified: lucene/java/branches/flex_1458/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestDistance.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestDistance.java?rev=930821&r1=930820&r2=930821&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestDistance.java (original)
+++ lucene/java/branches/flex_1458/contrib/spatial/src/test/org/apache/lucene/spatial/tier/TestDistance.java Mon Apr  5 08:07:18 2010
@@ -21,6 +21,7 @@ import java.io.IOException;
 import org.apache.lucene.analysis.WhitespaceAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericField;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.Term;
@@ -28,7 +29,6 @@ import org.apache.lucene.index.IndexRead
 import org.apache.lucene.search.QueryWrapperFilter;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.NumericUtils;
 import org.apache.lucene.store.RAMDirectory;
 
 public class TestDistance extends LuceneTestCase {
@@ -63,8 +63,8 @@ public class TestDistance extends Lucene
     doc.add(new Field("name", name,Field.Store.YES, Field.Index.ANALYZED));
     
     // convert the lat / long to lucene fields
-    doc.add(new Field(latField, NumericUtils.doubleToPrefixCoded(lat),Field.Store.YES, Field.Index.NOT_ANALYZED));
-    doc.add(new Field(lngField, NumericUtils.doubleToPrefixCoded(lng),Field.Store.YES, Field.Index.NOT_ANALYZED));
+    doc.add(new NumericField(latField, Integer.MAX_VALUE, Field.Store.YES, true).setDoubleValue(lat));
+    doc.add(new NumericField(lngField, Integer.MAX_VALUE,Field.Store.YES, true).setDoubleValue(lng));
     
     // add a default meta field to make searching all documents easy 
     doc.add(new Field("metafile", "doc",Field.Store.YES, Field.Index.ANALYZED));

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/NumericTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/NumericTokenStream.java?rev=930821&r1=930820&r2=930821&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/NumericTokenStream.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/NumericTokenStream.java Mon Apr  5 08:07:18 2010
@@ -17,12 +17,17 @@ package org.apache.lucene.analysis;
  * limitations under the License.
  */
 
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeImpl;
 import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.NumericUtils;
 import org.apache.lucene.document.NumericField; // for javadocs
 import org.apache.lucene.search.NumericRangeQuery; // for javadocs
 import org.apache.lucene.search.NumericRangeFilter; // for javadocs
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 
@@ -91,6 +96,88 @@ public final class NumericTokenStream ex
 
   /** The lower precision tokens gets this token type assigned. */
   public static final String TOKEN_TYPE_LOWER_PREC = "lowerPrecNumeric";
+  
+  /** <b>Expert:</b> Use this attribute to get the details of the currently generated token
+   * @lucene.experimental
+   * @since 3.1
+   */
+  public interface NumericTermAttribute extends Attribute {
+    /** Returns current shift value, undefined before first token */
+    int getShift();
+    /** Returns {@link NumericTokenStream}'s raw value as {@code long} */
+    long getRawValue();
+    /** Returns value size in bits (32 for {@code float}, {@code int}; 64 for {@code double}, {@code long}) */
+    int getValueSize();
+  }
+  
+  private static final class NumericAttributeFactory extends AttributeFactory {
+    private final AttributeFactory delegate;
+    private NumericTokenStream ts = null;
+
+    NumericAttributeFactory(AttributeFactory delegate) {
+      this.delegate = delegate;
+    }
+  
+    @Override
+    public AttributeImpl createAttributeInstance(Class<? extends Attribute> attClass) {
+      if (attClass == NumericTermAttribute.class)
+        return new NumericTermAttributeImpl(ts);
+      if (attClass.isAssignableFrom(CharTermAttribute.class) || attClass.isAssignableFrom(TermAttribute.class))
+        throw new IllegalArgumentException("NumericTokenStream does not support CharTermAttribute/TermAttribute.");
+      return delegate.createAttributeInstance(attClass);
+    }
+  }
+
+  private static final class NumericTermAttributeImpl extends AttributeImpl implements NumericTermAttribute,TermToBytesRefAttribute {
+    private final NumericTokenStream ts;
+    
+    public NumericTermAttributeImpl(NumericTokenStream ts) {
+      this.ts = ts;
+    }
+  
+    public int toBytesRef(BytesRef bytes) {
+      try {
+        assert ts.valSize == 64 || ts.valSize == 32;
+        return (ts.valSize == 64) ? 
+          NumericUtils.longToPrefixCoded(ts.value, ts.shift, bytes) :
+          NumericUtils.intToPrefixCoded((int) ts.value, ts.shift, bytes);
+      } catch (IllegalArgumentException iae) {
+        // return empty token before first
+        bytes.length = 0;
+        return 0;
+      }
+    }
+
+    public int getShift() { return ts.shift; }
+    public long getRawValue() { return ts.value; }
+    public int getValueSize() { return ts.valSize; }
+
+    @Override
+    public void clear() {
+      // this attribute has no contents to clear
+    }
+
+    @Override
+    public boolean equals(Object other) {
+      return other == this;
+    }
+
+    @Override
+    public int hashCode() {
+      return System.identityHashCode(this);
+    }
+    
+    @Override
+    public void copyTo(AttributeImpl target) {
+      // this attribute has no contents to copy
+    }
+    
+    @Override
+    public Object clone() {
+      // cannot throw CloneNotSupportedException (checked)
+      throw new UnsupportedOperationException();
+    }
+  }
 
   /**
    * Creates a token stream for numeric values using the default <code>precisionStep</code>
@@ -107,23 +194,15 @@ public final class NumericTokenStream ex
    * before using set a value using the various set<em>???</em>Value() methods.
    */
   public NumericTokenStream(final int precisionStep) {
-    super();
-    this.precisionStep = precisionStep;
-    if (precisionStep < 1)
-      throw new IllegalArgumentException("precisionStep must be >=1");
-  }
+    super(new NumericAttributeFactory(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY));
+    // we must do this after the super call :(
+    ((NumericAttributeFactory) getAttributeFactory()).ts = this;
+    addAttribute(NumericTermAttribute.class);
 
-  /**
-   * Expert: Creates a token stream for numeric values with the specified
-   * <code>precisionStep</code> using the given {@link AttributeSource}.
-   * The stream is not yet initialized,
-   * before using set a value using the various set<em>???</em>Value() methods.
-   */
-  public NumericTokenStream(AttributeSource source, final int precisionStep) {
-    super(source);
     this.precisionStep = precisionStep;
     if (precisionStep < 1)
       throw new IllegalArgumentException("precisionStep must be >=1");
+    shift = -precisionStep;
   }
 
   /**
@@ -134,10 +213,15 @@ public final class NumericTokenStream ex
    * before using set a value using the various set<em>???</em>Value() methods.
    */
   public NumericTokenStream(AttributeFactory factory, final int precisionStep) {
-    super(factory);
+    super(new NumericAttributeFactory(factory));
+    // we must do this after the super call :(
+    ((NumericAttributeFactory) getAttributeFactory()).ts = this;
+    addAttribute(NumericTermAttribute.class);
+
     this.precisionStep = precisionStep;
     if (precisionStep < 1)
       throw new IllegalArgumentException("precisionStep must be >=1");
+    shift = -precisionStep;
   }
 
   /**
@@ -149,7 +233,7 @@ public final class NumericTokenStream ex
   public NumericTokenStream setLongValue(final long value) {
     this.value = value;
     valSize = 64;
-    shift = 0;
+    shift = -precisionStep;
     return this;
   }
   
@@ -162,7 +246,7 @@ public final class NumericTokenStream ex
   public NumericTokenStream setIntValue(final int value) {
     this.value = value;
     valSize = 32;
-    shift = 0;
+    shift = -precisionStep;
     return this;
   }
   
@@ -175,7 +259,7 @@ public final class NumericTokenStream ex
   public NumericTokenStream setDoubleValue(final double value) {
     this.value = NumericUtils.doubleToSortableLong(value);
     valSize = 64;
-    shift = 0;
+    shift = -precisionStep;
     return this;
   }
   
@@ -188,7 +272,7 @@ public final class NumericTokenStream ex
   public NumericTokenStream setFloatValue(final float value) {
     this.value = NumericUtils.floatToSortableInt(value);
     valSize = 32;
-    shift = 0;
+    shift = -precisionStep;
     return this;
   }
   
@@ -196,37 +280,24 @@ public final class NumericTokenStream ex
   public void reset() {
     if (valSize == 0)
       throw new IllegalStateException("call set???Value() before usage");
-    shift = 0;
+    shift = -precisionStep;
   }
 
   @Override
   public boolean incrementToken() {
     if (valSize == 0)
       throw new IllegalStateException("call set???Value() before usage");
-    if (shift >= valSize)
+    shift += precisionStep;
+    if (shift >= valSize) {
+      // reset so the attribute still works after exhausted stream
+      shift -= precisionStep;
       return false;
+    }
 
     clearAttributes();
-    final char[] buffer;
-    switch (valSize) {
-      case 64:
-        buffer = termAtt.resizeTermBuffer(NumericUtils.BUF_SIZE_LONG);
-        termAtt.setTermLength(NumericUtils.longToPrefixCoded(value, shift, buffer));
-        break;
-      
-      case 32:
-        buffer = termAtt.resizeTermBuffer(NumericUtils.BUF_SIZE_INT);
-        termAtt.setTermLength(NumericUtils.intToPrefixCoded((int) value, shift, buffer));
-        break;
-      
-      default:
-        // should not happen
-        throw new IllegalArgumentException("valSize must be 32 or 64");
-    }
-    
+    // the TermToBytesRefAttribute is directly accessing shift & value.
     typeAtt.setType((shift == 0) ? TOKEN_TYPE_FULL_PREC : TOKEN_TYPE_LOWER_PREC);
     posIncrAtt.setPositionIncrement((shift == 0) ? 1 : 0);
-    shift += precisionStep;
     return true;
   }
   
@@ -238,12 +309,11 @@ public final class NumericTokenStream ex
   }
 
   // members
-  private final TermAttribute termAtt = addAttribute(TermAttribute.class);
   private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
   private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
   
-  private int shift = 0, valSize = 0; // valSize==0 means not initialized
+  int shift, valSize = 0; // valSize==0 means not initialized
   private final int precisionStep;
   
-  private long value = 0L;
+  long value = 0L;
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCache.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCache.java?rev=930821&r1=930820&r2=930821&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCache.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCache.java Mon Apr  5 08:07:18 2010
@@ -256,15 +256,14 @@ public interface FieldCache {
   };
 
   /**
-   * A parser instance for int values encoded by {@link NumericUtils#intToPrefixCoded(int)}, e.g. when indexed
+   * A parser instance for int values encoded by {@link NumericUtils}, e.g. when indexed
    * via {@link NumericField}/{@link NumericTokenStream}.
    */
   public static final IntParser NUMERIC_UTILS_INT_PARSER=new IntParser(){
-    public int parseInt(BytesRef val) {
-      final int shift = val.bytes[val.offset]-NumericUtils.SHIFT_START_INT;
-      if (shift>0 && shift<=31)
+    public int parseInt(BytesRef term) {
+      if (NumericUtils.getPrefixCodedIntShift(term) > 0)
         throw new FieldCacheImpl.StopFillCacheException();
-      return NumericUtils.prefixCodedToInt(val);
+      return NumericUtils.prefixCodedToInt(term);
     }
     protected Object readResolve() {
       return NUMERIC_UTILS_INT_PARSER;
@@ -281,8 +280,7 @@ public interface FieldCache {
    */
   public static final FloatParser NUMERIC_UTILS_FLOAT_PARSER=new FloatParser(){
     public float parseFloat(BytesRef term) {
-      final int shift = term.bytes[term.offset]-NumericUtils.SHIFT_START_INT;
-      if (shift>0 && shift<=31)
+      if (NumericUtils.getPrefixCodedIntShift(term) > 0)
         throw new FieldCacheImpl.StopFillCacheException();
       return NumericUtils.sortableIntToFloat(NumericUtils.prefixCodedToInt(term));
     }
@@ -296,13 +294,12 @@ public interface FieldCache {
   };
 
   /**
-   * A parser instance for long values encoded by {@link NumericUtils#longToPrefixCoded(long)}, e.g. when indexed
+   * A parser instance for long values encoded by {@link NumericUtils}, e.g. when indexed
    * via {@link NumericField}/{@link NumericTokenStream}.
    */
   public static final LongParser NUMERIC_UTILS_LONG_PARSER = new LongParser(){
     public long parseLong(BytesRef term) {
-      final int shift = term.bytes[term.offset]-NumericUtils.SHIFT_START_LONG;
-      if (shift>0 && shift<=63)
+      if (NumericUtils.getPrefixCodedLongShift(term) > 0)
         throw new FieldCacheImpl.StopFillCacheException();
       return NumericUtils.prefixCodedToLong(term);
     }
@@ -321,8 +318,7 @@ public interface FieldCache {
    */
   public static final DoubleParser NUMERIC_UTILS_DOUBLE_PARSER = new DoubleParser(){
     public double parseDouble(BytesRef term) {
-      final int shift = term.bytes[term.offset]-NumericUtils.SHIFT_START_LONG;
-      if (shift>0 && shift<=63)
+      if (NumericUtils.getPrefixCodedLongShift(term) > 0)
         throw new FieldCacheImpl.StopFillCacheException();
       return NumericUtils.sortableLongToDouble(NumericUtils.prefixCodedToLong(term));
     }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/NumericRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/NumericRangeQuery.java?rev=930821&r1=930820&r2=930821&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/NumericRangeQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/NumericRangeQuery.java Mon Apr  5 08:07:18 2010
@@ -379,9 +379,9 @@ public final class NumericRangeQuery<T e
    */
   private final class NumericRangeTermsEnum extends FilteredTermsEnum {
 
-    private final BytesRef currentLowerBound = new BytesRef(), currentUpperBound = new BytesRef();
+    private BytesRef currentLowerBound, currentUpperBound;
 
-    private final LinkedList<String> rangeBounds = new LinkedList<String>();
+    private final LinkedList<BytesRef> rangeBounds = new LinkedList<BytesRef>();
     private final Comparator<BytesRef> termComp;
 
     NumericRangeTermsEnum(final IndexReader reader) throws IOException {
@@ -414,7 +414,7 @@ public final class NumericRangeQuery<T e
           
           NumericUtils.splitLongRange(new NumericUtils.LongRangeBuilder() {
             @Override
-            public final void addRange(String minPrefixCoded, String maxPrefixCoded) {
+            public final void addRange(BytesRef minPrefixCoded, BytesRef maxPrefixCoded) {
               rangeBounds.add(minPrefixCoded);
               rangeBounds.add(maxPrefixCoded);
             }
@@ -449,7 +449,7 @@ public final class NumericRangeQuery<T e
           
           NumericUtils.splitIntRange(new NumericUtils.IntRangeBuilder() {
             @Override
-            public final void addRange(String minPrefixCoded, String maxPrefixCoded) {
+            public final void addRange(BytesRef minPrefixCoded, BytesRef maxPrefixCoded) {
               rangeBounds.add(minPrefixCoded);
               rangeBounds.add(maxPrefixCoded);
             }
@@ -470,11 +470,11 @@ public final class NumericRangeQuery<T e
       if (rangeBounds.size() >= 2) {
         assert rangeBounds.size() % 2 == 0;
 
-        this.currentLowerBound.copy(rangeBounds.removeFirst());
-        assert termComp.compare(currentUpperBound, currentLowerBound) <= 0 :
+        this.currentLowerBound = rangeBounds.removeFirst();
+        assert currentUpperBound == null || termComp.compare(currentUpperBound, currentLowerBound) <= 0 :
           "The current upper bound must be <= the new lower bound";
         
-        this.currentUpperBound.copy(rangeBounds.removeFirst());
+        this.currentUpperBound = rangeBounds.removeFirst();
         return currentLowerBound;
       }
       

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/NumericUtils.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/NumericUtils.java?rev=930821&r1=930820&r2=930821&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/NumericUtils.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/NumericUtils.java Mon Apr  5 08:07:18 2010
@@ -22,6 +22,8 @@ import org.apache.lucene.document.Numeri
 import org.apache.lucene.search.NumericRangeQuery; // for javadocs
 import org.apache.lucene.search.NumericRangeFilter; // for javadocs
 
+// TODO: Remove the commented out methods before release!
+
 /**
  * This is a helper class to generate prefix-encoded representations for numerical values
  * and supplies converters to represent float/double values as sortable integers/longs.
@@ -32,10 +34,10 @@ import org.apache.lucene.search.NumericR
  * more exactly. This reduces the number of terms dramatically.
  *
  * <p>This class generates terms to achieve this: First the numerical integer values need to
- * be converted to strings. For that integer values (32 bit or 64 bit) are made unsigned
- * and the bits are converted to ASCII chars with each 7 bit. The resulting string is
- * sortable like the original integer value. Each value is also prefixed
- * (in the first char) by the <code>shift</code> value (number of bits removed) used
+ * be converted to bytes. For that integer values (32 bit or 64 bit) are made unsigned
+ * and the bits are converted to ASCII chars with each 7 bit. The resulting byte[] is
+ * sortable like the original integer value (even using UTF-8 sort order). Each value is also
+ * prefixed (in the first char) by the <code>shift</code> value (number of bits removed) used
  * during encoding.
  *
  * <p>To also index floating point numbers, this class supplies two methods to convert them
@@ -51,13 +53,12 @@ import org.apache.lucene.search.NumericR
  * {@link NumericRangeQuery} and {@link NumericRangeFilter} implement the query part
  * for the same data types.
  *
- * <p>This class can also be used, to generate lexicographically sortable (according
- * {@link String#compareTo(String)}) representations of numeric data types for other
- * usages (e.g. sorting).
- *
- * @lucene.experimental
+ * <p>This class can also be used, to generate lexicographically sortable (according to
+ * {@link BytesRef#getUTF8SortedAsUTF16Comparator()}) representations of numeric data
+ * types for other usages (e.g. sorting).
  *
- * @since 2.9
+ * @lucene.internal
+ * @since 2.9, API changed non backwards-compliant in 3.1
  */
 public final class NumericUtils {
 
@@ -70,126 +71,150 @@ public final class NumericUtils {
   public static final int PRECISION_STEP_DEFAULT = 4;
   
   /**
-   * Expert: Longs are stored at lower precision by shifting off lower bits. The shift count is
-   * stored as <code>SHIFT_START_LONG+shift</code> in the first character
+   * Longs are stored at lower precision by shifting off lower bits. The shift count is
+   * stored as <code>SHIFT_START_LONG+shift</code> in the first byte
    */
-  public static final char SHIFT_START_LONG = (char)0x20;
+  public static final byte SHIFT_START_LONG = 0x20;
 
   /**
-   * Expert: The maximum term length (used for <code>char[]</code> buffer size)
+   * The maximum term length (used for <code>byte[]</code> buffer size)
    * for encoding <code>long</code> values.
-   * @see #longToPrefixCoded(long,int,char[])
+   * @see #longToPrefixCoded(long,int,BytesRef)
    */
   public static final int BUF_SIZE_LONG = 63/7 + 2;
 
   /**
-   * Expert: Integers are stored at lower precision by shifting off lower bits. The shift count is
-   * stored as <code>SHIFT_START_INT+shift</code> in the first character
+   * Integers are stored at lower precision by shifting off lower bits. The shift count is
+   * stored as <code>SHIFT_START_INT+shift</code> in the first byte
    */
-  public static final char SHIFT_START_INT  = (char)0x60;
+  public static final byte SHIFT_START_INT  = 0x60;
 
   /**
-   * Expert: The maximum term length (used for <code>char[]</code> buffer size)
+   * The maximum term length (used for <code>byte[]</code> buffer size)
    * for encoding <code>int</code> values.
-   * @see #intToPrefixCoded(int,int,char[])
+   * @see #intToPrefixCoded(int,int,BytesRef)
    */
   public static final int BUF_SIZE_INT = 31/7 + 2;
 
   /**
-   * Expert: Returns prefix coded bits after reducing the precision by <code>shift</code> bits.
+   * Returns prefix coded bits after reducing the precision by <code>shift</code> bits.
    * This is method is used by {@link NumericTokenStream}.
    * @param val the numeric value
    * @param shift how many bits to strip from the right
-   * @param buffer that will contain the encoded chars, must be at least of {@link #BUF_SIZE_LONG}
-   * length
-   * @return number of chars written to buffer
+   * @param bytes will contain the encoded value
+   * @return the hash code for indexing (TermsHash)
    */
-  public static int longToPrefixCoded(final long val, final int shift, final char[] buffer) {
+  public static int longToPrefixCoded(final long val, final int shift, final BytesRef bytes) {
     if (shift>63 || shift<0)
       throw new IllegalArgumentException("Illegal shift value, must be 0..63");
-    int nChars = (63-shift)/7 + 1, len = nChars+1;
-    buffer[0] = (char)(SHIFT_START_LONG + shift);
+    if (bytes.bytes == null) {
+      bytes.bytes = new byte[NumericUtils.BUF_SIZE_LONG];
+    } else if (bytes.bytes.length < NumericUtils.BUF_SIZE_LONG) {
+      bytes.grow(NumericUtils.BUF_SIZE_LONG);
+    }
+    int hash, nChars = (63-shift)/7 + 1;
+    bytes.length = nChars+1;
+    bytes.bytes[0] = (byte) (hash = (SHIFT_START_LONG + shift));
     long sortableBits = val ^ 0x8000000000000000L;
     sortableBits >>>= shift;
-    while (nChars>=1) {
-      // Store 7 bits per character for good efficiency when UTF-8 encoding.
-      // The whole number is right-justified so that lucene can prefix-encode
-      // the terms more efficiently.
-      buffer[nChars--] = (char)(sortableBits & 0x7f);
+    while (nChars > 0) {
+      // Store 7 bits per byte for compatibility
+      // with UTF-8 encoding of terms
+      bytes.bytes[nChars--] = (byte)(sortableBits & 0x7f);
       sortableBits >>>= 7;
     }
-    return len;
+    // calculate hash
+    for (int i = 1; i < bytes.length; i++) {
+      hash = 31*hash + bytes.bytes[i];
+    }
+    return hash;
   }
 
   /**
-   * Expert: Returns prefix coded bits after reducing the precision by <code>shift</code> bits.
+   * Returns prefix coded bits after reducing the precision by <code>shift</code> bits.
    * This is method is used by {@link LongRangeBuilder}.
    * @param val the numeric value
    * @param shift how many bits to strip from the right
-   */
+   * @deprecated This method is no longer needed!
+   *
+  @Deprecated
   public static String longToPrefixCoded(final long val, final int shift) {
-    final char[] buffer = new char[BUF_SIZE_LONG];
-    final int len = longToPrefixCoded(val, shift, buffer);
-    return new String(buffer, 0, len);
-  }
+    final BytesRef buffer = new BytesRef(BUF_SIZE_LONG);
+    longToPrefixCoded(val, shift, buffer);
+    return buffer.utf8ToString();
+  }*/
 
   /**
    * This is a convenience method, that returns prefix coded bits of a long without
    * reducing the precision. It can be used to store the full precision value as a
    * stored field in index.
    * <p>To decode, use {@link #prefixCodedToLong}.
-   */
+   * @deprecated This method is no longer needed!
+   *
+  @Deprecated
   public static String longToPrefixCoded(final long val) {
     return longToPrefixCoded(val, 0);
-  }
+  }*/
   
   /**
-   * Expert: Returns prefix coded bits after reducing the precision by <code>shift</code> bits.
+   * Returns prefix coded bits after reducing the precision by <code>shift</code> bits.
    * This is method is used by {@link NumericTokenStream}.
    * @param val the numeric value
    * @param shift how many bits to strip from the right
-   * @param buffer that will contain the encoded chars, must be at least of {@link #BUF_SIZE_INT}
-   * length
-   * @return number of chars written to buffer
+   * @param bytes will contain the encoded value
+   * @return the hash code for indexing (TermsHash)
    */
-  public static int intToPrefixCoded(final int val, final int shift, final char[] buffer) {
+  public static int intToPrefixCoded(final int val, final int shift, final BytesRef bytes) {
     if (shift>31 || shift<0)
       throw new IllegalArgumentException("Illegal shift value, must be 0..31");
-    int nChars = (31-shift)/7 + 1, len = nChars+1;
-    buffer[0] = (char)(SHIFT_START_INT + shift);
+    if (bytes.bytes == null) {
+      bytes.bytes = new byte[NumericUtils.BUF_SIZE_INT];
+    } else if (bytes.bytes.length < NumericUtils.BUF_SIZE_INT) {
+      bytes.grow(NumericUtils.BUF_SIZE_INT);
+    }
+    int hash, nChars = (31-shift)/7 + 1;
+    bytes.length = nChars+1;
+    bytes.bytes[0] = (byte) (hash = (SHIFT_START_INT + shift));
     int sortableBits = val ^ 0x80000000;
     sortableBits >>>= shift;
-    while (nChars>=1) {
-      // Store 7 bits per character for good efficiency when UTF-8 encoding.
-      // The whole number is right-justified so that lucene can prefix-encode
-      // the terms more efficiently.
-      buffer[nChars--] = (char)(sortableBits & 0x7f);
+    while (nChars > 0) {
+      // Store 7 bits per byte for compatibility
+      // with UTF-8 encoding of terms
+      bytes.bytes[nChars--] = (byte)(sortableBits & 0x7f);
       sortableBits >>>= 7;
     }
-    return len;
+    // calculate hash
+    for (int i = 1; i < bytes.length; i++) {
+      hash = 31*hash + bytes.bytes[i];
+    }
+    return hash;
   }
 
   /**
-   * Expert: Returns prefix coded bits after reducing the precision by <code>shift</code> bits.
+   * Returns prefix coded bits after reducing the precision by <code>shift</code> bits.
    * This is method is used by {@link IntRangeBuilder}.
    * @param val the numeric value
    * @param shift how many bits to strip from the right
-   */
+   * @deprecated This method is no longer needed!
+   *
+  @Deprecated
   public static String intToPrefixCoded(final int val, final int shift) {
-    final char[] buffer = new char[BUF_SIZE_INT];
-    final int len = intToPrefixCoded(val, shift, buffer);
-    return new String(buffer, 0, len);
-  }
+    final BytesRef buffer = new BytesRef(BUF_SIZE_INT);
+    intToPrefixCoded(val, shift, buffer);
+    return buffer.utf8ToString();
+  }*/
 
   /**
    * This is a convenience method, that returns prefix coded bits of an int without
    * reducing the precision. It can be used to store the full precision value as a
    * stored field in index.
    * <p>To decode, use {@link #prefixCodedToInt}.
-   */
+   * @deprecated This method is no longer needed!
+   *
+  @Deprecated
   public static String intToPrefixCoded(final int val) {
     return intToPrefixCoded(val, 0);
-  }
+  }*/
 
   /**
    * Returns a long from prefixCoded characters.
@@ -198,91 +223,97 @@ public final class NumericUtils {
    * @throws NumberFormatException if the supplied string is
    * not correctly prefix encoded.
    * @see #longToPrefixCoded(long)
-   */
+   * @deprecated This method is no longer needed!
+   *
+  @Deprecated
   public static long prefixCodedToLong(final String prefixCoded) {
-    final int shift = prefixCoded.charAt(0)-SHIFT_START_LONG;
-    if (shift>63 || shift<0)
-      throw new NumberFormatException("Invalid shift value in prefixCoded string (is encoded value really a LONG?)");
-    long sortableBits = 0L;
-    for (int i=1, len=prefixCoded.length(); i<len; i++) {
-      sortableBits <<= 7;
-      final char ch = prefixCoded.charAt(i);
-      if (ch>0x7f) {
-        throw new NumberFormatException(
-          "Invalid prefixCoded numerical value representation (char "+
-          Integer.toHexString(ch)+" at position "+i+" is invalid)"
-        );
-      }
-      sortableBits |= ch;
-    }
-    return (sortableBits << shift) ^ 0x8000000000000000L;
+    return prefixCodedToLong(new BytesRef(prefixCoded));
+  }*/
+
+  /**
+   * Returns the shift value from a prefix encoded {@code long}.
+   * @throws NumberFormatException if the supplied {@link BytesRef} is
+   * not correctly prefix encoded.
+   */
+  public static int getPrefixCodedLongShift(final BytesRef val) {
+    final int shift = val.bytes[val.offset] - SHIFT_START_LONG;
+    if (shift > 63 || shift < 0)
+      throw new NumberFormatException("Invalid shift value in prefixCoded bytes (is encoded value really an INT?)");
+    return shift;
   }
 
-  public static long prefixCodedToLong(final BytesRef term) {
-    final int shift = term.bytes[term.offset]-SHIFT_START_LONG;
-    if (shift>63 || shift<0)
-      throw new NumberFormatException("Invalid shift value in prefixCoded string (is encoded value really an INT?)");
+  /**
+   * Returns the shift value from a prefix encoded {@code int}.
+   * @throws NumberFormatException if the supplied {@link BytesRef} is
+   * not correctly prefix encoded.
+   */
+  public static int getPrefixCodedIntShift(final BytesRef val) {
+    final int shift = val.bytes[val.offset] - SHIFT_START_INT;
+    if (shift > 31 || shift < 0)
+      throw new NumberFormatException("Invalid shift value in prefixCoded bytes (is encoded value really an INT?)");
+    return shift;
+  }
+
+  /**
+   * Returns a long from prefixCoded bytes.
+   * Rightmost bits will be zero for lower precision codes.
+   * This method can be used to decode a term's value.
+   * @throws NumberFormatException if the supplied {@link BytesRef} is
+   * not correctly prefix encoded.
+   * @see #longToPrefixCoded(long,int,BytesRef)
+   */
+  public static long prefixCodedToLong(final BytesRef val) {
     long sortableBits = 0L;
-    final int limit = term.offset + term.length;
-    for (int i=term.offset+1; i<limit; i++) {
+    for (int i=val.offset+1, limit=val.offset+val.length; i<limit; i++) {
       sortableBits <<= 7;
-      final byte ch = term.bytes[i];
-      if (ch < 0) {
+      final byte b = val.bytes[i];
+      if (b < 0) {
         throw new NumberFormatException(
-          "Invalid prefixCoded numerical value representation (char "+
-          Integer.toHexString((int)(ch&0xff))+" at position "+(i-term.offset)+" is invalid)"
+          "Invalid prefixCoded numerical value representation (byte "+
+          Integer.toHexString(b&0xff)+" at position "+(i-val.offset)+" is invalid)"
         );
       }
-      sortableBits |= (long) ch;
+      sortableBits |= b;
     }
-    return (sortableBits << shift) ^ 0x8000000000000000L;
+    return (sortableBits << getPrefixCodedLongShift(val)) ^ 0x8000000000000000L;
   }
 
   /**
    * Returns an int from prefixCoded characters.
    * Rightmost bits will be zero for lower precision codes.
-   * This method can be used to decode e.g. a stored field.
+   * This method can be used to decode a term's value.
    * @throws NumberFormatException if the supplied string is
    * not correctly prefix encoded.
    * @see #intToPrefixCoded(int)
-   */
+   * @deprecated This method is no longer needed!
+   *
+  @Deprecated
   public static int prefixCodedToInt(final String prefixCoded) {
-    final int shift = prefixCoded.charAt(0)-SHIFT_START_INT;
-    if (shift>31 || shift<0)
-      throw new NumberFormatException("Invalid shift value in prefixCoded string (is encoded value really an INT?)");
-    int sortableBits = 0;
-    for (int i=1, len=prefixCoded.length(); i<len; i++) {
-      sortableBits <<= 7;
-      final char ch = prefixCoded.charAt(i);
-      if (ch>0x7f) {
-        throw new NumberFormatException(
-          "Invalid prefixCoded numerical value representation (char "+
-          Integer.toHexString(ch)+" at position "+i+" is invalid)"
-        );
-      }
-      sortableBits |= ch;
-    }
-    return (sortableBits << shift) ^ 0x80000000;
-  }
+    return prefixCodedToInt(new BytesRef(prefixCoded));
+  }*/
 
-  public static int prefixCodedToInt(final BytesRef term) {
-    final int shift = term.bytes[term.offset]-SHIFT_START_INT;
-    if (shift>31 || shift<0)
-      throw new NumberFormatException("Invalid shift value in prefixCoded string (is encoded value really an INT?)");
+  /**
+   * Returns an int from prefixCoded bytes.
+   * Rightmost bits will be zero for lower precision codes.
+   * This method can be used to decode a term's value.
+   * @throws NumberFormatException if the supplied {@link BytesRef} is
+   * not correctly prefix encoded.
+   * @see #intToPrefixCoded(int,int,BytesRef)
+   */
+  public static int prefixCodedToInt(final BytesRef val) {
     int sortableBits = 0;
-    final int limit = term.offset + term.length;
-    for (int i=term.offset+1; i<limit; i++) {
+    for (int i=val.offset+1, limit=val.offset+val.length; i<limit; i++) {
       sortableBits <<= 7;
-      final byte ch = term.bytes[i];
-      if (ch < 0) {
+      final byte b = val.bytes[i];
+      if (b < 0) {
         throw new NumberFormatException(
-          "Invalid prefixCoded numerical value representation (char "+
-          Integer.toHexString((int)(ch&0xff))+" at position "+(i-term.offset)+" is invalid)"
+          "Invalid prefixCoded numerical value representation (byte "+
+          Integer.toHexString(b&0xff)+" at position "+(i-val.offset)+" is invalid)"
         );
       }
-      sortableBits |= (int) ch;
+      sortableBits |= b;
     }
-    return (sortableBits << shift) ^ 0x80000000;
+    return (sortableBits << getPrefixCodedIntShift(val)) ^ 0x80000000;
   }
 
   /**
@@ -301,10 +332,12 @@ public final class NumericUtils {
   /**
    * Convenience method: this just returns:
    *   longToPrefixCoded(doubleToSortableLong(val))
-   */
+   * @deprecated This method is no longer needed!
+   *
+  @Deprecated
   public static String doubleToPrefixCoded(double val) {
     return longToPrefixCoded(doubleToSortableLong(val));
-  }
+  }*/
 
   /**
    * Converts a sortable <code>long</code> back to a <code>double</code>.
@@ -318,10 +351,12 @@ public final class NumericUtils {
   /**
    * Convenience method: this just returns:
    *    sortableLongToDouble(prefixCodedToLong(val))
-   */
+   * @deprecated This method is no longer needed!
+   *
+  @Deprecated
   public static double prefixCodedToDouble(String val) {
     return sortableLongToDouble(prefixCodedToLong(val));
-  }
+  }*/
 
   /**
    * Converts a <code>float</code> value to a sortable signed <code>int</code>.
@@ -339,10 +374,12 @@ public final class NumericUtils {
   /**
    * Convenience method: this just returns:
    *   intToPrefixCoded(floatToSortableInt(val))
-   */
+   * @deprecated This method is no longer needed!
+   *
+  @Deprecated
   public static String floatToPrefixCoded(float val) {
     return intToPrefixCoded(floatToSortableInt(val));
-  }
+  }*/
 
   /**
    * Converts a sortable <code>int</code> back to a <code>float</code>.
@@ -356,16 +393,18 @@ public final class NumericUtils {
   /**
    * Convenience method: this just returns:
    *    sortableIntToFloat(prefixCodedToInt(val))
-   */
+   * @deprecated This method is no longer needed!
+   *
+  @Deprecated
   public static float prefixCodedToFloat(String val) {
     return sortableIntToFloat(prefixCodedToInt(val));
-  }
+  }*/
 
   /**
-   * Expert: Splits a long range recursively.
+   * Splits a long range recursively.
    * You may implement a builder that adds clauses to a
    * {@link org.apache.lucene.search.BooleanQuery} for each call to its
-   * {@link LongRangeBuilder#addRange(String,String)}
+   * {@link LongRangeBuilder#addRange(BytesRef,BytesRef)}
    * method.
    * <p>This method is used by {@link NumericRangeQuery}.
    */
@@ -376,10 +415,10 @@ public final class NumericUtils {
   }
   
   /**
-   * Expert: Splits an int range recursively.
+   * Splits an int range recursively.
    * You may implement a builder that adds clauses to a
    * {@link org.apache.lucene.search.BooleanQuery} for each call to its
-   * {@link IntRangeBuilder#addRange(String,String)}
+   * {@link IntRangeBuilder#addRange(BytesRef,BytesRef)}
    * method.
    * <p>This method is used by {@link NumericRangeQuery}.
    */
@@ -452,10 +491,10 @@ public final class NumericUtils {
   }
 
   /**
-   * Expert: Callback for {@link #splitLongRange}.
+   * Callback for {@link #splitLongRange}.
    * You need to overwrite only one of the methods.
-   * <p><font color="red"><b>NOTE:</b> This is a very low-level interface,
-   * the method signatures may change in later versions.</font>
+   * @lucene.internal
+   * @since 2.9, API changed non backwards-compliant in 3.1
    */
   public static abstract class LongRangeBuilder {
     
@@ -463,7 +502,7 @@ public final class NumericUtils {
      * Overwrite this method, if you like to receive the already prefix encoded range bounds.
      * You can directly build classical (inclusive) range queries from them.
      */
-    public void addRange(String minPrefixCoded, String maxPrefixCoded) {
+    public void addRange(BytesRef minPrefixCoded, BytesRef maxPrefixCoded) {
       throw new UnsupportedOperationException();
     }
     
@@ -472,16 +511,19 @@ public final class NumericUtils {
      * You can use this for e.g. debugging purposes (print out range bounds).
      */
     public void addRange(final long min, final long max, final int shift) {
-      addRange(longToPrefixCoded(min, shift), longToPrefixCoded(max, shift));
+      final BytesRef minBytes = new BytesRef(BUF_SIZE_LONG), maxBytes = new BytesRef(BUF_SIZE_LONG);
+      longToPrefixCoded(min, shift, minBytes);
+      longToPrefixCoded(max, shift, maxBytes);
+      addRange(minBytes, maxBytes);
     }
   
   }
   
   /**
-   * Expert: Callback for {@link #splitIntRange}.
+   * Callback for {@link #splitIntRange}.
    * You need to overwrite only one of the methods.
-   * <p><font color="red"><b>NOTE:</b> This is a very low-level interface,
-   * the method signatures may change in later versions.</font>
+   * @lucene.internal
+   * @since 2.9, API changed non backwards-compliant in 3.1
    */
   public static abstract class IntRangeBuilder {
     
@@ -489,7 +531,7 @@ public final class NumericUtils {
      * Overwrite this method, if you like to receive the already prefix encoded range bounds.
      * You can directly build classical range (inclusive) queries from them.
      */
-    public void addRange(String minPrefixCoded, String maxPrefixCoded) {
+    public void addRange(BytesRef minPrefixCoded, BytesRef maxPrefixCoded) {
       throw new UnsupportedOperationException();
     }
     
@@ -498,7 +540,10 @@ public final class NumericUtils {
      * You can use this for e.g. debugging purposes (print out range bounds).
      */
     public void addRange(final int min, final int max, final int shift) {
-      addRange(intToPrefixCoded(min, shift), intToPrefixCoded(max, shift));
+      final BytesRef minBytes = new BytesRef(BUF_SIZE_INT), maxBytes = new BytesRef(BUF_SIZE_INT);
+      intToPrefixCoded(min, shift, minBytes);
+      intToPrefixCoded(max, shift, maxBytes);
+      addRange(minBytes, maxBytes);
     }
   
   }

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java?rev=930821&r1=930820&r2=930821&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java Mon Apr  5 08:07:18 2010
@@ -17,8 +17,9 @@ package org.apache.lucene.analysis;
  * limitations under the License.
  */
 
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.NumericUtils;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 
 public class TestNumericTokenStream extends BaseTokenStreamTestCase {
@@ -29,27 +30,47 @@ public class TestNumericTokenStream exte
   public void testLongStream() throws Exception {
     final NumericTokenStream stream=new NumericTokenStream().setLongValue(lvalue);
     // use getAttribute to test if attributes really exist, if not an IAE will be throwed
-    final TermAttribute termAtt = stream.getAttribute(TermAttribute.class);
+    final TermToBytesRefAttribute bytesAtt = stream.getAttribute(TermToBytesRefAttribute.class);
     final TypeAttribute typeAtt = stream.getAttribute(TypeAttribute.class);
+    final NumericTokenStream.NumericTermAttribute numericAtt = stream.getAttribute(NumericTokenStream.NumericTermAttribute.class);
+    final BytesRef bytes = new BytesRef();
+    stream.reset();
+    assertEquals(64, numericAtt.getValueSize());
+    assertEquals(lvalue, numericAtt.getRawValue());
     for (int shift=0; shift<64; shift+=NumericUtils.PRECISION_STEP_DEFAULT) {
       assertTrue("New token is available", stream.incrementToken());
-      assertEquals("Term is correctly encoded", NumericUtils.longToPrefixCoded(lvalue, shift), termAtt.term());
-      assertEquals("Type correct", (shift == 0) ? NumericTokenStream.TOKEN_TYPE_FULL_PREC : NumericTokenStream.TOKEN_TYPE_LOWER_PREC, typeAtt.type());
+      assertEquals("Shift value wrong", shift, numericAtt.getShift());
+      final int hash = bytesAtt.toBytesRef(bytes);
+      assertEquals("Hash incorrect", bytes.hashCode(), hash);
+      assertEquals("Term is incorrectly encoded", lvalue & ~((1L << shift) - 1L), NumericUtils.prefixCodedToLong(bytes));
+      assertEquals("Type incorrect", (shift == 0) ? NumericTokenStream.TOKEN_TYPE_FULL_PREC : NumericTokenStream.TOKEN_TYPE_LOWER_PREC, typeAtt.type());
     }
-    assertFalse("No more tokens available", stream.incrementToken());
+    assertFalse("More tokens available", stream.incrementToken());
+    stream.end();
+    stream.close();
   }
 
   public void testIntStream() throws Exception {
     final NumericTokenStream stream=new NumericTokenStream().setIntValue(ivalue);
     // use getAttribute to test if attributes really exist, if not an IAE will be throwed
-    final TermAttribute termAtt = stream.getAttribute(TermAttribute.class);
+    final TermToBytesRefAttribute bytesAtt = stream.getAttribute(TermToBytesRefAttribute.class);
     final TypeAttribute typeAtt = stream.getAttribute(TypeAttribute.class);
+    final NumericTokenStream.NumericTermAttribute numericAtt = stream.getAttribute(NumericTokenStream.NumericTermAttribute.class);
+    final BytesRef bytes = new BytesRef();
+    stream.reset();
+    assertEquals(32, numericAtt.getValueSize());
+    assertEquals(ivalue, numericAtt.getRawValue());
     for (int shift=0; shift<32; shift+=NumericUtils.PRECISION_STEP_DEFAULT) {
       assertTrue("New token is available", stream.incrementToken());
-      assertEquals("Term is correctly encoded", NumericUtils.intToPrefixCoded(ivalue, shift), termAtt.term());
-      assertEquals("Type correct", (shift == 0) ? NumericTokenStream.TOKEN_TYPE_FULL_PREC : NumericTokenStream.TOKEN_TYPE_LOWER_PREC, typeAtt.type());
+      assertEquals("Shift value wrong", shift, numericAtt.getShift());
+      final int hash = bytesAtt.toBytesRef(bytes);
+      assertEquals("Hash incorrect", bytes.hashCode(), hash);
+      assertEquals("Term is incorrectly encoded", ivalue & ~((1 << shift) - 1), NumericUtils.prefixCodedToInt(bytes));
+      assertEquals("Type incorrect", (shift == 0) ? NumericTokenStream.TOKEN_TYPE_FULL_PREC : NumericTokenStream.TOKEN_TYPE_LOWER_PREC, typeAtt.type());
     }
-    assertFalse("No more tokens available", stream.incrementToken());
+    assertFalse("More tokens available", stream.incrementToken());
+    stream.end();
+    stream.close();
   }
   
   public void testNotInitialized() throws Exception {

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java?rev=930821&r1=930820&r2=930821&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java Mon Apr  5 08:07:18 2010
@@ -24,11 +24,11 @@ import org.apache.lucene.document.Docume
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.NumericField;
 import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.index.IndexWriter.MaxFieldLength;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCaseJ4;
 import org.apache.lucene.util.NumericUtils;
 
@@ -333,9 +333,15 @@ public class TestNumericRangeQuery32 ext
       if (lower>upper) {
         int a=lower; lower=upper; upper=a;
       }
+      final BytesRef lowerBytes = new BytesRef(NumericUtils.BUF_SIZE_INT), upperBytes = new BytesRef(NumericUtils.BUF_SIZE_INT);
+      NumericUtils.intToPrefixCoded(lower, 0, lowerBytes);
+      NumericUtils.intToPrefixCoded(upper, 0, upperBytes);
+      // TODO: when new TermRange ctors with BytesRef available, use them and do not convert to string!
+      final String lowerString = lowerBytes.utf8ToString(), upperString = upperBytes.utf8ToString();
+
       // test inclusive range
       NumericRangeQuery<Integer> tq=NumericRangeQuery.newIntRange(field, precisionStep, lower, upper, true, true);
-      TermRangeQuery cq=new TermRangeQuery(field, NumericUtils.intToPrefixCoded(lower), NumericUtils.intToPrefixCoded(upper), true, true);
+      TermRangeQuery cq=new TermRangeQuery(field, lowerString, upperString, true, true);
       TopDocs tTopDocs = searcher.search(tq, 1);
       TopDocs cTopDocs = searcher.search(cq, 1);
       assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
@@ -343,7 +349,7 @@ public class TestNumericRangeQuery32 ext
       termCountC += cq.getTotalNumberOfTerms();
       // test exclusive range
       tq=NumericRangeQuery.newIntRange(field, precisionStep, lower, upper, false, false);
-      cq=new TermRangeQuery(field, NumericUtils.intToPrefixCoded(lower), NumericUtils.intToPrefixCoded(upper), false, false);
+      cq=new TermRangeQuery(field, lowerString, upperString, false, false);
       tTopDocs = searcher.search(tq, 1);
       cTopDocs = searcher.search(cq, 1);
       assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
@@ -351,7 +357,7 @@ public class TestNumericRangeQuery32 ext
       termCountC += cq.getTotalNumberOfTerms();
       // test left exclusive range
       tq=NumericRangeQuery.newIntRange(field, precisionStep, lower, upper, false, true);
-      cq=new TermRangeQuery(field, NumericUtils.intToPrefixCoded(lower), NumericUtils.intToPrefixCoded(upper), false, true);
+      cq=new TermRangeQuery(field, lowerString, upperString, false, true);
       tTopDocs = searcher.search(tq, 1);
       cTopDocs = searcher.search(cq, 1);
       assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
@@ -359,7 +365,7 @@ public class TestNumericRangeQuery32 ext
       termCountC += cq.getTotalNumberOfTerms();
       // test right exclusive range
       tq=NumericRangeQuery.newIntRange(field, precisionStep, lower, upper, true, false);
-      cq=new TermRangeQuery(field, NumericUtils.intToPrefixCoded(lower), NumericUtils.intToPrefixCoded(upper), true, false);
+      cq=new TermRangeQuery(field, lowerString, upperString, true, false);
       tTopDocs = searcher.search(tq, 1);
       cTopDocs = searcher.search(cq, 1);
       assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
@@ -558,7 +564,7 @@ public class TestNumericRangeQuery32 ext
     while (termEnum.next() != null) {
       final BytesRef t = termEnum.term();
       if (t != null) {
-        final int val = NumericUtils.prefixCodedToInt(t.utf8ToString());
+        final int val = NumericUtils.prefixCodedToInt(t);
         assertTrue("value not in bounds " + val + " >= " + lower + " && "
             + val + " <= " + upper, val >= lower && val <= upper);
         count++;

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java?rev=930821&r1=930820&r2=930821&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java Mon Apr  5 08:07:18 2010
@@ -26,6 +26,7 @@ import org.apache.lucene.document.Numeri
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCaseJ4;
 import org.apache.lucene.util.NumericUtils;
 
@@ -350,9 +351,15 @@ public class TestNumericRangeQuery64 ext
       if (lower>upper) {
         long a=lower; lower=upper; upper=a;
       }
+      final BytesRef lowerBytes = new BytesRef(NumericUtils.BUF_SIZE_LONG), upperBytes = new BytesRef(NumericUtils.BUF_SIZE_LONG);
+      NumericUtils.longToPrefixCoded(lower, 0, lowerBytes);
+      NumericUtils.longToPrefixCoded(upper, 0, upperBytes);
+      // TODO: when new TermRange ctors with BytesRef available, use them and do not convert to string!
+      final String lowerString = lowerBytes.utf8ToString(), upperString = upperBytes.utf8ToString();
+      
       // test inclusive range
       NumericRangeQuery<Long> tq=NumericRangeQuery.newLongRange(field, precisionStep, lower, upper, true, true);
-      TermRangeQuery cq=new TermRangeQuery(field, NumericUtils.longToPrefixCoded(lower), NumericUtils.longToPrefixCoded(upper), true, true);
+      TermRangeQuery cq=new TermRangeQuery(field, lowerString, upperString, true, true);
       TopDocs tTopDocs = searcher.search(tq, 1);
       TopDocs cTopDocs = searcher.search(cq, 1);
       assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
@@ -360,7 +367,7 @@ public class TestNumericRangeQuery64 ext
       termCountC += cq.getTotalNumberOfTerms();
       // test exclusive range
       tq=NumericRangeQuery.newLongRange(field, precisionStep, lower, upper, false, false);
-      cq=new TermRangeQuery(field, NumericUtils.longToPrefixCoded(lower), NumericUtils.longToPrefixCoded(upper), false, false);
+      cq=new TermRangeQuery(field, lowerString, upperString, false, false);
       tTopDocs = searcher.search(tq, 1);
       cTopDocs = searcher.search(cq, 1);
       assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
@@ -368,7 +375,7 @@ public class TestNumericRangeQuery64 ext
       termCountC += cq.getTotalNumberOfTerms();
       // test left exclusive range
       tq=NumericRangeQuery.newLongRange(field, precisionStep, lower, upper, false, true);
-      cq=new TermRangeQuery(field, NumericUtils.longToPrefixCoded(lower), NumericUtils.longToPrefixCoded(upper), false, true);
+      cq=new TermRangeQuery(field, lowerString, upperString, false, true);
       tTopDocs = searcher.search(tq, 1);
       cTopDocs = searcher.search(cq, 1);
       assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
@@ -376,7 +383,7 @@ public class TestNumericRangeQuery64 ext
       termCountC += cq.getTotalNumberOfTerms();
       // test right exclusive range
       tq=NumericRangeQuery.newLongRange(field, precisionStep, lower, upper, true, false);
-      cq=new TermRangeQuery(field, NumericUtils.longToPrefixCoded(lower), NumericUtils.longToPrefixCoded(upper), true, false);
+      cq=new TermRangeQuery(field, lowerString, upperString, true, false);
       tTopDocs = searcher.search(tq, 1);
       cTopDocs = searcher.search(cq, 1);
       assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/util/TestNumericUtils.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/util/TestNumericUtils.java?rev=930821&r1=930820&r2=930821&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/util/TestNumericUtils.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/util/TestNumericUtils.java Mon Apr  5 08:07:18 2010
@@ -25,33 +25,37 @@ public class TestNumericUtils extends Lu
 
   public void testLongConversionAndOrdering() throws Exception {
     // generate a series of encoded longs, each numerical one bigger than the one before
-    String last=null;
+    BytesRef last=null, act=new BytesRef(NumericUtils.BUF_SIZE_LONG);
     for (long l=-100000L; l<100000L; l++) {
-      String act=NumericUtils.longToPrefixCoded(l);
+      NumericUtils.longToPrefixCoded(l, 0, act);
       if (last!=null) {
         // test if smaller
-        assertTrue("actual bigger than last", last.compareTo(act) < 0 );
+        assertTrue("actual bigger than last (BytesRef)", BytesRef.getUTF8SortedAsUTF16Comparator().compare(last, act) < 0 );
+        assertTrue("actual bigger than last (as String)", last.utf8ToString().compareTo(act.utf8ToString()) < 0 );
       }
       // test is back and forward conversion works
       assertEquals("forward and back conversion should generate same long", l, NumericUtils.prefixCodedToLong(act));
       // next step
-      last=act;
+      last = act;
+      act = new BytesRef(NumericUtils.BUF_SIZE_LONG);
     }
   }
 
   public void testIntConversionAndOrdering() throws Exception {
     // generate a series of encoded ints, each numerical one bigger than the one before
-    String last=null;
+    BytesRef last=null, act=new BytesRef(NumericUtils.BUF_SIZE_INT);
     for (int i=-100000; i<100000; i++) {
-      String act=NumericUtils.intToPrefixCoded(i);
+      NumericUtils.intToPrefixCoded(i, 0, act);
       if (last!=null) {
         // test if smaller
-        assertTrue("actual bigger than last", last.compareTo(act) < 0 );
+        assertTrue("actual bigger than last (BytesRef)", BytesRef.getUTF8SortedAsUTF16Comparator().compare(last, act) < 0 );
+        assertTrue("actual bigger than last (as String)", last.utf8ToString().compareTo(act.utf8ToString()) < 0 );
       }
       // test is back and forward conversion works
       assertEquals("forward and back conversion should generate same int", i, NumericUtils.prefixCodedToInt(act));
       // next step
       last=act;
+      act = new BytesRef(NumericUtils.BUF_SIZE_INT);
     }
   }
 
@@ -60,10 +64,11 @@ public class TestNumericUtils extends Lu
       Long.MIN_VALUE, Long.MIN_VALUE+1, Long.MIN_VALUE+2, -5003400000000L,
       -4000L, -3000L, -2000L, -1000L, -1L, 0L, 1L, 10L, 300L, 50006789999999999L, Long.MAX_VALUE-2, Long.MAX_VALUE-1, Long.MAX_VALUE
     };
-    String[] prefixVals=new String[vals.length];
+    BytesRef[] prefixVals=new BytesRef[vals.length];
     
     for (int i=0; i<vals.length; i++) {
-      prefixVals[i]=NumericUtils.longToPrefixCoded(vals[i]);
+      prefixVals[i] = new BytesRef(NumericUtils.BUF_SIZE_LONG);
+      NumericUtils.longToPrefixCoded(vals[i], 0, prefixVals[i]);
       
       // check forward and back conversion
       assertEquals( "forward and back conversion should generate same long", vals[i], NumericUtils.prefixCodedToLong(prefixVals[i]) );
@@ -79,13 +84,15 @@ public class TestNumericUtils extends Lu
     
     // check sort order (prefixVals should be ascending)
     for (int i=1; i<prefixVals.length; i++) {
-      assertTrue( "check sort order", prefixVals[i-1].compareTo( prefixVals[i] ) < 0 );
+      assertTrue( "check sort order", BytesRef.getUTF8SortedAsUTF16Comparator().compare(prefixVals[i-1], prefixVals[i] ) < 0 );
     }
         
     // check the prefix encoding, lower precision should have the difference to original value equal to the lower removed bits
+    final BytesRef ref = new BytesRef(NumericUtils.BUF_SIZE_LONG);
     for (int i=0; i<vals.length; i++) {
       for (int j=0; j<64; j++) {
-        long prefixVal=NumericUtils.prefixCodedToLong(NumericUtils.longToPrefixCoded(vals[i], j));
+        NumericUtils.longToPrefixCoded(vals[i], j, ref);
+        long prefixVal=NumericUtils.prefixCodedToLong(ref);
         long mask=(1L << j) - 1L;
         assertEquals( "difference between prefix val and original value for "+vals[i]+" with shift="+j, vals[i] & mask, vals[i]-prefixVal );
       }
@@ -97,10 +104,11 @@ public class TestNumericUtils extends Lu
       Integer.MIN_VALUE, Integer.MIN_VALUE+1, Integer.MIN_VALUE+2, -64765767,
       -4000, -3000, -2000, -1000, -1, 0, 1, 10, 300, 765878989, Integer.MAX_VALUE-2, Integer.MAX_VALUE-1, Integer.MAX_VALUE
     };
-    String[] prefixVals=new String[vals.length];
+    BytesRef[] prefixVals=new BytesRef[vals.length];
     
     for (int i=0; i<vals.length; i++) {
-      prefixVals[i]=NumericUtils.intToPrefixCoded(vals[i]);
+      prefixVals[i] = new BytesRef(NumericUtils.BUF_SIZE_INT);
+      NumericUtils.intToPrefixCoded(vals[i], 0, prefixVals[i]);
       
       // check forward and back conversion
       assertEquals( "forward and back conversion should generate same int", vals[i], NumericUtils.prefixCodedToInt(prefixVals[i]) );
@@ -116,13 +124,15 @@ public class TestNumericUtils extends Lu
     
     // check sort order (prefixVals should be ascending)
     for (int i=1; i<prefixVals.length; i++) {
-      assertTrue( "check sort order", prefixVals[i-1].compareTo( prefixVals[i] ) < 0 );
+      assertTrue( "check sort order", BytesRef.getUTF8SortedAsUTF16Comparator().compare(prefixVals[i-1], prefixVals[i] ) < 0 );
     }
     
     // check the prefix encoding, lower precision should have the difference to original value equal to the lower removed bits
+    final BytesRef ref = new BytesRef(NumericUtils.BUF_SIZE_LONG);
     for (int i=0; i<vals.length; i++) {
       for (int j=0; j<32; j++) {
-        int prefixVal=NumericUtils.prefixCodedToInt(NumericUtils.intToPrefixCoded(vals[i], j));
+        NumericUtils.intToPrefixCoded(vals[i], j, ref);
+        int prefixVal=NumericUtils.prefixCodedToInt(ref);
         int mask=(1 << j) - 1;
         assertEquals( "difference between prefix val and original value for "+vals[i]+" with shift="+j, vals[i] & mask, vals[i]-prefixVal );
       }



Mime
View raw message