lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From uschind...@apache.org
Subject svn commit: r786470 [2/2] - in /lucene/java/trunk: ./ contrib/spatial/src/java/org/apache/lucene/spatial/ src/java/org/apache/lucene/analysis/ src/java/org/apache/lucene/document/ src/java/org/apache/lucene/search/ src/java/org/apache/lucene/util/ src/...
Date Fri, 19 Jun 2009 12:09:53 GMT
Added: lucene/java/trunk/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java?rev=786470&view=auto
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java (added)
+++ lucene/java/trunk/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java Fri Jun 19 12:09:52 2009
@@ -0,0 +1,431 @@
+package org.apache.lucene.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Random;
+
+import org.apache.lucene.analysis.NumericTokenStream;
+import org.apache.lucene.analysis.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriter.MaxFieldLength;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.NumericUtils;
+
+public class TestNumericRangeQuery32 extends LuceneTestCase {
+  // distance of entries
+  private static final int distance = 6666;
+  // shift the starting of the values to the left, to also have negative values:
+  private static final int startOffset = - 1 << 15;
+  // number of docs to generate for testing
+  private static final int noDocs = 10000;
+  
+  private static Field newField(String name, int precisionStep) {
+    NumericTokenStream stream = new NumericTokenStream(precisionStep);
+    stream.setUseNewAPI(true);
+    Field f=new Field(name, stream);
+    f.setOmitTermFreqAndPositions(true);
+    f.setOmitNorms(true);
+    return f;
+  }
+  
+  private static final RAMDirectory directory;
+  private static final IndexSearcher searcher;
+  static {
+    try {    
+      // set the theoretical maximum term count for 8bit (see docs for the number)
+      BooleanQuery.setMaxClauseCount(3*255*2 + 255);
+      
+      directory = new RAMDirectory();
+      IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(),
+      true, MaxFieldLength.UNLIMITED);
+      
+      Field
+        field8 = newField("field8", 8),
+        field4 = newField("field4", 4),
+        field2 = newField("field2", 2),
+        ascfield8 = newField("ascfield8", 8),
+        ascfield4 = newField("ascfield4", 4),
+        ascfield2 = newField("ascfield2", 2);
+      
+      // Add a series of noDocs docs with increasing int values
+      for (int l=0; l<noDocs; l++) {
+        Document doc=new Document();
+        // add fields, that have a distance to test general functionality
+        int val=distance*l+startOffset;
+        doc.add(new Field("value", Integer.toString(val), Field.Store.YES, Field.Index.NO));
+        ((NumericTokenStream)field8.tokenStreamValue()).setIntValue(val);
+        doc.add(field8);
+        ((NumericTokenStream)field4.tokenStreamValue()).setIntValue(val);
+        doc.add(field4);
+        ((NumericTokenStream)field2.tokenStreamValue()).setIntValue(val);
+        doc.add(field2);
+        // add ascending fields with a distance of 1, beginning at -noDocs/2 to test the correct splitting of range and inclusive/exclusive
+        val=l-(noDocs/2);
+        ((NumericTokenStream)ascfield8.tokenStreamValue()).setIntValue(val);
+        doc.add(ascfield8);
+        ((NumericTokenStream)ascfield4.tokenStreamValue()).setIntValue(val);
+        doc.add(ascfield4);
+        ((NumericTokenStream)ascfield2.tokenStreamValue()).setIntValue(val);
+        doc.add(ascfield2);
+        writer.addDocument(doc);
+      }
+    
+      writer.optimize();
+      writer.close();
+      searcher=new IndexSearcher(directory);
+    } catch (Exception e) {
+      throw new Error(e);
+    }
+  }
+  
+  /** test for both constant score and boolean query, the other tests only use the constant score mode */
+  private void testRange(int precisionStep) throws Exception {
+    String field="field"+precisionStep;
+    int count=3000;
+    int lower=(distance*3/2)+startOffset, upper=lower + count*distance + (distance/3);
+    NumericRangeQuery q = NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), true, true);
+    NumericRangeFilter f = NumericRangeFilter.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), true, true);
+    int lastTerms = 0;
+    for (byte i=0; i<3; i++) {
+      TopDocs topDocs;
+      int terms;
+      String type;
+      q.clearTotalNumberOfTerms();
+      f.clearTotalNumberOfTerms();
+      switch (i) {
+        case 0:
+          type = " (constant score)";
+          q.setConstantScoreRewrite(true);
+          topDocs = searcher.search(q, null, noDocs, Sort.INDEXORDER);
+          terms = q.getTotalNumberOfTerms();
+          break;
+        case 1:
+          type = " (boolean query)";
+          q.setConstantScoreRewrite(false);
+          topDocs = searcher.search(q, null, noDocs, Sort.INDEXORDER);
+          terms = q.getTotalNumberOfTerms();
+          break;
+        case 2:
+          type = " (filter)";
+          topDocs = searcher.search(new MatchAllDocsQuery(), f, noDocs, Sort.INDEXORDER);
+          terms = f.getTotalNumberOfTerms();
+          break;
+        default:
+          return;
+      }
+      System.out.println("Found "+terms+" distinct terms in range for field '"+field+"'"+type+".");
+      ScoreDoc[] sd = topDocs.scoreDocs;
+      assertNotNull(sd);
+      assertEquals("Score doc count"+type, count, sd.length );
+      Document doc=searcher.doc(sd[0].doc);
+      assertEquals("First doc"+type, 2*distance+startOffset, Integer.parseInt(doc.get("value")) );
+      doc=searcher.doc(sd[sd.length-1].doc);
+      assertEquals("Last doc"+type, (1+count)*distance+startOffset, Integer.parseInt(doc.get("value")) );
+      if (i>0) {
+        assertEquals("Distinct term number is equal for all query types", lastTerms, terms);
+      }
+      lastTerms = terms;
+    }
+  }
+
+  public void testRange_8bit() throws Exception {
+    testRange(8);
+  }
+  
+  public void testRange_4bit() throws Exception {
+    testRange(4);
+  }
+  
+  public void testRange_2bit() throws Exception {
+    testRange(2);
+  }
+  
+  public void testInverseRange() throws Exception {
+    NumericRangeFilter f = NumericRangeFilter.newIntRange("field8", 8, new Integer(1000), new Integer(-1000), true, true);
+    assertSame("A inverse range should return the EMPTY_DOCIDSET instance", DocIdSet.EMPTY_DOCIDSET, f.getDocIdSet(searcher.getIndexReader()));
+  }
+  
+  private void testLeftOpenRange(int precisionStep) throws Exception {
+    String field="field"+precisionStep;
+    int count=3000;
+    int upper=(count-1)*distance + (distance/3) + startOffset;
+    NumericRangeQuery q=NumericRangeQuery.newIntRange(field, precisionStep, null, new Integer(upper), true, true);
+    TopDocs topDocs = searcher.search(q, null, noDocs, Sort.INDEXORDER);
+    System.out.println("Found "+q.getTotalNumberOfTerms()+" distinct terms in left open range for field '"+field+"'.");
+    ScoreDoc[] sd = topDocs.scoreDocs;
+    assertNotNull(sd);
+    assertEquals("Score doc count", count, sd.length );
+    Document doc=searcher.doc(sd[0].doc);
+    assertEquals("First doc", startOffset, Integer.parseInt(doc.get("value")) );
+    doc=searcher.doc(sd[sd.length-1].doc);
+    assertEquals("Last doc", (count-1)*distance+startOffset, Integer.parseInt(doc.get("value")) );
+  }
+  
+  public void testLeftOpenRange_8bit() throws Exception {
+    testLeftOpenRange(8);
+  }
+  
+  public void testLeftOpenRange_4bit() throws Exception {
+    testLeftOpenRange(4);
+  }
+  
+  public void testLeftOpenRange_2bit() throws Exception {
+    testLeftOpenRange(2);
+  }
+  
+  private void testRightOpenRange(int precisionStep) throws Exception {
+    String field="field"+precisionStep;
+    int count=3000;
+    int lower=(count-1)*distance + (distance/3) +startOffset;
+    NumericRangeQuery q=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), null, true, true);
+    TopDocs topDocs = searcher.search(q, null, noDocs, Sort.INDEXORDER);
+    System.out.println("Found "+q.getTotalNumberOfTerms()+" distinct terms in right open range for field '"+field+"'.");
+    ScoreDoc[] sd = topDocs.scoreDocs;
+    assertNotNull(sd);
+    assertEquals("Score doc count", noDocs-count, sd.length );
+    Document doc=searcher.doc(sd[0].doc);
+    assertEquals("First doc", count*distance+startOffset, Integer.parseInt(doc.get("value")) );
+    doc=searcher.doc(sd[sd.length-1].doc);
+    assertEquals("Last doc", (noDocs-1)*distance+startOffset, Integer.parseInt(doc.get("value")) );
+  }
+  
+  public void testRightOpenRange_8bit() throws Exception {
+    testRightOpenRange(8);
+  }
+  
+  public void testRightOpenRange_4bit() throws Exception {
+    testRightOpenRange(4);
+  }
+  
+  public void testRightOpenRange_2bit() throws Exception {
+    testRightOpenRange(2);
+  }
+  
+  private void testRandomTrieAndClassicRangeQuery(int precisionStep) throws Exception {
+    final Random rnd=newRandom();
+    String field="field"+precisionStep;
+    int termCountT=0,termCountC=0;
+    for (int i=0; i<50; i++) {
+      int lower=(int)(rnd.nextDouble()*noDocs*distance)+startOffset;
+      int upper=(int)(rnd.nextDouble()*noDocs*distance)+startOffset;
+      if (lower>upper) {
+        int a=lower; lower=upper; upper=a;
+      }
+      // test inclusive range
+      NumericRangeQuery tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), true, true);
+      RangeQuery cq=new RangeQuery(field, NumericUtils.intToPrefixCoded(lower), NumericUtils.intToPrefixCoded(upper), true, true);
+      cq.setConstantScoreRewrite(true);
+      TopDocs tTopDocs = searcher.search(tq, 1);
+      TopDocs cTopDocs = searcher.search(cq, 1);
+      assertEquals("Returned count for NumericRangeQuery and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+      termCountT += tq.getTotalNumberOfTerms();
+      termCountC += cq.getTotalNumberOfTerms();
+      // test exclusive range
+      tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), false, false);
+      cq=new RangeQuery(field, NumericUtils.intToPrefixCoded(lower), NumericUtils.intToPrefixCoded(upper), false, false);
+      cq.setConstantScoreRewrite(true);
+      tTopDocs = searcher.search(tq, 1);
+      cTopDocs = searcher.search(cq, 1);
+      assertEquals("Returned count for NumericRangeQuery and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+      termCountT += tq.getTotalNumberOfTerms();
+      termCountC += cq.getTotalNumberOfTerms();
+      // test left exclusive range
+      tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), false, true);
+      cq=new RangeQuery(field, NumericUtils.intToPrefixCoded(lower), NumericUtils.intToPrefixCoded(upper), false, true);
+      cq.setConstantScoreRewrite(true);
+      tTopDocs = searcher.search(tq, 1);
+      cTopDocs = searcher.search(cq, 1);
+      assertEquals("Returned count for NumericRangeQuery and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+      termCountT += tq.getTotalNumberOfTerms();
+      termCountC += cq.getTotalNumberOfTerms();
+      // test right exclusive range
+      tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), true, false);
+      cq=new RangeQuery(field, NumericUtils.intToPrefixCoded(lower), NumericUtils.intToPrefixCoded(upper), true, false);
+      cq.setConstantScoreRewrite(true);
+      tTopDocs = searcher.search(tq, 1);
+      cTopDocs = searcher.search(cq, 1);
+      assertEquals("Returned count for NumericRangeQuery and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+      termCountT += tq.getTotalNumberOfTerms();
+      termCountC += cq.getTotalNumberOfTerms();
+    }
+    System.out.println("Average number of terms during random search on '" + field + "':");
+    System.out.println(" Trie query: " + (((double)termCountT)/(50*4)));
+    System.out.println(" Classical query: " + (((double)termCountC)/(50*4)));
+  }
+  
+  public void testRandomTrieAndClassicRangeQuery_8bit() throws Exception {
+    testRandomTrieAndClassicRangeQuery(8);
+  }
+  
+  public void testRandomTrieAndClassicRangeQuery_4bit() throws Exception {
+    testRandomTrieAndClassicRangeQuery(4);
+  }
+  
+  public void testRandomTrieAndClassicRangeQuery_2bit() throws Exception {
+    testRandomTrieAndClassicRangeQuery(2);
+  }
+  
+  private void testRangeSplit(int precisionStep) throws Exception {
+    final Random rnd=newRandom();
+    String field="ascfield"+precisionStep;
+    // 50 random tests
+    for (int i=0; i<50; i++) {
+      int lower=(int)(rnd.nextDouble()*noDocs - noDocs/2);
+      int upper=(int)(rnd.nextDouble()*noDocs - noDocs/2);
+      if (lower>upper) {
+        int a=lower; lower=upper; upper=a;
+      }
+      // test inclusive range
+      Query tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), true, true);
+      TopDocs tTopDocs = searcher.search(tq, 1);
+      assertEquals("Returned count of range query must be equal to inclusive range length", upper-lower+1, tTopDocs.totalHits );
+      // test exclusive range
+      tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), false, false);
+      tTopDocs = searcher.search(tq, 1);
+      assertEquals("Returned count of range query must be equal to exclusive range length", Math.max(upper-lower-1, 0), tTopDocs.totalHits );
+      // test left exclusive range
+      tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), false, true);
+      tTopDocs = searcher.search(tq, 1);
+      assertEquals("Returned count of range query must be equal to half exclusive range length", upper-lower, tTopDocs.totalHits );
+      // test right exclusive range
+      tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), true, false);
+      tTopDocs = searcher.search(tq, 1);
+      assertEquals("Returned count of range query must be equal to half exclusive range length", upper-lower, tTopDocs.totalHits );
+    }
+  }
+
+  public void testRangeSplit_8bit() throws Exception {
+    testRangeSplit(8);
+  }
+  
+  public void testRangeSplit_4bit() throws Exception {
+    testRangeSplit(4);
+  }
+  
+  public void testRangeSplit_2bit() throws Exception {
+    testRangeSplit(2);
+  }
+  
+  /** we fake a float test using int2float conversion of NumericUtils */
+  private void testFloatRange(int precisionStep) throws Exception {
+    final String field="ascfield"+precisionStep;
+    final int lower=-1000, upper=+2000;
+    
+    Query tq=NumericRangeQuery.newFloatRange(field, precisionStep,
+      new Float(NumericUtils.sortableIntToFloat(lower)), new Float(NumericUtils.sortableIntToFloat(upper)), true, true);
+    TopDocs tTopDocs = searcher.search(tq, 1);
+    assertEquals("Returned count of range query must be equal to inclusive range length", upper-lower+1, tTopDocs.totalHits );
+    
+    Filter tf=NumericRangeFilter.newFloatRange(field, precisionStep,
+      new Float(NumericUtils.sortableIntToFloat(lower)), new Float(NumericUtils.sortableIntToFloat(upper)), true, true);
+    tTopDocs = searcher.search(new MatchAllDocsQuery(), tf, 1);
+    assertEquals("Returned count of range filter must be equal to inclusive range length", upper-lower+1, tTopDocs.totalHits );
+  }
+
+  public void testFloatRange_8bit() throws Exception {
+    testFloatRange(8);
+  }
+  
+  public void testFloatRange_4bit() throws Exception {
+    testFloatRange(4);
+  }
+  
+  public void testFloatRange_2bit() throws Exception {
+    testFloatRange(2);
+  }
+  
+  private void testSorting(int precisionStep) throws Exception {
+    final Random rnd=newRandom();
+    String field="field"+precisionStep;
+    // 10 random tests, the index order is ascending,
+    // so using a reverse sort field should retun descending documents
+    for (int i=0; i<10; i++) {
+      int lower=(int)(rnd.nextDouble()*noDocs*distance)+startOffset;
+      int upper=(int)(rnd.nextDouble()*noDocs*distance)+startOffset;
+      if (lower>upper) {
+        int a=lower; lower=upper; upper=a;
+      }
+      Query tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), true, true);
+      TopDocs topDocs = searcher.search(tq, null, noDocs, new Sort(NumericUtils.getIntSortField(field, true)));
+      if (topDocs.totalHits==0) continue;
+      ScoreDoc[] sd = topDocs.scoreDocs;
+      assertNotNull(sd);
+      int last=Integer.parseInt(searcher.doc(sd[0].doc).get("value"));
+      for (int j=1; j<sd.length; j++) {
+        int act=Integer.parseInt(searcher.doc(sd[j].doc).get("value"));
+        assertTrue("Docs should be sorted backwards", last>act );
+        last=act;
+      }
+    }
+  }
+
+  public void testSorting_8bit() throws Exception {
+    testSorting(8);
+  }
+  
+  public void testSorting_4bit() throws Exception {
+    testSorting(4);
+  }
+  
+  public void testSorting_2bit() throws Exception {
+    testSorting(2);
+  }
+  
+  public void testEqualsAndHash() throws Exception {
+    QueryUtils.checkHashEquals(NumericRangeQuery.newIntRange("test1", 4, new Integer(10), new Integer(20), true, true));
+    QueryUtils.checkHashEquals(NumericRangeQuery.newIntRange("test2", 4, new Integer(10), new Integer(20), false, true));
+    QueryUtils.checkHashEquals(NumericRangeQuery.newIntRange("test3", 4, new Integer(10), new Integer(20), true, false));
+    QueryUtils.checkHashEquals(NumericRangeQuery.newIntRange("test4", 4, new Integer(10), new Integer(20), false, false));
+    QueryUtils.checkHashEquals(NumericRangeQuery.newIntRange("test5", 4, new Integer(10), null, true, true));
+    QueryUtils.checkHashEquals(NumericRangeQuery.newIntRange("test6", 4, null, new Integer(20), true, true));
+    QueryUtils.checkHashEquals(NumericRangeQuery.newIntRange("test7", 4, null, null, true, true));
+    QueryUtils.checkEqual(
+      NumericRangeQuery.newIntRange("test8", 4, new Integer(10), new Integer(20), true, true), 
+      NumericRangeQuery.newIntRange("test8", 4, new Integer(10), new Integer(20), true, true)
+    );
+    QueryUtils.checkUnequal(
+      NumericRangeQuery.newIntRange("test9", 4, new Integer(10), new Integer(20), true, true), 
+      NumericRangeQuery.newIntRange("test9", 8, new Integer(10), new Integer(20), true, true)
+    );
+    QueryUtils.checkUnequal(
+      NumericRangeQuery.newIntRange("test10a", 4, new Integer(10), new Integer(20), true, true), 
+      NumericRangeQuery.newIntRange("test10b", 4, new Integer(10), new Integer(20), true, true)
+    );
+    QueryUtils.checkUnequal(
+      NumericRangeQuery.newIntRange("test11", 4, new Integer(10), new Integer(20), true, true), 
+      NumericRangeQuery.newIntRange("test11", 4, new Integer(20), new Integer(10), true, true)
+    );
+    QueryUtils.checkUnequal(
+      NumericRangeQuery.newIntRange("test12", 4, new Integer(10), new Integer(20), true, true), 
+      NumericRangeQuery.newIntRange("test12", 4, new Integer(10), new Integer(20), false, true)
+    );
+    QueryUtils.checkUnequal(
+      NumericRangeQuery.newIntRange("test13", 4, new Integer(10), new Integer(20), true, true), 
+      NumericRangeQuery.newFloatRange("test13", 4, new Float(10f), new Float(20f), true, true)
+    );
+    // the following produces a hash collision, because Long and Integer have the same hashcode, so only test equality:
+    Query q1 = NumericRangeQuery.newIntRange("test14", 4, new Integer(10), new Integer(20), true, true);
+    Query q2 = NumericRangeQuery.newLongRange("test14", 4, new Long(10L), new Long(20L), true, true);
+    assertFalse(q1.equals(q2));
+    assertFalse(q2.equals(q1));
+  }
+  
+}

Propchange: lucene/java/trunk/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/trunk/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java?rev=786470&view=auto
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java (added)
+++ lucene/java/trunk/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java Fri Jun 19 12:09:52 2009
@@ -0,0 +1,427 @@
+package org.apache.lucene.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Random;
+
+import org.apache.lucene.analysis.NumericTokenStream;
+import org.apache.lucene.analysis.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriter.MaxFieldLength;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.NumericUtils;
+
+public class TestNumericRangeQuery64 extends LuceneTestCase {
+  // distance of entries
+  private static final long distance = 66666L;
+  // shift the starting of the values to the left, to also have negative values:
+  private static final long startOffset = - 1L << 31;
+  // number of docs to generate for testing
+  private static final int noDocs = 10000;
+  
+  private static Field newField(String name, int precisionStep) {
+    NumericTokenStream stream = new NumericTokenStream(precisionStep);
+    stream.setUseNewAPI(true);
+    Field f=new Field(name, stream);
+    f.setOmitTermFreqAndPositions(true);
+    f.setOmitNorms(true);
+    return f;
+  }
+  
+  private static final RAMDirectory directory;
+  private static final IndexSearcher searcher;
+  static {
+    try {
+      // set the theoretical maximum term count for 8bit (see docs for the number)
+      BooleanQuery.setMaxClauseCount(7*255*2 + 255);
+      
+      directory = new RAMDirectory();
+      IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(),
+      true, MaxFieldLength.UNLIMITED);
+      
+      Field
+        field8 = newField("field8", 8),
+        field4 = newField("field4", 4),
+        field2 = newField("field2", 2),
+        ascfield8 = newField("ascfield8", 8),
+        ascfield4 = newField("ascfield4", 4),
+        ascfield2 = newField("ascfield2", 2);
+      
+      // Add a series of noDocs docs with increasing long values
+      for (int l=0; l<noDocs; l++) {
+        Document doc=new Document();
+        // add fields, that have a distance to test general functionality
+        long val=distance*l+startOffset;
+        doc.add(new Field("value", Long.toString(val), Field.Store.YES, Field.Index.NO));
+        ((NumericTokenStream)field8.tokenStreamValue()).setLongValue(val);
+        doc.add(field8);
+        ((NumericTokenStream)field4.tokenStreamValue()).setLongValue(val);
+        doc.add(field4);
+        ((NumericTokenStream)field2.tokenStreamValue()).setLongValue(val);
+        doc.add(field2);
+        // add ascending fields with a distance of 1, beginning at -noDocs/2 to test the correct splitting of range and inclusive/exclusive
+        val=l-(noDocs/2);
+        ((NumericTokenStream)ascfield8.tokenStreamValue()).setLongValue(val);
+        doc.add(ascfield8);
+        ((NumericTokenStream)ascfield4.tokenStreamValue()).setLongValue(val);
+        doc.add(ascfield4);
+        ((NumericTokenStream)ascfield2.tokenStreamValue()).setLongValue(val);
+        doc.add(ascfield2);
+        writer.addDocument(doc);
+      }
+    
+      writer.optimize();
+      writer.close();
+      searcher=new IndexSearcher(directory);
+    } catch (Exception e) {
+      throw new Error(e);
+    }
+  }
+  
+  /** test for constant score + boolean query + filter, the other tests only use the constant score mode */
+  private void testRange(int precisionStep) throws Exception {
+    String field="field"+precisionStep;
+    int count=3000;
+    long lower=(distance*3/2)+startOffset, upper=lower + count*distance + (distance/3);
+    NumericRangeQuery q = NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), true, true);
+    NumericRangeFilter f = NumericRangeFilter.newLongRange(field, precisionStep, new Long(lower), new Long(upper), true, true);
+    int lastTerms = 0;
+    for (byte i=0; i<3; i++) {
+      TopDocs topDocs;
+      int terms;
+      String type;
+      q.clearTotalNumberOfTerms();
+      f.clearTotalNumberOfTerms();
+      switch (i) {
+        case 0:
+          type = " (constant score)";
+          q.setConstantScoreRewrite(true);
+          topDocs = searcher.search(q, null, noDocs, Sort.INDEXORDER);
+          terms = q.getTotalNumberOfTerms();
+          break;
+        case 1:
+          type = " (boolean query)";
+          q.setConstantScoreRewrite(false);
+          topDocs = searcher.search(q, null, noDocs, Sort.INDEXORDER);
+          terms = q.getTotalNumberOfTerms();
+          break;
+        case 2:
+          type = " (filter)";
+          topDocs = searcher.search(new MatchAllDocsQuery(), f, noDocs, Sort.INDEXORDER);
+          terms = f.getTotalNumberOfTerms();
+          break;
+        default:
+          return;
+      }
+      System.out.println("Found "+terms+" distinct terms in range for field '"+field+"'"+type+".");
+      ScoreDoc[] sd = topDocs.scoreDocs;
+      assertNotNull(sd);
+      assertEquals("Score doc count"+type, count, sd.length );
+      Document doc=searcher.doc(sd[0].doc);
+      assertEquals("First doc"+type, 2*distance+startOffset, Long.parseLong(doc.get("value")) );
+      doc=searcher.doc(sd[sd.length-1].doc);
+      assertEquals("Last doc"+type, (1+count)*distance+startOffset, Long.parseLong(doc.get("value")) );
+      if (i>0) {
+        assertEquals("Distinct term number is equal for all query types", lastTerms, terms);
+      }
+      lastTerms = terms;
+    }
+  }
+
+  public void testRange_8bit() throws Exception {
+    testRange(8);
+  }
+  
+  public void testRange_4bit() throws Exception {
+    testRange(4);
+  }
+  
+  public void testRange_2bit() throws Exception {
+    testRange(2);
+  }
+  
+  public void testInverseRange() throws Exception {
+    NumericRangeFilter f = NumericRangeFilter.newLongRange("field8", 8, new Long(1000L), new Long(-1000L), true, true);
+    assertSame("A inverse range should return the EMPTY_DOCIDSET instance", DocIdSet.EMPTY_DOCIDSET, f.getDocIdSet(searcher.getIndexReader()));
+  }
+  
+  private void testLeftOpenRange(int precisionStep) throws Exception {
+    String field="field"+precisionStep;
+    int count=3000;
+    long upper=(count-1)*distance + (distance/3) + startOffset;
+    NumericRangeQuery q=NumericRangeQuery.newLongRange(field, precisionStep, null, new Long(upper), true, true);
+    TopDocs topDocs = searcher.search(q, null, noDocs, Sort.INDEXORDER);
+    System.out.println("Found "+q.getTotalNumberOfTerms()+" distinct terms in left open range for field '"+field+"'.");
+    ScoreDoc[] sd = topDocs.scoreDocs;
+    assertNotNull(sd);
+    assertEquals("Score doc count", count, sd.length );
+    Document doc=searcher.doc(sd[0].doc);
+    assertEquals("First doc", startOffset, Long.parseLong(doc.get("value")) );
+    doc=searcher.doc(sd[sd.length-1].doc);
+    assertEquals("Last doc", (count-1)*distance+startOffset, Long.parseLong(doc.get("value")) );
+  }
+  
+  public void testLeftOpenRange_8bit() throws Exception {
+    testLeftOpenRange(8);
+  }
+  
+  public void testLeftOpenRange_4bit() throws Exception {
+    testLeftOpenRange(4);
+  }
+  
+  public void testLeftOpenRange_2bit() throws Exception {
+    testLeftOpenRange(2);
+  }
+  
+  private void testRightOpenRange(int precisionStep) throws Exception {
+    String field="field"+precisionStep;
+    int count=3000;
+    long lower=(count-1)*distance + (distance/3) +startOffset;
+    NumericRangeQuery q=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), null, true, true);
+    TopDocs topDocs = searcher.search(q, null, noDocs, Sort.INDEXORDER);
+    System.out.println("Found "+q.getTotalNumberOfTerms()+" distinct terms in right open range for field '"+field+"'.");
+    ScoreDoc[] sd = topDocs.scoreDocs;
+    assertNotNull(sd);
+    assertEquals("Score doc count", noDocs-count, sd.length );
+    Document doc=searcher.doc(sd[0].doc);
+    assertEquals("First doc", count*distance+startOffset, Long.parseLong(doc.get("value")) );
+    doc=searcher.doc(sd[sd.length-1].doc);
+    assertEquals("Last doc", (noDocs-1)*distance+startOffset, Long.parseLong(doc.get("value")) );
+  }
+  
+  public void testRightOpenRange_8bit() throws Exception {
+    testRightOpenRange(8);
+  }
+  
+  public void testRightOpenRange_4bit() throws Exception {
+    testRightOpenRange(4);
+  }
+  
+  public void testRightOpenRange_2bit() throws Exception {
+    testRightOpenRange(2);
+  }
+  
+  private void testRandomTrieAndClassicRangeQuery(int precisionStep) throws Exception {
+    final Random rnd=newRandom();
+    String field="field"+precisionStep;
+    int termCountT=0,termCountC=0;
+    for (int i=0; i<50; i++) {
+      long lower=(long)(rnd.nextDouble()*noDocs*distance)+startOffset;
+      long upper=(long)(rnd.nextDouble()*noDocs*distance)+startOffset;
+      if (lower>upper) {
+        long a=lower; lower=upper; upper=a;
+      }
+      // test inclusive range
+      NumericRangeQuery tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), true, true);
+      RangeQuery cq=new RangeQuery(field, NumericUtils.longToPrefixCoded(lower), NumericUtils.longToPrefixCoded(upper), true, true);
+      cq.setConstantScoreRewrite(true);
+      TopDocs tTopDocs = searcher.search(tq, 1);
+      TopDocs cTopDocs = searcher.search(cq, 1);
+      assertEquals("Returned count for NumericRangeQuery and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+      termCountT += tq.getTotalNumberOfTerms();
+      termCountC += cq.getTotalNumberOfTerms();
+      // test exclusive range
+      tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), false, false);
+      cq=new RangeQuery(field, NumericUtils.longToPrefixCoded(lower), NumericUtils.longToPrefixCoded(upper), false, false);
+      cq.setConstantScoreRewrite(true);
+      tTopDocs = searcher.search(tq, 1);
+      cTopDocs = searcher.search(cq, 1);
+      assertEquals("Returned count for NumericRangeQuery and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+      termCountT += tq.getTotalNumberOfTerms();
+      termCountC += cq.getTotalNumberOfTerms();
+      // test left exclusive range
+      tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), false, true);
+      cq=new RangeQuery(field, NumericUtils.longToPrefixCoded(lower), NumericUtils.longToPrefixCoded(upper), false, true);
+      cq.setConstantScoreRewrite(true);
+      tTopDocs = searcher.search(tq, 1);
+      cTopDocs = searcher.search(cq, 1);
+      assertEquals("Returned count for NumericRangeQuery and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+      termCountT += tq.getTotalNumberOfTerms();
+      termCountC += cq.getTotalNumberOfTerms();
+      // test right exclusive range
+      tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), true, false);
+      cq=new RangeQuery(field, NumericUtils.longToPrefixCoded(lower), NumericUtils.longToPrefixCoded(upper), true, false);
+      cq.setConstantScoreRewrite(true);
+      tTopDocs = searcher.search(tq, 1);
+      cTopDocs = searcher.search(cq, 1);
+      assertEquals("Returned count for NumericRangeQuery and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+      termCountT += tq.getTotalNumberOfTerms();
+      termCountC += cq.getTotalNumberOfTerms();
+    }
+    System.out.println("Average number of terms during random search on '" + field + "':");
+    System.out.println(" Trie query: " + (((double)termCountT)/(50*4)));
+    System.out.println(" Classical query: " + (((double)termCountC)/(50*4)));
+  }
+  
+  public void testRandomTrieAndClassicRangeQuery_8bit() throws Exception {
+    testRandomTrieAndClassicRangeQuery(8);
+  }
+  
+  public void testRandomTrieAndClassicRangeQuery_4bit() throws Exception {
+    testRandomTrieAndClassicRangeQuery(4);
+  }
+  
+  public void testRandomTrieAndClassicRangeQuery_2bit() throws Exception {
+    testRandomTrieAndClassicRangeQuery(2);
+  }
+  
+  private void testRangeSplit(int precisionStep) throws Exception {
+    final Random rnd=newRandom();
+    String field="ascfield"+precisionStep;
+    // 50 random tests
+    for (int i=0; i<50; i++) {
+      long lower=(long)(rnd.nextDouble()*noDocs - noDocs/2);
+      long upper=(long)(rnd.nextDouble()*noDocs - noDocs/2);
+      if (lower>upper) {
+        long a=lower; lower=upper; upper=a;
+      }
+      // test inclusive range
+      Query tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), true, true);
+      TopDocs tTopDocs = searcher.search(tq, 1);
+      assertEquals("Returned count of range query must be equal to inclusive range length", upper-lower+1, tTopDocs.totalHits );
+      // test exclusive range
+      tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), false, false);
+      tTopDocs = searcher.search(tq, 1);
+      assertEquals("Returned count of range query must be equal to exclusive range length", Math.max(upper-lower-1, 0), tTopDocs.totalHits );
+      // test left exclusive range
+      tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), false, true);
+      tTopDocs = searcher.search(tq, 1);
+      assertEquals("Returned count of range query must be equal to half exclusive range length", upper-lower, tTopDocs.totalHits );
+      // test right exclusive range
+      tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), true, false);
+      tTopDocs = searcher.search(tq, 1);
+      assertEquals("Returned count of range query must be equal to half exclusive range length", upper-lower, tTopDocs.totalHits );
+    }
+  }
+
+  public void testRangeSplit_8bit() throws Exception {
+    testRangeSplit(8);
+  }
+  
+  public void testRangeSplit_4bit() throws Exception {
+    testRangeSplit(4);
+  }
+  
+  public void testRangeSplit_2bit() throws Exception {
+    testRangeSplit(2);
+  }
+  
+  /** we fake a double test using long2double conversion of NumericUtils */
+  private void testDoubleRange(int precisionStep) throws Exception {
+    final String field="ascfield"+precisionStep;
+    final long lower=-1000L, upper=+2000L;
+    
+    Query tq=NumericRangeQuery.newDoubleRange(field, precisionStep,
+      new Double(NumericUtils.sortableLongToDouble(lower)), new Double(NumericUtils.sortableLongToDouble(upper)), true, true);
+    TopDocs tTopDocs = searcher.search(tq, 1);
+    assertEquals("Returned count of range query must be equal to inclusive range length", upper-lower+1, tTopDocs.totalHits );
+    
+    Filter tf=NumericRangeFilter.newDoubleRange(field, precisionStep,
+      new Double(NumericUtils.sortableLongToDouble(lower)), new Double(NumericUtils.sortableLongToDouble(upper)), true, true);
+    tTopDocs = searcher.search(new MatchAllDocsQuery(), tf, 1);
+    assertEquals("Returned count of range filter must be equal to inclusive range length", upper-lower+1, tTopDocs.totalHits );
+  }
+
+  public void testDoubleRange_8bit() throws Exception {
+    testDoubleRange(8);
+  }
+  
+  public void testDoubleRange_4bit() throws Exception {
+    testDoubleRange(4);
+  }
+  
+  public void testDoubleRange_2bit() throws Exception {
+    testDoubleRange(2);
+  }
+  
+  private void testSorting(int precisionStep) throws Exception {
+    final Random rnd=newRandom();
+    String field="field"+precisionStep;
+    // 10 random tests, the index order is ascending,
+    // so using a reverse sort field should retun descending documents
+    for (int i=0; i<10; i++) {
+      long lower=(long)(rnd.nextDouble()*noDocs*distance)+startOffset;
+      long upper=(long)(rnd.nextDouble()*noDocs*distance)+startOffset;
+      if (lower>upper) {
+        long a=lower; lower=upper; upper=a;
+      }
+      Query tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), true, true);
+      TopDocs topDocs = searcher.search(tq, null, noDocs, new Sort(NumericUtils.getLongSortField(field, true)));
+      if (topDocs.totalHits==0) continue;
+      ScoreDoc[] sd = topDocs.scoreDocs;
+      assertNotNull(sd);
+      long last=Long.parseLong(searcher.doc(sd[0].doc).get("value"));
+      for (int j=1; j<sd.length; j++) {
+        long act=Long.parseLong(searcher.doc(sd[j].doc).get("value"));
+        assertTrue("Docs should be sorted backwards", last>act );
+        last=act;
+      }
+    }
+  }
+
+  public void testSorting_8bit() throws Exception {
+    testSorting(8);
+  }
+  
+  public void testSorting_4bit() throws Exception {
+    testSorting(4);
+  }
+  
+  public void testSorting_2bit() throws Exception {
+    testSorting(2);
+  }
+  
+  public void testEqualsAndHash() throws Exception {
+    QueryUtils.checkHashEquals(NumericRangeQuery.newLongRange("test1", 4, new Long(10L), new Long(20L), true, true));
+    QueryUtils.checkHashEquals(NumericRangeQuery.newLongRange("test2", 4, new Long(10L), new Long(20L), false, true));
+    QueryUtils.checkHashEquals(NumericRangeQuery.newLongRange("test3", 4, new Long(10L), new Long(20L), true, false));
+    QueryUtils.checkHashEquals(NumericRangeQuery.newLongRange("test4", 4, new Long(10L), new Long(20L), false, false));
+    QueryUtils.checkHashEquals(NumericRangeQuery.newLongRange("test5", 4, new Long(10L), null, true, true));
+    QueryUtils.checkHashEquals(NumericRangeQuery.newLongRange("test6", 4, null, new Long(20L), true, true));
+    QueryUtils.checkHashEquals(NumericRangeQuery.newLongRange("test7", 4, null, null, true, true));
+    QueryUtils.checkEqual(
+      NumericRangeQuery.newLongRange("test8", 4, new Long(10L), new Long(20L), true, true), 
+      NumericRangeQuery.newLongRange("test8", 4, new Long(10L), new Long(20L), true, true)
+    );
+    QueryUtils.checkUnequal(
+      NumericRangeQuery.newLongRange("test9", 4, new Long(10L), new Long(20L), true, true), 
+      NumericRangeQuery.newLongRange("test9", 8, new Long(10L), new Long(20L), true, true)
+    );
+    QueryUtils.checkUnequal(
+      NumericRangeQuery.newLongRange("test10a", 4, new Long(10L), new Long(20L), true, true), 
+      NumericRangeQuery.newLongRange("test10b", 4, new Long(10L), new Long(20L), true, true)
+    );
+    QueryUtils.checkUnequal(
+      NumericRangeQuery.newLongRange("test11", 4, new Long(10L), new Long(20L), true, true), 
+      NumericRangeQuery.newLongRange("test11", 4, new Long(20L), new Long(10L), true, true)
+    );
+    QueryUtils.checkUnequal(
+      NumericRangeQuery.newLongRange("test12", 4, new Long(10L), new Long(20L), true, true), 
+      NumericRangeQuery.newLongRange("test12", 4, new Long(10L), new Long(20L), false, true)
+    );
+    QueryUtils.checkUnequal(
+      NumericRangeQuery.newLongRange("test13", 4, new Long(10L), new Long(20L), true, true), 
+      NumericRangeQuery.newFloatRange("test13", 4, new Float(10f), new Float(20f), true, true)
+    );
+     // difference to int range is tested in TestNumericRangeQuery32
+  }
+  
+}

Propchange: lucene/java/trunk/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: lucene/java/trunk/src/test/org/apache/lucene/util/TestNumericUtils.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/util/TestNumericUtils.java?rev=786470&view=auto
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/util/TestNumericUtils.java (added)
+++ lucene/java/trunk/src/test/org/apache/lucene/util/TestNumericUtils.java Fri Jun 19 12:09:52 2009
@@ -0,0 +1,339 @@
+package org.apache.lucene.util;
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.OpenBitSet;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+
+public class TestNumericUtils extends LuceneTestCase {
+
+  public void testLongConversionAndOrdering() throws Exception {
+    // generate a series of encoded longs, each numerical one bigger than the one before
+    String last=null;
+    for (long l=-100000L; l<100000L; l++) {
+      String act=NumericUtils.longToPrefixCoded(l);
+      if (last!=null) {
+        // test if smaller
+        assertTrue("actual bigger than last", last.compareTo(act) < 0 );
+      }
+      // test is back and forward conversion works
+      assertEquals("forward and back conversion should generate same long", l, NumericUtils.prefixCodedToLong(act));
+      // next step
+      last=act;
+    }
+  }
+
+  public void testIntConversionAndOrdering() throws Exception {
+    // generate a series of encoded ints, each numerical one bigger than the one before
+    String last=null;
+    for (int i=-100000; i<100000; i++) {
+      String act=NumericUtils.intToPrefixCoded(i);
+      if (last!=null) {
+        // test if smaller
+        assertTrue("actual bigger than last", last.compareTo(act) < 0 );
+      }
+      // test is back and forward conversion works
+      assertEquals("forward and back conversion should generate same int", i, NumericUtils.prefixCodedToInt(act));
+      // next step
+      last=act;
+    }
+  }
+
+  public void testLongSpecialValues() throws Exception {
+    long[] vals=new long[]{
+      Long.MIN_VALUE, Long.MIN_VALUE+1, Long.MIN_VALUE+2, -5003400000000L,
+      -4000L, -3000L, -2000L, -1000L, -1L, 0L, 1L, 10L, 300L, 50006789999999999L, Long.MAX_VALUE-2, Long.MAX_VALUE-1, Long.MAX_VALUE
+    };
+    String[] prefixVals=new String[vals.length];
+    
+    for (int i=0; i<vals.length; i++) {
+      prefixVals[i]=NumericUtils.longToPrefixCoded(vals[i]);
+      
+      // check forward and back conversion
+      assertEquals( "forward and back conversion should generate same long", vals[i], NumericUtils.prefixCodedToLong(prefixVals[i]) );
+
+      // test if decoding values as int fails correctly
+      try {
+        NumericUtils.prefixCodedToInt(prefixVals[i]);
+        fail("decoding a prefix coded long value as int should fail");
+      } catch (NumberFormatException e) {
+        // worked
+      }
+    }
+    
+    // check sort order (prefixVals should be ascending)
+    for (int i=1; i<prefixVals.length; i++) {
+      assertTrue( "check sort order", prefixVals[i-1].compareTo( prefixVals[i] ) < 0 );
+    }
+        
+    // check the prefix encoding, lower precision should have the difference to original value equal to the lower removed bits
+    for (int i=0; i<vals.length; i++) {
+      for (int j=0; j<64; j++) {
+        long prefixVal=NumericUtils.prefixCodedToLong(NumericUtils.longToPrefixCoded(vals[i], j));
+        long mask=(1L << j) - 1L;
+        assertEquals( "difference between prefix val and original value for "+vals[i]+" with shift="+j, vals[i] & mask, vals[i]-prefixVal );
+      }
+    }
+  }
+
+  public void testIntSpecialValues() throws Exception {
+    int[] vals=new int[]{
+      Integer.MIN_VALUE, Integer.MIN_VALUE+1, Integer.MIN_VALUE+2, -64765767,
+      -4000, -3000, -2000, -1000, -1, 0, 1, 10, 300, 765878989, Integer.MAX_VALUE-2, Integer.MAX_VALUE-1, Integer.MAX_VALUE
+    };
+    String[] prefixVals=new String[vals.length];
+    
+    for (int i=0; i<vals.length; i++) {
+      prefixVals[i]=NumericUtils.intToPrefixCoded(vals[i]);
+      
+      // check forward and back conversion
+      assertEquals( "forward and back conversion should generate same int", vals[i], NumericUtils.prefixCodedToInt(prefixVals[i]) );
+      
+      // test if decoding values as long fails correctly
+      try {
+        NumericUtils.prefixCodedToLong(prefixVals[i]);
+        fail("decoding a prefix coded int value as long should fail");
+      } catch (NumberFormatException e) {
+        // worked
+      }
+    }
+    
+    // check sort order (prefixVals should be ascending)
+    for (int i=1; i<prefixVals.length; i++) {
+      assertTrue( "check sort order", prefixVals[i-1].compareTo( prefixVals[i] ) < 0 );
+    }
+    
+    // check the prefix encoding, lower precision should have the difference to original value equal to the lower removed bits
+    for (int i=0; i<vals.length; i++) {
+      for (int j=0; j<32; j++) {
+        int prefixVal=NumericUtils.prefixCodedToInt(NumericUtils.intToPrefixCoded(vals[i], j));
+        int mask=(1 << j) - 1;
+        assertEquals( "difference between prefix val and original value for "+vals[i]+" with shift="+j, vals[i] & mask, vals[i]-prefixVal );
+      }
+    }
+  }
+
+  public void testDoubles() throws Exception {
+    double[] vals=new double[]{
+      Double.NEGATIVE_INFINITY, -2.3E25, -1.0E15, -1.0, -1.0E-1, -1.0E-2, -0.0, 
+      +0.0, 1.0E-2, 1.0E-1, 1.0, 1.0E15, 2.3E25, Double.POSITIVE_INFINITY
+    };
+    long[] longVals=new long[vals.length];
+    
+    // check forward and back conversion
+    for (int i=0; i<vals.length; i++) {
+      longVals[i]=NumericUtils.doubleToSortableLong(vals[i]);
+      assertTrue( "forward and back conversion should generate same double", Double.compare(vals[i], NumericUtils.sortableLongToDouble(longVals[i]))==0 );
+    }
+    
+    // check sort order (prefixVals should be ascending)
+    for (int i=1; i<longVals.length; i++) {
+      assertTrue( "check sort order", longVals[i-1] < longVals[i] );
+    }
+  }
+
+  public void testFloats() throws Exception {
+    float[] vals=new float[]{
+      Float.NEGATIVE_INFINITY, -2.3E25f, -1.0E15f, -1.0f, -1.0E-1f, -1.0E-2f, -0.0f, 
+      +0.0f, 1.0E-2f, 1.0E-1f, 1.0f, 1.0E15f, 2.3E25f, Float.POSITIVE_INFINITY
+    };
+    int[] intVals=new int[vals.length];
+    
+    // check forward and back conversion
+    for (int i=0; i<vals.length; i++) {
+      intVals[i]=NumericUtils.floatToSortableInt(vals[i]);
+      assertTrue( "forward and back conversion should generate same double", Float.compare(vals[i], NumericUtils.sortableIntToFloat(intVals[i]))==0 );
+    }
+    
+    // check sort order (prefixVals should be ascending)
+    for (int i=1; i<intVals.length; i++) {
+      assertTrue( "check sort order", intVals[i-1] < intVals[i] );
+    }
+  }
+  
+  // INFO: Tests for trieCodeLong()/trieCodeInt() not needed because implicitely tested by range filter tests
+  
+  /** Note: The neededBounds iterator must be unsigned (easier understanding what's happening) */
+  protected void assertLongRangeSplit(final long lower, final long upper, int precisionStep,
+    final boolean useBitSet, final Iterator neededBounds
+  ) throws Exception {
+    final OpenBitSet bits=useBitSet ? new OpenBitSet(upper-lower+1) : null;
+    
+    NumericUtils.splitLongRange(new NumericUtils.LongRangeBuilder() {
+      //@Override
+      public void addRange(long min, long max, int shift) {
+        assertTrue("min, max should be inside bounds", min>=lower && min<=upper && max>=lower && max<=upper);
+        if (useBitSet) for (long l=min; l<=max; l++) {
+          assertFalse("ranges should not overlap", bits.getAndSet(l-lower) );
+        }
+        // make unsigned longs for easier display and understanding
+        min ^= 0x8000000000000000L;
+        max ^= 0x8000000000000000L;
+        //System.out.println("new Long(0x"+Long.toHexString(min>>>shift)+"L),new Long(0x"+Long.toHexString(max>>>shift)+"L),");
+        assertEquals( "inner min bound", ((Long)neededBounds.next()).longValue(), min>>>shift);
+        assertEquals( "inner max bound", ((Long)neededBounds.next()).longValue(), max>>>shift);
+      }
+    }, precisionStep, lower, upper);
+    
+    if (useBitSet) {
+      // after flipping all bits in the range, the cardinality should be zero
+      bits.flip(0,upper-lower+1);
+      assertTrue("The sub-range concenated should match the whole range", bits.isEmpty());
+    }
+  }
+  
+  public void testSplitLongRange() throws Exception {
+    // a hard-coded "standard" range
+    assertLongRangeSplit(-5000L, 9500L, 4, true, Arrays.asList(new Long[]{
+      new Long(0x7fffffffffffec78L),new Long(0x7fffffffffffec7fL),
+      new Long(0x8000000000002510L),new Long(0x800000000000251cL),
+      new Long(0x7fffffffffffec8L), new Long(0x7fffffffffffecfL),
+      new Long(0x800000000000250L), new Long(0x800000000000250L),
+      new Long(0x7fffffffffffedL),  new Long(0x7fffffffffffefL),
+      new Long(0x80000000000020L),  new Long(0x80000000000024L),
+      new Long(0x7ffffffffffffL),   new Long(0x8000000000001L)
+    }).iterator());
+    
+    // the same with no range splitting
+    assertLongRangeSplit(-5000L, 9500L, 64, true, Arrays.asList(new Long[]{
+      new Long(0x7fffffffffffec78L),new Long(0x800000000000251cL)
+    }).iterator());
+    
+    // this tests optimized range splitting, if one of the inner bounds
+    // is also the bound of the next lower precision, it should be used completely
+    assertLongRangeSplit(0L, 1024L+63L, 4, true, Arrays.asList(new Long[]{
+      new Long(0x800000000000040L), new Long(0x800000000000043L),
+      new Long(0x80000000000000L),  new Long(0x80000000000003L)
+    }).iterator());
+    
+    // the full long range should only consist of a lowest precision range; no bitset testing here, as too much memory needed :-)
+    assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 8, false, Arrays.asList(new Long[]{
+      new Long(0x00L),new Long(0xffL)
+    }).iterator());
+
+    // the same with precisionStep=4
+    assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 4, false, Arrays.asList(new Long[]{
+      new Long(0x0L),new Long(0xfL)
+    }).iterator());
+
+    // the same with precisionStep=2
+    assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 2, false, Arrays.asList(new Long[]{
+      new Long(0x0L),new Long(0x3L)
+    }).iterator());
+
+    // the same with precisionStep=1
+    assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 1, false, Arrays.asList(new Long[]{
+      new Long(0x0L),new Long(0x1L)
+    }).iterator());
+
+    // a inverse range should produce no sub-ranges
+    assertLongRangeSplit(9500L, -5000L, 4, false, Collections.EMPTY_LIST.iterator());    
+
+    // a 0-length range should reproduce the range itsself
+    assertLongRangeSplit(9500L, 9500L, 4, false, Arrays.asList(new Long[]{
+      new Long(0x800000000000251cL),new Long(0x800000000000251cL)
+    }).iterator());
+  }
+
+  /** Note: The neededBounds iterator must be unsigned (easier understanding what's happening) */
+  protected void assertIntRangeSplit(final int lower, final int upper, int precisionStep,
+    final boolean useBitSet, final Iterator neededBounds
+  ) throws Exception {
+    final OpenBitSet bits=useBitSet ? new OpenBitSet(upper-lower+1) : null;
+    
+    NumericUtils.splitIntRange(new NumericUtils.IntRangeBuilder() {
+      //@Override
+      public void addRange(int min, int max, int shift) {
+        assertTrue("min, max should be inside bounds", min>=lower && min<=upper && max>=lower && max<=upper);
+        if (useBitSet) for (int i=min; i<=max; i++) {
+          assertFalse("ranges should not overlap", bits.getAndSet(i-lower) );
+        }
+        // make unsigned ints for easier display and understanding
+        min ^= 0x80000000;
+        max ^= 0x80000000;
+        //System.out.println("new Integer(0x"+Integer.toHexString(min>>>shift)+"),new Integer(0x"+Integer.toHexString(max>>>shift)+"),");
+        assertEquals( "inner min bound", ((Integer)neededBounds.next()).intValue(), min>>>shift);
+        assertEquals( "inner max bound", ((Integer)neededBounds.next()).intValue(), max>>>shift);
+      }
+    }, precisionStep, lower, upper);
+    
+    if (useBitSet) {
+      // after flipping all bits in the range, the cardinality should be zero
+      bits.flip(0,upper-lower+1);
+      assertTrue("The sub-range concenated should match the whole range", bits.isEmpty());
+    }
+  }
+  
+  public void testSplitIntRange() throws Exception {
+    // a hard-coded "standard" range
+    assertIntRangeSplit(-5000, 9500, 4, true, Arrays.asList(new Integer[]{
+      new Integer(0x7fffec78),new Integer(0x7fffec7f),
+      new Integer(0x80002510),new Integer(0x8000251c),
+      new Integer(0x7fffec8), new Integer(0x7fffecf),
+      new Integer(0x8000250), new Integer(0x8000250),
+      new Integer(0x7fffed),  new Integer(0x7fffef),
+      new Integer(0x800020),  new Integer(0x800024),
+      new Integer(0x7ffff),   new Integer(0x80001)
+    }).iterator());
+    
+    // the same with no range splitting
+    assertIntRangeSplit(-5000, 9500, 32, true, Arrays.asList(new Integer[]{
+      new Integer(0x7fffec78),new Integer(0x8000251c)
+    }).iterator());
+    
+    // this tests optimized range splitting, if one of the inner bounds
+    // is also the bound of the next lower precision, it should be used completely
+    assertIntRangeSplit(0, 1024+63, 4, true, Arrays.asList(new Integer[]{
+      new Integer(0x8000040), new Integer(0x8000043),
+      new Integer(0x800000),  new Integer(0x800003)
+    }).iterator());
+    
+    // the full int range should only consist of a lowest precision range; no bitset testing here, as too much memory needed :-)
+    assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 8, false, Arrays.asList(new Integer[]{
+      new Integer(0x00),new Integer(0xff)
+    }).iterator());
+
+    // the same with precisionStep=4
+    assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 4, false, Arrays.asList(new Integer[]{
+      new Integer(0x0),new Integer(0xf)
+    }).iterator());
+
+    // the same with precisionStep=2
+    assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 2, false, Arrays.asList(new Integer[]{
+      new Integer(0x0),new Integer(0x3)
+    }).iterator());
+
+    // the same with precisionStep=1
+    assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 1, false, Arrays.asList(new Integer[]{
+      new Integer(0x0),new Integer(0x1)
+    }).iterator());
+
+    // a inverse range should produce no sub-ranges
+    assertIntRangeSplit(9500, -5000, 4, false, Collections.EMPTY_LIST.iterator());    
+
+    // a 0-length range should reproduce the range itsself
+    assertIntRangeSplit(9500, 9500, 4, false, Arrays.asList(new Integer[]{
+      new Integer(0x8000251c),new Integer(0x8000251c)
+    }).iterator());
+  }
+
+}

Propchange: lucene/java/trunk/src/test/org/apache/lucene/util/TestNumericUtils.java
------------------------------------------------------------------------------
    svn:eol-style = native



Mime
View raw message