lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1294876 - in /lucene/dev/branches/branch_3x: ./ lucene/ lucene/core/src/java/org/apache/lucene/search/ lucene/core/src/test/org/apache/lucene/search/ lucene/test-framework/src/java/org/apache/lucene/util/
Date Tue, 28 Feb 2012 22:44:42 GMT
Author: mikemccand
Date: Tue Feb 28 22:44:41 2012
New Revision: 1294876

URL: http://svn.apache.org/viewvc?rev=1294876&view=rev
Log:
LUCENE-3824: don't do pointless by-value cmp in TermOrdVal/DocValuesComparator.setBottom

Modified:
    lucene/dev/branches/branch_3x/   (props changed)
    lucene/dev/branches/branch_3x/lucene/   (props changed)
    lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
    lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java
    lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestSort.java
    lucene/dev/branches/branch_3x/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java

Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java?rev=1294876&r1=1294875&r2=1294876&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
Tue Feb 28 22:44:41 2012
@@ -886,31 +886,17 @@ public abstract class FieldComparator<T>
     @Override
     public int compareBottom(int doc) {
       assert bottomSlot != -1;
+      final int docOrd = this.order[doc];
       if (bottomSameReader) {
         // ord is precisely comparable, even in the equal case
-        return bottomOrd - this.order[doc];
+        return bottomOrd - docOrd;
+      } else if (bottomOrd >= docOrd) {
+        // the equals case always means bottom is > doc
+        // (because we set bottomOrd to the lower bound in
+        // setBottom):
+        return 1;
       } else {
-        // ord is only approx comparable: if they are not
-        // equal, we can use that; if they are equal, we
-        // must fallback to compare by value
-        final int order = this.order[doc];
-        final int cmp = bottomOrd - order;
-        if (cmp != 0) {
-          return cmp;
-        }
-
-        final String val2 = lookup[order];
-        if (bottomValue == null) {
-          if (val2 == null) {
-            return 0;
-          }
-          // bottom wins
-          return -1;
-        } else if (val2 == null) {
-          // doc wins
-          return 1;
-        }
-        return bottomValue.compareTo(val2);
+        return -1;
       }
     }
 

Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java?rev=1294876&r1=1294875&r2=1294876&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java
Tue Feb 28 22:44:41 2012
@@ -68,7 +68,7 @@ public abstract class TopFieldCollector 
       if (queueFull) {
         if ((reverseMul * comparator.compareBottom(doc)) <= 0) {
           // since docs are visited in doc Id order, if compare is 0, it means
-          // this document is largest than anything else in the queue, and
+          // this document is larger than anything else in the queue, and
           // therefore not competitive.
           return;
         }

Modified: lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestSort.java?rev=1294876&r1=1294875&r2=1294876&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestSort.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestSort.java
Tue Feb 28 22:44:41 2012
@@ -22,9 +22,14 @@ import java.io.Serializable;
 import java.text.Collator;
 import java.util.ArrayList;
 import java.util.BitSet;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Locale;
+import java.util.Random;
+import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
@@ -41,9 +46,10 @@ import org.apache.lucene.index.Term;
 import org.apache.lucene.queryParser.ParseException;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.FieldValueHitQueue.Entry;
-import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.util.DocIdBitSet;
+import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
 import org.junit.BeforeClass;
@@ -1293,4 +1299,141 @@ public class TestSort extends LuceneTest
     return data[doc][2]==null ? missingVal : Integer.parseInt(data[doc][2]);
   }
 
+  private static class RandomFilter extends Filter {
+    private final Random random;
+    private float density;
+    private final List<String> docValues;
+    public final List<String> matchValues = Collections.synchronizedList(new ArrayList<String>());
+
+    // density should be 0.0 ... 1.0
+    public RandomFilter(Random random, float density, List<String> docValues) {
+      this.random = random;
+      this.density = density;
+      this.docValues = docValues;
+    }
+
+    @Override
+    public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
+      final int maxDoc = reader.maxDoc();
+      final int[] docIDToID = FieldCache.DEFAULT.getInts(reader, "id");
+      final FixedBitSet bits = new FixedBitSet(maxDoc);
+      for(int docID=0;docID<maxDoc;docID++) {
+        if (random.nextFloat() <= density) {
+          bits.set(docID);
+          //System.out.println("  acc id=" + idSource.getInt(docID) + " docID=" + docID);
+          matchValues.add(docValues.get(docIDToID[docID]));
+        }
+      }
+
+      return bits;
+    }
+  }
+
+  public void testRandomStringSort() throws Exception {
+    final int NUM_DOCS = atLeast(100);
+    final Directory dir = newDirectory();
+    final RandomIndexWriter writer = new RandomIndexWriter(random, dir);
+    final boolean allowDups = random.nextBoolean();
+    final Set<String> seen = new HashSet<String>();
+    final int maxLength = _TestUtil.nextInt(random, 5, 100);
+    if (VERBOSE) {
+      System.out.println("TEST: NUM_DOCS=" + NUM_DOCS + " maxLength=" + maxLength + " allowDups="
+ allowDups);
+    }
+    int numDocs = 0;
+    final List<String> docValues = new ArrayList<String>();
+    // TODO: deletions
+    while (numDocs < NUM_DOCS) {
+      final String s;
+      if (random.nextBoolean()) {
+        s = _TestUtil.randomSimpleString(random, maxLength);
+      } else {
+        s = _TestUtil.randomUnicodeString(random, maxLength);
+      }
+
+      if (!allowDups) {
+        if (seen.contains(s)) {
+          continue;
+        }
+        seen.add(s);
+      }
+
+      if (VERBOSE) {
+        System.out.println("  " + numDocs + ": s=" + s);
+      }
+      
+      final Document doc = new Document();
+      doc.add(newField("string", s, Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS));
+      doc.add(newField("id", ""+numDocs, Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS));
+      docValues.add(s);
+      writer.addDocument(doc);
+      numDocs++;
+
+      if (random.nextInt(40) == 17) {
+        // force flush
+        writer.getReader().close();
+      }
+    }
+
+    final IndexReader r = writer.getReader();
+    writer.close();
+    if (VERBOSE) {
+      System.out.println("  reader=" + r);
+    }
+    
+    final IndexSearcher s = newSearcher(r, false);
+    final int ITERS = atLeast(100);
+    for(int iter=0;iter<ITERS;iter++) {
+      final boolean reverse = random.nextBoolean();
+      final TopFieldDocs hits;
+      final SortField sf = new SortField("string", SortField.STRING, reverse);
+      final Sort sort = new Sort(sf);
+      final int hitCount = _TestUtil.nextInt(random, 1, r.maxDoc() + 20);
+      final RandomFilter f = new RandomFilter(random, random.nextFloat(), docValues);
+      if (random.nextBoolean()) {
+        hits = s.search(new ConstantScoreQuery(f),
+                        hitCount,
+                        sort);
+      } else {
+        hits = s.search(new MatchAllDocsQuery(),
+                        f,
+                        hitCount,
+                        sort);
+      }
+
+      if (VERBOSE) {
+        System.out.println("\nTEST: iter=" + iter + " " + hits.totalHits + " hits; topN="
+ hitCount + "; reverse=" + reverse);
+      }
+
+      // Compute expected results:
+      Collections.sort(f.matchValues);
+      if (reverse) {
+        Collections.reverse(f.matchValues);
+      }
+      final List<String> expected = f.matchValues;
+      if (VERBOSE) {
+        System.out.println("  expected:");
+        for(int idx=0;idx<expected.size();idx++) {
+          System.out.println("    " + idx + ": " + expected.get(idx));
+          if (idx == hitCount-1) {
+            break;
+          }
+        }
+      }
+      
+      if (VERBOSE) {
+        System.out.println("  actual:");
+        for(int hitIDX=0;hitIDX<hits.scoreDocs.length;hitIDX++) {
+          final FieldDoc fd = (FieldDoc) hits.scoreDocs[hitIDX];
+          System.out.println("    " + hitIDX + ": " + fd.fields[0]);
+        }
+      }
+      for(int hitIDX=0;hitIDX<hits.scoreDocs.length;hitIDX++) {
+        final FieldDoc fd = (FieldDoc) hits.scoreDocs[hitIDX];
+        assertEquals(expected.get(hitIDX), (String) fd.fields[0]);
+      }
+    }
+
+    r.close();
+    dir.close();
+  }
 }

Modified: lucene/dev/branches/branch_3x/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java?rev=1294876&r1=1294875&r2=1294876&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java
Tue Feb 28 22:44:41 2012
@@ -207,8 +207,8 @@ public class _TestUtil {
     return buf.toString();
   }
 
-  public static String randomSimpleString(Random r) {
-    final int end = r.nextInt(10);
+  public static String randomSimpleString(Random r, int maxLength) {
+    final int end = r.nextInt(maxLength);
     if (end == 0) {
       // allow 0 length
       return "";
@@ -220,6 +220,10 @@ public class _TestUtil {
     return new String(buffer, 0, end);
   }
 
+  public static String randomSimpleString(Random r) {
+    return randomSimpleString(r, 10);
+  }
+
   /** Returns random string, including full unicode range. */
   public static String randomUnicodeString(Random r) {
     return randomUnicodeString(r, 20);



Mime
View raw message