lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From uschind...@apache.org
Subject svn commit: r1173701 - in /lucene/dev/branches/branch_3x/lucene: ./ src/java/org/apache/lucene/search/ src/java/org/apache/lucene/util/ src/test/org/apache/lucene/search/
Date Wed, 21 Sep 2011 15:07:05 GMT
Author: uschindler
Date: Wed Sep 21 15:07:04 2011
New Revision: 1173701

URL: http://svn.apache.org/viewvc?rev=1173701&view=rev
Log:
LUCENE-3390: Corrected handling of missing values when two parallel searches using different
missing values for sorting: the missing value was populated directly into the FieldCache arrays
during sorting, leading to concurrency issues.

Added:
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/Bits.java
      - copied unchanged from r1173312, lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/Bits.java
Modified:
    lucene/dev/branches/branch_3x/lucene/CHANGES.txt
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCache.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldComparator.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/SortField.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/BitVector.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/FixedBitSet.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/OpenBitSet.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestFieldCache.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSort.java

Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1173701&r1=1173700&r2=1173701&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Wed Sep 21 15:07:04 2011
@@ -5,6 +5,19 @@ http://s.apache.org/luceneversions
 
 ======================= Lucene 3.5.0 =======================
 
+Changes in backwards compatibility policy
+
+* LUCENE-3390: The first approach in Lucene 3.4.0 for missing values
+  support for sorting had a design problem that made the missing value
+  be populated directly into the FieldCache arrays during sorting,
+  leading to concurrency issues. To fix this behaviour, the method
+  signatures had to be changed:
+  - FieldCache.getUnValuedDocs() returns the interface Bits instead DocIdSet
+  - FieldComparator.setMissingValue() was removed and added to
+    constructor
+  As this is expert API, most code will not be affected.
+  (Uwe Schindler, Doron Cohen, Mike McCandless)
+
 Bug fixes
 
 * SOLR-2762: (backport form 4.x line): FSTLookup could return duplicate 
@@ -25,6 +38,11 @@ Bug fixes
   QueryWrapperFilter and similar classes to get a top-level DocIdSet.
   (Dan C., Uwe Schindler)
 
+* LUCENE-3390: Corrected handling of missing values when two parallel searches
+  using different missing values for sorting: the missing value was populated
+  directly into the FieldCache arrays during sorting, leading to concurrency
+  issues.  (Uwe Schindler, Doron Cohen, Mike McCandless)
+
 New Features
 
 Optimizations

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCache.java?rev=1173701&r1=1173700&r2=1173701&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCache.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCache.java
Wed Sep 21 15:07:04 2011
@@ -18,6 +18,7 @@ package org.apache.lucene.search;
  */
 
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.NumericUtils;
 import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.document.NumericField; // for javadocs
@@ -315,7 +316,7 @@ public interface FieldCache {
    * <code>reader.maxDoc()</code>, with turned on bits for each docid that 
    * does not have a value for this field.
    */
-  public DocIdSet getUnValuedDocs (IndexReader reader, String field) 
+  public Bits getUnValuedDocs (IndexReader reader, String field) 
   throws IOException;
   
   /** Checks the internal cache for an appropriate entry, and if none is

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1173701&r1=1173700&r2=1173701&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
Wed Sep 21 15:07:04 2011
@@ -30,9 +30,8 @@ import org.apache.lucene.index.IndexRead
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermDocs;
 import org.apache.lucene.index.TermEnum;
-import org.apache.lucene.util.BitVector;
-import org.apache.lucene.util.DocIdBitSet;
-import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.StringHelper;
 import org.apache.lucene.util.FieldCacheSanityChecker;
 
@@ -413,6 +412,11 @@ class FieldCacheImpl implements FieldCac
       return retArray;
     }
   }
+  
+  public Bits getUnValuedDocs(IndexReader reader, String field)
+      throws IOException {
+    return (Bits) caches.get(UnValuedDocsCache.class).get(reader, new Entry(field, null));
+  }
 
   static final class UnValuedDocsCache extends Cache {
     UnValuedDocsCache(FieldCache wrapper) {
@@ -422,34 +426,35 @@ class FieldCacheImpl implements FieldCac
     @Override
     protected Object createValue(IndexReader reader, Entry entryKey)
     throws IOException {
-      Entry entry = entryKey;
-      String field = entry.field;
-      
-      if (reader.maxDoc() == reader.docFreq(new Term(field))) {
-        return DocIdSet.EMPTY_DOCIDSET;
-      }
-      
-      OpenBitSet res = new OpenBitSet(reader.maxDoc());
-      TermDocs termDocs = reader.termDocs();
-      TermEnum termEnum = reader.terms (new Term (field));
+      final Entry entry = entryKey;
+      final String field = entry.field;      
+      final FixedBitSet res = new FixedBitSet(reader.maxDoc());
+      final TermDocs termDocs = reader.termDocs();
+      final TermEnum termEnum = reader.terms(new Term(field));
       try {
         do {
-          Term term = termEnum.term();
-          if (term==null || term.field() != field) break;
-          termDocs.seek (termEnum);
+          final Term term = termEnum.term();
+          if (term == null || term.field() != field) break;
+          termDocs.seek(termEnum);
           while (termDocs.next()) {
-            res.fastSet(termDocs.doc());
+            res.set(termDocs.doc());
           }
         } while (termEnum.next());
       } finally {
         termDocs.close();
         termEnum.close();
       }
+      final int numSet = res.cardinality();
+      if (numSet >= reader.numDocs()) {
+        // The cardinality of the BitSet is numDocs if all documents have a value.
+        // As deleted docs are not in TermDocs, this is always true
+        assert numSet == reader.numDocs();
+        return new Bits.MatchNoBits(reader.maxDoc());
+      }
       res.flip(0, reader.maxDoc());
       return res;
     }
   }
-  
 
   // inherit javadocs
   public float[] getFloats (IndexReader reader, String field)
@@ -736,10 +741,5 @@ class FieldCacheImpl implements FieldCac
   public PrintStream getInfoStream() {
     return infoStream;
   }
-  
-  public DocIdSet getUnValuedDocs(IndexReader reader, String field)
-      throws IOException {
-    return (DocIdSet) caches.get(UnValuedDocsCache.class).get(reader, new Entry(field, null));
 
-  }
 }
 

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldComparator.java?rev=1173701&r1=1173700&r2=1173701&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldComparator.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldComparator.java
Wed Sep 21 15:07:04 2011
@@ -29,7 +29,7 @@ import org.apache.lucene.search.FieldCac
 import org.apache.lucene.search.FieldCache.IntParser;
 import org.apache.lucene.search.FieldCache.ShortParser;
 import org.apache.lucene.search.FieldCache.StringIndex;
-import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.Bits;
 
 /**
  * Expert: a FieldComparator compares hits so as to determine their
@@ -84,14 +84,6 @@ import org.apache.lucene.util.OpenBitSet
  */
 public abstract class FieldComparator<T> {
 
-  protected T missingValue = null;
-  
-  /** Set a default sorting value for documents which lacks one */
-  public FieldComparator<T> setMissingValue(T missingValue) {
-    this.missingValue = missingValue;
-    return this;
-  }
-  
   /**
    * Compare hit at slot1 with hit at slot2.
    * 
@@ -191,18 +183,46 @@ public abstract class FieldComparator<T>
     }
   }
 
+  public static abstract class NumericComparator<T extends Number> extends FieldComparator<T>
{
+    protected final T missingValue;
+    protected final String field;
+
+    protected Bits unvaluedDocs = null;
+    
+    public NumericComparator(String field, T missingValue) {
+      this.field = field;
+      this.missingValue = missingValue;
+    }
+    
+    @Override
+    public void setNextReader(IndexReader reader, int docBase) throws IOException {
+      if (missingValue != null) {
+        unvaluedDocs = FieldCache.DEFAULT.getUnValuedDocs(reader, field);
+        // optimization to remove unneeded checks on the bit interface:
+        if (unvaluedDocs instanceof Bits.MatchNoBits)
+          unvaluedDocs = null;
+      } else {
+        unvaluedDocs = null;
+      }
+    }
+    
+  }
+
   /** Parses field's values as byte (using {@link
    *  FieldCache#getBytes} and sorts by ascending value */
-  public static final class ByteComparator extends FieldComparator<Byte> {
+  public static final class ByteComparator extends NumericComparator<Byte> {
     private final byte[] values;
     private byte[] currentReaderValues;
-    private final String field;
     private ByteParser parser;
     private byte bottom;
 
     ByteComparator(int numHits, String field, FieldCache.Parser parser) {
+      this(numHits, field, parser, null);
+    }
+    
+    ByteComparator(int numHits, String field, FieldCache.Parser parser, Byte missingValue)
{
+      super(field, missingValue);
       values = new byte[numHits];
-      this.field = field;
       this.parser = (ByteParser) parser;
     }
 
@@ -213,24 +233,24 @@ public abstract class FieldComparator<T>
 
     @Override
     public int compareBottom(int doc) {
-      return bottom - currentReaderValues[doc];
+      byte v2 = currentReaderValues[doc];
+      if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc))
+        v2 = missingValue;
+      return bottom - v2;
     }
 
     @Override
     public void copy(int slot, int doc) {
-      values[slot] = currentReaderValues[doc];
+      byte v2 = currentReaderValues[doc];
+      if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc))
+        v2 = missingValue;
+      values[slot] = v2;
     }
 
     @Override
     public void setNextReader(IndexReader reader, int docBase) throws IOException {
       currentReaderValues = FieldCache.DEFAULT.getBytes(reader, field, parser);
-      if (missingValue != null) {
-        DocIdSetIterator iterator = FieldCache.DEFAULT.getUnValuedDocs(reader, field).iterator();
-        final byte byteValue = missingValue.byteValue();
-        for (int doc=iterator.nextDoc(); doc!=DocIdSetIterator.NO_MORE_DOCS; doc=iterator.nextDoc())
{
-          currentReaderValues[doc] = byteValue;
-        }
-      }
+      super.setNextReader(reader, docBase);
     }
     
     @Override
@@ -292,16 +312,19 @@ public abstract class FieldComparator<T>
 
   /** Parses field's values as double (using {@link
    *  FieldCache#getDoubles} and sorts by ascending value */
-  public static final class DoubleComparator extends FieldComparator<Double> {
+  public static final class DoubleComparator extends NumericComparator<Double> {
     private final double[] values;
     private double[] currentReaderValues;
-    private final String field;
     private DoubleParser parser;
     private double bottom;
 
     DoubleComparator(int numHits, String field, FieldCache.Parser parser) {
+      this(numHits, field, parser, null);
+    }
+    
+    DoubleComparator(int numHits, String field, FieldCache.Parser parser, Double missingValue)
{
+      super(field, missingValue);
       values = new double[numHits];
-      this.field = field;
       this.parser = (DoubleParser) parser;
     }
 
@@ -320,7 +343,9 @@ public abstract class FieldComparator<T>
 
     @Override
     public int compareBottom(int doc) {
-      final double v2 = currentReaderValues[doc];
+      double v2 = currentReaderValues[doc];
+      if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc))
+        v2 = missingValue;
       if (bottom > v2) {
         return 1;
       } else if (bottom < v2) {
@@ -332,19 +357,16 @@ public abstract class FieldComparator<T>
 
     @Override
     public void copy(int slot, int doc) {
-      values[slot] = currentReaderValues[doc];
+      double v2 = currentReaderValues[doc];
+      if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc))
+        v2 = missingValue;
+      values[slot] = v2;
     }
 
     @Override
     public void setNextReader(IndexReader reader, int docBase) throws IOException {
       currentReaderValues = FieldCache.DEFAULT.getDoubles(reader, field, parser);
-      if (missingValue != null) {
-        DocIdSetIterator iterator = FieldCache.DEFAULT.getUnValuedDocs(reader, field).iterator();
-        final double doubleValue = missingValue.doubleValue();
-        for (int doc=iterator.nextDoc(); doc!=DocIdSetIterator.NO_MORE_DOCS; doc=iterator.nextDoc())
{
-          currentReaderValues[doc] = doubleValue;
-        }
-      }
+      super.setNextReader(reader, docBase);
     }
     
     @Override
@@ -360,16 +382,19 @@ public abstract class FieldComparator<T>
 
   /** Parses field's values as float (using {@link
    *  FieldCache#getFloats} and sorts by ascending value */
-  public static final class FloatComparator extends FieldComparator<Float> {
+  public static final class FloatComparator extends NumericComparator<Float> {
     private final float[] values;
     private float[] currentReaderValues;
-    private final String field;
     private FloatParser parser;
     private float bottom;
 
     FloatComparator(int numHits, String field, FieldCache.Parser parser) {
+      this(numHits, field, parser, null);
+    }
+    
+    FloatComparator(int numHits, String field, FieldCache.Parser parser, Float missingValue)
{
+      super(field, missingValue);
       values = new float[numHits];
-      this.field = field;
       this.parser = (FloatParser) parser;
     }
 
@@ -392,7 +417,9 @@ public abstract class FieldComparator<T>
     public int compareBottom(int doc) {
       // TODO: are there sneaky non-branch ways to compute
       // sign of float?
-      final float v2 = currentReaderValues[doc];
+      float v2 = currentReaderValues[doc];
+      if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc))
+        v2 = missingValue;
       if (bottom > v2) {
         return 1;
       } else if (bottom < v2) {
@@ -404,19 +431,16 @@ public abstract class FieldComparator<T>
 
     @Override
     public void copy(int slot, int doc) {
-      values[slot] = currentReaderValues[doc];
+      float v2 = currentReaderValues[doc];
+      if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc))
+        v2 = missingValue;
+      values[slot] = v2;
     }
 
     @Override
     public void setNextReader(IndexReader reader, int docBase) throws IOException {
       currentReaderValues = FieldCache.DEFAULT.getFloats(reader, field, parser);
-      if (missingValue != null) {
-        DocIdSetIterator iterator = FieldCache.DEFAULT.getUnValuedDocs(reader, field).iterator();
-        final float floatValue = missingValue.floatValue();
-        for (int doc=iterator.nextDoc(); doc!=DocIdSetIterator.NO_MORE_DOCS; doc=iterator.nextDoc())
{
-          currentReaderValues[doc] = floatValue;
-        }
-      }
+      super.setNextReader(reader, docBase);
     }
     
     @Override
@@ -432,16 +456,19 @@ public abstract class FieldComparator<T>
 
   /** Parses field's values as int (using {@link
    *  FieldCache#getInts} and sorts by ascending value */
-  public static final class IntComparator extends FieldComparator<Integer> {
+  public static final class IntComparator extends NumericComparator<Integer> {
     private final int[] values;
     private int[] currentReaderValues;
-    private final String field;
     private IntParser parser;
     private int bottom;                           // Value of bottom of queue
 
     IntComparator(int numHits, String field, FieldCache.Parser parser) {
+      this(numHits, field, parser, null);
+    }
+    
+    IntComparator(int numHits, String field, FieldCache.Parser parser, Integer missingValue)
{
+      super(field, missingValue);
       values = new int[numHits];
-      this.field = field;
       this.parser = (IntParser) parser;
     }
 
@@ -468,7 +495,9 @@ public abstract class FieldComparator<T>
       // -1/+1/0 sign
       // Cannot return bottom - values[slot2] because that
       // may overflow
-      final int v2 = currentReaderValues[doc];
+      int v2 = currentReaderValues[doc];
+      if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc))
+        v2 = missingValue;
       if (bottom > v2) {
         return 1;
       } else if (bottom < v2) {
@@ -480,19 +509,16 @@ public abstract class FieldComparator<T>
 
     @Override
     public void copy(int slot, int doc) {
-      values[slot] = currentReaderValues[doc];
+      int v2 = currentReaderValues[doc];
+      if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc))
+        v2 = missingValue;
+      values[slot] = v2;
     }
 
     @Override
     public void setNextReader(IndexReader reader, int docBase) throws IOException {
       currentReaderValues = FieldCache.DEFAULT.getInts(reader, field, parser);
-      if (missingValue != null) {
-        DocIdSetIterator iterator = FieldCache.DEFAULT.getUnValuedDocs(reader, field).iterator();
-        final int intValue = missingValue.intValue();
-        for (int doc=iterator.nextDoc(); doc!=DocIdSetIterator.NO_MORE_DOCS; doc=iterator.nextDoc())
{
-          currentReaderValues[doc] = intValue;
-        }
-      }
+      super.setNextReader(reader, docBase);
     }
     
     @Override
@@ -508,16 +534,19 @@ public abstract class FieldComparator<T>
 
   /** Parses field's values as long (using {@link
    *  FieldCache#getLongs} and sorts by ascending value */
-  public static final class LongComparator extends FieldComparator<Long> {
+  public static final class LongComparator extends NumericComparator<Long> {
     private final long[] values;
     private long[] currentReaderValues;
-    private final String field;
     private LongParser parser;
     private long bottom;
 
     LongComparator(int numHits, String field, FieldCache.Parser parser) {
+      this(numHits, field, parser, null);
+    }
+    
+    LongComparator(int numHits, String field, FieldCache.Parser parser, Long missingValue)
{
+      super(field, missingValue);
       values = new long[numHits];
-      this.field = field;
       this.parser = (LongParser) parser;
     }
 
@@ -540,7 +569,9 @@ public abstract class FieldComparator<T>
     public int compareBottom(int doc) {
       // TODO: there are sneaky non-branch ways to compute
       // -1/+1/0 sign
-      final long v2 = currentReaderValues[doc];
+      long v2 = currentReaderValues[doc];
+      if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc))
+        v2 = missingValue;
       if (bottom > v2) {
         return 1;
       } else if (bottom < v2) {
@@ -552,19 +583,16 @@ public abstract class FieldComparator<T>
 
     @Override
     public void copy(int slot, int doc) {
-      values[slot] = currentReaderValues[doc];
+      long v2 = currentReaderValues[doc];
+      if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc))
+        v2 = missingValue;
+      values[slot] = v2;
     }
 
     @Override
     public void setNextReader(IndexReader reader, int docBase) throws IOException {
       currentReaderValues = FieldCache.DEFAULT.getLongs(reader, field, parser);
-      if (missingValue != null) {
-        DocIdSetIterator iterator = FieldCache.DEFAULT.getUnValuedDocs(reader, field).iterator();
-        final long longValue = missingValue.longValue();
-        for (int doc=iterator.nextDoc(); doc!=DocIdSetIterator.NO_MORE_DOCS; doc=iterator.nextDoc())
{
-          currentReaderValues[doc] = longValue;
-        }
-      }
+      super.setNextReader(reader, docBase);
     }
     
     @Override
@@ -648,16 +676,19 @@ public abstract class FieldComparator<T>
 
   /** Parses field's values as short (using {@link
    *  FieldCache#getShorts} and sorts by ascending value */
-  public static final class ShortComparator extends FieldComparator<Short> {
+  public static final class ShortComparator extends NumericComparator<Short> {
     private final short[] values;
     private short[] currentReaderValues;
-    private final String field;
     private ShortParser parser;
     private short bottom;
 
     ShortComparator(int numHits, String field, FieldCache.Parser parser) {
+      this(numHits, field, parser, null);
+    }
+    
+    ShortComparator(int numHits, String field, FieldCache.Parser parser, Short missingValue)
{
+      super(field, missingValue);
       values = new short[numHits];
-      this.field = field;
       this.parser = (ShortParser) parser;
     }
 
@@ -668,24 +699,24 @@ public abstract class FieldComparator<T>
 
     @Override
     public int compareBottom(int doc) {
-      return bottom - currentReaderValues[doc];
+      short v2 = currentReaderValues[doc];
+      if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc))
+        v2 = missingValue;
+      return bottom - v2;
     }
 
     @Override
     public void copy(int slot, int doc) {
-      values[slot] = currentReaderValues[doc];
+      short v2 = currentReaderValues[doc];
+      if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc))
+        v2 = missingValue;
+      values[slot] = v2;
     }
 
     @Override
     public void setNextReader(IndexReader reader, int docBase) throws IOException {
       currentReaderValues = FieldCache.DEFAULT.getShorts(reader, field, parser);
-      if (missingValue != null) {
-        DocIdSetIterator iterator = FieldCache.DEFAULT.getUnValuedDocs(reader, field).iterator();
-        final short shortValue = missingValue.shortValue();
-        for (int doc=iterator.nextDoc(); doc!=DocIdSetIterator.NO_MORE_DOCS; doc=iterator.nextDoc())
{
-          currentReaderValues[doc] = shortValue;
-        }
-      }
+      super.setNextReader(reader, docBase);
     }
     
     @Override

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/SortField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/SortField.java?rev=1173701&r1=1173700&r2=1173701&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/SortField.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/SortField.java
Wed Sep 21 15:07:04 2011
@@ -403,22 +403,22 @@ implements Serializable {
       return new FieldComparator.DocComparator(numHits);
 
     case SortField.INT:
-      return new FieldComparator.IntComparator(numHits, field, parser).setMissingValue((Integer)
missingValue);
+      return new FieldComparator.IntComparator(numHits, field, parser, (Integer) missingValue);
 
     case SortField.FLOAT:
-      return new FieldComparator.FloatComparator(numHits, field, parser).setMissingValue((Float)
missingValue);
+      return new FieldComparator.FloatComparator(numHits, field, parser, (Float) missingValue);
 
     case SortField.LONG:
-      return new FieldComparator.LongComparator(numHits, field, parser).setMissingValue((Long)
missingValue);
+      return new FieldComparator.LongComparator(numHits, field, parser, (Long) missingValue);
 
     case SortField.DOUBLE:
-      return new FieldComparator.DoubleComparator(numHits, field, parser).setMissingValue((Double)
missingValue);
+      return new FieldComparator.DoubleComparator(numHits, field, parser, (Double) missingValue);
 
     case SortField.BYTE:
-      return new FieldComparator.ByteComparator(numHits, field, parser).setMissingValue((Byte)
missingValue);
+      return new FieldComparator.ByteComparator(numHits, field, parser, (Byte) missingValue);
 
     case SortField.SHORT:
-      return new FieldComparator.ShortComparator(numHits, field, parser).setMissingValue((Short)
missingValue);
+      return new FieldComparator.ShortComparator(numHits, field, parser, (Short) missingValue);
 
     case SortField.CUSTOM:
       assert comparatorSource != null;

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/BitVector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/BitVector.java?rev=1173701&r1=1173700&r2=1173701&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/BitVector.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/BitVector.java Wed
Sep 21 15:07:04 2011
@@ -34,7 +34,7 @@ import org.apache.lucene.store.IndexOutp
  *
  *  @lucene.internal
  */
-public final class BitVector implements Cloneable {
+public final class BitVector implements Cloneable, Bits {
 
   private byte[] bits;
   private int size;
@@ -120,6 +120,12 @@ public final class BitVector implements 
     return size;
   }
 
+  /** Returns the number of bits in this vector.  This is also one greater than
+    the number of the largest valid bit number. */
+  public final int length() {
+    return size;
+  }
+
   /** Returns the total number of one bits in this vector.  This is efficiently
     computed and cached, so that, if the vector is not changed, no
     recomputation is done for repeated calls. */

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/FixedBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/FixedBitSet.java?rev=1173701&r1=1173700&r2=1173701&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/FixedBitSet.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/FixedBitSet.java
Wed Sep 21 15:07:04 2011
@@ -36,7 +36,7 @@ import org.apache.lucene.search.DocIdSet
  * @lucene.internal
  **/
 
-public final class FixedBitSet extends DocIdSet {
+public final class FixedBitSet extends DocIdSet implements Bits {
   private final long[] bits;
   private int numBits;
 

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/OpenBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/OpenBitSet.java?rev=1173701&r1=1173700&r2=1173701&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/OpenBitSet.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/OpenBitSet.java Wed
Sep 21 15:07:04 2011
@@ -75,7 +75,7 @@ Test system: AMD Opteron, 64 bit linux, 
 </table>
  */
 
-public class OpenBitSet extends DocIdSet implements Cloneable, Serializable {
+public class OpenBitSet extends DocIdSet implements Cloneable, Serializable, Bits {
   protected long[] bits;
   protected int wlen;   // number of words (elements) used in the array
 
@@ -137,6 +137,10 @@ public class OpenBitSet extends DocIdSet
       return capacity();
   }
 
+  public int length() {
+    return bits.length << 6;
+  }
+
   /** Returns true if there are no set bits */
   public boolean isEmpty() { return cardinality()==0; }
 

Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestFieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestFieldCache.java?rev=1173701&r1=1173700&r2=1173701&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestFieldCache.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestFieldCache.java
Wed Sep 21 15:07:04 2011
@@ -22,6 +22,7 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.LuceneTestCase;
 import java.io.IOException;
 import java.io.ByteArrayOutputStream;
@@ -52,6 +53,9 @@ public class TestFieldCache extends Luce
       doc.add(newField("theShort", String.valueOf(theShort--), Field.Store.NO, Field.Index.NOT_ANALYZED));
       doc.add(newField("theInt", String.valueOf(theInt--), Field.Store.NO, Field.Index.NOT_ANALYZED));
       doc.add(newField("theFloat", String.valueOf(theFloat--), Field.Store.NO, Field.Index.NOT_ANALYZED));
+      if (i%2 == 0) {
+        doc.add(newField("sparse", String.valueOf(i), Field.Store.NO, Field.Index.NOT_ANALYZED));
+      }
       writer.addDocument(doc);
     }
     writer.close();
@@ -133,5 +137,21 @@ public class TestFieldCache extends Luce
       assertTrue(floats[i] + " does not equal: " + (Float.MAX_VALUE - i), floats[i] == (Float.MAX_VALUE
- i));
 
     }
+    
+    Bits unvalued = cache.getUnValuedDocs(reader, "theLong");
+    assertSame("Second request to cache return same array", unvalued, cache.getUnValuedDocs(reader,
"theLong"));
+    assertTrue("unvalued(theLong) must be class Bits.MatchNoBits", unvalued instanceof Bits.MatchNoBits);
+    assertTrue("unvalued(theLong) Size: " + unvalued.length() + " is not: " + NUM_DOCS, unvalued.length()
== NUM_DOCS);
+    for (int i = 0; i < unvalued.length(); i++) {
+      assertFalse(unvalued.get(i));
+    }
+    
+    unvalued = cache.getUnValuedDocs(reader, "sparse");
+    assertSame("Second request to cache return same array", unvalued, cache.getUnValuedDocs(reader,
"sparse"));
+    assertFalse("unvalued(sparse) must not be class Bits.MatchNoBits", unvalued instanceof
Bits.MatchNoBits);
+    assertTrue("unvalued(sparse) Size: " + unvalued.length() + " is not: " + NUM_DOCS, unvalued.length()
== NUM_DOCS);
+    for (int i = 0; i < unvalued.length(); i++) {
+      assertEquals(i%2 != 0, unvalued.get(i));
+    }
   }
 }

Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSort.java?rev=1173701&r1=1173700&r2=1173701&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSort.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSort.java Wed
Sep 21 15:07:04 2011
@@ -1248,4 +1248,46 @@ public class TestSort extends LuceneTest
     reader.close();
     indexStore.close();
   }
+  
+  /** LUCENE-3390: expose bug in first round of that issue */
+  public void testSimultaneousSorts() throws IOException {
+    Sort sortMin = new Sort(new SortField ("int", SortField.INT, false).setMissingValue(new
Integer(Integer.MIN_VALUE)));
+    Sort sortMax = new Sort(new SortField ("int", SortField.INT, false).setMissingValue(new
Integer(Integer.MAX_VALUE)));
+    Sort sortMinRev = new Sort(new SortField ("int", SortField.INT, true).setMissingValue(new
Integer(Integer.MIN_VALUE)));
+    Sort sortMaxRev = new Sort(new SortField ("int", SortField.INT, true).setMissingValue(new
Integer(Integer.MAX_VALUE)));
+    
+    int ndocs = full.maxDoc();
+    TopFieldCollector collectorMin = TopFieldCollector.create(sortMin, ndocs, false, false,
false, true);
+    TopFieldCollector collectorMax = TopFieldCollector.create(sortMax, ndocs, false, false,
false, true);
+    TopFieldCollector collectorMinRev = TopFieldCollector.create(sortMinRev, ndocs, false,
false, false, true);
+    TopFieldCollector collectorMaxRev = TopFieldCollector.create(sortMaxRev, ndocs, false,
false, false, true);
+    full.search(new MatchAllDocsQuery(), MultiCollector.wrap(collectorMin, collectorMax,
collectorMinRev, collectorMaxRev));
+    
+    assertIntResultsOrder(collectorMin, ndocs, false, Integer.MIN_VALUE);
+    assertIntResultsOrder(collectorMax, ndocs, false, Integer.MAX_VALUE);
+    assertIntResultsOrder(collectorMinRev, ndocs, true, Integer.MIN_VALUE);
+    assertIntResultsOrder(collectorMaxRev, ndocs, true, Integer.MAX_VALUE);
+  }
+
+  private void assertIntResultsOrder(TopFieldCollector collector, int ndocs, boolean reverse,
int missingVal) {
+    ScoreDoc[] fdocs = collector.topDocs().scoreDocs;
+    assertEquals("wrong number of docs collected", ndocs, fdocs.length);
+    int b = dataIntVal(fdocs[0].doc, missingVal);
+    for (int i=1; i<fdocs.length; i++) {
+      int a = b;
+      b = dataIntVal(fdocs[i].doc, missingVal);
+      if (reverse) {
+        // reverse of natural int order: descending
+        assertTrue(a >= b);
+      } else {
+        // natural int order: ascending
+        assertTrue( b >= a);
+      }
+    }
+  }
+
+  private int dataIntVal(int doc, int missingVal) {
+    return data[doc][2]==null ? missingVal : Integer.parseInt(data[doc][2]);
+  }
+
 }



Mime
View raw message