lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sim...@apache.org
Subject svn commit: r1131057 - in /lucene/dev/branches/docvalues/lucene/src: java/org/apache/lucene/index/values/ java/org/apache/lucene/search/ test-framework/org/apache/lucene/util/ test/org/apache/lucene/index/values/
Date Fri, 03 Jun 2011 14:33:19 GMT
Author: simonw
Date: Fri Jun  3 14:33:19 2011
New Revision: 1131057

URL: http://svn.apache.org/viewvc?rev=1131057&view=rev
Log:
LUCENE-3108: Cut over to consistent default values and added support for entire long range

Modified:
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java
    lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldComparator.java
    lucene/dev/branches/docvalues/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
    lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestDocValues.java
    lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java?rev=1131057&r1=1131056&r2=1131057&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java Fri Jun  3 14:33:19 2011
@@ -24,7 +24,6 @@ import java.util.Comparator;
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.values.DocValues.MissingValue;
 import org.apache.lucene.index.values.DocValues.SortedSource;
 import org.apache.lucene.index.values.DocValues.Source;
 import org.apache.lucene.index.values.DocValues.SourceEnum;
@@ -225,14 +224,13 @@ public final class Bytes {
 
     @Override
     public DocValuesEnum getEnum(AttributeSource attrSource) throws IOException {
-      final MissingValue missing = getMissing();
       return new SourceEnum(attrSource, type(), this, maxDoc()) {
         @Override
         public int advance(int target) throws IOException {
           if (target >= numDocs) {
             return pos = NO_MORE_DOCS;
           }
-          while (source.getBytes(target, bytesRef) == missing.bytesValue) {
+          while (source.getBytes(target, bytesRef).length == 0) {
             if (++target >= numDocs) {
               return pos = NO_MORE_DOCS;
             }
@@ -251,7 +249,6 @@ public final class Bytes {
     protected final static int PAGED_BYTES_BITS = 15;
     private final PagedBytes pagedBytes;
     protected final PagedBytes.Reader data;
-    protected final LookupResult lookupResult = new LookupResult();
     private final Comparator<BytesRef> comp;
 
     protected BytesBaseSortedSource(IndexInput datIn, IndexInput idxIn,
@@ -271,7 +268,8 @@ public final class Bytes {
 
     @Override
     public BytesRef getByOrd(int ord, BytesRef bytesRef) {
-      return ord == 0 ? null : deref(--ord, bytesRef);
+      assert ord >= 0;
+      return deref(ord, bytesRef);
     }
 
     protected void closeIndexInput() throws IOException {
@@ -297,10 +295,11 @@ public final class Bytes {
      */
     protected abstract BytesRef deref(int ord, BytesRef bytesRef);
 
-    protected LookupResult binarySearch(BytesRef b, BytesRef bytesRef, int low,
+    protected int binarySearch(BytesRef b, BytesRef bytesRef, int low,
         int high) {
+      int mid = 0;
       while (low <= high) {
-        int mid = (low + high) >>> 1;
+        mid = (low + high) >>> 1;
         deref(mid, bytesRef);
         final int cmp = comp.compare(bytesRef, b);
         if (cmp < 0) {
@@ -308,20 +307,15 @@ public final class Bytes {
         } else if (cmp > 0) {
           high = mid - 1;
         } else {
-          lookupResult.ord = mid + 1;
-          lookupResult.found = true;
-          return lookupResult;
+          return mid;
         }
       }
       assert comp.compare(bytesRef, b) != 0;
-      lookupResult.ord = low;
-      lookupResult.found = false;
-      return lookupResult;
+      return -(low + 1);
     }
 
     @Override
     public DocValuesEnum getEnum(AttributeSource attrSource) throws IOException {
-      final MissingValue missing = getMissing();
       return new SourceEnum(attrSource, type(), this, maxDoc()) {
 
         @Override
@@ -329,7 +323,7 @@ public final class Bytes {
           if (target >= numDocs) {
             return pos = NO_MORE_DOCS;
           }
-          while (source.getBytes(target, bytesRef) == missing.bytesValue) {
+          while (source.getBytes(target, bytesRef).length == 0) {
             if (++target >= numDocs) {
               return pos = NO_MORE_DOCS;
             }

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java?rev=1131057&r1=1131056&r2=1131057&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java Fri Jun  3 14:33:19 2011
@@ -169,8 +169,6 @@ public abstract class DocValues implemen
    * {@link Source} defines 3 {@link ValueType} //TODO finish this
    */
   public static abstract class Source {
-    // TODO we might need a close method here to null out the internal used arrays?!
-    protected final MissingValue missingValue = new MissingValue();
 
     /**
      * Returns a <tt>long</tt> for the given document id or throws an
@@ -179,8 +177,6 @@ public abstract class DocValues implemen
      * 
      * @throws UnsupportedOperationException
      *           if this source doesn't support <tt>long</tt> values.
-     * @see MissingValue
-     * @see #getMissing()
      */
     public long getInt(int docID) {
       throw new UnsupportedOperationException("ints are not supported");
@@ -193,8 +189,6 @@ public abstract class DocValues implemen
      * 
      * @throws UnsupportedOperationException
      *           if this source doesn't support <tt>double</tt> values.
-     * @see MissingValue
-     * @see #getMissing()
      */
     public double getFloat(int docID) {
       throw new UnsupportedOperationException("floats are not supported");
@@ -207,8 +201,6 @@ public abstract class DocValues implemen
      * 
      * @throws UnsupportedOperationException
      *           if this source doesn't support <tt>byte[]</tt> values.
-     * @see MissingValue
-     * @see #getMissing()
      */
     public BytesRef getBytes(int docID, BytesRef ref) {
       throw new UnsupportedOperationException("bytes are not supported");
@@ -230,17 +222,6 @@ public abstract class DocValues implemen
     }
 
     /**
-     * Returns a {@link MissingValue} instance for this {@link Source}.
-     * Depending on the type of this {@link Source} consumers of the API should
-     * check if the value returned from on of the getter methods represents a
-     * value for a missing document or rather a value for a document no value
-     * was specified during indexing.
-     */
-    public MissingValue getMissing() {
-      return missingValue;
-    }
-
-    /**
      * Returns the {@link ValueType} of this source.
      * 
      * @return the {@link ValueType} of this source.
@@ -310,7 +291,13 @@ public abstract class DocValues implemen
 
     @Override
     public BytesRef getBytes(int docID, BytesRef bytesRef) {
-      return getByOrd(ord(docID), bytesRef);
+      final int ord = ord(docID);
+      if (ord < 0) {
+        bytesRef.length = 0;
+      } else {
+        getByOrd(ord , bytesRef);
+      }
+      return bytesRef;
     }
 
     /**
@@ -323,22 +310,18 @@ public abstract class DocValues implemen
     /** Returns value for specified ord. */
     public abstract BytesRef getByOrd(int ord, BytesRef bytesRef);
 
-    public static class LookupResult {
-      /** <code>true</code> iff the values was found */
-      public boolean found;
-      /**
-       * the ordinal of the value if found or the ordinal of the value if it
-       * would be present in the source
-       */
-      public int ord;
-    }
 
     /**
-     * Finds the largest ord whose value is less or equal to the requested
-     * value. If {@link LookupResult#found} is true, then ord is an exact match.
-     * The returned {@link LookupResult} may be reused across calls.
+     * Finds the ordinal whose value is greater or equal to the given value.
+     * 
+     * @return the given values ordinal if found or otherwise
+     *         <code>(-(ord)-1)</code>, defined as the ordinal of the first
+     *         element that is greater than the given value. This guarantees
+     *         that the return value will always be &gt;= 0 if the given value
+     *         is found.
+     * 
      */
-    public final LookupResult getByValue(BytesRef value) {
+    public final int getByValue(BytesRef value) {
       return getByValue(value, new BytesRef());
     }
 
@@ -350,35 +333,12 @@ public abstract class DocValues implemen
      * @param tmpRef
      *          a temporary {@link BytesRef} instance used to compare internal
      *          values to the given value. Must not be <code>null</code>
-     * @return the {@link LookupResult}
+     * @return the given values ordinal if found or otherwise
+     *         <code>(-(ord)-1)</code>, defined as the ordinal of the first
+     *         element that is greater than the given value. This guarantees
+     *         that the return value will always be &gt;= 0 if the given value
+     *         is found.
      */
-    public abstract LookupResult getByValue(BytesRef value, BytesRef tmpRef);
+    public abstract int getByValue(BytesRef value, BytesRef tmpRef);
   }
-
-  /**
-   * {@link MissingValue} is used by {@link Source} implementations to define an
-   * Implementation dependent value for documents that had no value assigned
-   * during indexing. Its purpose is similar to a default value but since the a
-   * missing value across {@link ValueType} and its implementations can be highly
-   * dynamic the actual values are not constant but defined per {@link Source}
-   * through the {@link MissingValue} struct. The actual value used to indicate
-   * a missing value can even changed within the same field from one segment to
-   * another. Certain {@link Ints} implementations for instance use a value
-   * outside of value set as the missing value.
-   */
-  public final static class MissingValue {
-    public long longValue;
-    public double doubleValue;
-    public BytesRef bytesValue;
-
-    /**
-     * Copies the values from the given {@link MissingValue}.
-     */
-    public final void copy(MissingValue values) {
-      longValue = values.longValue;
-      doubleValue = values.doubleValue;
-      bytesValue = values.bytesValue;
-    }
-  }
-
 }

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java?rev=1131057&r1=1131056&r2=1131057&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java Fri Jun  3 14:33:19 2011
@@ -161,7 +161,8 @@ class FixedDerefBytesImpl {
       public BytesRef getBytes(int docID, BytesRef bytesRef) {
         final int id = (int) index.get(docID);
         if (id == 0) {
-          return null;
+          bytesRef.length = 0;
+          return bytesRef;
         }
         return data.fillSlice(bytesRef, ((id - 1) * size), size);
       }

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java?rev=1131057&r1=1131056&r2=1131057&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java Fri Jun  3 14:33:19 2011
@@ -199,11 +199,11 @@ class FixedSortedBytesImpl {
 
       @Override
       public int ord(int docID) {
-        return (int) index.get(docID);
+        return (int) index.get(docID) -1;
       }
 
       @Override
-      public LookupResult getByValue(BytesRef bytes, BytesRef tmpRef) {
+      public int getByValue(BytesRef bytes, BytesRef tmpRef) {
         return binarySearch(bytes, tmpRef, 0, numValue - 1);
       }
 

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java?rev=1131057&r1=1131056&r2=1131057&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java Fri Jun  3 14:33:19 2011
@@ -155,7 +155,6 @@ class FixedStraightBytesImpl {
           throws IOException {
         super(datIn, null, new PagedBytes(PAGED_BYTES_BITS), size * maxDoc);
         this.size = size;
-        this.missingValue.bytesValue = new BytesRef(size);
         this.maxDoc = maxDoc;
       }
 

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java?rev=1131057&r1=1131056&r2=1131057&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java Fri Jun  3 14:33:19 2011
@@ -17,7 +17,6 @@ package org.apache.lucene.index.values;
  * limitations under the License.
  */
 import java.io.IOException;
-import java.nio.ByteBuffer;
 import java.util.Collection;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -46,9 +45,9 @@ public class Floats {
   static final int VERSION_START = 0;
   static final int VERSION_CURRENT = VERSION_START;
   private static final int INT_DEFAULT = Float
-      .floatToRawIntBits(Float.NEGATIVE_INFINITY);
+      .floatToRawIntBits(0.0f);
   private static final long LONG_DEFAULT = Double
-      .doubleToRawLongBits(Double.NEGATIVE_INFINITY);
+      .doubleToRawLongBits(0.0d);
 
   
   public static Writer getWriter(Directory dir, String id, int precisionBytes,
@@ -298,7 +297,6 @@ public class Floats {
 
       Source4(final float[] values ) throws IOException {
         this.values = values;
-        missingValue.doubleValue = Float.NEGATIVE_INFINITY;
       }
 
       @Override
@@ -309,17 +307,11 @@ public class Floats {
       @Override
       public DocValuesEnum getEnum(AttributeSource attrSource)
           throws IOException {
-        final MissingValue missing = getMissing();
         return new SourceEnum(attrSource, ValueType.FLOAT_32, this, maxDoc) {
           @Override
           public int advance(int target) throws IOException {
             if (target >= numDocs)
               return pos = NO_MORE_DOCS;
-            while (missing.doubleValue == source.getFloat(target)) {
-              if (++target >= numDocs) {
-                return pos = NO_MORE_DOCS;
-              }
-            }
             floatsRef.floats[floatsRef.offset] = source.getFloat(target);
             return pos = target;
           }
@@ -337,7 +329,6 @@ public class Floats {
 
       Source8(final double[] values) throws IOException {
         this.values = values;
-        missingValue.doubleValue = Double.NEGATIVE_INFINITY;
       }
 
       @Override
@@ -348,17 +339,11 @@ public class Floats {
       @Override
       public DocValuesEnum getEnum(AttributeSource attrSource)
           throws IOException {
-        final MissingValue missing = getMissing();
         return new SourceEnum(attrSource, type(), this, maxDoc) {
           @Override
           public int advance(int target) throws IOException {
             if (target >= numDocs)
               return pos = NO_MORE_DOCS;
-            while (missing.doubleValue == source.getFloat(target)) {
-              if (++target >= numDocs) {
-                return pos = NO_MORE_DOCS;
-              }
-            }
             floatsRef.floats[floatsRef.offset] = source.getFloat(target);
             return pos = target;
           }
@@ -406,11 +391,7 @@ public class Floats {
       if (target >= maxDoc)
         return pos = NO_MORE_DOCS;
       dataIn.seek(fp + (target * precision));
-      int intBits;
-      while ((intBits = dataIn.readInt()) == INT_DEFAULT) {
-        if (++target >= maxDoc)
-          return pos = NO_MORE_DOCS;
-      }
+      final int intBits = dataIn.readInt();
       floatsRef.floats[0] = Float.intBitsToFloat(intBits);
       floatsRef.offset = 0;
       return pos = target;
@@ -443,13 +424,8 @@ public class Floats {
         return pos = NO_MORE_DOCS;
       }
       dataIn.seek(fp + (target * precision));
-      long value;
-      while ((value = dataIn.readLong()) == LONG_DEFAULT) {
-        if (++target >= maxDoc)
-          return pos = NO_MORE_DOCS;
-      }
-      floatsRef.floats[0] = Double.longBitsToDouble(value);
-      floatsRef.offset = 0;
+      final long value = dataIn.readLong();
+      floatsRef.floats[floatsRef.offset] = Double.longBitsToDouble(value);
       return pos = target;
     }
 

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java?rev=1131057&r1=1131056&r2=1131057&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java Fri Jun  3 14:33:19 2011
@@ -200,7 +200,6 @@ public class MultiDocValues extends DocV
         assert docValuesIdx[idx] != null;
         try {
           current = docValuesIdx[idx].docValues.getSource();
-          missingValue.copy(current.getMissing());
         } catch (IOException e) {
           throw new RuntimeException("load failed", e); // TODO how should we
           // handle this
@@ -243,18 +242,19 @@ public class MultiDocValues extends DocV
 
     @Override
     public BytesRef getBytes(int docID, BytesRef ref) {
-      return this.missingValue.bytesValue;
+      ref.length = 0;
+      return ref;
 
     }
 
     @Override
     public double getFloat(int docID) {
-      return missingValue.doubleValue;
+      return 0d;
     }
 
     @Override
     public long getInt(int docID) {
-      return missingValue.longValue;
+      return 0;
     }
 
     @Override

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java?rev=1131057&r1=1131056&r2=1131057&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java Fri Jun  3 14:33:19 2011
@@ -29,7 +29,6 @@ import org.apache.lucene.util.AttributeS
 import org.apache.lucene.util.CodecUtil;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LongsRef;
-import org.apache.lucene.util.OpenBitSet;
 import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util.packed.PackedInts;
 
@@ -40,7 +39,9 @@ import org.apache.lucene.util.packed.Pac
  * */
 class PackedIntsImpl {
 
-  private static final String CODEC_NAME = "PackedInts";
+  private static final String CODEC_NAME = "Ints";
+  private static final byte PACKED = 0x00;
+  private static final byte FIXED = 0x01;
 
   static final int VERSION_START = 0;
   static final int VERSION_CURRENT = VERSION_START;
@@ -54,7 +55,6 @@ class PackedIntsImpl {
     private long maxValue;
     private boolean started;
     private final String id;
-    private final OpenBitSet defaultValues = new OpenBitSet(1);
     private int lastDocId = -1;
     private IndexOutput datOut;
 
@@ -93,13 +93,11 @@ class PackedIntsImpl {
           maxValue = v;
         }
       }
-      defaultValues.set(docID);
       lastDocId = docID;
 
       if (docID >= docToValue.length) {
         final long len = docToValue.length;
         docToValue = ArrayUtil.grow(docToValue, 1 + docID);
-        defaultValues.ensureCapacity(docToValue.length);
         bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_LONG
             * ((docToValue.length) - len));
       }
@@ -112,40 +110,51 @@ class PackedIntsImpl {
         if (!started) {
           minValue = maxValue = 0;
         }
-        // TODO -- long can't work right since it's signed
-        datOut.writeLong(minValue);
-        // write a default value to recognize docs without a value for that
-        // field
-        final long defaultValue = ++maxValue - minValue;
-        datOut.writeLong(defaultValue);
-        PackedInts.Writer w = PackedInts.getWriter(datOut, docCount,
-            PackedInts.bitsRequired(maxValue - minValue));
-        final int firstDoc = defaultValues.nextSetBit(0);
-        lastDocId++;
-        if (firstDoc != -1) { 
-          for (int i = 0; i < firstDoc; i++) {
-            w.add(defaultValue); // fill with defaults until first bit set
-          }
-
-          for (int i = firstDoc; i < lastDocId;) {
-            w.add(docToValue[i] - minValue);
-            final int nextValue = defaultValues.nextSetBit(++i);
-            for (; i < nextValue; i++) {
-              w.add(defaultValue); // fill all gaps
-            }
-          }
+        // if we exceed the range of positive longs we must switch to fixed ints
+        if ((maxValue - minValue) < (((long)1) << 63) && (maxValue - minValue) > 0) {
+          writePackedInts(docCount);
+        } else {
+          writeFixedInts(docCount);
         }
-        for (int i = lastDocId; i < docCount; i++) {
-          w.add(defaultValue);
-        }
-        w.finish();
+
       } finally {
         datOut.close();
         bytesUsed
             .addAndGet(-(RamUsageEstimator.NUM_BYTES_LONG * docToValue.length));
         docToValue = null;
       }
+    }
 
+    private void writeFixedInts(int docCount) throws IOException {
+      datOut.writeByte(FIXED);
+      datOut.writeInt(docCount);
+      for (int i = 0; i < docToValue.length; i++) {
+        datOut.writeLong(docToValue[i]); // write full array - we use 0 as default
+      }
+      for (int i = docToValue.length; i < docCount; i++) {
+        datOut.writeLong(0); // fill with defaults values
+      }
+    }
+
+    private void writePackedInts(int docCount) throws IOException {
+      datOut.writeByte(PACKED);
+      // TODO -- long can't work right since it's signed
+      datOut.writeLong(minValue);
+      // write a default value to recognize docs without a value for that
+      // field
+      final long defaultValue = maxValue>= 0 && minValue <=0 ? 0-minValue : ++maxValue-minValue;
+      datOut.writeLong(defaultValue);
+      PackedInts.Writer w = PackedInts.getWriter(datOut, docCount,
+          PackedInts.bitsRequired(maxValue-minValue));
+      final int limit = docToValue.length > docCount ? docCount : docToValue.length;
+      for (int i = 0; i < limit; i++) {
+        w.add(docToValue[i] == 0 ? defaultValue : docToValue[i] - minValue);
+      }
+      for (int i = limit; i < docCount; i++) {
+        w.add(defaultValue);
+      }
+      
+      w.finish();
     }
 
     @Override
@@ -175,6 +184,7 @@ class PackedIntsImpl {
    */
   static class IntsReader extends DocValues {
     private final IndexInput datIn;
+    private final boolean packed;
 
     protected IntsReader(Directory dir, String id) throws IOException {
       datIn = dir.openInput(IndexFileNames.segmentFileName(id, "",
@@ -182,6 +192,7 @@ class PackedIntsImpl {
       boolean success = false;
       try {
         CodecUtil.checkHeader(datIn, CODEC_NAME, VERSION_START, VERSION_START);
+        packed = PACKED == datIn.readByte();
         success = true;
       } finally {
         if (!success) {
@@ -196,20 +207,69 @@ class PackedIntsImpl {
      */
     @Override
     public Source load() throws IOException {
-      return new IntsSource((IndexInput) datIn.clone());
+      final IndexInput input = (IndexInput) datIn.clone();
+      boolean success = false;
+      try {
+        final Source source = packed ? new PackedIntsSource(input)
+            : new FixedIntsSource(input);
+        success = true;
+        return source;
+      } finally {
+        if (!success) {
+          IOUtils.closeSafely(true, datIn);
+        }
+      }
     }
+    
+    private static class FixedIntsSource extends Source {
+      private final long[] values;
+      public FixedIntsSource(IndexInput dataIn) throws IOException {
+        dataIn.seek(CodecUtil.headerLength(CODEC_NAME) + 1);
+        final int numDocs = dataIn.readInt();
+        values = new long[numDocs];
+        for (int i = 0; i < values.length; i++) {
+          values[i] = dataIn.readLong();
+        }
+      }
+      
+      @Override
+      public long getInt(int docID) {
+        assert docID >= 0 && docID < values.length;
+        return values[docID];
+      }
 
-    private static class IntsSource extends Source {
+      @Override
+      public ValueType type() {
+        return ValueType.INTS;
+      }
+
+      @Override
+      public DocValuesEnum getEnum(AttributeSource attrSource)
+          throws IOException {
+        return new SourceEnum(attrSource, type(), this, values.length) {
+          
+          @Override
+          public int advance(int target) throws IOException {
+            if (target >= numDocs)
+              return pos = NO_MORE_DOCS;
+            intsRef.ints[intsRef.offset] = values[target];
+            return pos = target;
+          }
+        };
+      }
+      
+    }
+
+    private static class PackedIntsSource extends Source {
       private final long minValue;
       private final long defaultValue;
       private final PackedInts.Reader values;
 
-      public IntsSource(IndexInput dataIn) throws IOException {
-        dataIn.seek(CodecUtil.headerLength(CODEC_NAME));
+      public PackedIntsSource(IndexInput dataIn) throws IOException {
+        dataIn.seek(CodecUtil.headerLength(CODEC_NAME) + 1);
         minValue = dataIn.readLong();
         defaultValue = dataIn.readLong();
         values = PackedInts.getReader(dataIn);
-        missingValue.longValue = minValue + defaultValue;
       }
 
       @Override
@@ -218,23 +278,18 @@ class PackedIntsImpl {
         // on each get? must push minValue down, and make
         // PackedInts implement Ints.Source
         assert docID >= 0;
-        return minValue + values.get(docID);
+        final long value = values.get(docID);
+        return value == defaultValue ? 0 : minValue + value;
       }
 
       @Override
       public DocValuesEnum getEnum(AttributeSource attrSource)
           throws IOException {
-        final MissingValue missing = getMissing();
         return new SourceEnum(attrSource, type(), this, values.size()) {
           @Override
           public int advance(int target) throws IOException {
             if (target >= numDocs)
               return pos = NO_MORE_DOCS;
-            while (source.getInt(target) == missing.longValue) {
-              if (++target >= numDocs) {
-                return pos = NO_MORE_DOCS;
-              }
-            }
             intsRef.ints[intsRef.offset] = source.getInt(target);
             return pos = target;
           }
@@ -255,7 +310,18 @@ class PackedIntsImpl {
 
     @Override
     public DocValuesEnum getEnum(AttributeSource source) throws IOException {
-      return new IntsEnumImpl(source, (IndexInput) datIn.clone());
+      final IndexInput input = (IndexInput) datIn.clone();
+      boolean success = false;
+      try {
+        DocValuesEnum inst = packed ? new PackedIntsEnumImpl(source, input)
+            : new FixedIntsEnumImpl(source, input);
+        success = true;
+        return inst;
+      } finally {
+        if (!success) {
+          IOUtils.closeSafely(true, input);
+        }
+      }
     }
 
     @Override
@@ -265,7 +331,7 @@ class PackedIntsImpl {
 
   }
 
-  private static final class IntsEnumImpl extends DocValuesEnum {
+  private static final class PackedIntsEnumImpl extends DocValuesEnum {
     private final PackedInts.ReaderIterator ints;
     private long minValue;
     private final IndexInput dataIn;
@@ -273,12 +339,12 @@ class PackedIntsImpl {
     private final int maxDoc;
     private int pos = -1;
 
-    private IntsEnumImpl(AttributeSource source, IndexInput dataIn)
+    private PackedIntsEnumImpl(AttributeSource source, IndexInput dataIn)
         throws IOException {
       super(source, ValueType.INTS);
       intsRef.offset = 0;
       this.dataIn = dataIn;
-      dataIn.seek(CodecUtil.headerLength(CODEC_NAME));
+      dataIn.seek(CodecUtil.headerLength(CODEC_NAME) + 1);
       minValue = dataIn.readLong();
       defaultValue = dataIn.readLong();
       this.ints = PackedInts.getReaderIterator(dataIn);
@@ -296,15 +362,54 @@ class PackedIntsImpl {
       if (target >= maxDoc) {
         return pos = NO_MORE_DOCS;
       }
-      long val = ints.advance(target);
-      while (val == defaultValue) {
-        if (++target >= maxDoc) {
-          return pos = NO_MORE_DOCS;
-        }
-        val = ints.advance(target);
+      final long val = ints.advance(target);
+      intsRef.ints[intsRef.offset] = val == defaultValue ? 0 : minValue + val;
+      return pos = target;
+    }
+
+    @Override
+    public int docID() {
+      return pos;
+    }
+
+    @Override
+    public int nextDoc() throws IOException {
+      if (pos >= maxDoc) {
+        return pos = NO_MORE_DOCS;
+      }
+      return advance(pos + 1);
+    }
+  }
+  
+  private static final class FixedIntsEnumImpl extends DocValuesEnum {
+    private final IndexInput dataIn;
+    private final int maxDoc;
+    private int pos = -1;
+
+    private FixedIntsEnumImpl(AttributeSource source, IndexInput dataIn)
+        throws IOException {
+      super(source, ValueType.INTS);
+      intsRef.offset = 0;
+      this.dataIn = dataIn;
+      dataIn.seek(CodecUtil.headerLength(CODEC_NAME) + 1);
+      maxDoc = dataIn.readInt();
+    }
+
+    @Override
+    public void close() throws IOException {
+      dataIn.close();
+    }
+
+    @Override
+    public int advance(int target) throws IOException {
+      if (target >= maxDoc) {
+        return pos = NO_MORE_DOCS;
+      }
+      assert target > pos;
+      if (target > pos+1) {
+        dataIn.seek(dataIn.getFilePointer() + ((target - pos - 1) * 8));
       }
-      intsRef.ints[0] = minValue + val;
-      intsRef.offset = 0; // can we skip this?
+      intsRef.ints[intsRef.offset] = dataIn.readLong();
       return pos = target;
     }
 
@@ -321,4 +426,5 @@ class PackedIntsImpl {
       return advance(pos + 1);
     }
   }
+ 
 }
\ No newline at end of file

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java?rev=1131057&r1=1131056&r2=1131057&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java Fri Jun  3 14:33:19 2011
@@ -227,7 +227,8 @@ class VarDerefBytesImpl {
       @Override
       public BytesRef getBytes(int docID, BytesRef bytesRef) {
         long address = index.get(docID);
-        return address == 0 ? null : data.fillSliceWithPrefix(bytesRef,
+        bytesRef.length = 0;
+        return address == 0 ? bytesRef : data.fillSliceWithPrefix(bytesRef,
             --address);
       }
 

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java?rev=1131057&r1=1131056&r2=1131057&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java Fri Jun  3 14:33:19 2011
@@ -191,11 +191,11 @@ class VarSortedBytesImpl {
 
       @Override
       public int ord(int docID) {
-        return (int) docToOrdIndex.get(docID);
+        return (int) docToOrdIndex.get(docID) - 1;
       }
 
       @Override
-      public LookupResult getByValue(BytesRef bytes, BytesRef tmpRef) {
+      public int getByValue(BytesRef bytes, BytesRef tmpRef) {
         return binarySearch(bytes, tmpRef, 0, valueCount - 1);
       }
 

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java?rev=1131057&r1=1131056&r2=1131057&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java Fri Jun  3 14:33:19 2011
@@ -137,7 +137,6 @@ class VarStraightBytesImpl {
       public Source(IndexInput datIn, IndexInput idxIn) throws IOException {
         super(datIn, idxIn, new PagedBytes(PAGED_BYTES_BITS), idxIn.readVLong());
         addresses = PackedInts.getReader(idxIn);
-        missingValue.bytesValue = new BytesRef(0); // empty
       }
 
       @Override

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldComparator.java?rev=1131057&r1=1131056&r2=1131057&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldComparator.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldComparator.java Fri Jun  3 14:33:19 2011
@@ -21,7 +21,6 @@ import java.io.IOException;
 
 import org.apache.lucene.index.IndexReader.AtomicReaderContext;
 import org.apache.lucene.index.values.DocValues;
-import org.apache.lucene.index.values.DocValues.MissingValue;
 import org.apache.lucene.index.values.DocValues.Source;
 import org.apache.lucene.search.FieldCache.DocTerms;
 import org.apache.lucene.search.FieldCache.DocTermsIndex;
@@ -337,7 +336,6 @@ public abstract class FieldComparator {
     private final String field;
     private double bottom;
     private final float missingValue;
-    private MissingValue missing;
 
     FloatDocValuesComparator(int numHits, String field, Float missingValue) {
       values = new double[numHits];
@@ -372,8 +370,7 @@ public abstract class FieldComparator {
 
     @Override
     public void copy(int slot, int doc) {
-      final double value = currentReaderValues.getFloat(doc);
-      values[slot] = value == missing.doubleValue ? missingValue : value; 
+      values[slot] = currentReaderValues.getFloat(doc); 
     }
 
     @Override
@@ -381,7 +378,6 @@ public abstract class FieldComparator {
       final DocValues docValues = context.reader.docValues(field);
       if (docValues != null) {
         currentReaderValues = docValues.getSource(); 
-        missing = currentReaderValues.getMissing();
       }
       return this;
     }
@@ -612,7 +608,6 @@ public abstract class FieldComparator {
     private final String field;
     private long bottom;
     private int missingValue;
-    private MissingValue missing;
 
     IntDocValuesComparator(int numHits, String field, Integer missingValue) {
       values = new long[numHits];
@@ -651,8 +646,7 @@ public abstract class FieldComparator {
 
     @Override
     public void copy(int slot, int doc) {
-      final long value = currentReaderValues.getInt(doc);
-      values[slot] = value == missing.longValue ? missingValue : value; 
+      values[slot] = currentReaderValues.getInt(doc); 
     }
 
     @Override
@@ -660,7 +654,6 @@ public abstract class FieldComparator {
       DocValues docValues = context.reader.docValues(field);
       if (docValues != null) {
         currentReaderValues = docValues.getSource();
-        missing = currentReaderValues.getMissing();
       }
       return this;
     }

Modified: lucene/dev/branches/docvalues/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java?rev=1131057&r1=1131056&r2=1131057&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java Fri Jun  3 14:33:19 2011
@@ -287,35 +287,46 @@ public class _TestUtil {
     return sb.toString();
   }
   
-  public static String randomUnicodeString(Random r, int minLength, int maxLength) {
-    if(minLength > maxLength)
-      throw new IllegalArgumentException("minLength must be >= maxLength");
-    final boolean lenEqual = minLength==maxLength;
-    final int end = lenEqual?minLength:minLength + r.nextInt(maxLength-minLength+1);
-    if (end == 0) {
-      // allow 0 length
-      return "";
-    }
+  /** Returns random string, with a given UTF-8 byte length*/
+  public static String randomFixedByteLengthUnicodeString(Random r, int length) {
     
-    // TODO(simonw): check this
-    final int fixedPlane = 5;//minLength % 5;
-    final char[] buffer = new char[end];
-    for (int i = 0; i < end; i++) {
-      int t = lenEqual? fixedPlane: r.nextInt(5);
-      //buffer[i] = (char) (97 + r.nextInt(26));
-      if (0 == t && i < end - 1 && !lenEqual) {
+    final char[] buffer = new char[length*3];
+    int bytes = length;
+    int i = 0;
+    for (; i < buffer.length && bytes != 0; i++) {
+      int t;
+      if (bytes >= 4) {
+        t = r.nextInt(5);
+      } else if (bytes >= 3) {
+        t = r.nextInt(4);
+      } else if (bytes >= 2) {
+        t = r.nextInt(2);
+      } else {
+        t = 0;
+      }
+      if (t == 0) {
+        buffer[i] = (char) r.nextInt(0x80);
+        bytes--;
+      } else if (1 == t) {
+        buffer[i] = (char) nextInt(r, 0x80, 0x7ff);
+        bytes -= 2;
+      } else if (2 == t) {
+        buffer[i] = (char) nextInt(r, 0x800, 0xd7ff);
+        bytes -= 3;
+      } else if (3 == t) {
+        buffer[i] = (char) nextInt(r, 0xe000, 0xffff);
+        bytes -= 3;
+      } else if (4 == t) {
         // Make a surrogate pair
         // High surrogate
         buffer[i++] = (char) nextInt(r, 0xd800, 0xdbff);
         // Low surrogate
         buffer[i] = (char) nextInt(r, 0xdc00, 0xdfff);
+        bytes -= 4;
       }
-      else if (t <= 1) buffer[i] = (char) r.nextInt(0x80);
-      else if (2 == t) buffer[i] = (char) nextInt(r, 0x80, 0x800);
-      else if (3 == t) buffer[i] = (char) nextInt(r, 0x800, 0xd7ff);
-      else if (4 == t) buffer[i] = (char) nextInt(r, 0xe000, 0xffff);
+
     }
-    return new String(buffer, 0, end);
+    return new String(buffer, 0, i);
   }
 
   public static CodecProvider alwaysCodec(final Codec c) {

Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestDocValues.java?rev=1131057&r1=1131056&r2=1131057&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestDocValues.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestDocValues.java Fri Jun  3 14:33:19 2011
@@ -50,7 +50,7 @@ public class TestDocValues extends Lucen
     runTestBytes(Bytes.Mode.SORTED, true);
     runTestBytes(Bytes.Mode.SORTED, false);
   }
-
+  
   public void runTestBytes(final Bytes.Mode mode, final boolean fixedSize)
       throws IOException {
 
@@ -64,20 +64,14 @@ public class TestDocValues extends Lucen
     Writer w = Bytes.getWriter(dir, "test", mode, comp, fixedSize, trackBytes);
     int maxDoc = 220;
     final String[] values = new String[maxDoc];
-    final int lenMin, lenMax;
-    if (fixedSize) {
-      lenMin = lenMax = 3 + random.nextInt(7);
-    } else {
-      lenMin = 1;
-      lenMax = 15 + random.nextInt(6);
-    }
+    final int fixedLength = 3 + random.nextInt(7);
     for (int i = 0; i < 100; i++) {
       final String s;
       if (i > 0 && random.nextInt(5) <= 2) {
         // use prior value
         s = values[2 * random.nextInt(i)];
       } else {
-        s = _TestUtil.randomUnicodeString(random, lenMin, lenMax);
+        s = _TestUtil.randomFixedByteLengthUnicodeString(random, fixedSize? fixedLength : 1 + random.nextInt(39));
       }
       values[2 * i] = s;
 
@@ -89,7 +83,7 @@ public class TestDocValues extends Lucen
 
     DocValues r = Bytes.getValues(dir, "test", mode, fixedSize, maxDoc);
     for (int iter = 0; iter < 2; iter++) {
-      DocValuesEnum bytesEnum = r.getEnum();
+      DocValuesEnum bytesEnum = getEnum(r);
       assertNotNull("enum is null", bytesEnum);
       BytesRef ref = bytesEnum.bytes();
 
@@ -125,10 +119,10 @@ public class TestDocValues extends Lucen
         if (ss != null) {
           assertEquals("doc " + idx, values[idx], ss.getByOrd(ss.ord(idx),
               bytesRef).utf8ToString());
-          DocValues.SortedSource.LookupResult result = ss
+         int ord = ss
               .getByValue(new BytesRef(values[idx]));
-          assertTrue(result.found);
-          assertEquals(ss.ord(idx), result.ord);
+          assertTrue(ord >= 0);
+          assertEquals(ss.ord(idx), ord);
         }
       }
 
@@ -136,39 +130,36 @@ public class TestDocValues extends Lucen
       if (mode == Bytes.Mode.SORTED) {
         final int numValues = ss.getValueCount();
         for (int i = 0; i < 1000; i++) {
-          BytesRef bytesValue = new BytesRef(_TestUtil.randomUnicodeString(
-              random, lenMin, lenMax));
-          SortedSource.LookupResult result = ss.getByValue(bytesValue);
-          if (result.found) {
-            assert result.ord > 0;
+          BytesRef bytesValue = new BytesRef(_TestUtil.randomFixedByteLengthUnicodeString(random, fixedSize? fixedLength : 1 + random.nextInt(39)));
+          int ord = ss.getByValue(bytesValue);
+          if (ord >= 0) {
             assertTrue(bytesValue
-                .bytesEquals(ss.getByOrd(result.ord, bytesRef)));
+                .bytesEquals(ss.getByOrd(ord, bytesRef)));
             int count = 0;
             for (int k = 0; k < 100; k++) {
               if (bytesValue.utf8ToString().equals(values[2 * k])) {
-                assertEquals(ss.ord(2 * k), result.ord);
+                assertEquals(ss.ord(2 * k), ord);
                 count++;
               }
             }
             assertTrue(count > 0);
           } else {
-            assert result.ord >= 0;
-            if (result.ord == 0) {
+            assert ord < 0;
+            int insertIndex = (-ord)-1;
+            if (insertIndex == 0) {
               final BytesRef firstRef = ss.getByOrd(1, bytesRef);
               // random string was before our first
               assertTrue(firstRef.compareTo(bytesValue) > 0);
-            } else if (result.ord == numValues) {
-              final BytesRef lastRef = ss.getByOrd(numValues, bytesRef);
+            } else if (insertIndex == numValues) {
+              final BytesRef lastRef = ss.getByOrd(numValues-1, bytesRef);
               // random string was after our last
               assertTrue(lastRef.compareTo(bytesValue) < 0);
             } else {
-              // random string fell between two of our values
-              final BytesRef before = (BytesRef) ss.getByOrd(result.ord,
-                  bytesRef).clone();
-              final BytesRef after = ss.getByOrd(result.ord + 1, bytesRef);
-              assertTrue(before.compareTo(bytesValue) < 0);
-              assertTrue(bytesValue.compareTo(after) < 0);
-
+              final BytesRef before = (BytesRef) ss.getByOrd(insertIndex-1, bytesRef)
+              .clone();
+              BytesRef after = ss.getByOrd(insertIndex, bytesRef);
+              assertTrue(comp.compare(before, bytesValue) < 0);
+              assertTrue(comp.compare(bytesValue, after) < 0);
             }
           }
         }
@@ -180,67 +171,75 @@ public class TestDocValues extends Lucen
   }
 
   public void testInts() throws IOException {
-    long maxV = 1;
-    final int NUM_VALUES = 777 + random.nextInt(777);
-    final long[] values = new long[NUM_VALUES];
-    for (int rx = 1; rx < 63; rx++, maxV *= 2) {
-      Directory dir = newDirectory();
-      final AtomicLong trackBytes = new AtomicLong(0);
-      Writer w = Ints.getWriter(dir, "test", false, trackBytes);
-      for (int i = 0; i < NUM_VALUES; i++) {
-        final long v = random.nextLong() % (1 + maxV);
-        values[i] = v;
-        w.add(i, v);
-      }
-      final int additionalDocs = 1 + random.nextInt(9);
-      w.finish(NUM_VALUES + additionalDocs);
-      assertEquals(0, trackBytes.get());
-
-
-      DocValues r = Ints.getValues(dir, "test", false);
-      for (int iter = 0; iter < 2; iter++) {
-        Source s = getSource(r);
-        for (int i = 0; i < NUM_VALUES; i++) {
-          final long v = s.getInt(i);
-          assertEquals("index " + i, values[i], v);
+    long[] maxMin = new long[] { 
+        Long.MIN_VALUE, Long.MAX_VALUE,
+        1, Long.MAX_VALUE,
+        0, Long.MAX_VALUE,
+        -1, Long.MAX_VALUE,
+        Long.MIN_VALUE, -1,
+        random.nextInt(), random.nextInt() };
+    for (int j = 0; j < maxMin.length; j+=2) {
+      long maxV = 1;
+      final int NUM_VALUES = 777 + random.nextInt(777);
+      final long[] values = new long[NUM_VALUES];
+      for (int rx = 1; rx < 63; rx++, maxV *= 2) {
+        Directory dir = newDirectory();
+        final AtomicLong trackBytes = new AtomicLong(0);
+        Writer w = Ints.getWriter(dir, "test", false, trackBytes);
+        values[0] = maxMin[j];
+        w.add(0, values[0]);
+        values[1] = maxMin[j+1];
+        w.add(1, values[1]);
+        for (int i = 2; i < NUM_VALUES; i++) {
+          final long v = random.nextLong() % (1 + maxV);
+          values[i] = v;
+          w.add(i, v);
         }
-      }
-
-      for (int iter = 0; iter < 2; iter++) {
-        DocValuesEnum iEnum = r.getEnum();
-        LongsRef ints = iEnum.getInt();
-        for (int i = 0; i < NUM_VALUES; i++) {
-          assertEquals(i, iEnum.nextDoc());
-          assertEquals(values[i], ints.get());
-        }
-        if (iEnum.docID() < NUM_VALUES - 1) {
-          assertEquals(NUM_VALUES - 1, iEnum.advance(NUM_VALUES - 1));
-        }
-        for (int i = NUM_VALUES; i < NUM_VALUES + additionalDocs; i++) {
-          assertEquals(DocValuesEnum.NO_MORE_DOCS, iEnum.nextDoc());
+        final int additionalDocs = 1 + random.nextInt(9);
+        w.finish(NUM_VALUES + additionalDocs);
+        assertEquals(0, trackBytes.get());
+
+        DocValues r = Ints.getValues(dir, "test", false);
+        for (int iter = 0; iter < 2; iter++) {
+          Source s = getSource(r);
+          for (int i = 0; i < NUM_VALUES; i++) {
+            final long v = s.getInt(i);
+            assertEquals("index " + i, values[i], v);
+          }
         }
 
-        iEnum.close();
-      }
-
-      for (int iter = 0; iter < 2; iter++) {
-        DocValuesEnum iEnum = r.getEnum();
-        LongsRef ints = iEnum.getInt();
-        for (int i = 0; i < NUM_VALUES; i += 1 + random.nextInt(25)) {
-          assertEquals(i, iEnum.advance(i));
-          assertEquals(values[i], ints.get());
-        }
-        if (iEnum.docID() < NUM_VALUES - 1) {
-          assertEquals(NUM_VALUES - 1, iEnum.advance(NUM_VALUES - 1));
-        }
-        for (int i = NUM_VALUES; i < NUM_VALUES + additionalDocs; i++) {
+        for (int iter = 0; iter < 2; iter++) {
+          DocValuesEnum iEnum = getEnum(r);
+          LongsRef ints = iEnum.getInt();
+          for (int i = 0; i < NUM_VALUES + additionalDocs; i++) {
+            assertEquals(i, iEnum.nextDoc());
+            if (i < NUM_VALUES) {
+              assertEquals(values[i], ints.get());
+            } else {
+              assertEquals(0, ints.get());
+            }
+          }
           assertEquals(DocValuesEnum.NO_MORE_DOCS, iEnum.nextDoc());
+          iEnum.close();
         }
 
-        iEnum.close();
+        for (int iter = 0; iter < 2; iter++) {
+          DocValuesEnum iEnum = getEnum(r);
+          LongsRef ints = iEnum.getInt();
+          for (int i = 0; i < NUM_VALUES + additionalDocs; i += 1 + random.nextInt(25)) {
+            assertEquals(i, iEnum.advance(i));
+            if (i < NUM_VALUES) {
+              assertEquals(values[i], ints.get());
+            } else {
+              assertEquals(0, ints.get());
+            }
+          }
+          assertEquals(DocValuesEnum.NO_MORE_DOCS, iEnum.advance(NUM_VALUES + additionalDocs));
+          iEnum.close();
+        }
+        r.close();
+        dir.close();
       }
-      r.close();
-      dir.close();
     }
   }
 
@@ -273,27 +272,31 @@ public class TestDocValues extends Lucen
     }
 
     for (int iter = 0; iter < 2; iter++) {
-      DocValuesEnum fEnum = r.getEnum();
+      DocValuesEnum fEnum = getEnum(r);
       FloatsRef floats = fEnum.getFloat();
-      for (int i = 0; i < NUM_VALUES; i++) {
+      for (int i = 0; i < NUM_VALUES + additionalValues; i++) {
         assertEquals(i, fEnum.nextDoc());
-        assertEquals(values[i], floats.get(), delta);
-      }
-      for (int i = NUM_VALUES; i < NUM_VALUES + additionalValues; i++) {
-        assertEquals(DocValuesEnum.NO_MORE_DOCS, fEnum.nextDoc());
+        if (i < NUM_VALUES) {
+          assertEquals(values[i], floats.get(), delta);
+        } else {
+          assertEquals(0.0d, floats.get(), delta);
+        }
       }
+      assertEquals(DocValuesEnum.NO_MORE_DOCS, fEnum.nextDoc());
       fEnum.close();
     }
     for (int iter = 0; iter < 2; iter++) {
-      DocValuesEnum fEnum = r.getEnum();
+      DocValuesEnum fEnum = getEnum(r);
       FloatsRef floats = fEnum.getFloat();
-      for (int i = 0; i < NUM_VALUES; i += 1 + random.nextInt(25)) {
+      for (int i = 0; i < NUM_VALUES + additionalValues; i += 1 + random.nextInt(25)) {
         assertEquals(i, fEnum.advance(i));
-        assertEquals(values[i], floats.get(), delta);
-      }
-      for (int i = NUM_VALUES; i < NUM_VALUES + additionalValues; i++) {
-        assertEquals(DocValuesEnum.NO_MORE_DOCS, fEnum.advance(i));
+        if (i < NUM_VALUES) {
+          assertEquals(values[i], floats.get(), delta);
+        } else {
+          assertEquals(0.0d, floats.get(), delta);
+        }
       }
+      assertEquals(DocValuesEnum.NO_MORE_DOCS, fEnum.advance(NUM_VALUES + additionalValues));
       fEnum.close();
     }
 
@@ -304,6 +307,10 @@ public class TestDocValues extends Lucen
   public void testFloats8() throws IOException {
     runTestFloats(8, 0.0);
   }
+  
+  private DocValuesEnum getEnum(DocValues values) throws IOException {
+    return random.nextBoolean() ? values.getEnum() : getSource(values).getEnum();
+  }
 
   private Source getSource(DocValues values) throws IOException {
     // getSource uses cache internally

Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java?rev=1131057&r1=1131056&r2=1131057&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java Fri Jun  3 14:33:19 2011
@@ -41,7 +41,6 @@ import org.apache.lucene.index.MultiPerD
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.index.codecs.PerDocValues;
-import org.apache.lucene.index.values.DocValues.MissingValue;
 import org.apache.lucene.index.values.DocValues.Source;
 import org.apache.lucene.queryParser.ParseException;
 import org.apache.lucene.queryParser.QueryParser;
@@ -197,16 +196,22 @@ public class TestDocValuesIndexing exten
     assertEquals(valuesPerIndex * 2, w.maxDoc());
 
     // check values
-
+    
     IndexReader merged = IndexReader.open(w, true);
     DocValuesEnum vE_1 = getValuesEnum(getDocValues(r_1, first.name()));
     DocValuesEnum vE_2 = getValuesEnum(getDocValues(r_2, second.name()));
     DocValuesEnum vE_1_merged = getValuesEnum(getDocValues(merged, first.name()));
     DocValuesEnum vE_2_merged = getValuesEnum(getDocValues(merged, second
         .name()));
-    if (second == ValueType.BYTES_VAR_STRAIGHT || second == ValueType.BYTES_FIXED_STRAIGHT) {
+    switch (second) { // these variants don't advance over missing values
+    case BYTES_FIXED_STRAIGHT:
+    case BYTES_VAR_STRAIGHT:
+    case FLOAT_32:
+    case FLOAT_64:
+    case INTS:  
       assertEquals(msg, valuesPerIndex-1, vE_2_merged.advance(valuesPerIndex-1));
     }
+    
     for (int i = 0; i < valuesPerIndex; i++) {
       assertEquals(msg, i, vE_1.nextDoc());
       assertEquals(msg, i, vE_1_merged.nextDoc());
@@ -263,15 +268,14 @@ public class TestDocValuesIndexing exten
         assertNotNull(intsReader);
 
         Source ints = getSource(intsReader);
-        MissingValue missing = ints.getMissing();
 
         for (int i = 0; i < base; i++) {
           long value = ints.getInt(i);
-          assertEquals("index " + i, missing.longValue, value);
+          assertEquals("index " + i, 0, value);
         }
 
         DocValuesEnum intsEnum = getValuesEnum(intsReader);
-        assertTrue(intsEnum.advance(0) >= base);
+        assertTrue(intsEnum.advance(base) >= base);
 
         intsEnum = getValuesEnum(intsReader);
         LongsRef enumRef = intsEnum.getInt();
@@ -283,8 +287,8 @@ public class TestDocValuesIndexing exten
           }
           assertEquals("advance failed at index: " + i + " of " + r.numDocs()
               + " docs", i, intsEnum.advance(i));
-          assertEquals(expected, enumRef.get());
           assertEquals(expected, ints.getInt(i));
+          assertEquals(expected, enumRef.get());
 
         }
       }
@@ -294,15 +298,13 @@ public class TestDocValuesIndexing exten
         DocValues floatReader = getDocValues(r, val.name());
         assertNotNull(floatReader);
         Source floats = getSource(floatReader);
-        MissingValue missing = floats.getMissing();
-
         for (int i = 0; i < base; i++) {
           double value = floats.getFloat(i);
           assertEquals(val + " failed for doc: " + i + " base: " + base,
-              missing.doubleValue, value, 0.0d);
+              0.0d, value, 0.0d);
         }
         DocValuesEnum floatEnum = getValuesEnum(floatReader);
-        assertTrue(floatEnum.advance(0) >= base);
+        assertTrue(floatEnum.advance(base) >= base);
 
         floatEnum = getValuesEnum(floatReader);
         FloatsRef enumRef = floatEnum.getFloat();
@@ -358,7 +360,6 @@ public class TestDocValuesIndexing exten
       byte upto = 0;
 
       // test the filled up slots for correctness
-      MissingValue missing = bytes.getMissing();
       for (int i = 0; i < base; i++) {
 
         BytesRef br = bytes.getBytes(i, new BytesRef());
@@ -369,18 +370,14 @@ public class TestDocValuesIndexing exten
         case BYTES_FIXED_STRAIGHT:
           // fixed straight returns bytesref with zero bytes all of fixed
           // length
-          if (missing.bytesValue != null) {
-            assertNotNull("expected none null - " + msg, br);
-            if (br.length != 0) {
-              assertEquals("expected zero bytes of length " + bytesSize + " - "
-                  + msg, bytesSize, br.length);
-              for (int j = 0; j < br.length; j++) {
-                assertEquals("Byte at index " + j + " doesn't match - " + msg,
-                    0, br.bytes[br.offset + j]);
-              }
+          assertNotNull("expected none null - " + msg, br);
+          if (br.length != 0) {
+            assertEquals("expected zero bytes of length " + bytesSize + " - "
+                + msg, bytesSize, br.length);
+            for (int j = 0; j < br.length; j++) {
+              assertEquals("Byte at index " + j + " doesn't match - " + msg, 0,
+                  br.bytes[br.offset + j]);
             }
-          } else {
-            assertNull("expected null - " + msg + " " + br, br);
           }
           break;
         case BYTES_VAR_SORTED:
@@ -388,13 +385,21 @@ public class TestDocValuesIndexing exten
         case BYTES_VAR_DEREF:
         case BYTES_FIXED_DEREF:
         default:
-          assertNull("expected null - " + msg + " " + br, br);
+          assertNotNull("expected none null - " + msg, br);
+          assertEquals(0, br.length);
           // make sure we advance at least until base
           DocValuesEnum bytesEnum = getValuesEnum(bytesReader);
+          try {
+          
           final int advancedTo = bytesEnum.advance(0);
           assertTrue(byteIndexValue.name() + " advanced failed base:" + base
               + " advancedTo: " + advancedTo, base <= advancedTo);
+          }catch(Throwable e) {
+            final int advancedTo = bytesEnum.advance(0);
+            assertTrue(byteIndexValue.name() + " advanced failed base:" + base
+                + " advancedTo: " + advancedTo, base <= advancedTo);
 
+          }
         }
       }
 



Mime
View raw message