lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jpou...@apache.org
Subject svn commit: r1494753 - in /lucene/dev/branches/branch_4x: lucene/ lucene/core/src/java/org/apache/lucene/search/ lucene/core/src/java/org/apache/lucene/util/packed/ lucene/core/src/test/org/apache/lucene/search/ lucene/core/src/test/org/apache/lucene/u...
Date Wed, 19 Jun 2013 19:44:13 GMT
Author: jpountz
Date: Wed Jun 19 19:44:13 2013
New Revision: 1494753

URL: http://svn.apache.org/r1494753
Log:
LUCENE-5063: Compress integer and long field caches and deprecate FieldCache.get(Byte|Short)s
and related classes/methods.

Modified:
    lucene/dev/branches/branch_4x/lucene/CHANGES.txt
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCache.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/SortField.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
    lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ByteFieldSource.java
    lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ShortFieldSource.java
    lucene/dev/branches/branch_4x/solr/CHANGES.txt
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/ByteField.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/ShortField.java

Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1494753&r1=1494752&r2=1494753&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Wed Jun 19 19:44:13 2013
@@ -71,6 +71,12 @@ Changes in backwards compatibility polic
   on segment size and noCFSRatio. The default implemantion was pulled up from
   TieredMergePolicy. (Simon Willnauer)
 
+* LUCENE-5063: FieldCache.get(Bytes|Shorts), SortField.Type.(BYTE|SHORT) and
+  FieldCache.DEFAULT_(BYTE|SHORT|INT|LONG|FLOAT|DOUBLE)_PARSER are now
+  deprecated. These methods/types assume that data is stored as strings although
+  Lucene has much better support for numeric data through (Int|Long)Field,
+  NumericRangeQuery and FieldCache.get(Int|Long)s. (Adrien Grand)
+
 Bug Fixes
 
 * LUCENE-4997: Internal test framework's tests are sensitive to previous 
@@ -190,6 +196,9 @@ New Features
 * LUCENE-5025: FST's Builder can now handle more than 2.1 billion
   "tail nodes" while building a minimal FST.  (Aaron Binns, Adrien
   Grand, Mike McCandless)
+
+* LUCENE-5063: FieldCache.DEFAULT.get(Ints|Longs) now uses bit-packing to save
+  memory. (Adrien Grand)
   
 Build
 

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCache.java?rev=1494753&r1=1494752&r2=1494753&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCache.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCache.java
Wed Jun 19 19:44:13 2013
@@ -181,6 +181,7 @@ public interface FieldCache {
   /** Interface to parse bytes from document fields.
    * @see FieldCache#getBytes(AtomicReader, String, FieldCache.ByteParser, boolean)
    */
+  @Deprecated
   public interface ByteParser extends Parser {
     /** Return a single Byte representation of this field's value. */
     public byte parseByte(BytesRef term);
@@ -189,6 +190,7 @@ public interface FieldCache {
   /** Interface to parse shorts from document fields.
    * @see FieldCache#getShorts(AtomicReader, String, FieldCache.ShortParser, boolean)
    */
+  @Deprecated
   public interface ShortParser extends Parser {
     /** Return a short representation of this field's value. */
     public short parseShort(BytesRef term);
@@ -230,6 +232,7 @@ public interface FieldCache {
   public static FieldCache DEFAULT = new FieldCacheImpl();
 
   /** The default parser for byte values, which are encoded by {@link Byte#toString(byte)}
*/
+  @Deprecated
   public static final ByteParser DEFAULT_BYTE_PARSER = new ByteParser() {
     @Override
     public byte parseByte(BytesRef term) {
@@ -250,6 +253,7 @@ public interface FieldCache {
   };
 
   /** The default parser for short values, which are encoded by {@link Short#toString(short)}
*/
+  @Deprecated
   public static final ShortParser DEFAULT_SHORT_PARSER = new ShortParser() {
     @Override
     public short parseShort(BytesRef term) {
@@ -271,6 +275,7 @@ public interface FieldCache {
   };
 
   /** The default parser for int values, which are encoded by {@link Integer#toString(int)}
*/
+  @Deprecated
   public static final IntParser DEFAULT_INT_PARSER = new IntParser() {
     @Override
     public int parseInt(BytesRef term) {
@@ -293,6 +298,7 @@ public interface FieldCache {
   };
 
   /** The default parser for float values, which are encoded by {@link Float#toString(float)}
*/
+  @Deprecated
   public static final FloatParser DEFAULT_FLOAT_PARSER = new FloatParser() {
     @Override
     public float parseFloat(BytesRef term) {
@@ -315,6 +321,7 @@ public interface FieldCache {
   };
 
   /** The default parser for long values, which are encoded by {@link Long#toString(long)}
*/
+  @Deprecated
   public static final LongParser DEFAULT_LONG_PARSER = new LongParser() {
     @Override
     public long parseLong(BytesRef term) {
@@ -337,6 +344,7 @@ public interface FieldCache {
   };
 
   /** The default parser for double values, which are encoded by {@link Double#toString(double)}
*/
+  @Deprecated
   public static final DoubleParser DEFAULT_DOUBLE_PARSER = new DoubleParser() {
     @Override
     public double parseDouble(BytesRef term) {
@@ -459,7 +467,9 @@ public interface FieldCache {
    *        also be computed and stored in the FieldCache.
    * @return The values in the given field for each document.
    * @throws IOException  If any error occurs.
+   * @deprecated (4.4) Index as a numeric field using {@link IntField} and then use {@link
#getInts(AtomicReader, String, boolean)} instead.
    */
+  @Deprecated
   public Bytes getBytes(AtomicReader reader, String field, boolean setDocsWithField) throws
IOException;
 
   /** Checks the internal cache for an appropriate entry, and if none is found,
@@ -473,7 +483,9 @@ public interface FieldCache {
    *        also be computed and stored in the FieldCache.
    * @return The values in the given field for each document.
    * @throws IOException  If any error occurs.
+   * @deprecated (4.4) Index as a numeric field using {@link IntField} and then use {@link
#getInts(AtomicReader, String, boolean)} instead.
    */
+  @Deprecated
   public Bytes getBytes(AtomicReader reader, String field, ByteParser parser, boolean setDocsWithField)
throws IOException;
 
   /** Checks the internal cache for an appropriate entry, and if none is
@@ -486,7 +498,9 @@ public interface FieldCache {
    *        also be computed and stored in the FieldCache.
    * @return The values in the given field for each document.
    * @throws IOException  If any error occurs.
+   * @deprecated (4.4) Index as a numeric field using {@link IntField} and then use {@link
#getInts(AtomicReader, String, boolean)} instead.
    */
+  @Deprecated
   public Shorts getShorts (AtomicReader reader, String field, boolean setDocsWithField) throws
IOException;
 
   /** Checks the internal cache for an appropriate entry, and if none is found,
@@ -500,7 +514,9 @@ public interface FieldCache {
    *        also be computed and stored in the FieldCache.
    * @return The values in the given field for each document.
    * @throws IOException  If any error occurs.
+   * @deprecated (4.4) Index as a numeric field using {@link IntField} and then use {@link
#getInts(AtomicReader, String, boolean)} instead.
    */
+  @Deprecated
   public Shorts getShorts (AtomicReader reader, String field, ShortParser parser, boolean
setDocsWithField) throws IOException;
   
   /** Checks the internal cache for an appropriate entry, and if none is

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1494753&r1=1494752&r2=1494753&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
Wed Jun 19 19:44:13 2013
@@ -38,7 +38,6 @@ import org.apache.lucene.index.SortedDoc
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.FieldCacheSanityChecker;
@@ -574,15 +573,19 @@ class FieldCacheImpl implements FieldCac
   }
 
   static class IntsFromArray extends Ints {
-    private final int[] values;
+    private final PackedInts.Reader values;
+    private final int minValue;
 
-    public IntsFromArray(int[] values) {
+    public IntsFromArray(PackedInts.Reader values, int minValue) {
+      assert values.getBitsPerValue() <= 32;
       this.values = values;
+      this.minValue = minValue;
     }
     
     @Override
     public int get(int docID) {
-      return values[docID];
+      final long delta = values.get(docID);
+      return minValue + (int) delta;
     }
   }
 
@@ -598,6 +601,15 @@ class FieldCacheImpl implements FieldCac
     }
   }
 
+  private static class GrowableWriterAndMinValue {
+    GrowableWriterAndMinValue(GrowableWriter array, long minValue) {
+      this.writer = array;
+      this.minValue = minValue;
+    }
+    public GrowableWriter writer;
+    public long minValue;
+  }
+
   static final class IntCache extends Cache {
     IntCache(FieldCacheImpl wrapper) {
       super(wrapper);
@@ -621,11 +633,12 @@ class FieldCacheImpl implements FieldCac
         }
       }
 
-      final HoldsOneThing<int[]> valuesRef = new HoldsOneThing<int[]>();
+      final HoldsOneThing<GrowableWriterAndMinValue> valuesRef = new HoldsOneThing<GrowableWriterAndMinValue>();
 
       Uninvert u = new Uninvert() {
+          private int minValue;
           private int currentValue;
-          private int[] values;
+          private GrowableWriter values;
 
           @Override
           public void visitTerm(BytesRef term) {
@@ -635,16 +648,29 @@ class FieldCacheImpl implements FieldCac
               // (which will hit a NumberFormatException
               // when we first try the DEFAULT_INT_PARSER),
               // we don't double-alloc:
-              values = new int[reader.maxDoc()];
-              valuesRef.set(values);
+              int startBitsPerValue;
+              // Make sure than missing values (0) can be stored without resizing
+              if (currentValue < 0) {
+                minValue = currentValue;
+                startBitsPerValue = PackedInts.bitsRequired((-minValue) & 0xFFFFFFFFL);
+              } else {
+                minValue = 0;
+                startBitsPerValue = PackedInts.bitsRequired(currentValue);
+              }
+              startBitsPerValue = Math.max(startBitsPerValue, 4);
+              values = new GrowableWriter(startBitsPerValue, reader.maxDoc(), PackedInts.FAST);
+              if (minValue != 0) {
+                values.fill(0, values.size(), (-minValue) & 0xFFFFFFFFL); // default
value must be 0
+              }
+              valuesRef.set(new GrowableWriterAndMinValue(values, minValue));
             }
           }
 
           @Override
           public void visitDoc(int docID) {
-            values[docID] = currentValue;
+            values.set(docID, (currentValue - minValue) & 0xFFFFFFFFL);
           }
-          
+
           @Override
           protected TermsEnum termsEnum(Terms terms) throws IOException {
             return parser.termsEnum(terms);
@@ -656,11 +682,11 @@ class FieldCacheImpl implements FieldCac
       if (setDocsWithField) {
         wrapper.setDocsWithField(reader, key.field, u.docsWithField);
       }
-      int[] values = valuesRef.get();
+      GrowableWriterAndMinValue values = valuesRef.get();
       if (values == null) {
-        values = new int[reader.maxDoc()];
+        return new IntsFromArray(new PackedInts.NullReader(reader.maxDoc()), 0);
       }
-      return new IntsFromArray(values);
+      return new IntsFromArray(values.writer.getMutable(), (int) values.minValue);
     }
   }
 
@@ -879,15 +905,17 @@ class FieldCacheImpl implements FieldCac
   }
 
   static class LongsFromArray extends Longs {
-    private final long[] values;
+    private final PackedInts.Reader values;
+    private final long minValue;
 
-    public LongsFromArray(long[] values) {
+    public LongsFromArray(PackedInts.Reader values, long minValue) {
       this.values = values;
+      this.minValue = minValue;
     }
     
     @Override
     public long get(int docID) {
-      return values[docID];
+      return minValue + values.get(docID);
     }
   }
 
@@ -914,11 +942,12 @@ class FieldCacheImpl implements FieldCac
         }
       }
 
-      final HoldsOneThing<long[]> valuesRef = new HoldsOneThing<long[]>();
+      final HoldsOneThing<GrowableWriterAndMinValue> valuesRef = new HoldsOneThing<GrowableWriterAndMinValue>();
 
       Uninvert u = new Uninvert() {
+          private long minValue;
           private long currentValue;
-          private long[] values;
+          private GrowableWriter values;
 
           @Override
           public void visitTerm(BytesRef term) {
@@ -928,14 +957,27 @@ class FieldCacheImpl implements FieldCac
               // (which will hit a NumberFormatException
               // when we first try the DEFAULT_INT_PARSER),
               // we don't double-alloc:
-              values = new long[reader.maxDoc()];
-              valuesRef.set(values);
+              int startBitsPerValue;
+              // Make sure than missing values (0) can be stored without resizing
+              if (currentValue < 0) {
+                minValue = currentValue;
+                startBitsPerValue = minValue == Long.MIN_VALUE ? 64 : PackedInts.bitsRequired(-minValue);
+              } else {
+                minValue = 0;
+                startBitsPerValue = PackedInts.bitsRequired(currentValue);
+              }
+              startBitsPerValue = Math.max(startBitsPerValue, 4);
+              values = new GrowableWriter(startBitsPerValue, reader.maxDoc(), PackedInts.FAST);
+              if (minValue != 0) {
+                values.fill(0, values.size(), -minValue); // default value must be 0
+              }
+              valuesRef.set(new GrowableWriterAndMinValue(values, minValue));
             }
           }
 
           @Override
           public void visitDoc(int docID) {
-            values[docID] = currentValue;
+            values.set(docID, currentValue - minValue);
           }
           
           @Override
@@ -949,11 +991,11 @@ class FieldCacheImpl implements FieldCac
       if (setDocsWithField) {
         wrapper.setDocsWithField(reader, key.field, u.docsWithField);
       }
-      long[] values = valuesRef.get();
+      GrowableWriterAndMinValue values = valuesRef.get();
       if (values == null) {
-        values = new long[reader.maxDoc()];
+        return new LongsFromArray(new PackedInts.NullReader(reader.maxDoc()), 0L);
       }
-      return new LongsFromArray(values);
+      return new LongsFromArray(values.writer.getMutable(), values.minValue);
     }
   }
 
@@ -1146,7 +1188,6 @@ class FieldCacheImpl implements FieldCac
       final PagedBytes bytes = new PagedBytes(15);
 
       int startTermsBPV;
-      int startNumUniqueTerms;
 
       final int termCountHardLimit;
       if (maxDoc == Integer.MAX_VALUE) {
@@ -1170,15 +1211,11 @@ class FieldCacheImpl implements FieldCac
           }
 
           startTermsBPV = PackedInts.bitsRequired(numUniqueTerms);
-
-          startNumUniqueTerms = (int) numUniqueTerms;
         } else {
           startTermsBPV = 1;
-          startNumUniqueTerms = 1;
         }
       } else {
         startTermsBPV = 1;
-        startNumUniqueTerms = 1;
       }
 
       MonotonicAppendingLongBuffer termOrdToBytesOffset = new MonotonicAppendingLongBuffer();

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java?rev=1494753&r1=1494752&r2=1494753&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java
Wed Jun 19 19:44:13 2013
@@ -199,6 +199,7 @@ public abstract class FieldCacheRangeFil
    * byte fields containing exactly one numeric term in the field. The range can be half-open
by setting one
    * of the values to <code>null</code>.
    */
+  @Deprecated
   public static FieldCacheRangeFilter<Byte> newByteRange(String field, Byte lowerVal,
Byte upperVal, boolean includeLower, boolean includeUpper) {
     return newByteRange(field, null, lowerVal, upperVal, includeLower, includeUpper);
   }
@@ -208,6 +209,7 @@ public abstract class FieldCacheRangeFil
    * byte fields containing exactly one numeric term in the field. The range can be half-open
by setting one
    * of the values to <code>null</code>.
    */
+  @Deprecated
   public static FieldCacheRangeFilter<Byte> newByteRange(String field, FieldCache.ByteParser
parser, Byte lowerVal, Byte upperVal, boolean includeLower, boolean includeUpper) {
     return new FieldCacheRangeFilter<Byte>(field, parser, lowerVal, upperVal, includeLower,
includeUpper) {
       @Override
@@ -250,6 +252,7 @@ public abstract class FieldCacheRangeFil
    * short fields containing exactly one numeric term in the field. The range can be half-open
by setting one
    * of the values to <code>null</code>.
    */
+  @Deprecated
   public static FieldCacheRangeFilter<Short> newShortRange(String field, Short lowerVal,
Short upperVal, boolean includeLower, boolean includeUpper) {
     return newShortRange(field, null, lowerVal, upperVal, includeLower, includeUpper);
   }
@@ -259,6 +262,7 @@ public abstract class FieldCacheRangeFil
    * short fields containing exactly one numeric term in the field. The range can be half-open
by setting one
    * of the values to <code>null</code>.
    */
+  @Deprecated
   public static FieldCacheRangeFilter<Short> newShortRange(String field, FieldCache.ShortParser
parser, Short lowerVal, Short upperVal, boolean includeLower, boolean includeUpper) {
     return new FieldCacheRangeFilter<Short>(field, parser, lowerVal, upperVal, includeLower,
includeUpper) {
       @Override

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java?rev=1494753&r1=1494752&r2=1494753&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
Wed Jun 19 19:44:13 2013
@@ -220,6 +220,7 @@ public abstract class FieldComparator<T>
 
   /** Parses field's values as byte (using {@link
    *  FieldCache#getBytes} and sorts by ascending value */
+  @Deprecated
   public static final class ByteComparator extends NumericComparator<Byte> {
     private final byte[] values;
     private final ByteParser parser;
@@ -441,6 +442,7 @@ public abstract class FieldComparator<T>
 
   /** Parses field's values as short (using {@link
    *  FieldCache#getShorts} and sorts by ascending value */
+  @Deprecated
   public static final class ShortComparator extends NumericComparator<Short> {
     private final short[] values;
     private final ShortParser parser;

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/SortField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/SortField.java?rev=1494753&r1=1494752&r2=1494753&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/SortField.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/SortField.java
Wed Jun 19 19:44:13 2013
@@ -73,6 +73,7 @@ public class SortField {
 
     /** Sort using term values as encoded Shorts.  Sort values are Short and
      * lower values are at the front. */
+    @Deprecated
     SHORT,
 
     /** Sort using a custom Comparator.  Sort values are any Comparable and
@@ -81,6 +82,7 @@ public class SortField {
 
     /** Sort using term values as encoded Bytes.  Sort values are Byte and
      * lower values are at the front. */
+    @Deprecated
     BYTE,
 
     /** Sort using term values as Strings, but comparing by

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java?rev=1494753&r1=1494752&r2=1494753&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java
Wed Jun 19 19:44:13 2013
@@ -25,20 +25,30 @@ import org.apache.lucene.util.RamUsageEs
 /**     
  * Implements {@link PackedInts.Mutable}, but grows the
  * bit count of the underlying packed ints on-demand.
+ * <p>Beware that this class will accept to set negative values but in order
+ * to do this, it will grow the number of bits per value to 64.
  *
  * <p>@lucene.internal</p>
  */
-
 public class GrowableWriter implements PackedInts.Mutable {
 
-  private long currentMaxValue;
+  private long currentMask;
   private PackedInts.Mutable current;
   private final float acceptableOverheadRatio;
 
+  /**
+   * @param startBitsPerValue       the initial number of bits per value, may grow depending
on the data
+   * @param valueCount              the number of values
+   * @param acceptableOverheadRatio an acceptable overhead ratio
+   */
   public GrowableWriter(int startBitsPerValue, int valueCount, float acceptableOverheadRatio)
{
     this.acceptableOverheadRatio = acceptableOverheadRatio;
     current = PackedInts.getMutable(valueCount, startBitsPerValue, this.acceptableOverheadRatio);
-    currentMaxValue = PackedInts.maxValue(current.getBitsPerValue());
+    currentMask = mask(current.getBitsPerValue());
+  }
+
+  private static long mask(int bitsPerValue) {
+    return bitsPerValue == 64 ? ~0L : PackedInts.maxValue(bitsPerValue);
   }
 
   @Override
@@ -71,16 +81,16 @@ public class GrowableWriter implements P
   }
 
   private void ensureCapacity(long value) {
-    assert value >= 0;
-    if (value <= currentMaxValue) {
+    if ((value & currentMask) == value) {
       return;
     }
-    final int bitsRequired = PackedInts.bitsRequired(value);
+    final int bitsRequired = value < 0 ? 64 : PackedInts.bitsRequired(value);
+    assert bitsRequired > current.getBitsPerValue();
     final int valueCount = size();
     PackedInts.Mutable next = PackedInts.getMutable(valueCount, bitsRequired, acceptableOverheadRatio);
     PackedInts.copy(current, 0, next, 0, valueCount, PackedInts.DEFAULT_BUFFER_SIZE);
     current = next;
-    currentMaxValue = PackedInts.maxValue(current.getBitsPerValue());
+    currentMask = mask(current.getBitsPerValue());
   }
 
   @Override
@@ -110,6 +120,10 @@ public class GrowableWriter implements P
   public int set(int index, long[] arr, int off, int len) {
     long max = 0;
     for (int i = off, end = off + len; i < end; ++i) {
+      // bitwise or is nice because either all values are positive and the
+      // or-ed result will require as many bits per value as the max of the
+      // values, or one of them is negative and the result will be negative,
+      // forcing GrowableWriter to use 64 bits per value
       max |= arr[i];
     }
     ensureCapacity(max);

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java?rev=1494753&r1=1494752&r2=1494753&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
Wed Jun 19 19:44:13 2013
@@ -23,23 +23,33 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.LinkedHashSet;
 import java.util.List;
-import java.util.Locale;
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.LongField;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.document.StoredField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.*;
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.DocTermOrds;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.SlowCompositeReaderWrapper;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.FieldCache.Bytes;
 import org.apache.lucene.search.FieldCache.Doubles;
 import org.apache.lucene.search.FieldCache.Floats;
@@ -693,4 +703,97 @@ public class TestFieldCache extends Luce
     ir.close();
     dir.close();
   }
+
+  // Make sure that the use of GrowableWriter doesn't prevent from using the full long range
+  public void testLongFieldCache() throws IOException {
+    Directory dir = newDirectory();
+    IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+    cfg.setMergePolicy(newLogMergePolicy());
+    RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+    Document doc = new Document();
+    LongField field = new LongField("f", 0L, Store.YES);
+    doc.add(field);
+    final long[] values = new long[_TestUtil.nextInt(random(), 1, 10)];
+    for (int i = 0; i < values.length; ++i) {
+      final long v;
+      switch (random().nextInt(10)) {
+        case 0:
+          v = Long.MIN_VALUE;
+          break;
+        case 1:
+          v = 0;
+          break;
+        case 2:
+          v = Long.MAX_VALUE;
+          break;
+        default:
+          v = _TestUtil.nextLong(random(), -10, 10);
+          break;
+      }
+      values[i] = v;
+      if (v == 0 && random().nextBoolean()) {
+        // missing
+        iw.addDocument(new Document());
+      } else {
+        field.setLongValue(v);
+        iw.addDocument(doc);
+      }
+    }
+    iw.forceMerge(1);
+    final DirectoryReader reader = iw.getReader();
+    final FieldCache.Longs longs = FieldCache.DEFAULT.getLongs(getOnlySegmentReader(reader),
"f", false);
+    for (int i = 0; i < values.length; ++i) {
+      assertEquals(values[i], longs.get(i));
+    }
+    reader.close();
+    iw.close();
+    dir.close();
+  }
+
+  // Make sure that the use of GrowableWriter doesn't prevent from using the full int range
+  public void testIntFieldCache() throws IOException {
+    Directory dir = newDirectory();
+    IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+    cfg.setMergePolicy(newLogMergePolicy());
+    RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+    Document doc = new Document();
+    IntField field = new IntField("f", 0, Store.YES);
+    doc.add(field);
+    final int[] values = new int[_TestUtil.nextInt(random(), 1, 10)];
+    for (int i = 0; i < values.length; ++i) {
+      final int v;
+      switch (random().nextInt(10)) {
+        case 0:
+          v = Integer.MIN_VALUE;
+          break;
+        case 1:
+          v = 0;
+          break;
+        case 2:
+          v = Integer.MAX_VALUE;
+          break;
+        default:
+          v = _TestUtil.nextInt(random(), -10, 10);
+          break;
+      }
+      values[i] = v;
+      if (v == 0 && random().nextBoolean()) {
+        // missing
+        iw.addDocument(new Document());
+      } else {
+        field.setIntValue(v);
+        iw.addDocument(doc);
+      }
+    }
+    iw.forceMerge(1);
+    final DirectoryReader reader = iw.getReader();
+    final FieldCache.Ints ints = FieldCache.DEFAULT.getInts(getOnlySegmentReader(reader),
"f", false);
+    for (int i = 0; i < values.length; ++i) {
+      assertEquals(values[i], ints.get(i));
+    }
+    reader.close();
+    iw.close();
+    dir.close();
+  }
+
 }

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java?rev=1494753&r1=1494752&r2=1494753&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
Wed Jun 19 19:44:13 2013
@@ -650,8 +650,11 @@ public class TestPackedInts extends Luce
     wrt.set(99, (1 << 23) - 1);
     assertEquals(1 << 10, wrt.get(valueCount - 1));
     wrt.set(1, Long.MAX_VALUE);
+    wrt.set(2, -3);
+    assertEquals(64, wrt.getBitsPerValue());
     assertEquals(1 << 10, wrt.get(valueCount - 1));
     assertEquals(Long.MAX_VALUE, wrt.get(1));
+    assertEquals(-3L, wrt.get(2));
     assertEquals(2, wrt.get(4));
     assertEquals((1 << 23) - 1, wrt.get(99));
     assertEquals(10, wrt.get(7));

Modified: lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ByteFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ByteFieldSource.java?rev=1494753&r1=1494752&r2=1494753&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ByteFieldSource.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ByteFieldSource.java
Wed Jun 19 19:44:13 2013
@@ -30,7 +30,7 @@ import org.apache.lucene.search.FieldCac
  *
  *
  */
-
+@Deprecated
 public class ByteFieldSource extends FieldCacheSource {
 
   private final FieldCache.ByteParser parser;

Modified: lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ShortFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ShortFieldSource.java?rev=1494753&r1=1494752&r2=1494753&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ShortFieldSource.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ShortFieldSource.java
Wed Jun 19 19:44:13 2013
@@ -29,6 +29,7 @@ import org.apache.lucene.search.FieldCac
  * using <code>getShorts()</code>
  * and makes those values available as other numeric types, casting as needed.
  **/
+@Deprecated
 public class ShortFieldSource extends FieldCacheSource {
 
   final FieldCache.ShortParser parser;

Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1494753&r1=1494752&r2=1494753&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Wed Jun 19 19:44:13 2013
@@ -46,6 +46,10 @@ Upgrading from Solr 4.3.0
 * SOLR-4778: The signature of LogWatcher.registerListener has changed, from
   (ListenerConfig, CoreContainer) to (ListenerConfig).  Users implementing their
   own LogWatcher classes will need to change their code accordingly.
+
+* LUCENE-5063: ByteField and ShortField have been deprecated and will be removed
+  in 5.0. If you are still using these field types, you should migrate your
+  fields to TrieIntField.
   
 Detailed Change List
 ----------------------

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/ByteField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/ByteField.java?rev=1494753&r1=1494752&r2=1494753&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/ByteField.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/ByteField.java
Wed Jun 19 19:44:13 2013
@@ -43,7 +43,9 @@ import java.util.Map;
  * </ul>
  *
  * @see Byte
+ * @deprecated Use {@link TrieIntField} instead.
  */
+@Deprecated
 public class ByteField extends PrimitiveFieldType {
   @Override
   protected void init(IndexSchema schema, Map<String, String> args) {

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/ShortField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/ShortField.java?rev=1494753&r1=1494752&r2=1494753&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/ShortField.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/ShortField.java
Wed Jun 19 19:44:13 2013
@@ -44,7 +44,9 @@ import java.util.Map;
  * </ul>
  *
  * @see Short
+ * @deprecated Use {@link TrieIntField} instead.
  **/
+@Deprecated
 public class ShortField extends PrimitiveFieldType {
   @Override
   protected void init(IndexSchema schema, Map<String, String> args) {



Mime
View raw message