lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jpou...@apache.org
Subject svn commit: r1503580 - in /lucene/dev/branches/branch_4x: ./ lucene/ lucene/core/ lucene/core/src/java/org/apache/lucene/index/ lucene/core/src/java/org/apache/lucene/search/ lucene/core/src/java/org/apache/lucene/util/packed/ lucene/core/src/test/org/...
Date Tue, 16 Jul 2013 07:30:39 GMT
Author: jpountz
Date: Tue Jul 16 07:30:38 2013
New Revision: 1503580

URL: http://svn.apache.org/r1503580
Log:
LUCENE-5113: Added (Monotonic)AppendingLongBuffer.freeze to pack the pending values.

Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/lucene/   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MergeState.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
    lucene/dev/branches/branch_4x/lucene/misc/   (props changed)
    lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java
    lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingMergePolicy.java

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MergeState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MergeState.java?rev=1503580&r1=1503579&r2=1503580&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MergeState.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MergeState.java
Tue Jul 16 07:30:38 2013
@@ -77,6 +77,7 @@ public class MergeState {
           ++del;
         }
       }
+      docMap.freeze();
       final int numDeletedDocs = del;
       assert docMap.size() == maxDoc;
       return new DocMap() {

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java?rev=1503580&r1=1503579&r2=1503580&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
Tue Jul 16 07:30:38 2013
@@ -328,6 +328,11 @@ public class MultiDocValues {
         }
         globalOrd++;
       }
+      subIndexes.freeze();
+      globalOrdDeltas.freeze();
+      for (int i = 0; i < ordDeltas.length; ++i) {
+        ordDeltas[i].freeze();
+      }
     }
     
     /** 

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1503580&r1=1503579&r2=1503580&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
Tue Jul 16 07:30:38 2013
@@ -1249,6 +1249,7 @@ class FieldCacheImpl implements FieldCac
           termOrd++;
         }
       }
+      termOrdToBytesOffset.freeze();
 
       // maybe an int-only impl?
       return new SortedDocValuesImpl(bytes.freeze(true), termOrdToBytesOffset, docToTermOrd.getMutable(),
termOrd);

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java?rev=1503580&r1=1503579&r2=1503580&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java
Tue Jul 16 07:30:38 2013
@@ -37,7 +37,7 @@ abstract class AbstractAppendingLongBuff
   PackedInts.Reader[] deltas;
   private long deltasBytes;
   int valuesOff;
-  final long[] pending;
+  long[] pending;
   int pendingOff;
 
   AbstractAppendingLongBuffer(int initialBlockCount, int pageSize) {
@@ -50,13 +50,27 @@ abstract class AbstractAppendingLongBuff
     pendingOff = 0;
   }
 
+  final int pageSize() {
+    return pageMask + 1;
+  }
+
   /** Get the number of values that have been added to the buffer. */
   public final long size() {
-    return valuesOff * (long) pending.length + pendingOff;
+    long size = pendingOff;
+    if (valuesOff > 0) {
+      size += deltas[valuesOff - 1].size();
+    }
+    if (valuesOff > 1) {
+      size += (long) (valuesOff - 1) * pageSize();
+    }
+    return size;
   }
 
   /** Append a value to this buffer. */
   public final void add(long l) {
+    if (pending == null) {
+      throw new IllegalStateException("This buffer is frozen");
+    }
     if (pendingOff == pending.length) {
       // check size
       if (deltas.length == valuesOff) {
@@ -64,9 +78,7 @@ abstract class AbstractAppendingLongBuff
         grow(newLength);
       }
       packPendingValues();
-      if (deltas[valuesOff] != null) {
-        deltasBytes += deltas[valuesOff].ramBytesUsed();
-      }
+      deltasBytes += deltas[valuesOff].ramBytesUsed();
       ++valuesOff;
       // reset pending buffer
       pendingOff = 0;
@@ -99,13 +111,15 @@ abstract class AbstractAppendingLongBuff
 
     long[] currentValues;
     int vOff, pOff;
+    int currentCount; // number of entries of the current page
 
     Iterator() {
       vOff = pOff = 0;
       if (valuesOff == 0) {
         currentValues = pending;
+        currentCount = pendingOff;
       } else {
-        currentValues = new long[pending.length];
+        currentValues = new long[deltas[0].size()];
         fillValues();
       }
     }
@@ -114,18 +128,20 @@ abstract class AbstractAppendingLongBuff
 
     /** Whether or not there are remaining values. */
     public final boolean hasNext() {
-      return vOff < valuesOff || (vOff == valuesOff && pOff < pendingOff);
+      return pOff < currentCount;
     }
 
     /** Return the next long in the buffer. */
     public final long next() {
       assert hasNext();
       long result = currentValues[pOff++];
-      if (pOff == pending.length) {
+      if (pOff == currentCount) {
         vOff += 1;
         pOff = 0;
         if (vOff <= valuesOff) {
           fillValues();
+        } else {
+          currentCount = 0;
         }
       }
       return result;
@@ -136,7 +152,9 @@ abstract class AbstractAppendingLongBuff
   long baseRamBytesUsed() {
     return RamUsageEstimator.NUM_BYTES_OBJECT_HEADER
         + 3 * RamUsageEstimator.NUM_BYTES_OBJECT_REF // the 3 arrays
-        + 2 * RamUsageEstimator.NUM_BYTES_INT; // the 2 offsets
+        + 2 * RamUsageEstimator.NUM_BYTES_INT // the 2 offsets
+        + 2 * RamUsageEstimator.NUM_BYTES_INT // pageShift, pageMask
+        + RamUsageEstimator.NUM_BYTES_LONG; // deltasBytes
   }
 
   /**
@@ -145,13 +163,25 @@ abstract class AbstractAppendingLongBuff
   public long ramBytesUsed() {
     // TODO: this is called per-doc-per-norms/dv-field, can we optimize this?
     long bytesUsed = RamUsageEstimator.alignObjectSize(baseRamBytesUsed())
-        + 2 * RamUsageEstimator.NUM_BYTES_INT // pageShift, pageMask
-        + RamUsageEstimator.NUM_BYTES_LONG // valuesBytes
-        + RamUsageEstimator.sizeOf(pending)
+        + (pending != null ? RamUsageEstimator.sizeOf(pending) : 0L)
         + RamUsageEstimator.sizeOf(minValues)
         + RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long)
RamUsageEstimator.NUM_BYTES_OBJECT_REF * deltas.length); // values
 
     return bytesUsed + deltasBytes;
   }
 
+  /** Pack all pending values in this buffer. Subsequent calls to {@link #add(long)} will
fail. */
+  public void freeze() {
+    if (pendingOff > 0) {
+      if (deltas.length == valuesOff) {
+        grow(valuesOff + 1); // don't oversize!
+      }
+      packPendingValues();
+      deltasBytes += deltas[valuesOff].ramBytesUsed();
+      ++valuesOff;
+      pendingOff = 0;
+    }
+    pending = null;
+  }
+
 }

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java?rev=1503580&r1=1503579&r2=1503580&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java
Tue Jul 16 07:30:38 2013
@@ -17,7 +17,6 @@ package org.apache.lucene.util.packed;
  * limitations under the License.
  */
 
-import java.util.Arrays;
 
 /**
  * Utility class to buffer a list of signed longs in memory. This class only
@@ -52,8 +51,6 @@ public final class AppendingLongBuffer e
 
   @Override
   void packPendingValues() {
-    assert pendingOff == pending.length;
-
     // compute max delta
     long minValue = pending[0];
     long maxValue = pending[0];
@@ -64,7 +61,9 @@ public final class AppendingLongBuffer e
     final long delta = maxValue - minValue;
 
     minValues[valuesOff] = minValue;
-    if (delta != 0) {
+    if (delta == 0) {
+      deltas[valuesOff] = new PackedInts.NullReader(pendingOff);
+    } else {
       // build a new packed reader
       final int bitsRequired = delta < 0 ? 64 : PackedInts.bitsRequired(delta);
       for (int i = 0; i < pendingOff; ++i) {
@@ -95,13 +94,13 @@ public final class AppendingLongBuffer e
     void fillValues() {
       if (vOff == valuesOff) {
         currentValues = pending;
-      } else if (deltas[vOff] == null) {
-        Arrays.fill(currentValues, minValues[vOff]);
+        currentCount = pendingOff;
       } else {
-        for (int k = 0; k < pending.length; ) {
-          k += deltas[vOff].get(k, currentValues, k, pending.length - k);
+        currentCount = deltas[vOff].size();
+        for (int k = 0; k < currentCount; ) {
+          k += deltas[vOff].get(k, currentValues, k, currentCount - k);
         }
-        for (int k = 0; k < pending.length; ++k) {
+        for (int k = 0; k < currentCount; ++k) {
           currentValues[k] += minValues[vOff];
         }
       }

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java?rev=1503580&r1=1503579&r2=1503580&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java
Tue Jul 16 07:30:38 2013
@@ -43,7 +43,7 @@ public final class MonotonicAppendingLon
    *  @param pageSize         the size of a single page */
   public MonotonicAppendingLongBuffer(int initialPageCount, int pageSize) {
     super(initialPageCount, pageSize);
-    averages = new float[pending.length];
+    averages = new float[pageSize];
   }
 
   /** Create an {@link MonotonicAppendingLongBuffer} with initialPageCount=16
@@ -74,16 +74,15 @@ public final class MonotonicAppendingLon
 
   @Override
   void packPendingValues() {
-    assert pendingOff == pending.length;
-
+    assert pendingOff > 0;
     minValues[valuesOff] = pending[0];
-    averages[valuesOff] = (float) (pending[pending.length - 1] - pending[0]) / (pending.length
- 1);
+    averages[valuesOff] = pendingOff == 1 ? 0 : (float) (pending[pendingOff - 1] - pending[0])
/ (pendingOff - 1);
 
-    for (int i = 0; i < pending.length; ++i) {
+    for (int i = 0; i < pendingOff; ++i) {
       pending[i] = zigZagEncode(pending[i] - minValues[valuesOff] - (long) (averages[valuesOff]
* (long) i));
     }
     long maxDelta = 0;
-    for (int i = 0; i < pending.length; ++i) {
+    for (int i = 0; i < pendingOff; ++i) {
       if (pending[i] < 0) {
         maxDelta = -1;
         break;
@@ -91,7 +90,9 @@ public final class MonotonicAppendingLon
         maxDelta = Math.max(maxDelta, pending[i]);
       }
     }
-    if (maxDelta != 0) {
+    if (maxDelta == 0) {
+      deltas[valuesOff] = new  PackedInts.NullReader(pendingOff);
+    } else {
       final int bitsRequired = maxDelta < 0 ? 64 : PackedInts.bitsRequired(maxDelta);
       final PackedInts.Mutable mutable = PackedInts.getMutable(pendingOff, bitsRequired,
PackedInts.COMPACT);
       for (int i = 0; i < pendingOff; ) {
@@ -118,15 +119,13 @@ public final class MonotonicAppendingLon
     void fillValues() {
       if (vOff == valuesOff) {
         currentValues = pending;
-      } else if (deltas[vOff] == null) {
-        for (int k = 0; k < pending.length; ++k) {
-          currentValues[k] = minValues[vOff] + (long) (averages[vOff] * (long) k);
-        }
+        currentCount = pendingOff;
       } else {
-        for (int k = 0; k < pending.length; ) {
-          k += deltas[vOff].get(k, currentValues, k, pending.length - k);
+        currentCount = deltas[vOff].size();
+        for (int k = 0; k < currentCount; ) {
+          k += deltas[vOff].get(k, currentValues, k, currentCount - k);
         }
-        for (int k = 0; k < pending.length; ++k) {
+        for (int k = 0; k < currentCount; ++k) {
           currentValues[k] = minValues[vOff] + (long) (averages[vOff] * (long) k) + zigZagDecode(currentValues[k]);
         }
       }

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java?rev=1503580&r1=1503579&r2=1503580&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
Tue Jul 16 07:30:38 2013
@@ -18,12 +18,14 @@ package org.apache.lucene.util.packed;
  */
 
 import java.io.IOException;
+import java.util.Arrays;
 
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.store.DataInput;
 import org.apache.lucene.store.DataOutput;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.LongsRef;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /**
  * Simplistic compression for array of unsigned long values.
@@ -703,7 +705,8 @@ public class PackedInts {
 
     @Override
     public int get(int index, long[] arr, int off, int len) {
-      return 0;
+      Arrays.fill(arr, off, off + len, 0);
+      return len;
     }
 
     @Override
@@ -718,7 +721,7 @@ public class PackedInts {
 
     @Override
     public long ramBytesUsed() {
-      return 0;
+      return RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_OBJECT_HEADER
+ RamUsageEstimator.NUM_BYTES_INT);
     }
 
     @Override

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java?rev=1503580&r1=1503579&r2=1503580&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
Tue Jul 16 07:30:38 2013
@@ -958,6 +958,14 @@ public class TestPackedInts extends Luce
           buf.add(arr[i]);
         }
         assertEquals(arr.length, buf.size());
+        if (random().nextBoolean()) {
+          buf.freeze();
+          if (random().nextBoolean()) {
+            // Make sure double freeze doesn't break anything
+            buf.freeze();
+          }
+        }
+        assertEquals(arr.length, buf.size());
         final AbstractAppendingLongBuffer.Iterator it = buf.iterator();
         for (int i = 0; i < arr.length; ++i) {
           if (random().nextBoolean()) {
@@ -973,8 +981,7 @@ public class TestPackedInts extends Luce
   
         final long expectedBytesUsed = RamUsageEstimator.sizeOf(buf);
         final long computedBytesUsed = buf.ramBytesUsed();
-        assertEquals("got " + computedBytesUsed + ", expected: " + expectedBytesUsed,
-            expectedBytesUsed, computedBytesUsed);
+        assertEquals(expectedBytesUsed, computedBytesUsed);
       }
     }
   }

Modified: lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java?rev=1503580&r1=1503579&r2=1503580&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java
Tue Jul 16 07:30:38 2013
@@ -191,6 +191,7 @@ public abstract class Sorter {
     for (int i = 0; i < maxDoc; ++i) {
       newToOld.add(docs[i]);
     }
+    newToOld.freeze();
 
     for (int i = 0; i < maxDoc; ++i) {
       docs[(int) newToOld.get(i)] = i;
@@ -200,6 +201,7 @@ public abstract class Sorter {
     for (int i = 0; i < maxDoc; ++i) {
       oldToNew.add(docs[i]);
     }
+    oldToNew.freeze();
     
     return new Sorter.DocMap() {
 

Modified: lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingMergePolicy.java?rev=1503580&r1=1503579&r2=1503580&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingMergePolicy.java
(original)
+++ lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingMergePolicy.java
Tue Jul 16 07:30:38 2013
@@ -107,6 +107,7 @@ public final class SortingMergePolicy ex
           }
         }
       }
+      deletes.freeze();
       return deletes;
     }
 



Mime
View raw message