lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sh...@apache.org
Subject svn commit: r1498804 [5/8] - in /lucene/dev/branches/lucene4258: lucene/ lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/ lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/ lucene/analysis/common/src/java/org/apache/l...
Date Tue, 02 Jul 2013 07:12:03 GMT
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/fst/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/fst/package.html?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/fst/package.html (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/fst/package.html Tue Jul  2 07:12:00 2013
@@ -43,7 +43,7 @@ FST Construction example:
     String inputValues[] = {"cat", "dog", "dogs"};
     long outputValues[] = {5, 7, 12};
     
-    PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
+    PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
     Builder<Long> builder = new Builder<Long>(INPUT_TYPE.BYTE1, outputs);
     BytesRef scratchBytes = new BytesRef();
     IntsRef scratchInts = new IntsRef();
@@ -60,8 +60,7 @@ Retrieval by key:
 </pre>
 Retrieval by value:
 <pre class="prettyprint">
-    // Only works because outputs are also in sorted order, and
-    // we passed 'true' for sharing to PositiveIntOutputs.getSingleton
+    // Only works because outputs are also in sorted order
     IntsRef key = Util.getByOutput(fst, 12);
     System.out.println(Util.toBytesRef(key, scratchBytes).utf8ToString()); // dogs
 </pre>
@@ -77,7 +76,6 @@ Iterate over key-value pairs in sorted o
 </pre>
 N-shortest paths by weight:
 <pre class="prettyprint">
-    // Only works because we passed 'true' for sharing to PositiveIntOutputs.getSingleton
     Comparator&lt;Long&gt; comparator = new Comparator&lt;Long&gt;() {
       public int compare(Long left, Long right) {
         return left.compareTo(right);

Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java Tue Jul  2 07:12:00 2013
@@ -17,6 +17,8 @@ package org.apache.lucene.util.packed;
  * limitations under the License.
  */
 
+import static org.apache.lucene.util.packed.PackedInts.checkBlockSize;
+
 import java.util.Arrays;
 
 import org.apache.lucene.util.ArrayUtil;
@@ -25,33 +27,37 @@ import org.apache.lucene.util.RamUsageEs
 /** Common functionality shared by {@link AppendingLongBuffer} and {@link MonotonicAppendingLongBuffer}. */
 abstract class AbstractAppendingLongBuffer {
 
-  static final int BLOCK_BITS = 10;
-  static final int MAX_PENDING_COUNT = 1 << BLOCK_BITS;
-  static final int BLOCK_MASK = MAX_PENDING_COUNT - 1;
+  static final int MIN_PAGE_SIZE = 64;
+  // More than 1M doesn't really makes sense with these appending buffers
+  // since their goal is to try to have small numbers of bits per value
+  static final int MAX_PAGE_SIZE = 1 << 20;
 
+  final int pageShift, pageMask;
   long[] minValues;
   PackedInts.Reader[] deltas;
   private long deltasBytes;
   int valuesOff;
-  long[] pending;
+  final long[] pending;
   int pendingOff;
 
-  AbstractAppendingLongBuffer(int initialBlockCount) {
-    minValues = new long[16];
-    deltas = new PackedInts.Reader[16];
-    pending = new long[MAX_PENDING_COUNT];
+  AbstractAppendingLongBuffer(int initialBlockCount, int pageSize) {
+    minValues = new long[initialBlockCount];
+    deltas = new PackedInts.Reader[initialBlockCount];
+    pending = new long[pageSize];
+    pageShift = checkBlockSize(pageSize, MIN_PAGE_SIZE, MAX_PAGE_SIZE);
+    pageMask = pageSize - 1;
     valuesOff = 0;
     pendingOff = 0;
   }
 
   /** Get the number of values that have been added to the buffer. */
   public final long size() {
-    return valuesOff * (long) MAX_PENDING_COUNT + pendingOff;
+    return valuesOff * (long) pending.length + pendingOff;
   }
 
   /** Append a value to this buffer. */
   public final void add(long l) {
-    if (pendingOff == MAX_PENDING_COUNT) {
+    if (pendingOff == pending.length) {
       // check size
       if (deltas.length == valuesOff) {
         final int newLength = ArrayUtil.oversize(valuesOff + 1, 8);
@@ -80,8 +86,8 @@ abstract class AbstractAppendingLongBuff
     if (index < 0 || index >= size()) {
       throw new IndexOutOfBoundsException("" + index);
     }
-    int block = (int) (index >> BLOCK_BITS);
-    int element = (int) (index & BLOCK_MASK);
+    final int block = (int) (index >> pageShift);
+    final int element = (int) (index & pageMask);
     return get(block, element);
   }
 
@@ -99,7 +105,7 @@ abstract class AbstractAppendingLongBuff
       if (valuesOff == 0) {
         currentValues = pending;
       } else {
-        currentValues = new long[MAX_PENDING_COUNT];
+        currentValues = new long[pending.length];
         fillValues();
       }
     }
@@ -115,7 +121,7 @@ abstract class AbstractAppendingLongBuff
     public final long next() {
       assert hasNext();
       long result = currentValues[pOff++];
-      if (pOff == MAX_PENDING_COUNT) {
+      if (pOff == pending.length) {
         vOff += 1;
         pOff = 0;
         if (vOff <= valuesOff) {
@@ -139,6 +145,7 @@ abstract class AbstractAppendingLongBuff
   public long ramBytesUsed() {
     // TODO: this is called per-doc-per-norms/dv-field, can we optimize this?
     long bytesUsed = RamUsageEstimator.alignObjectSize(baseRamBytesUsed())
+        + 2 * RamUsageEstimator.NUM_BYTES_INT // pageShift, pageMask
         + RamUsageEstimator.NUM_BYTES_LONG // valuesBytes
         + RamUsageEstimator.sizeOf(pending)
         + RamUsageEstimator.sizeOf(minValues)

Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/AbstractBlockPackedWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/AbstractBlockPackedWriter.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/AbstractBlockPackedWriter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/AbstractBlockPackedWriter.java Tue Jul  2 07:12:00 2013
@@ -17,6 +17,8 @@ package org.apache.lucene.util.packed;
  * limitations under the License.
  */
 
+import static org.apache.lucene.util.packed.PackedInts.checkBlockSize;
+
 import java.io.IOException;
 import java.util.Arrays;
 
@@ -24,22 +26,11 @@ import org.apache.lucene.store.DataOutpu
 
 abstract class AbstractBlockPackedWriter {
 
+  static final int MIN_BLOCK_SIZE = 64;
   static final int MAX_BLOCK_SIZE = 1 << (30 - 3);
   static final int MIN_VALUE_EQUALS_0 = 1 << 0;
   static final int BPV_SHIFT = 1;
 
-  static void checkBlockSize(int blockSize) {
-    if (blockSize <= 0 || blockSize > MAX_BLOCK_SIZE) {
-      throw new IllegalArgumentException("blockSize must be > 0 and < " + MAX_BLOCK_SIZE + ", got " + blockSize);
-    }
-    if (blockSize < 64) {
-      throw new IllegalArgumentException("blockSize must be >= 64, got " + blockSize);
-    }
-    if ((blockSize & (blockSize - 1)) != 0) {
-      throw new IllegalArgumentException("blockSize must be a power of two, got " + blockSize);
-    }
-  }
-
   static long zigZagEncode(long n) {
     return (n >> 63) ^ (n << 1);
   }
@@ -66,7 +57,7 @@ abstract class AbstractBlockPackedWriter
    * @param blockSize the number of values of a single block, must be a multiple of <tt>64</tt>
    */
   public AbstractBlockPackedWriter(DataOutput out, int blockSize) {
-    checkBlockSize(blockSize);
+    checkBlockSize(blockSize, MIN_BLOCK_SIZE, MAX_BLOCK_SIZE);
     reset(out);
     values = new long[blockSize];
   }

Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java Tue Jul  2 07:12:00 2013
@@ -27,9 +27,16 @@ import java.util.Arrays;
  */
 public final class AppendingLongBuffer extends AbstractAppendingLongBuffer {
 
-  /** Sole constructor. */
+  /** @param initialPageCount the initial number of pages
+   *  @param pageSize         the size of a single page */
+  public AppendingLongBuffer(int initialPageCount, int pageSize) {
+    super(initialPageCount, pageSize);
+  }
+
+  /** Create an {@link AppendingLongBuffer} with initialPageCount=16 and
+   *  pageSize=1024. */
   public AppendingLongBuffer() {
-    super(16);
+    this(16, 1024);
   }
 
   @Override
@@ -43,8 +50,9 @@ public final class AppendingLongBuffer e
     }
   }
 
+  @Override
   void packPendingValues() {
-    assert pendingOff == MAX_PENDING_COUNT;
+    assert pendingOff == pending.length;
 
     // compute max delta
     long minValue = pending[0];
@@ -71,6 +79,7 @@ public final class AppendingLongBuffer e
   }
 
   /** Return an iterator over the values of this buffer. */
+  @Override
   public Iterator iterator() {
     return new Iterator();
   }
@@ -78,20 +87,21 @@ public final class AppendingLongBuffer e
   /** A long iterator. */
   public final class Iterator extends AbstractAppendingLongBuffer.Iterator {
 
-    private Iterator() {
+    Iterator() {
       super();
     }
 
+    @Override
     void fillValues() {
       if (vOff == valuesOff) {
         currentValues = pending;
       } else if (deltas[vOff] == null) {
         Arrays.fill(currentValues, minValues[vOff]);
       } else {
-        for (int k = 0; k < MAX_PENDING_COUNT; ) {
-          k += deltas[vOff].get(k, currentValues, k, MAX_PENDING_COUNT - k);
+        for (int k = 0; k < pending.length; ) {
+          k += deltas[vOff].get(k, currentValues, k, pending.length - k);
         }
-        for (int k = 0; k < MAX_PENDING_COUNT; ++k) {
+        for (int k = 0; k < pending.length; ++k) {
           currentValues[k] += minValues[vOff];
         }
       }

Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java Tue Jul  2 07:12:00 2013
@@ -17,11 +17,14 @@ package org.apache.lucene.util.packed;
  * limitations under the License.
  */
 
+import static org.apache.lucene.util.packed.AbstractBlockPackedWriter.BPV_SHIFT;
+import static org.apache.lucene.util.packed.AbstractBlockPackedWriter.MAX_BLOCK_SIZE;
+import static org.apache.lucene.util.packed.AbstractBlockPackedWriter.MIN_BLOCK_SIZE;
+import static org.apache.lucene.util.packed.AbstractBlockPackedWriter.MIN_VALUE_EQUALS_0;
 import static org.apache.lucene.util.packed.BlockPackedReaderIterator.readVLong;
 import static org.apache.lucene.util.packed.BlockPackedReaderIterator.zigZagDecode;
-import static org.apache.lucene.util.packed.BlockPackedWriter.BPV_SHIFT;
-import static org.apache.lucene.util.packed.BlockPackedWriter.MIN_VALUE_EQUALS_0;
-import static org.apache.lucene.util.packed.BlockPackedWriter.checkBlockSize;
+import static org.apache.lucene.util.packed.PackedInts.checkBlockSize;
+import static org.apache.lucene.util.packed.PackedInts.numBlocks;
 
 import java.io.IOException;
 
@@ -40,14 +43,10 @@ public final class BlockPackedReader {
 
   /** Sole constructor. */
   public BlockPackedReader(IndexInput in, int packedIntsVersion, int blockSize, long valueCount, boolean direct) throws IOException {
-    checkBlockSize(blockSize);
     this.valueCount = valueCount;
-    blockShift = Integer.numberOfTrailingZeros(blockSize);
+    blockShift = checkBlockSize(blockSize, MIN_BLOCK_SIZE, MAX_BLOCK_SIZE);
     blockMask = blockSize - 1;
-    final int numBlocks = (int) (valueCount / blockSize) + (valueCount % blockSize == 0 ? 0 : 1);
-    if ((long) numBlocks * blockSize < valueCount) {
-      throw new IllegalArgumentException("valueCount is too large for this block size");
-    }
+    final int numBlocks = numBlocks(valueCount, blockSize);
     long[] minValues = null;
     subReaders = new PackedInts.Reader[numBlocks];
     for (int i = 0; i < numBlocks; ++i) {

Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReaderIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReaderIterator.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReaderIterator.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReaderIterator.java Tue Jul  2 07:12:00 2013
@@ -17,9 +17,13 @@ package org.apache.lucene.util.packed;
  * limitations under the License.
  */
 
-import static org.apache.lucene.util.packed.BlockPackedWriter.BPV_SHIFT;
-import static org.apache.lucene.util.packed.BlockPackedWriter.MIN_VALUE_EQUALS_0;
-import static org.apache.lucene.util.packed.BlockPackedWriter.checkBlockSize;
+import static org.apache.lucene.util.packed.AbstractBlockPackedWriter.BPV_SHIFT;
+import static org.apache.lucene.util.packed.AbstractBlockPackedWriter.MAX_BLOCK_SIZE;
+import static org.apache.lucene.util.packed.AbstractBlockPackedWriter.MIN_BLOCK_SIZE;
+import static org.apache.lucene.util.packed.AbstractBlockPackedWriter.MIN_VALUE_EQUALS_0;
+import static org.apache.lucene.util.packed.BlockPackedReaderIterator.readVLong;
+import static org.apache.lucene.util.packed.BlockPackedReaderIterator.zigZagDecode;
+import static org.apache.lucene.util.packed.PackedInts.checkBlockSize;
 
 import java.io.EOFException;
 import java.io.IOException;
@@ -87,7 +91,7 @@ public final class BlockPackedReaderIter
    *                  been used to write the stream
    */
   public BlockPackedReaderIterator(DataInput in, int packedIntsVersion, int blockSize, long valueCount) {
-    checkBlockSize(blockSize);
+    checkBlockSize(blockSize, MIN_BLOCK_SIZE, MAX_BLOCK_SIZE);
     this.packedIntsVersion = packedIntsVersion;
     this.blockSize = blockSize;
     this.values = new long[blockSize];

Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java Tue Jul  2 07:12:00 2013
@@ -37,14 +37,22 @@ public final class MonotonicAppendingLon
     return (n >> 63) ^ (n << 1);
   }
 
-  private float[] averages;
+  float[] averages;
 
-  /** Sole constructor. */
+  /** @param initialPageCount the initial number of pages
+   *  @param pageSize         the size of a single page */
+  public MonotonicAppendingLongBuffer(int initialPageCount, int pageSize) {
+    super(initialPageCount, pageSize);
+    averages = new float[pending.length];
+  }
+
+  /** Create an {@link MonotonicAppendingLongBuffer} with initialPageCount=16
+   *  and pageSize=1024. */
   public MonotonicAppendingLongBuffer() {
-    super(16);
-    averages = new float[16];
+    this(16, 1024);
   }
-  
+
+  @Override
   long get(int block, int element) {
     if (block == valuesOff) {
       return pending[element];
@@ -66,16 +74,16 @@ public final class MonotonicAppendingLon
 
   @Override
   void packPendingValues() {
-    assert pendingOff == MAX_PENDING_COUNT;
+    assert pendingOff == pending.length;
 
     minValues[valuesOff] = pending[0];
-    averages[valuesOff] = (float) (pending[BLOCK_MASK] - pending[0]) / BLOCK_MASK;
+    averages[valuesOff] = (float) (pending[pending.length - 1] - pending[0]) / (pending.length - 1);
 
-    for (int i = 0; i < MAX_PENDING_COUNT; ++i) {
+    for (int i = 0; i < pending.length; ++i) {
       pending[i] = zigZagEncode(pending[i] - minValues[valuesOff] - (long) (averages[valuesOff] * (long) i));
     }
     long maxDelta = 0;
-    for (int i = 0; i < MAX_PENDING_COUNT; ++i) {
+    for (int i = 0; i < pending.length; ++i) {
       if (pending[i] < 0) {
         maxDelta = -1;
         break;
@@ -94,6 +102,7 @@ public final class MonotonicAppendingLon
   }
 
   /** Return an iterator over the values of this buffer. */
+  @Override
   public Iterator iterator() {
     return new Iterator();
   }
@@ -105,18 +114,19 @@ public final class MonotonicAppendingLon
       super();
     }
 
+    @Override
     void fillValues() {
       if (vOff == valuesOff) {
         currentValues = pending;
       } else if (deltas[vOff] == null) {
-        for (int k = 0; k < MAX_PENDING_COUNT; ++k) {
+        for (int k = 0; k < pending.length; ++k) {
           currentValues[k] = minValues[vOff] + (long) (averages[vOff] * (long) k);
         }
       } else {
-        for (int k = 0; k < MAX_PENDING_COUNT; ) {
-          k += deltas[vOff].get(k, currentValues, k, MAX_PENDING_COUNT - k);
+        for (int k = 0; k < pending.length; ) {
+          k += deltas[vOff].get(k, currentValues, k, pending.length - k);
         }
-        for (int k = 0; k < MAX_PENDING_COUNT; ++k) {
+        for (int k = 0; k < pending.length; ++k) {
           currentValues[k] = minValues[vOff] + (long) (averages[vOff] * (long) k) + zigZagDecode(currentValues[k]);
         }
       }

Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicBlockPackedReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicBlockPackedReader.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicBlockPackedReader.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicBlockPackedReader.java Tue Jul  2 07:12:00 2013
@@ -17,8 +17,11 @@ package org.apache.lucene.util.packed;
  * limitations under the License.
  */
 
-import static org.apache.lucene.util.packed.AbstractBlockPackedWriter.checkBlockSize;
+import static org.apache.lucene.util.packed.AbstractBlockPackedWriter.MAX_BLOCK_SIZE;
+import static org.apache.lucene.util.packed.AbstractBlockPackedWriter.MIN_BLOCK_SIZE;
 import static org.apache.lucene.util.packed.BlockPackedReaderIterator.zigZagDecode;
+import static org.apache.lucene.util.packed.PackedInts.checkBlockSize;
+import static org.apache.lucene.util.packed.PackedInts.numBlocks;
 
 import java.io.IOException;
 
@@ -39,14 +42,10 @@ public final class MonotonicBlockPackedR
 
   /** Sole constructor. */
   public MonotonicBlockPackedReader(IndexInput in, int packedIntsVersion, int blockSize, long valueCount, boolean direct) throws IOException {
-    checkBlockSize(blockSize);
     this.valueCount = valueCount;
-    blockShift = Integer.numberOfTrailingZeros(blockSize);
+    blockShift = checkBlockSize(blockSize, MIN_BLOCK_SIZE, MAX_BLOCK_SIZE);
     blockMask = blockSize - 1;
-    final int numBlocks = (int) (valueCount / blockSize) + (valueCount % blockSize == 0 ? 0 : 1);
-    if ((long) numBlocks * blockSize < valueCount) {
-      throw new IllegalArgumentException("valueCount is too large for this block size");
-    }
+    final int numBlocks = numBlocks(valueCount, blockSize);
     minValues = new long[numBlocks];
     averages = new float[numBlocks];
     subReaders = new PackedInts.Reader[numBlocks];

Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java Tue Jul  2 07:12:00 2013
@@ -213,6 +213,11 @@ public class PackedInts {
       this.format = format;
       this.bitsPerValue = bitsPerValue;
     }
+
+    @Override
+    public String toString() {
+      return "FormatAndBits(format=" + format + " bitsPerValue=" + bitsPerValue + ")";
+    }
   }
 
   /**
@@ -1198,33 +1203,39 @@ public class PackedInts {
       for (int i = 0; i < len; ++i) {
         dest.set(destPos++, src.get(srcPos++));
       }
-    } else {
+    } else if (len > 0) {
       // use bulk operations
-      long[] buf = new long[Math.min(capacity, len)];
-      int remaining = 0;
-      while (len > 0) {
-        final int read = src.get(srcPos, buf, remaining, Math.min(len, buf.length - remaining));
-        assert read > 0;
-        srcPos += read;
-        len -= read;
-        remaining += read;
-        final int written = dest.set(destPos, buf, 0, remaining);
-        assert written > 0;
-        destPos += written;
-        if (written < remaining) {
-          System.arraycopy(buf, written, buf, 0, remaining - written);
-        }
-        remaining -= written;
-      }
-      while (remaining > 0) {
-        final int written = dest.set(destPos, buf, 0, remaining);
-        destPos += written;
-        remaining -= written;
-        System.arraycopy(buf, written, buf, 0, remaining);
+      final long[] buf = new long[Math.min(capacity, len)];
+      copy(src, srcPos, dest, destPos, len, buf);
+    }
+  }
+
+  /** Same as {@link #copy(Reader, int, Mutable, int, int, int)} but using a pre-allocated buffer. */
+  static void copy(Reader src, int srcPos, Mutable dest, int destPos, int len, long[] buf) {
+    assert buf.length > 0;
+    int remaining = 0;
+    while (len > 0) {
+      final int read = src.get(srcPos, buf, remaining, Math.min(len, buf.length - remaining));
+      assert read > 0;
+      srcPos += read;
+      len -= read;
+      remaining += read;
+      final int written = dest.set(destPos, buf, 0, remaining);
+      assert written > 0;
+      destPos += written;
+      if (written < remaining) {
+        System.arraycopy(buf, written, buf, 0, remaining - written);
       }
+      remaining -= written;
+    }
+    while (remaining > 0) {
+      final int written = dest.set(destPos, buf, 0, remaining);
+      destPos += written;
+      remaining -= written;
+      System.arraycopy(buf, written, buf, 0, remaining);
     }
   }
-  
+
   /**
    * Expert: reads only the metadata from a stream. This is useful to later
    * restore a stream or open a direct reader via 
@@ -1261,4 +1272,26 @@ public class PackedInts {
     }    
   }
 
-}
\ No newline at end of file
+  /** Check that the block size is a power of 2, in the right bounds, and return
+   *  its log in base 2. */
+  static int checkBlockSize(int blockSize, int minBlockSize, int maxBlockSize) {
+    if (blockSize < minBlockSize || blockSize > maxBlockSize) {
+      throw new IllegalArgumentException("blockSize must be >= " + minBlockSize + " and <= " + maxBlockSize + ", got " + blockSize);
+    }
+    if ((blockSize & (blockSize - 1)) != 0) {
+      throw new IllegalArgumentException("blockSize must be a power of two, got " + blockSize);
+    }
+    return Integer.numberOfTrailingZeros(blockSize);
+  }
+
+  /** Return the number of blocks required to store <code>size</code> values on
+   *  <code>blockSize</code>. */
+  static int numBlocks(long size, int blockSize) {
+    final int numBlocks = (int) (size / blockSize) + (size % blockSize == 0 ? 0 : 1);
+    if ((long) numBlocks * blockSize < size) {
+      throw new IllegalArgumentException("size is too large for this block size");
+    }
+    return numBlocks;
+  }
+
+}

Added: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/PagedGrowableWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/PagedGrowableWriter.java?rev=1498804&view=auto
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/PagedGrowableWriter.java (added)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/PagedGrowableWriter.java Tue Jul  2 07:12:00 2013
@@ -0,0 +1,136 @@
+package org.apache.lucene.util.packed;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static org.apache.lucene.util.packed.PackedInts.checkBlockSize;
+import static org.apache.lucene.util.packed.PackedInts.numBlocks;
+
+/**
+ * A {@link PagedGrowableWriter}. This class slices data into fixed-size blocks
+ * which have independent numbers of bits per value and grow on-demand.
+ * <p>You should use this class instead of {@link AppendingLongBuffer} only when
+ * you need random write-access. Otherwise this class will likely be slower and
+ * less memory-efficient.
+ * @lucene.internal
+ */
+public final class PagedGrowableWriter {
+
+  static final int MIN_BLOCK_SIZE = 1 << 6;
+  static final int MAX_BLOCK_SIZE = 1 << 30;
+
+  final long size;
+  final int pageShift;
+  final int pageMask;
+  final GrowableWriter[] subWriters;
+  final int startBitsPerValue;
+  final float acceptableOverheadRatio;
+
+  /**
+   * Create a new {@link PagedGrowableWriter} instance.
+   *
+   * @param size the number of values to store.
+   * @param pageSize the number of values per page
+   * @param startBitsPerValue the initial number of bits per value
+   * @param acceptableOverheadRatio an acceptable overhead ratio
+   */
+  public PagedGrowableWriter(long size, int pageSize,
+      int startBitsPerValue, float acceptableOverheadRatio) {
+    this(size, pageSize, startBitsPerValue, acceptableOverheadRatio, true);
+  }
+
+  PagedGrowableWriter(long size, int pageSize,int startBitsPerValue, float acceptableOverheadRatio, boolean fillPages) {
+    this.size = size;
+    this.startBitsPerValue = startBitsPerValue;
+    this.acceptableOverheadRatio = acceptableOverheadRatio;
+    pageShift = checkBlockSize(pageSize, MIN_BLOCK_SIZE, MAX_BLOCK_SIZE);
+    pageMask = pageSize - 1;
+    final int numPages = numBlocks(size, pageSize);
+    subWriters = new GrowableWriter[numPages];
+    if (fillPages) {
+      for (int i = 0; i < numPages; ++i) {
+        // do not allocate for more entries than necessary on the last page
+        final int valueCount = i == numPages - 1 ? lastPageSize(size) : pageSize;
+        subWriters[i] = new GrowableWriter(startBitsPerValue, valueCount, acceptableOverheadRatio);
+      }
+    }
+  }
+
+  private int lastPageSize(long size) {
+    final int sz = indexInPage(size);
+    return sz == 0 ? pageSize() : sz;
+  }
+
+  private int pageSize() {
+    return pageMask + 1;
+  }
+
+  /** The number of values. */
+  public long size() {
+    return size;
+  }
+
+  int pageIndex(long index) {
+    return (int) (index >>> pageShift);
+  }
+
+  int indexInPage(long index) {
+    return (int) index & pageMask;
+  }
+
+  /** Get value at <code>index</code>. */
+  public long get(long index) {
+    assert index >= 0 && index < size;
+    final int pageIndex = pageIndex(index);
+    final int indexInPage = indexInPage(index);
+    return subWriters[pageIndex].get(indexInPage);
+  }
+
+  /** Set value at <code>index</code>. */
+  public void set(long index, long value) {
+    assert index >= 0 && index < size;
+    final int pageIndex = pageIndex(index);
+    final int indexInPage = indexInPage(index);
+    subWriters[pageIndex].set(indexInPage, value);
+  }
+
+  /** Create a new {@link PagedGrowableWriter} of size <code>newSize</code>
+   *  based on the content of this buffer. This method is much more efficient
+   *  than creating a new {@link PagedGrowableWriter} and copying values one by
+   *  one. */
+  public PagedGrowableWriter resize(long newSize) {
+    final PagedGrowableWriter newWriter = new PagedGrowableWriter(newSize, pageSize(), startBitsPerValue, acceptableOverheadRatio, false);
+    final int numCommonPages = Math.min(newWriter.subWriters.length, subWriters.length);
+    final long[] copyBuffer = new long[1024];
+    for (int i = 0; i < newWriter.subWriters.length; ++i) {
+      final int valueCount = i == newWriter.subWriters.length - 1 ? lastPageSize(newSize) : pageSize();
+      final int bpv = i < numCommonPages ? subWriters[i].getBitsPerValue() : startBitsPerValue;
+      newWriter.subWriters[i] = new GrowableWriter(bpv, valueCount, acceptableOverheadRatio);
+      if (i < numCommonPages) {
+        final int copyLength = Math.min(valueCount, subWriters[i].size());
+        PackedInts.copy(subWriters[i], 0, newWriter.subWriters[i].getMutable(), 0, copyLength, copyBuffer);
+      }
+    }
+    return newWriter;
+  }
+
+  @Override
+  public String toString() {
+    return getClass().getSimpleName() + "(size=" + size() + ",pageSize=" + pageSize() + ")";
+  }
+
+}

Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/package.html?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/package.html (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/util/packed/package.html Tue Jul  2 07:12:00 2013
@@ -47,6 +47,11 @@
         <li>Same as PackedInts.Mutable but grows the number of bits per values when needed.</li>
         <li>Useful to build a PackedInts.Mutable from a read-once stream of longs.</li>
     </ul></li>
+    <li><b>{@link org.apache.lucene.util.packed.PagedGrowableWriter}</b><ul>
+        <li>Slices data into fixed-size blocks stored in GrowableWriters.</li>
+        <li>Supports more than 2B values.</li>
+        <li>You should use AppendingLongBuffer instead if you don't need random write access.</li>
+    </ul></li>
     <li><b>{@link org.apache.lucene.util.packed.AppendingLongBuffer}</b><ul>
         <li>Can store any sequence of longs.</li>
         <li>Compression is good when values are close to each other.</li>

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestAtomicUpdate.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestAtomicUpdate.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestAtomicUpdate.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestAtomicUpdate.java Tue Jul  2 07:12:00 2013
@@ -25,18 +25,7 @@ import org.apache.lucene.store.*;
 import org.apache.lucene.util.*;
 
 public class TestAtomicUpdate extends LuceneTestCase {
-  private static final class MockIndexWriter extends IndexWriter {
-    public MockIndexWriter(Directory dir, IndexWriterConfig conf) throws IOException {
-      super(dir, conf);
-    }
-
-    @Override
-    boolean testPoint(String name) {
-      if (LuceneTestCase.random().nextInt(4) == 2)
-        Thread.yield();
-      return true;
-    }
-  }
+  
 
   private static abstract class TimedThread extends Thread {
     volatile boolean failed;
@@ -124,7 +113,7 @@ public class TestAtomicUpdate extends Lu
         TEST_VERSION_CURRENT, new MockAnalyzer(random()))
         .setMaxBufferedDocs(7);
     ((TieredMergePolicy) conf.getMergePolicy()).setMaxMergeAtOnce(3);
-    IndexWriter writer = new MockIndexWriter(directory, conf);
+    IndexWriter writer = RandomIndexWriter.mockIndexWriter(directory, conf, random());
 
     // Establish a base index of 100 docs:
     for(int i=0;i<100;i++) {

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java Tue Jul  2 07:12:00 2013
@@ -658,8 +658,7 @@ public class TestCodecs extends LuceneTe
     final int termIndexInterval = _TestUtil.nextInt(random(), 13, 27);
     final Codec codec = Codec.getDefault();
     final SegmentInfo si = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, SEGMENT, 10000, false, codec, null, null);
-    final SegmentWriteState state = 
-        new SegmentWriteState(InfoStream.getDefault(), dir, si, 0, fieldInfos, termIndexInterval, null, null, newIOContext(random()));
+    final SegmentWriteState state = new SegmentWriteState(InfoStream.getDefault(), dir, si, termIndexInterval, fieldInfos, termIndexInterval, null, null, newIOContext(random()));
 
     final FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(state);
     Arrays.sort(fields);

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java Tue Jul  2 07:12:00 2013
@@ -58,6 +58,9 @@ public class TestConcurrentMergeSchedule
         boolean isClose = false;
         StackTraceElement[] trace = new Exception().getStackTrace();
         for (int i = 0; i < trace.length; i++) {
+          if (isDoFlush && isClose) {
+            break;
+          }
           if ("flush".equals(trace[i].getMethodName())) {
             isDoFlush = true;
           }

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java Tue Jul  2 07:12:00 2013
@@ -112,12 +112,7 @@ public class TestCustomNorms extends Luc
     }
 
     @Override
-    public ExactSimScorer exactSimScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public SloppySimScorer sloppySimScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
+    public SimScorer simScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
       throw new UnsupportedOperationException();
     }
   }

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterStallControl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterStallControl.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterStallControl.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterStallControl.java Tue Jul  2 07:12:00 2013
@@ -339,6 +339,7 @@ public class TestDocumentsWriterStallCon
       for (Thread thread : threads) {
         if (thread.getState() != state) {
           done = false;
+          break;
         }
       }
       if (done) {

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldReplacements.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldReplacements.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldReplacements.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldReplacements.java Tue Jul  2 07:12:00 2013
@@ -773,8 +773,9 @@ public class TestFieldReplacements exten
   }
   
   public void testReplaceLayers() throws IOException {
-    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
-        TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+    IndexWriterConfig indexWriterConfig = newIndexWriterConfig(
+        TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+    IndexWriter writer = new IndexWriter(dir, indexWriterConfig);
     
     FieldType fieldType = new FieldType();
     fieldType.setIndexed(true);
@@ -784,6 +785,7 @@ public class TestFieldReplacements exten
     
     Document doc0 = new Document();
     doc0.add(new StoredField("f1", "a", fieldType));
+    doc0.add(new StoredField("f2", "a", fieldType));
     writer.addDocument(doc0);
 
     // add f2:b
@@ -791,7 +793,7 @@ public class TestFieldReplacements exten
     fields1.add(new StoredField("f2", "b", fieldType));
     writer.updateFields(Operation.ADD_FIELDS, new Term("f1", "a"), fields1);
     
-    // remove f2:b and add f2:c
+    // remove f2:a and f2:b, add f2:c
     Document fields2 = new Document();
     fields2.add(new StoredField("f2", "c", fieldType));
     writer.updateFields(Operation.REPLACE_FIELDS, new Term("f2", "b"), fields2);
@@ -801,12 +803,17 @@ public class TestFieldReplacements exten
     fields3.add(new StoredField("f2", "d", fieldType));
     writer.updateFields(Operation.ADD_FIELDS, new Term("f2", "b"), fields3);
     
+    // do nothing since f2:a was removed
+    writer.deleteDocuments(new Term("f2", "a"));
+    
     writer.close();
     
     DirectoryReader directoryReader = DirectoryReader.open(dir);
     final AtomicReader atomicReader = directoryReader.leaves().get(0).reader();
     printField(atomicReader, "f1");
     
+    assertEquals("wrong number of documents", 1, directoryReader.numDocs());
+    
     // check indexed fields
     final DocsAndPositionsEnum termPositionsA = atomicReader
         .termPositionsEnum(new Term("f1", "a"));
@@ -816,6 +823,12 @@ public class TestFieldReplacements exten
     assertEquals("wrong doc id", DocIdSetIterator.NO_MORE_DOCS,
         termPositionsA.nextDoc());
     
+    final DocsAndPositionsEnum termPositionsA2 = atomicReader
+        .termPositionsEnum(new Term("f2", "a"));
+    assertNotNull("no positions for term", termPositionsA2);
+    assertEquals("wrong doc id", DocIdSetIterator.NO_MORE_DOCS,
+        termPositionsA2.nextDoc());
+    
     final DocsAndPositionsEnum termPositionsB = atomicReader
         .termPositionsEnum(new Term("f2", "b"));
     assertNotNull("no positions for term", termPositionsB);
@@ -826,6 +839,7 @@ public class TestFieldReplacements exten
         .termPositionsEnum(new Term("f2", "c"));
     assertNotNull("no positions for term", termPositionsC);
     assertEquals("wrong doc id", 0, termPositionsC.nextDoc());
+    // 100000 == 2 * StackedDocsEnum.STACKED_SEGMENT_POSITION_INCREMENT
     assertEquals("wrong position", 100000, termPositionsC.nextPosition());
     assertEquals("wrong doc id", DocIdSetIterator.NO_MORE_DOCS,
         termPositionsC.nextDoc());
@@ -872,7 +886,7 @@ public class TestFieldReplacements exten
   }
   
   public void printIndexes() throws IOException {
-    File outDir = new File("D:/temp/ifu/compare/scenario/b");
+    File outDir = new File("D:/temp/ifu/compare/scenario/a");
     outDir.mkdirs();
     
     for (int i = 0; i < 42; i++) {

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Tue Jul  2 07:12:00 2013
@@ -1183,6 +1183,46 @@ public class TestIndexWriter extends Luc
     t.join();
     assertFalse(t.failed);
   }
+  
+  /** testThreadInterruptDeadlock but with 2 indexer threads */
+  public void testTwoThreadsInterruptDeadlock() throws Exception {
+    IndexerThreadInterrupt t1 = new IndexerThreadInterrupt();
+    t1.setDaemon(true);
+    t1.start();
+    
+    IndexerThreadInterrupt t2 = new IndexerThreadInterrupt();
+    t2.setDaemon(true);
+    t2.start();
+
+    // Force class loader to load ThreadInterruptedException
+    // up front... else we can see a false failure if 2nd
+    // interrupt arrives while class loader is trying to
+    // init this class (in servicing a first interrupt):
+    assertTrue(new ThreadInterruptedException(new InterruptedException()).getCause() instanceof InterruptedException);
+
+    // issue 300 interrupts to child thread
+    final int numInterrupts = atLeast(300);
+    int i = 0;
+    while(i < numInterrupts) {
+      // TODO: would be nice to also sometimes interrupt the
+      // CMS merge threads too ...
+      Thread.sleep(10);
+      IndexerThreadInterrupt t = random().nextBoolean() ? t1 : t2;
+      if (t.allowInterrupt) {
+        i++;
+        t.interrupt();
+      }
+      if (!t1.isAlive() && !t2.isAlive()) {
+        break;
+      }
+    }
+    t1.finish = true;
+    t2.finish = true;
+    t1.join();
+    t2.join();
+    assertFalse(t1.failed);
+    assertFalse(t2.failed);
+  }
 
 
   public void testIndexStoreCombos() throws Exception {

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Tue Jul  2 07:12:00 2013
@@ -25,6 +25,8 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Random;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -302,6 +304,69 @@ public class TestIndexWriterDelete exten
     modifier.close();
     dir.close();
   }
+  
+  
+  public void testDeleteAllNoDeadLock() throws IOException, InterruptedException {
+    Directory dir = newDirectory();
+    final RandomIndexWriter modifier = new RandomIndexWriter(random(), dir); 
+    int numThreads = atLeast(2);
+    Thread[] threads = new Thread[numThreads];
+    final CountDownLatch latch = new CountDownLatch(1);
+    final CountDownLatch doneLatch = new CountDownLatch(numThreads);
+    for (int i = 0; i < numThreads; i++) {
+      final int offset = i;
+      threads[i] = new Thread() {
+        @Override
+        public void run() {
+          int id = offset * 1000;
+          int value = 100;
+          try {
+            latch.await();
+            for (int i = 0; i < 1000; i++) {
+              Document doc = new Document();
+              doc.add(newTextField("content", "aaa", Field.Store.NO));
+              doc.add(newStringField("id", String.valueOf(id++), Field.Store.YES));
+              doc.add(newStringField("value", String.valueOf(value), Field.Store.NO));
+              doc.add(new NumericDocValuesField("dv", value));
+              modifier.addDocument(doc);
+              if (VERBOSE) {
+                System.out.println("\tThread["+offset+"]: add doc: " + id);
+              }
+            }
+          } catch (Exception e) {
+            throw new RuntimeException(e);
+          } finally {
+            doneLatch.countDown();
+            if (VERBOSE) {
+              System.out.println("\tThread["+offset+"]: done indexing" );
+            }
+          }
+        }
+      };
+      threads[i].start();
+    }
+    latch.countDown();
+    while(!doneLatch.await(1, TimeUnit.MILLISECONDS)) {
+      modifier.deleteAll();
+      if (VERBOSE) {
+        System.out.println("del all");
+      }
+    }
+    
+    modifier.deleteAll();
+    for (Thread thread : threads) {
+      thread.join();
+    }
+    
+    modifier.close();
+    DirectoryReader reader = DirectoryReader.open(dir);
+    assertEquals(reader.maxDoc(), 0);
+    assertEquals(reader.numDocs(), 0);
+    assertEquals(reader.numDeletedDocs(), 0);
+    reader.close();
+
+    dir.close();
+  }
 
   // test rollback of deleteAll()
   public void testDeleteAllRollback() throws IOException {

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Tue Jul  2 07:12:00 2013
@@ -210,15 +210,10 @@ public class TestIndexWriterExceptions e
 
   ThreadLocal<Thread> doFail = new ThreadLocal<Thread>();
 
-  private class MockIndexWriter extends IndexWriter {
+  private class TestPoint1 implements RandomIndexWriter.TestPoint {
     Random r = new Random(random().nextLong());
-
-    public MockIndexWriter(Directory dir, IndexWriterConfig conf) throws IOException {
-      super(dir, conf);
-    }
-
     @Override
-    boolean testPoint(String name) {
+    public void apply(String name) {
       if (doFail.get() != null && !name.equals("startDoFlush") && r.nextInt(40) == 17) {
         if (VERBOSE) {
           System.out.println(Thread.currentThread().getName() + ": NOW FAIL: " + name);
@@ -226,7 +221,6 @@ public class TestIndexWriterExceptions e
         }
         throw new RuntimeException(Thread.currentThread().getName() + ": intentionally failing at " + name);
       }
-      return true;
     }
   }
 
@@ -238,8 +232,9 @@ public class TestIndexWriterExceptions e
 
     MockAnalyzer analyzer = new MockAnalyzer(random());
     analyzer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
-    MockIndexWriter writer  = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)
-        .setRAMBufferSizeMB(0.1).setMergeScheduler(new ConcurrentMergeScheduler()));
+    
+    IndexWriter writer  = RandomIndexWriter.mockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)
+        .setRAMBufferSizeMB(0.1).setMergeScheduler(new ConcurrentMergeScheduler()), new TestPoint1());
     ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions();
     //writer.setMaxBufferedDocs(10);
     if (VERBOSE) {
@@ -281,8 +276,8 @@ public class TestIndexWriterExceptions e
     Directory dir = newDirectory();
     MockAnalyzer analyzer = new MockAnalyzer(random());
     analyzer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
-    MockIndexWriter writer  = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)
-        .setRAMBufferSizeMB(0.2).setMergeScheduler(new ConcurrentMergeScheduler()));
+    IndexWriter writer  = RandomIndexWriter.mockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)
+        .setRAMBufferSizeMB(0.2).setMergeScheduler(new ConcurrentMergeScheduler()), new TestPoint1());
     ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions();
     //writer.setMaxBufferedDocs(10);
     writer.commit();
@@ -324,19 +319,13 @@ public class TestIndexWriterExceptions e
   }
 
   // LUCENE-1198
-  private static final class MockIndexWriter2 extends IndexWriter {
-
-    public MockIndexWriter2(Directory dir, IndexWriterConfig conf) throws IOException {
-      super(dir, conf);
-    }
-
+  private static final class TestPoint2 implements RandomIndexWriter.TestPoint {
     boolean doFail;
 
     @Override
-    boolean testPoint(String name) {
+    public void apply(String name) {
       if (doFail && name.equals("DocumentsWriterPerThread addDocument start"))
         throw new RuntimeException("intentionally failing");
-      return true;
     }
   }
 
@@ -367,11 +356,12 @@ public class TestIndexWriterExceptions e
 
   public void testExceptionDocumentsWriterInit() throws IOException {
     Directory dir = newDirectory();
-    MockIndexWriter2 w = new MockIndexWriter2(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+    TestPoint2 testPoint = new TestPoint2();
+    IndexWriter w = RandomIndexWriter.mockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())), testPoint);
     Document doc = new Document();
     doc.add(newTextField("field", "a field", Field.Store.YES));
     w.addDocument(doc);
-    w.doFail = true;
+    testPoint.doFail = true;
     try {
       w.addDocument(doc);
       fail("did not hit exception");
@@ -385,7 +375,7 @@ public class TestIndexWriterExceptions e
   // LUCENE-1208
   public void testExceptionJustBeforeFlush() throws IOException {
     Directory dir = newDirectory();
-    MockIndexWriter w = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMaxBufferedDocs(2));
+    IndexWriter w = RandomIndexWriter.mockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMaxBufferedDocs(2), new TestPoint1());
     Document doc = new Document();
     doc.add(newTextField("field", "a field", Field.Store.YES));
     w.addDocument(doc);
@@ -412,22 +402,15 @@ public class TestIndexWriterExceptions e
     dir.close();
   }
 
-  private static final class MockIndexWriter3 extends IndexWriter {
-
-    public MockIndexWriter3(Directory dir, IndexWriterConfig conf) throws IOException {
-      super(dir, conf);
-    }
-
+  private static final class TestPoint3 implements RandomIndexWriter.TestPoint {
     boolean doFail;
     boolean failed;
-
     @Override
-    boolean testPoint(String name) {
+    public void apply(String name) {
       if (doFail && name.equals("startMergeInit")) {
         failed = true;
         throw new RuntimeException("intentionally failing");
       }
-      return true;
     }
   }
 
@@ -441,8 +424,9 @@ public class TestIndexWriterExceptions e
     cms.setSuppressExceptions();
     conf.setMergeScheduler(cms);
     ((LogMergePolicy) conf.getMergePolicy()).setMergeFactor(2);
-    MockIndexWriter3 w = new MockIndexWriter3(dir, conf);
-    w.doFail = true;
+    TestPoint3 testPoint = new TestPoint3();
+    IndexWriter w = RandomIndexWriter.mockIndexWriter(dir, conf, testPoint);
+    testPoint.doFail = true;
     Document doc = new Document();
     doc.add(newTextField("field", "a field", Field.Store.YES));
     for(int i=0;i<10;i++)
@@ -453,7 +437,7 @@ public class TestIndexWriterExceptions e
       }
 
     ((ConcurrentMergeScheduler) w.getConfig().getMergeScheduler()).sync();
-    assertTrue(w.failed);
+    assertTrue(testPoint.failed);
     w.close();
     dir.close();
   }
@@ -555,10 +539,15 @@ public class TestIndexWriterExceptions e
         boolean sawAppend = false;
         boolean sawFlush = false;
         for (int i = 0; i < trace.length; i++) {
-          if (FreqProxTermsWriterPerField.class.getName().equals(trace[i].getClassName()) && "flush".equals(trace[i].getMethodName()))
+          if (sawAppend && sawFlush) {
+            break;
+          }
+          if (FreqProxTermsWriterPerField.class.getName().equals(trace[i].getClassName()) && "flush".equals(trace[i].getMethodName())) {
             sawAppend = true;
-          if ("flush".equals(trace[i].getMethodName()))
+          }
+          if ("flush".equals(trace[i].getMethodName())) {
             sawFlush = true;
+          }
         }
 
         if (sawAppend && sawFlush && count++ >= 30) {
@@ -892,12 +881,18 @@ public class TestIndexWriterExceptions e
       boolean isDelete = false;
       boolean isInGlobalFieldMap = false;
       for (int i = 0; i < trace.length; i++) {
-        if (SegmentInfos.class.getName().equals(trace[i].getClassName()) && stage.equals(trace[i].getMethodName()))
+        if (isCommit && isDelete && isInGlobalFieldMap) {
+          break;
+        }
+        if (SegmentInfos.class.getName().equals(trace[i].getClassName()) && stage.equals(trace[i].getMethodName())) {
           isCommit = true;
-        if (MockDirectoryWrapper.class.getName().equals(trace[i].getClassName()) && "deleteFile".equals(trace[i].getMethodName()))
+        }
+        if (MockDirectoryWrapper.class.getName().equals(trace[i].getClassName()) && "deleteFile".equals(trace[i].getMethodName())) {
           isDelete = true;
-        if (SegmentInfos.class.getName().equals(trace[i].getClassName()) && "writeGlobalFieldMap".equals(trace[i].getMethodName()))
+        }
+        if (SegmentInfos.class.getName().equals(trace[i].getClassName()) && "writeGlobalFieldMap".equals(trace[i].getMethodName())) {
           isInGlobalFieldMap = true;
+        }
           
       }
       if (isInGlobalFieldMap && dontFailDuringGlobalFieldMap) {
@@ -1014,29 +1009,26 @@ public class TestIndexWriterExceptions e
   }
 
   // LUCENE-1347
-  private static final class MockIndexWriter4 extends IndexWriter {
-
-    public MockIndexWriter4(Directory dir, IndexWriterConfig conf) throws IOException {
-      super(dir, conf);
-    }
+  private static final class TestPoint4 implements RandomIndexWriter.TestPoint {
 
     boolean doFail;
 
     @Override
-    boolean testPoint(String name) {
+    public void apply(String name) {
       if (doFail && name.equals("rollback before checkpoint"))
         throw new RuntimeException("intentionally failing");
-      return true;
     }
   }
 
   // LUCENE-1347
   public void testRollbackExceptionHang() throws Throwable {
     Directory dir = newDirectory();
-    MockIndexWriter4 w = new MockIndexWriter4(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+    TestPoint4 testPoint = new TestPoint4();
+    IndexWriter w = RandomIndexWriter.mockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())), testPoint);
+    
 
     addDoc(w);
-    w.doFail = true;
+    testPoint.doFail = true;
     try {
       w.rollback();
       fail("did not hit intentional RuntimeException");
@@ -1044,7 +1036,7 @@ public class TestIndexWriterExceptions e
       // expected
     }
 
-    w.doFail = false;
+    testPoint.doFail = false;
     w.rollback();
     dir.close();
   }
@@ -1342,6 +1334,7 @@ public class TestIndexWriterExceptions e
       for (int i = 0; i < trace.length; i++) {
         if (TermVectorsConsumer.class.getName().equals(trace[i].getClassName()) && stage.equals(trace[i].getMethodName())) {
           fail = true;
+          break;
         }
       }
       

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java Tue Jul  2 07:12:00 2013
@@ -18,10 +18,11 @@ package org.apache.lucene.index;
  *
  */
 
-import java.io.BufferedInputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.PrintStream;
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
@@ -49,10 +50,6 @@ public class TestIndexWriterOnJRECrash e
   
   @Override @Nightly
   public void testNRTThreads() throws Exception {
-    String vendor = Constants.JAVA_VENDOR;
-    assumeTrue(vendor + " JRE not supported.", 
-        vendor.startsWith("Oracle") || vendor.startsWith("Sun") || vendor.startsWith("Apple"));
-    
     // if we are not the fork
     if (System.getProperty("tests.crashmode") == null) {
       // try up to 10 times to create an index
@@ -112,18 +109,40 @@ public class TestIndexWriterOnJRECrash e
     pb.directory(tempDir);
     pb.redirectErrorStream(true);
     Process p = pb.start();
-    InputStream is = p.getInputStream();
-    BufferedInputStream isl = new BufferedInputStream(is);
-    byte buffer[] = new byte[1024];
-    int len = 0;
-    if (VERBOSE) System.err.println(">>> Begin subprocess output");
-    while ((len = isl.read(buffer)) != -1) {
-      if (VERBOSE) {
-        System.err.write(buffer, 0, len);
-      }
-    }
-    if (VERBOSE) System.err.println("<<< End subprocess output");
+
+    // We pump everything to stderr.
+    PrintStream childOut = System.err; 
+    Thread stdoutPumper = ThreadPumper.start(p.getInputStream(), childOut);
+    Thread stderrPumper = ThreadPumper.start(p.getErrorStream(), childOut);
+    if (VERBOSE) childOut.println(">>> Begin subprocess output");
     p.waitFor();
+    stdoutPumper.join();
+    stderrPumper.join();
+    if (VERBOSE) childOut.println("<<< End subprocess output");
+  }
+
+  /** A pipe thread. It'd be nice to reuse guava's implementation for this... */
+  static class ThreadPumper {
+    public static Thread start(final InputStream from, final OutputStream to) {
+      Thread t = new Thread() {
+        @Override
+        public void run() {
+          try {
+            byte [] buffer = new byte [1024];
+            int len;
+            while ((len = from.read(buffer)) != -1) {
+              if (VERBOSE) {
+                to.write(buffer, 0, len);
+              }
+            }
+          } catch (IOException e) {
+            System.err.println("Couldn't pipe from the forked process: " + e.toString());
+          }
+        }
+      };
+      t.start();
+      return t;
+    }
   }
   
   /**
@@ -155,20 +174,40 @@ public class TestIndexWriterOnJRECrash e
     }
     return false;
   }
-  
+
   /**
    * currently, this only works/tested on Sun and IBM.
    */
   public void crashJRE() {
-    try {
-      Class<?> clazz = Class.forName("sun.misc.Unsafe");
-      // we should use getUnsafe instead, harmony implements it, etc.
-      Field field = clazz.getDeclaredField("theUnsafe");
-      field.setAccessible(true);
-      Object o = field.get(null);
-      Method m = clazz.getMethod("putAddress", long.class, long.class);
-      m.invoke(o, 0L, 0L);
-    } catch (Exception e) { e.printStackTrace(); }
-    fail();
+    final String vendor = Constants.JAVA_VENDOR;
+    final boolean supportsUnsafeNpeDereference = 
+        vendor.startsWith("Oracle") || 
+        vendor.startsWith("Sun") || 
+        vendor.startsWith("Apple");
+
+      try {
+        if (supportsUnsafeNpeDereference) {
+          try {
+            Class<?> clazz = Class.forName("sun.misc.Unsafe");
+            Field field = clazz.getDeclaredField("theUnsafe");
+            field.setAccessible(true);
+            Object o = field.get(null);
+            Method m = clazz.getMethod("putAddress", long.class, long.class);
+            m.invoke(o, 0L, 0L);
+          } catch (Throwable e) {
+            System.out.println("Couldn't kill the JVM via Unsafe.");
+            e.printStackTrace(System.out); 
+          }
+        }
+
+        // Fallback attempt to Runtime.halt();
+        Runtime.getRuntime().halt(-1);
+      } catch (Exception e) {
+        System.out.println("Couldn't kill the JVM.");
+        e.printStackTrace(System.out); 
+      }
+
+      // We couldn't get the JVM to crash for some reason.
+      fail();
   }
 }

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java Tue Jul  2 07:12:00 2013
@@ -358,6 +358,9 @@ public class TestIndexWriterWithThreads 
         boolean sawClose = false;
         boolean sawMerge = false;
         for (int i = 0; i < trace.length; i++) {
+          if (sawAbortOrFlushDoc && sawMerge && sawClose) {
+            break;
+          }
           if ("abort".equals(trace[i].getMethodName()) ||
               "finishDocument".equals(trace[i].getMethodName())) {
             sawAbortOrFlushDoc = true;
@@ -370,8 +373,9 @@ public class TestIndexWriterWithThreads 
           }
         }
         if (sawAbortOrFlushDoc && !sawClose && !sawMerge) {
-          if (onlyOnce)
+          if (onlyOnce) {
             doFail = false;
+          }
           //System.out.println(Thread.currentThread().getName() + ": now fail");
           //new Throwable().printStackTrace(System.out);
           throw new IOException("now failing on purpose");

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestNorms.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestNorms.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestNorms.java Tue Jul  2 07:12:00 2013
@@ -179,12 +179,7 @@ public class TestNorms extends LuceneTes
     }
 
     @Override
-    public ExactSimScorer exactSimScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public SloppySimScorer sloppySimScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
+    public SimScorer simScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
       throw new UnsupportedOperationException();
     }
   } 

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java Tue Jul  2 07:12:00 2013
@@ -47,21 +47,16 @@ public class TestStressIndexing2 extends
   static int maxBufferedDocs=3;
   static int seed=0;
 
-  public class MockIndexWriter extends IndexWriter {
-
-    public MockIndexWriter(Directory dir, IndexWriterConfig conf) throws IOException {
-      super(dir, conf);
-    }
+  public final class YieldTestPoint implements RandomIndexWriter.TestPoint {
 
     @Override
-    boolean testPoint(String name) {
+    public void apply(String name) {
       //      if (name.equals("startCommit")) {
       if (random().nextInt(4) == 2)
         Thread.yield();
-      return true;
     }
   }
-  
+//  
   public void testRandomIWReader() throws Throwable {
     Directory dir = newDirectory();
     
@@ -151,9 +146,9 @@ public class TestStressIndexing2 extends
   
   public DocsAndWriter indexRandomIWReader(int nThreads, int iterations, int range, Directory dir) throws IOException, InterruptedException {
     Map<String,Document> docs = new HashMap<String,Document>();
-    IndexWriter w = new MockIndexWriter(dir, newIndexWriterConfig(
+    IndexWriter w = RandomIndexWriter.mockIndexWriter(dir, newIndexWriterConfig(
         TEST_VERSION_CURRENT, new MockAnalyzer(random())).setOpenMode(OpenMode.CREATE).setRAMBufferSizeMB(
-                                                                                                  0.1).setMaxBufferedDocs(maxBufferedDocs).setMergePolicy(newLogMergePolicy()));
+            0.1).setMaxBufferedDocs(maxBufferedDocs).setMergePolicy(newLogMergePolicy()), new YieldTestPoint());
     w.commit();
     LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy();
     lmp.setUseCompoundFile(false);
@@ -202,10 +197,10 @@ public class TestStressIndexing2 extends
   public Map<String,Document> indexRandom(int nThreads, int iterations, int range, Directory dir, int maxThreadStates,
                                           boolean doReaderPooling) throws IOException, InterruptedException {
     Map<String,Document> docs = new HashMap<String,Document>();
-    IndexWriter w = new MockIndexWriter(dir, newIndexWriterConfig(
+    IndexWriter w = RandomIndexWriter.mockIndexWriter(dir, newIndexWriterConfig(
         TEST_VERSION_CURRENT, new MockAnalyzer(random())).setOpenMode(OpenMode.CREATE)
              .setRAMBufferSizeMB(0.1).setMaxBufferedDocs(maxBufferedDocs).setIndexerThreadPool(new ThreadAffinityDocumentsWriterThreadPool(maxThreadStates))
-             .setReaderPooling(doReaderPooling).setMergePolicy(newLogMergePolicy()));
+             .setReaderPooling(doReaderPooling).setMergePolicy(newLogMergePolicy()), new YieldTestPoint());
     LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy();
     lmp.setUseCompoundFile(false);
     lmp.setMergeFactor(mergeFactor);

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java Tue Jul  2 07:12:00 2013
@@ -110,12 +110,7 @@ public class TestUniqueTermCount extends
     }
 
     @Override
-    public ExactSimScorer exactSimScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public SloppySimScorer sloppySimScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
+    public SimScorer simScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
       throw new UnsupportedOperationException();
     }
   }

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java Tue Jul  2 07:12:00 2013
@@ -270,12 +270,7 @@ final class JustCompileSearch {
     }
 
     @Override
-    public ExactSimScorer exactSimScorer(SimWeight stats, AtomicReaderContext context) {
-      throw new UnsupportedOperationException(UNSUPPORTED_MSG);
-    }
-
-    @Override
-    public SloppySimScorer sloppySimScorer(SimWeight stats, AtomicReaderContext context) {
+    public SimScorer simScorer(SimWeight stats, AtomicReaderContext context) {
       throw new UnsupportedOperationException(UNSUPPORTED_MSG);
     }
 

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java Tue Jul  2 07:12:00 2013
@@ -109,18 +109,8 @@ public class TestConjunctions extends Lu
     }
 
     @Override
-    public ExactSimScorer exactSimScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
-      return new ExactSimScorer() {
-        @Override
-        public float score(int doc, int freq) {
-          return freq;
-        }
-      };
-    }
-
-    @Override
-    public SloppySimScorer sloppySimScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
-      return new SloppySimScorer() {
+    public SimScorer simScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
+      return new SimScorer() {
         @Override
         public float score(int doc, float freq) {
           return freq;

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/TestDocValuesScoring.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/TestDocValuesScoring.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/TestDocValuesScoring.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/TestDocValuesScoring.java Tue Jul  2 07:12:00 2013
@@ -156,34 +156,11 @@ public class TestDocValuesScoring extend
     }
 
     @Override
-    public ExactSimScorer exactSimScorer(SimWeight stats, AtomicReaderContext context) throws IOException {
-      final ExactSimScorer sub = sim.exactSimScorer(stats, context);
-      final FieldCache.Floats values = FieldCache.DEFAULT.getFloats(context.reader(), boostField, false);
-
-      return new ExactSimScorer() {
-        @Override
-        public float score(int doc, int freq) {
-          return values.get(doc) * sub.score(doc, freq);
-        }
-
-        @Override
-        public Explanation explain(int doc, Explanation freq) {
-          Explanation boostExplanation = new Explanation(values.get(doc), "indexDocValue(" + boostField + ")");
-          Explanation simExplanation = sub.explain(doc, freq);
-          Explanation expl = new Explanation(boostExplanation.getValue() * simExplanation.getValue(), "product of:");
-          expl.addDetail(boostExplanation);
-          expl.addDetail(simExplanation);
-          return expl;
-        }
-      };
-    }
-
-    @Override
-    public SloppySimScorer sloppySimScorer(SimWeight stats, AtomicReaderContext context) throws IOException {
-      final SloppySimScorer sub = sim.sloppySimScorer(stats, context);
+    public SimScorer simScorer(SimWeight stats, AtomicReaderContext context) throws IOException {
+      final SimScorer sub = sim.simScorer(stats, context);
       final FieldCache.Floats values = FieldCache.DEFAULT.getFloats(context.reader(), boostField, false);
       
-      return new SloppySimScorer() {
+      return new SimScorer() {
         @Override
         public float score(int doc, float freq) {
           return values.get(doc) * sub.score(doc, freq);

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java Tue Jul  2 07:12:00 2013
@@ -37,7 +37,7 @@ import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermContext;
 import org.apache.lucene.search.BooleanQuery.BooleanWeight;
 import org.apache.lucene.search.similarities.DefaultSimilarity;
-import org.apache.lucene.search.similarities.Similarity.ExactSimScorer;
+import org.apache.lucene.search.similarities.Similarity.SimScorer;
 import org.apache.lucene.search.similarities.Similarity.SimWeight;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
@@ -274,7 +274,7 @@ public class TestMinShouldMatch2 extends
     final int maxDoc;
 
     final Set<Long> ords = new HashSet<Long>();
-    final ExactSimScorer[] sims;
+    final SimScorer[] sims;
     final int minNrShouldMatch;
     
     double score = Float.NaN;
@@ -285,7 +285,7 @@ public class TestMinShouldMatch2 extends
       this.maxDoc = reader.maxDoc();
       BooleanQuery bq = (BooleanQuery) weight.getQuery();
       this.minNrShouldMatch = bq.getMinimumNumberShouldMatch();
-      this.sims = new ExactSimScorer[(int)dv.getValueCount()];
+      this.sims = new SimScorer[(int)dv.getValueCount()];
       for (BooleanClause clause : bq.getClauses()) {
         assert !clause.isProhibited();
         assert !clause.isRequired();
@@ -300,7 +300,7 @@ public class TestMinShouldMatch2 extends
                         searcher.termStatistics(term, context));
           w.getValueForNormalization(); // ignored
           w.normalize(1F, 1F);
-          sims[(int)ord] = weight.similarity.exactSimScorer(w, reader.getContext());
+          sims[(int)ord] = weight.similarity.simScorer(w, reader.getContext());
         }
       }
     }

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java Tue Jul  2 07:12:00 2013
@@ -148,7 +148,7 @@ final class JustCompileSearchSpans {
   static final class JustCompileSpanScorer extends SpanScorer {
 
     protected JustCompileSpanScorer(Spans spans, Weight weight,
-        Similarity.SloppySimScorer docScorer) throws IOException {
+        Similarity.SimScorer docScorer) throws IOException {
       super(spans, weight, docScorer);
     }
 

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java Tue Jul  2 07:12:00 2013
@@ -379,11 +379,11 @@ public class TestPayloadSpans extends Lu
     PayloadSpanUtil psu = new PayloadSpanUtil(searcher.getTopReaderContext());
     
     Collection<byte[]> payloads = psu.getPayloadsForQuery(new TermQuery(new Term(PayloadHelper.FIELD, "rr")));
-    if(VERBOSE)
+    if(VERBOSE) {
       System.out.println("Num payloads:" + payloads.size());
-    for (final byte [] bytes : payloads) {
-      if(VERBOSE)
+      for (final byte [] bytes : payloads) {
         System.out.println(new String(bytes, "UTF-8"));
+      }
     }
     reader.close();
     directory.close();
@@ -451,12 +451,12 @@ public class TestPayloadSpans extends Lu
         System.out.println("\nSpans Dump --");
       if (spans.isPayloadAvailable()) {
         Collection<byte[]> payload = spans.getPayload();
-        if(VERBOSE)
+        if(VERBOSE) {
           System.out.println("payloads for span:" + payload.size());
-        for (final byte [] bytes : payload) {
-          if(VERBOSE)
+          for (final byte [] bytes : payload) {
             System.out.println("doc:" + spans.doc() + " s:" + spans.start() + " e:" + spans.end() + " "
               + new String(bytes, "UTF-8"));
+          }
         }
 
         assertEquals(numPayloads[cnt],payload.size());

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java Tue Jul  2 07:12:00 2013
@@ -55,7 +55,7 @@ public class TestMockDirectoryWrapper ex
   public void testDiskFull() throws IOException {
     // test writeBytes
     MockDirectoryWrapper dir = newMockDirectory();
-    dir.setMaxSizeInBytes(2);
+    dir.setMaxSizeInBytes(3);
     final byte[] bytes = new byte[] { 1, 2};
     IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT);
     out.writeBytes(bytes, bytes.length); // first write should succeed
@@ -73,7 +73,7 @@ public class TestMockDirectoryWrapper ex
     
     // test copyBytes
     dir = newMockDirectory();
-    dir.setMaxSizeInBytes(2);
+    dir.setMaxSizeInBytes(3);
     out = dir.createOutput("foo", IOContext.DEFAULT);
     out.copyBytes(new ByteArrayDataInput(bytes), bytes.length); // first copy should succeed
     // flush() to ensure the written bytes are not buffered and counted

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/util/TestTimSorter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/util/TestTimSorter.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/util/TestTimSorter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/util/TestTimSorter.java Tue Jul  2 07:12:00 2013
@@ -25,7 +25,7 @@ public class TestTimSorter extends BaseS
 
   @Override
   public Sorter newSorter(Entry[] arr) {
-    return new ArrayTimSorter<Entry>(arr, ArrayUtil.<Entry>naturalComparator(), random().nextInt(arr.length));
+    return new ArrayTimSorter<Entry>(arr, ArrayUtil.<Entry>naturalComparator(), _TestUtil.nextInt(random(), 0, arr.length));
   }
 
 }

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java Tue Jul  2 07:12:00 2013
@@ -34,7 +34,7 @@ import org.apache.lucene.util.packed.Pac
 import org.junit.Ignore;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 
-@Ignore("Requires tons of heap to run (10G works)")
+@Ignore("Requires tons of heap to run (420G works)")
 @TimeoutSuite(millis = 100 * TimeUnits.HOUR)
 public class Test2BFST extends LuceneTestCase {
 
@@ -50,12 +50,12 @@ public class Test2BFST extends LuceneTes
     for(int doPackIter=0;doPackIter<2;doPackIter++) {
       boolean doPack = doPackIter == 1;
 
-      // Build FST w/ NoOutputs and stop when nodeCount > 3B
+      // Build FST w/ NoOutputs and stop when nodeCount > 2.2B
       if (!doPack) {
         System.out.println("\nTEST: 3B nodes; doPack=false output=NO_OUTPUTS");
         Outputs<Object> outputs = NoOutputs.getSingleton();
         Object NO_OUTPUT = outputs.getNoOutput();
-        final Builder<Object> b = new Builder<Object>(FST.INPUT_TYPE.BYTE1, 0, 0, false, false, Integer.MAX_VALUE, outputs,
+        final Builder<Object> b = new Builder<Object>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs,
                                                       null, doPack, PackedInts.COMPACT, true, 15);
 
         int count = 0;
@@ -72,7 +72,7 @@ public class Test2BFST extends LuceneTes
           if (count % 100000 == 0) {
             System.out.println(count + ": " + b.fstSizeInBytes() + " bytes; " + b.getTotStateCount() + " nodes");
           }
-          if (b.getTotStateCount() > LIMIT) {
+          if (b.getTotStateCount() > Integer.MAX_VALUE + 100L * 1024 * 1024) {
             break;
           }
           nextInput(r, ints2);



Mime
View raw message