lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rm...@apache.org
Subject svn commit: r1631928 [3/5] - in /lucene/dev/branches/lucene5969: ./ dev-tools/ dev-tools/idea/.idea/libraries/ lucene/ lucene/analysis/ lucene/analysis/common/ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ lucene/analysis/common/...
Date Wed, 15 Oct 2014 01:26:31 GMT
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java Wed Oct 15 01:26:26 2014
@@ -22,7 +22,6 @@ import java.util.List;
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesConsumer;
-import org.apache.lucene.codecs.FieldInfosWriter;
 import org.apache.lucene.codecs.FieldsConsumer;
 import org.apache.lucene.codecs.NormsConsumer;
 import org.apache.lucene.codecs.StoredFieldsWriter;
@@ -147,8 +146,7 @@ final class SegmentMerger {
     }
     
     // write the merged infos
-    FieldInfosWriter fieldInfosWriter = codec.fieldInfosFormat().getFieldInfosWriter();
-    fieldInfosWriter.write(directory, mergeState.segmentInfo, "", mergeState.mergeFieldInfos, context);
+    codec.fieldInfosFormat().write(directory, mergeState.segmentInfo, "", mergeState.mergeFieldInfos, context);
 
     return mergeState;
   }

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java Wed Oct 15 01:26:26 2014
@@ -182,7 +182,7 @@ public final class SegmentReader extends
       // updates always outside of CFS
       FieldInfosFormat fisFormat = si.info.getCodec().fieldInfosFormat();
       final String segmentSuffix = Long.toString(si.getFieldInfosGen(), Character.MAX_RADIX);
-      return fisFormat.getFieldInfosReader().read(si.info.dir, si.info, segmentSuffix, IOContext.READONCE);
+      return fisFormat.read(si.info.dir, si.info, segmentSuffix, IOContext.READONCE);
     }
   }
   

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java Wed Oct 15 01:26:26 2014
@@ -142,7 +142,7 @@ public class TieredMergePolicy extends M
       throw new IllegalArgumentException("maxMergedSegmentMB must be >=0 (got " + v + ")");
     }
     v *= 1024 * 1024;
-    maxMergedSegmentBytes = (v > Long.MAX_VALUE) ? Long.MAX_VALUE : (long) v;
+    maxMergedSegmentBytes = v > Long.MAX_VALUE ? Long.MAX_VALUE : (long) v;
     return this;
   }
 
@@ -183,7 +183,7 @@ public class TieredMergePolicy extends M
       throw new IllegalArgumentException("floorSegmentMB must be >= 0.0 (got " + v + ")");
     }
     v *= 1024 * 1024;
-    floorSegmentBytes = (v > Long.MAX_VALUE) ? Long.MAX_VALUE : (long) v;
+    floorSegmentBytes = v > Long.MAX_VALUE ? Long.MAX_VALUE : (long) v;
     return this;
   }
 
@@ -314,8 +314,12 @@ public class TieredMergePolicy extends M
     // If we have too-large segments, grace them out
     // of the maxSegmentCount:
     int tooBigCount = 0;
-    while (tooBigCount < infosSorted.size() && size(infosSorted.get(tooBigCount), writer) >= maxMergedSegmentBytes/2.0) {
-      totIndexBytes -= size(infosSorted.get(tooBigCount), writer);
+    while (tooBigCount < infosSorted.size()) {
+      long segBytes = size(infosSorted.get(tooBigCount), writer);
+      if (segBytes < maxMergedSegmentBytes/2.0) {
+        break;
+      }
+      totIndexBytes -= segBytes;
       tooBigCount++;
     }
 
@@ -351,7 +355,7 @@ public class TieredMergePolicy extends M
       for(int idx = tooBigCount; idx<infosSorted.size(); idx++) {
         final SegmentCommitInfo info = infosSorted.get(idx);
         if (merging.contains(info)) {
-          mergingBytes += info.sizeInBytes();
+          mergingBytes += size(info, writer);
         } else if (!toBeMerged.contains(info)) {
           eligible.add(info);
         }
@@ -400,6 +404,10 @@ public class TieredMergePolicy extends M
             totAfterMergeBytes += segBytes;
           }
 
+          // We should never see an empty candidate: we iterated over maxMergeAtOnce
+          // segments, and already pre-excluded the too-large segments:
+          assert candidate.size() > 0;
+
           final MergeScore score = score(candidate, hitTooLarge, mergingBytes, writer);
           if (verbose(writer)) {
             message("  maybe=" + writer.segString(candidate) + " score=" + score.getScore() + " " + score.getExplanation() + " tooLarge=" + hitTooLarge + " size=" + String.format(Locale.ROOT, "%.3f MB", totAfterMergeBytes/1024./1024.), writer);

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java Wed Oct 15 01:26:26 2014
@@ -31,7 +31,7 @@ import org.apache.lucene.index.LeafReade
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.Accountables;
 import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.WAH8DocIdSet;
+import org.apache.lucene.util.RoaringDocIdSet;
 
 /**
  * Wraps another {@link Filter}'s result and caches it.  The purpose is to allow
@@ -86,12 +86,10 @@ public class CachingWrapperFilter extend
   }
   
   /**
-   * Default cache implementation: uses {@link WAH8DocIdSet}.
+   * Default cache implementation: uses {@link RoaringDocIdSet}.
    */
   protected DocIdSet cacheImpl(DocIdSetIterator iterator, LeafReader reader) throws IOException {
-    WAH8DocIdSet.Builder builder = new WAH8DocIdSet.Builder();
-    builder.add(iterator);
-    return builder.build();
+    return new RoaringDocIdSet.Builder(reader.maxDoc()).add(iterator).build();
   }
 
   // for testing

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java Wed Oct 15 01:26:26 2014
@@ -254,12 +254,12 @@ public class FilteredQuery extends Query
    * jumping past the target document. When both land on the same document, it's
    * collected.
    */
-  private static class LeapFrogScorer extends Scorer {
+  private static final class LeapFrogScorer extends Scorer {
     private final DocIdSetIterator secondary;
     private final DocIdSetIterator primary;
     private final Scorer scorer;
-    protected int primaryDoc = -1;
-    protected int secondaryDoc = -1;
+    private int primaryDoc = -1;
+    private int secondaryDoc = -1;
 
     protected LeapFrogScorer(Weight weight, DocIdSetIterator primary, DocIdSetIterator secondary, Scorer scorer) {
       super(weight);
@@ -324,26 +324,6 @@ public class FilteredQuery extends Query
     }
   }
   
-  // TODO once we have way to figure out if we use RA or LeapFrog we can remove this scorer
-  private static final class PrimaryAdvancedLeapFrogScorer extends LeapFrogScorer {
-    private final int firstFilteredDoc;
-
-    protected PrimaryAdvancedLeapFrogScorer(Weight weight, int firstFilteredDoc, DocIdSetIterator filterIter, Scorer other) {
-      super(weight, filterIter, other, other);
-      this.firstFilteredDoc = firstFilteredDoc;
-      this.primaryDoc = firstFilteredDoc; // initialize to prevent and advance call to move it further
-    }
-
-    @Override
-    protected int primaryNext() throws IOException {
-      if (secondaryDoc != -1) {
-        return super.primaryNext();
-      } else {
-        return firstFilteredDoc;
-      }
-    }
-  }
-  
   /** Rewrites the query. If the wrapped is an instance of
    * {@link MatchAllDocsQuery} it returns a {@link ConstantScoreQuery}. Otherwise
    * it returns a new {@code FilteredQuery} wrapping the rewritten query. */
@@ -421,7 +401,7 @@ public class FilteredQuery extends Query
    * A {@link FilterStrategy} that conditionally uses a random access filter if
    * the given {@link DocIdSet} supports random access (returns a non-null value
    * from {@link DocIdSet#bits()}) and
-   * {@link RandomAccessFilterStrategy#useRandomAccess(Bits, int)} returns
+   * {@link RandomAccessFilterStrategy#useRandomAccess(Bits, long)} returns
    * <code>true</code>. Otherwise this strategy falls back to a "zig-zag join" (
    * {@link FilteredQuery#LEAP_FROG_FILTER_FIRST_STRATEGY}) strategy.
    * 
@@ -515,7 +495,7 @@ public class FilteredQuery extends Query
    * A {@link FilterStrategy} that conditionally uses a random access filter if
    * the given {@link DocIdSet} supports random access (returns a non-null value
    * from {@link DocIdSet#bits()}) and
-   * {@link RandomAccessFilterStrategy#useRandomAccess(Bits, int)} returns
+   * {@link RandomAccessFilterStrategy#useRandomAccess(Bits, long)} returns
    * <code>true</code>. Otherwise this strategy falls back to a "zig-zag join" (
    * {@link FilteredQuery#LEAP_FROG_FILTER_FIRST_STRATEGY}) strategy .
    */
@@ -528,25 +508,18 @@ public class FilteredQuery extends Query
         // this means the filter does not accept any documents.
         return null;
       }  
-
-      final int firstFilterDoc = filterIter.nextDoc();
-      if (firstFilterDoc == DocIdSetIterator.NO_MORE_DOCS) {
-        return null;
-      }
       
       final Bits filterAcceptDocs = docIdSet.bits();
       // force if RA is requested
-      final boolean useRandomAccess = filterAcceptDocs != null && useRandomAccess(filterAcceptDocs, firstFilterDoc);
+      final boolean useRandomAccess = filterAcceptDocs != null && useRandomAccess(filterAcceptDocs, filterIter.cost());
       if (useRandomAccess) {
         // if we are using random access, we return the inner scorer, just with other acceptDocs
         return weight.scorer(context, filterAcceptDocs);
       } else {
-        assert firstFilterDoc > -1;
         // we are gonna advance() this scorer, so we set inorder=true/toplevel=false
         // we pass null as acceptDocs, as our filter has already respected acceptDocs, no need to do twice
         final Scorer scorer = weight.scorer(context, null);
-        // TODO once we have way to figure out if we use RA or LeapFrog we can remove this scorer
-        return (scorer == null) ? null : new PrimaryAdvancedLeapFrogScorer(weight, firstFilterDoc, filterIter, scorer);
+        return (scorer == null) ? null : new LeapFrogScorer(weight, filterIter, scorer, scorer);
       }
     }
     
@@ -557,14 +530,14 @@ public class FilteredQuery extends Query
      * However, when the filter is very sparse, it can be faster to execute the query+filter
      * as a conjunction in some cases.
      * 
-     * The default implementation returns <code>true</code> if the first document accepted by the
-     * filter is < 100.
+     * The default implementation returns <code>true</code> if the filter matches more than 1%
+     * of documents
      * 
      * @lucene.internal
      */
-    protected boolean useRandomAccess(Bits bits, int firstFilterDoc) {
-      //TODO once we have a cost API on filters and scorers we should rethink this heuristic
-      return firstFilterDoc < 100;
+    protected boolean useRandomAccess(Bits bits, long filterCost) {
+      // if the filter matches more than 1% of documents, we use random-access
+      return filterCost * 100 > bits.length();
     }
   }
   

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java Wed Oct 15 01:26:26 2014
@@ -153,26 +153,30 @@ public class SparseFixedBitSet extends D
     final int i4096 = i >>> 12;
     final long index = indices[i4096];
     final int i64 = i >>> 6;
-    if (index == 0) {
+    if ((index & (1L << i64)) != 0) {
+      // in that case the sub 64-bits block we are interested in already exists,
+      // we just need to set a bit in an existing long: the number of ones on
+      // the right of i64 gives us the index of the long we need to update
+      bits[i4096][Long.bitCount(index & ((1L << i64) - 1))] |= 1L << i; // shifts are mod 64 in java
+    } else if (index == 0) {
       // if the index is 0, it means that we just found a block of 4096 bits
       // that has no bit that is set yet. So let's initialize a new block:
-      indices[i4096] = 1L << i64; // shifts are mod 64 in java
-      assert bits[i4096] == null;
-      bits[i4096] = new long[] { 1L << i }; // shifts are mod 64 in java
-      ++nonZeroLongCount;
-      ramBytesUsed += SINGLE_ELEMENT_ARRAY_BYTES_USED;
-    } else if ((index & (1L << i64)) == 0) {
+      insertBlock(i4096, i64, i);
+    } else {
       // in that case we found a block of 4096 bits that has some values, but
       // the sub-block of 64 bits that we are interested in has no value yet,
       // so we need to insert a new long
       insertLong(i4096, i64, i, index);
-    } else {
-      // in that case the sub 64-bits block we are interested in already exists,
-      // we just need to set a bit in an existing long: the number of ones on
-      // the right of i64 gives us the index of the long we need to update
-      bits[i4096][Long.bitCount(index & ((1L << i64) - 1))] |= 1L << i; // shifts are mod 64 in java
     }
   }
+  
+  private void insertBlock(int i4096, int i64, int i) {
+    indices[i4096] = 1L << i64; // shifts are mod 64 in java
+    assert bits[i4096] == null;
+    bits[i4096] = new long[] { 1L << i }; // shifts are mod 64 in java
+    ++nonZeroLongCount;
+    ramBytesUsed += SINGLE_ELEMENT_ARRAY_BYTES_USED;
+  }
 
   private void insertLong(int i4096, int i64, int i, long index) {
     indices[i4096] |= 1L << i64; // shifts are mod 64 in java
@@ -244,7 +248,7 @@ public class SparseFixedBitSet extends D
 
     @Override
     public int nextDoc() throws IOException {
-      if (doc == NO_MORE_DOCS || ++doc >= length) {
+      if (++doc >= length) {
         return doc = NO_MORE_DOCS;
       }
       return currentOrNextDoc();
@@ -253,22 +257,17 @@ public class SparseFixedBitSet extends D
     private int currentOrNextDoc() {
       final int i4096 = doc >>> 12;
       final long index = indices[i4096];
-      if (index == 0) {
+      int i64 = doc >>> 6;
+      long indexBits = index >>> i64;
+      if (indexBits == 0) {
         // if the index is zero, it means that there is no value in the
         // current block, so return the first document of the next block
+        // or
+        // if neither the i64-th bit or any other bit on its left is set then
+        // it means that there are no more documents in this block, go to the
+        // next one
         return firstDoc(i4096 + 1);
       } else {
-        // now we are on a block that contains at least one document
-        assert Long.bitCount(index) <= bits[i4096].length;
-        int i64 = doc >>> 6;
-        long indexBits = index >>> i64; // shifts are mod 64 in java
-        if (indexBits == 0) {
-          // if neither the i64-th bit or any other bit on its left is set then
-          // it means that there are no more documents in this block, go to the
-          // next one
-          return firstDoc(i4096 + 1);
-        }
-
         // We know we still have some 64-bits blocks that have bits set, let's
         // advance to the next one by skipping trailing zeros of the index
         int i1 = doc & 0x3F;

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/StringHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/StringHelper.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/StringHelper.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/StringHelper.java Wed Oct 15 01:26:26 2014
@@ -162,10 +162,7 @@ public abstract class StringHelper {
     if (prop != null) {
       // So if there is a test failure that relied on hash
       // order, we remain reproducible based on the test seed:
-      if (prop.length() > 8) {
-        prop = prop.substring(prop.length()-8);
-      }
-      GOOD_FAST_HASH_SEED = (int) Long.parseLong(prop, 16);
+      GOOD_FAST_HASH_SEED = prop.hashCode();
     } else {
       GOOD_FAST_HASH_SEED = (int) System.currentTimeMillis();
     }

Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/Version.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/Version.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/Version.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/util/Version.java Wed Oct 15 01:26:26 2014
@@ -370,6 +370,13 @@ public final class Version {
         }
     }
   }
+  
+  /** Returns a new version based on raw numbers
+   *
+   *  @lucene.internal */
+  public static final Version fromBits(int major, int minor, int bugfix) {
+    return new Version(major, minor, bugfix);
+  }
 
   /** Major version, the difference between stable and trunk */
   public final int major;

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java Wed Oct 15 01:26:26 2014
@@ -28,13 +28,16 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.LuceneTestCase.Monster;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
+import org.apache.lucene.util.TestUtil;
 import org.apache.lucene.util.TimeUnits;
 
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 
 @SuppressCodecs({"SimpleText", "Memory", "Direct"})
-@TimeoutSuite(millis = 80 * TimeUnits.HOUR)
-@Monster("takes ~ 45 minutes")
+@TimeoutSuite(millis = 80 * TimeUnits.HOUR) // effectively no limit
+// The six hour time was achieved on a Linux 3.13 system with these specs:
+// 3-core AMD at 2.5Ghz, 12 GB RAM, 5GB test heap, 2 test JVMs, 2TB SATA.
+@Monster("takes ~ 6 hours if the heap is 5gb")
 public class Test2BBinaryDocValues extends LuceneTestCase {
   
   // indexes IndexWriter.MAX_DOCS docs with a fixed binary field
@@ -50,7 +53,8 @@ public class Test2BBinaryDocValues exten
         .setRAMBufferSizeMB(256.0)
         .setMergeScheduler(new ConcurrentMergeScheduler())
         .setMergePolicy(newLogMergePolicy(false, 10))
-        .setOpenMode(IndexWriterConfig.OpenMode.CREATE));
+        .setOpenMode(IndexWriterConfig.OpenMode.CREATE)
+        .setCodec(TestUtil.getDefaultCodec()));
 
     Document doc = new Document();
     byte bytes[] = new byte[4];
@@ -109,7 +113,8 @@ public class Test2BBinaryDocValues exten
         .setRAMBufferSizeMB(256.0)
         .setMergeScheduler(new ConcurrentMergeScheduler())
         .setMergePolicy(newLogMergePolicy(false, 10))
-        .setOpenMode(IndexWriterConfig.OpenMode.CREATE));
+        .setOpenMode(IndexWriterConfig.OpenMode.CREATE)
+        .setCodec(TestUtil.getDefaultCodec()));
 
     Document doc = new Document();
     byte bytes[] = new byte[4];

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BNumericDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BNumericDocValues.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BNumericDocValues.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BNumericDocValues.java Wed Oct 15 01:26:26 2014
@@ -23,6 +23,7 @@ import org.apache.lucene.document.Numeri
 import org.apache.lucene.store.BaseDirectoryWrapper;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
 import org.apache.lucene.util.TimeUnits;
 import org.apache.lucene.util.LuceneTestCase.Monster;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
@@ -30,8 +31,10 @@ import org.apache.lucene.util.LuceneTest
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 
 @SuppressCodecs({"SimpleText", "Memory", "Direct"})
-@TimeoutSuite(millis = 80 * TimeUnits.HOUR)
-@Monster("takes ~ 30 minutes")
+@TimeoutSuite(millis = 8 * TimeUnits.HOUR)
+// The two hour time was achieved on a Linux 3.13 system with these specs:
+// 3-core AMD at 2.5Ghz, 12 GB RAM, 5GB test heap, 2 test JVMs, 2TB SATA.
+@Monster("takes ~ 2 hours if the heap is 5gb")
 public class Test2BNumericDocValues extends LuceneTestCase {
   
   // indexes IndexWriter.MAX_DOCS docs with an increasing dv field
@@ -47,7 +50,8 @@ public class Test2BNumericDocValues exte
         .setRAMBufferSizeMB(256.0)
         .setMergeScheduler(new ConcurrentMergeScheduler())
         .setMergePolicy(newLogMergePolicy(false, 10))
-        .setOpenMode(IndexWriterConfig.OpenMode.CREATE));
+        .setOpenMode(IndexWriterConfig.OpenMode.CREATE)
+        .setCodec(TestUtil.getDefaultCodec()));
 
     Document doc = new Document();
     NumericDocValuesField dvField = new NumericDocValuesField("dv", 0);

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BPositions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BPositions.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BPositions.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BPositions.java Wed Oct 15 01:26:26 2014
@@ -28,6 +28,7 @@ import org.apache.lucene.document.TextFi
 import org.apache.lucene.store.BaseDirectoryWrapper;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
 import org.apache.lucene.util.TimeUnits;
 import org.apache.lucene.util.LuceneTestCase.Monster;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
@@ -39,7 +40,6 @@ import com.carrotsearch.randomizedtestin
  * @lucene.experimental
  */
 @SuppressCodecs({ "SimpleText", "Memory", "Direct" })
-@TimeoutSuite(millis = 4 * TimeUnits.HOUR)
 @Monster("uses lots of space and takes a few minutes")
 public class Test2BPositions extends LuceneTestCase {
 
@@ -55,7 +55,8 @@ public class Test2BPositions extends Luc
         .setRAMBufferSizeMB(256.0)
         .setMergeScheduler(new ConcurrentMergeScheduler())
         .setMergePolicy(newLogMergePolicy(false, 10))
-        .setOpenMode(IndexWriterConfig.OpenMode.CREATE));
+        .setOpenMode(IndexWriterConfig.OpenMode.CREATE)
+        .setCodec(TestUtil.getDefaultCodec()));
 
     MergePolicy mp = w.getConfig().getMergePolicy();
     if (mp instanceof LogByteSizeMergePolicy) {

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BPostingsBytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BPostingsBytes.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BPostingsBytes.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BPostingsBytes.java Wed Oct 15 01:26:26 2014
@@ -30,6 +30,7 @@ import org.apache.lucene.index.FieldInfo
 import org.apache.lucene.store.BaseDirectoryWrapper;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
 import org.apache.lucene.util.TimeUnits;
 import org.apache.lucene.util.LuceneTestCase.Monster;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
@@ -42,7 +43,6 @@ import com.carrotsearch.randomizedtestin
  * @lucene.experimental
  */
 @SuppressCodecs({ "SimpleText", "Memory", "Direct" })
-@TimeoutSuite(millis = 4 * TimeUnits.HOUR)
 @Monster("takes ~20GB-30GB of space and 10 minutes, and more heap space sometimes")
 public class Test2BPostingsBytes extends LuceneTestCase {
 
@@ -58,7 +58,8 @@ public class Test2BPostingsBytes extends
         .setRAMBufferSizeMB(256.0)
         .setMergeScheduler(new ConcurrentMergeScheduler())
         .setMergePolicy(newLogMergePolicy(false, 10))
-        .setOpenMode(IndexWriterConfig.OpenMode.CREATE));
+        .setOpenMode(IndexWriterConfig.OpenMode.CREATE)
+        .setCodec(TestUtil.getDefaultCodec()));
 
     MergePolicy mp = w.getConfig().getMergePolicy();
     if (mp instanceof LogByteSizeMergePolicy) {

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValues.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValues.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValues.java Wed Oct 15 01:26:26 2014
@@ -24,6 +24,7 @@ import org.apache.lucene.store.BaseDirec
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
 import org.apache.lucene.util.TimeUnits;
 import org.apache.lucene.util.LuceneTestCase.Monster;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
@@ -31,8 +32,10 @@ import org.apache.lucene.util.LuceneTest
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 
 @SuppressCodecs({"SimpleText", "Memory", "Direct"})
-@TimeoutSuite(millis = 80 * TimeUnits.HOUR)
-@Monster("very slow")
+@TimeoutSuite(millis = 80 * TimeUnits.HOUR) // effectively no limit
+// The six hour time was achieved on a Linux 3.13 system with these specs:
+// 3-core AMD at 2.5Ghz, 12 GB RAM, 5GB test heap, 2 test JVMs, 2TB SATA.
+@Monster("Takes ~ 6 hours if the heap is 5gb")
 public class Test2BSortedDocValues extends LuceneTestCase {
   
   // indexes Integer.MAX_VALUE docs with a fixed binary field
@@ -48,7 +51,8 @@ public class Test2BSortedDocValues exten
         .setRAMBufferSizeMB(256.0)
         .setMergeScheduler(new ConcurrentMergeScheduler())
         .setMergePolicy(newLogMergePolicy(false, 10))
-        .setOpenMode(IndexWriterConfig.OpenMode.CREATE));
+        .setOpenMode(IndexWriterConfig.OpenMode.CREATE)
+        .setCodec(TestUtil.getDefaultCodec()));
 
     Document doc = new Document();
     byte bytes[] = new byte[2];
@@ -103,7 +107,8 @@ public class Test2BSortedDocValues exten
         .setRAMBufferSizeMB(256.0)
         .setMergeScheduler(new ConcurrentMergeScheduler())
         .setMergePolicy(newLogMergePolicy(false, 10))
-        .setOpenMode(IndexWriterConfig.OpenMode.CREATE));
+        .setOpenMode(IndexWriterConfig.OpenMode.CREATE)
+        .setCodec(TestUtil.getDefaultCodec()));
 
     Document doc = new Document();
     byte bytes[] = new byte[4];

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java Wed Oct 15 01:26:26 2014
@@ -55,9 +55,8 @@ import com.carrotsearch.randomizedtestin
 //   ant test -Dtests.monster=true -Dtests.heapsize=8g -Dtests.codec=Lucene50 -Dtestcase=Test2BTerms
 //
 @SuppressCodecs({ "SimpleText", "Memory", "Direct" })
-@Monster("very slow, use 8g heap")
-@TimeoutSuite(millis = 6 * TimeUnits.HOUR)
-@SuppressSysoutChecks(bugUrl="We.print.lots.o.stuff.on.purpose")
+@Monster("very slow, use 5g minimum heap")
+@TimeoutSuite(millis = 80 * TimeUnits.HOUR) // effectively no limit
 public class Test2BTerms extends LuceneTestCase {
 
   private final static int TOKEN_LEN = 5;
@@ -185,7 +184,8 @@ public class Test2BTerms extends LuceneT
                                       .setRAMBufferSizeMB(256.0)
                                       .setMergeScheduler(new ConcurrentMergeScheduler())
                                       .setMergePolicy(newLogMergePolicy(false, 10))
-                                      .setOpenMode(IndexWriterConfig.OpenMode.CREATE));
+                                      .setOpenMode(IndexWriterConfig.OpenMode.CREATE)
+                                      .setCodec(TestUtil.getDefaultCodec()));
 
       MergePolicy mp = w.getConfig().getMergePolicy();
       if (mp instanceof LogByteSizeMergePolicy) {
@@ -261,7 +261,7 @@ public class Test2BTerms extends LuceneT
   private void testSavedTerms(IndexReader r, List<BytesRef> terms) throws IOException {
     System.out.println("TEST: run " + terms.size() + " terms on reader=" + r);
     IndexSearcher s = newSearcher(r);
-    Collections.shuffle(terms);
+    Collections.shuffle(terms, random());
     TermsEnum termsEnum = MultiFields.getTerms(r, "field").iterator(null);
     boolean failed = false;
     for(int iter=0;iter<10*terms.size();iter++) {

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexReaderClose.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexReaderClose.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexReaderClose.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexReaderClose.java Wed Oct 15 01:26:26 2014
@@ -122,7 +122,7 @@ public class TestIndexReaderClose extend
       leafReader.addCoreClosedListener(listeners.get(random().nextInt(listeners.size())));
     }
     final int removed = random().nextInt(numListeners);
-    Collections.shuffle(listeners);
+    Collections.shuffle(listeners, random());
     for (int i = 0; i < removed; ++i) {
       leafReader.removeCoreClosedListener(listeners.get(i));
     }

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Wed Oct 15 01:26:26 2014
@@ -35,6 +35,8 @@ import org.apache.lucene.analysis.MockTo
 import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
@@ -46,6 +48,7 @@ import org.apache.lucene.document.Sorted
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.PhraseQuery;
@@ -2118,6 +2121,48 @@ public class TestIndexWriterExceptions e
     dir.close();
   }
   
+  // kind of slow, but omits positions, so just CPU
+  @Nightly
+  public void testTooManyTokens() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
+    Document doc = new Document();
+    FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
+    ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
+    doc.add(new Field("foo", new TokenStream() {
+      CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+      PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
+      long num = 0;
+      
+      @Override
+      public boolean incrementToken() throws IOException {
+        if (num == Integer.MAX_VALUE + 1) {
+          return false;
+        }
+        clearAttributes();
+        if (num == 0) {
+          posIncAtt.setPositionIncrement(1);
+        } else {
+          posIncAtt.setPositionIncrement(0);
+        }
+        termAtt.append("a");
+        num++;
+        if (VERBOSE && num % 1000000 == 0) {
+          System.out.println("indexed: " + num);
+        }
+        return true;
+      }
+    }, ft));
+    try {
+      iw.addDocument(doc);
+      fail("didn't hit exception");
+    } catch (IllegalArgumentException expected) {
+      assertTrue(expected.getMessage().contains("too many tokens"));
+    }
+    iw.close();
+    dir.close();
+  }
+  
   public void testExceptionDuringRollback() throws Exception {
     // currently: fail in two different places
     final String messageToFailOn = random().nextBoolean() ? 

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMaxDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMaxDocs.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMaxDocs.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMaxDocs.java Wed Oct 15 01:26:26 2014
@@ -34,10 +34,12 @@ import org.apache.lucene.util.TimeUnits;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 
 @SuppressCodecs({ "SimpleText", "Memory", "Direct" })
-@TimeoutSuite(millis = 6 * TimeUnits.HOUR)
+@TimeoutSuite(millis = 8 * TimeUnits.HOUR)
 public class TestIndexWriterMaxDocs extends LuceneTestCase {
 
-  @Monster("takes a long time")
+  // The two hour time was achieved on a Linux 3.13 system with these specs:
+  // 3-core AMD at 2.5Ghz, 12 GB RAM, 5GB test heap, 2 test JVMs, 2TB SATA.
+  @Monster("takes over two hours")
   public void testExactlyAtTrueLimit() throws Exception {
     Directory dir = newFSDirectory(createTempDir("2BDocs3"));
     IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(null));

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java Wed Oct 15 01:26:26 2014
@@ -331,7 +331,7 @@ public class TestIndexWriterThreadsToSeg
           String segName = IndexFileNames.parseSegmentName(fileName);
           if (segSeen.contains(segName) == false) {
             segSeen.add(segName);
-            SegmentInfo si = TestUtil.getDefaultCodec().segmentInfoFormat().getSegmentInfoReader().read(dir, segName, IOContext.DEFAULT);
+            SegmentInfo si = TestUtil.getDefaultCodec().segmentInfoFormat().read(dir, segName, IOContext.DEFAULT);
             si.setCodec(codec);
             SegmentCommitInfo sci = new SegmentCommitInfo(si, 0, -1, -1, -1);
             SegmentReader sr = new SegmentReader(sci, IOContext.DEFAULT);

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java Wed Oct 15 01:26:26 2014
@@ -23,9 +23,9 @@ import org.apache.lucene.analysis.MockAn
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.SerialMergeScheduler;
 import org.apache.lucene.index.SlowCompositeReaderWrapper;
@@ -35,7 +35,7 @@ import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.WAH8DocIdSet;
+import org.apache.lucene.util.RoaringDocIdSet;
 
 public class TestCachingWrapperFilter extends LuceneTestCase {
   Directory dir;
@@ -241,7 +241,7 @@ public class TestCachingWrapperFilter ex
       if (originalSet.isCacheable()) {
         assertEquals("Cached DocIdSet must be of same class like uncached, if cacheable", originalSet.getClass(), cachedSet.getClass());
       } else {
-        assertTrue("Cached DocIdSet must be an WAH8DocIdSet if the original one was not cacheable", cachedSet instanceof WAH8DocIdSet || cachedSet == null);
+        assertTrue("Cached DocIdSet must be a RoaringDocIdSet if the original one was not cacheable", cachedSet instanceof RoaringDocIdSet || cachedSet == null);
       }
     }
   }

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java Wed Oct 15 01:26:26 2014
@@ -34,7 +34,7 @@ import org.apache.lucene.search.BooleanC
 import org.apache.lucene.search.FilteredQuery.FilterStrategy;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.DocIdBitSet;
+import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;
@@ -101,10 +101,10 @@ public class TestFilteredQuery extends L
       @Override
       public DocIdSet getDocIdSet (LeafReaderContext context, Bits acceptDocs) {
         if (acceptDocs == null) acceptDocs = new Bits.MatchAllBits(5);
-        BitSet bitset = new BitSet(5);
+        FixedBitSet bitset = new FixedBitSet(context.reader().maxDoc());
         if (acceptDocs.get(1)) bitset.set(1);
         if (acceptDocs.get(3)) bitset.set(3);
-        return new DocIdBitSet(bitset);
+        return bitset;
       }
     };
   }
@@ -183,9 +183,9 @@ public class TestFilteredQuery extends L
       @Override
       public DocIdSet getDocIdSet (LeafReaderContext context, Bits acceptDocs) {
         assertNull("acceptDocs should be null, as we have an index without deletions", acceptDocs);
-        BitSet bitset = new BitSet(5);
-        bitset.set(0, 5);
-        return new DocIdBitSet(bitset);
+        FixedBitSet bitset = new FixedBitSet(context.reader().maxDoc());
+        bitset.set(0, Math.min(5, bitset.length()));
+        return bitset;
       }
     };
   }
@@ -387,7 +387,7 @@ public class TestFilteredQuery extends L
     if (useRandomAccess) {
       return new FilteredQuery.RandomAccessFilterStrategy() {
         @Override
-        protected boolean useRandomAccess(Bits bits, int firstFilterDoc) {
+        protected boolean useRandomAccess(Bits bits, long filterCost) {
           return true;
         }
       };

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestSameScoresWithThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestSameScoresWithThreads.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestSameScoresWithThreads.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestSameScoresWithThreads.java Wed Oct 15 01:26:26 2014
@@ -93,7 +93,7 @@ public class TestSameScoresWithThreads e
                 startingGun.await();
                 for(int i=0;i<20;i++) {
                   List<Map.Entry<BytesRef,TopDocs>> shuffled = new ArrayList<>(answers.entrySet());
-                  Collections.shuffle(shuffled);
+                  Collections.shuffle(shuffled, random());
                   for(Map.Entry<BytesRef,TopDocs> ent : shuffled) {
                     TopDocs actual = s.search(new TermQuery(new Term("body", ent.getKey())), 100);
                     TopDocs expected = ent.getValue();

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java Wed Oct 15 01:26:26 2014
@@ -1,22 +1,21 @@
 package org.apache.lucene.search;
 
-import org.apache.lucene.document.Field;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.DocIdBitSet;
-import org.apache.lucene.util.LuceneTestCase;
-
-import java.util.BitSet;
 import java.io.IOException;
+import java.util.BitSet;
 
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.Term;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.Document;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.LuceneTestCase;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -38,7 +37,7 @@ import org.apache.lucene.document.Docume
 public class TestScorerPerf extends LuceneTestCase {
   boolean validate = true;  // set to false when doing performance testing
 
-  BitSet[] sets;
+  FixedBitSet[] sets;
   Term[] terms;
   IndexSearcher s;
   IndexReader r;
@@ -81,16 +80,16 @@ public class TestScorerPerf extends Luce
   }
 
 
-  public BitSet randBitSet(int sz, int numBitsToSet) {
-    BitSet set = new BitSet(sz);
+  public FixedBitSet randBitSet(int sz, int numBitsToSet) {
+    FixedBitSet set = new FixedBitSet(sz);
     for (int i=0; i<numBitsToSet; i++) {
       set.set(random().nextInt(sz));
     }
     return set;
   }
 
-  public BitSet[] randBitSets(int numSets, int setSize) {
-    BitSet[] sets = new BitSet[numSets];
+  public FixedBitSet[] randBitSets(int numSets, int setSize) {
+    FixedBitSet[] sets = new FixedBitSet[numSets];
     for (int i=0; i<sets.length; i++) {
       sets[i] = randBitSet(setSize, random().nextInt(setSize));
     }
@@ -123,9 +122,9 @@ public class TestScorerPerf extends Luce
 
 
   public static class MatchingHitCollector extends CountingHitCollector {
-    BitSet answer;
+    FixedBitSet answer;
     int pos=-1;
-    public MatchingHitCollector(BitSet answer) {
+    public MatchingHitCollector(FixedBitSet answer) {
       this.answer = answer;
     }
 
@@ -140,18 +139,18 @@ public class TestScorerPerf extends Luce
   }
 
 
-  BitSet addClause(BooleanQuery bq, BitSet result) {
-    final BitSet rnd = sets[random().nextInt(sets.length)];
+  FixedBitSet addClause(BooleanQuery bq, FixedBitSet result) {
+    final FixedBitSet rnd = sets[random().nextInt(sets.length)];
     Query q = new ConstantScoreQuery(new Filter() {
       @Override
       public DocIdSet getDocIdSet (LeafReaderContext context, Bits acceptDocs) {
         assertNull("acceptDocs should be null, as we have an index without deletions", acceptDocs);
-        return new DocIdBitSet(rnd);
+        return rnd;
       }
     });
     bq.add(q, BooleanClause.Occur.MUST);
     if (validate) {
-      if (result==null) result = (BitSet)rnd.clone();
+      if (result==null) result = rnd.clone();
       else result.and(rnd);
     }
     return result;
@@ -164,7 +163,7 @@ public class TestScorerPerf extends Luce
     for (int i=0; i<iter; i++) {
       int nClauses = random().nextInt(maxClauses-1)+2; // min 2 clauses
       BooleanQuery bq = new BooleanQuery();
-      BitSet result=null;
+      FixedBitSet result=null;
       for (int j=0; j<nClauses; j++) {
         result = addClause(bq,result);
       }
@@ -188,7 +187,7 @@ public class TestScorerPerf extends Luce
     for (int i=0; i<iter; i++) {
       int oClauses = random().nextInt(maxOuterClauses-1)+2;
       BooleanQuery oq = new BooleanQuery();
-      BitSet result=null;
+      FixedBitSet result=null;
 
       for (int o=0; o<oClauses; o++) {
 

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/util/TestDocIdSetBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/util/TestDocIdSetBuilder.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/util/TestDocIdSetBuilder.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/util/TestDocIdSetBuilder.java Wed Oct 15 01:26:26 2014
@@ -52,7 +52,7 @@ public class TestDocIdSetBuilder extends
     final FixedBitSet ref = new FixedBitSet(maxDoc);
     for (int i = 0; i < numIterators; ++i) {
       final int baseInc = 200000 + random().nextInt(10000);
-      WAH8DocIdSet.Builder b = new WAH8DocIdSet.Builder();
+      RoaringDocIdSet.Builder b = new RoaringDocIdSet.Builder(maxDoc);
       for (int doc = random().nextInt(100); doc < maxDoc; doc += baseInc + random().nextInt(10000)) {
         b.add(doc);
         ref.set(doc);
@@ -73,11 +73,11 @@ public class TestDocIdSetBuilder extends
       // try upgrades
       final int doc = random().nextInt(maxDoc);
       ref.set(doc);
-      builder.or(new WAH8DocIdSet.Builder().add(doc).build().iterator());
+      builder.or(new RoaringDocIdSet.Builder(maxDoc).add(doc).build().iterator());
     }
     for (int i = 0; i < numIterators; ++i) {
       final int baseInc = 2 + random().nextInt(10000);
-      WAH8DocIdSet.Builder b = new WAH8DocIdSet.Builder();
+      RoaringDocIdSet.Builder b = new RoaringDocIdSet.Builder(maxDoc);
       for (int doc = random().nextInt(10000); doc < maxDoc; doc += baseInc + random().nextInt(2000)) {
         b.add(doc);
         ref.set(doc);

Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/util/TestSparseFixedBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/util/TestSparseFixedBitSet.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/util/TestSparseFixedBitSet.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/util/TestSparseFixedBitSet.java Wed Oct 15 01:26:26 2014
@@ -34,14 +34,14 @@ public class TestSparseFixedBitSet exten
     for (int doc = bs.nextSetBit(0); doc != -1; doc = bs.nextSetBit(doc + 1)) {
       buffer.add(doc);
       if (buffer.size() >= 100000) {
-        Collections.shuffle(buffer);
+        Collections.shuffle(buffer, random());
         for (int i : buffer) {
           set.set(i);
         }
         buffer.clear();
       }
     }
-    Collections.shuffle(buffer);
+    Collections.shuffle(buffer, random());
     for (int i : buffer) {
       set.set(i);
     }

Modified: lucene/dev/branches/lucene5969/lucene/ivy-versions.properties
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/ivy-versions.properties?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/ivy-versions.properties (original)
+++ lucene/dev/branches/lucene5969/lucene/ivy-versions.properties Wed Oct 15 01:26:26 2014
@@ -7,7 +7,7 @@
 /cglib/cglib-nodep = 2.2
 /com.adobe.xmp/xmpcore = 5.1.2
 
-com.carrotsearch.randomizedtesting.version = 2.1.6
+com.carrotsearch.randomizedtesting.version = 2.1.9
 /com.carrotsearch.randomizedtesting/junit4-ant = ${com.carrotsearch.randomizedtesting.version}
 /com.carrotsearch.randomizedtesting/randomizedtesting-runner = ${com.carrotsearch.randomizedtesting.version}
 
@@ -35,7 +35,7 @@ com.google.inject.guice.version = 3.0
 /com.googlecode.concurrentlinkedhashmap/concurrentlinkedhashmap-lru = 1.2
 /com.googlecode.juniversalchardet/juniversalchardet = 1.0.3
 /com.googlecode.mp4parser/isoparser = 1.0.2
-/com.ibm.icu/icu4j = 53.1
+/com.ibm.icu/icu4j = 54.1
 /com.pff/java-libpst = 0.8.1
 /com.spatial4j/spatial4j = 0.4.1
 

Modified: lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyFieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyFieldInfosFormat.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyFieldInfosFormat.java (original)
+++ lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyFieldInfosFormat.java Wed Oct 15 01:26:26 2014
@@ -21,8 +21,6 @@ import java.io.IOException;
 import java.util.Random;
 
 import org.apache.lucene.codecs.FieldInfosFormat;
-import org.apache.lucene.codecs.FieldInfosReader;
-import org.apache.lucene.codecs.FieldInfosWriter;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.store.Directory;
@@ -36,35 +34,17 @@ class CrankyFieldInfosFormat extends Fie
     this.delegate = delegate;
     this.random = random;
   }
-  
+
   @Override
-  public FieldInfosReader getFieldInfosReader() throws IOException {
-    return delegate.getFieldInfosReader();
+  public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) throws IOException {
+    return delegate.read(directory, segmentInfo, segmentSuffix, iocontext);
   }
 
   @Override
-  public FieldInfosWriter getFieldInfosWriter() throws IOException {
+  public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
     if (random.nextInt(100) == 0) {
       throw new IOException("Fake IOException from FieldInfosFormat.getFieldInfosWriter()");
     }
-    return new CrankyFieldInfosWriter(delegate.getFieldInfosWriter(), random);
-  }
-  
-  static class CrankyFieldInfosWriter extends FieldInfosWriter {
-    final FieldInfosWriter delegate;
-    final Random random;
-    
-    CrankyFieldInfosWriter(FieldInfosWriter delegate, Random random) {
-      this.delegate = delegate;
-      this.random = random;
-    }
-
-    @Override
-    public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
-      if (random.nextInt(100) == 0) {
-        throw new IOException("Fake IOException from FieldInfosWriter.write()");
-      }
-      delegate.write(directory, segmentInfo, segmentSuffix, infos, context);
-    }
+    delegate.write(directory, segmentInfo, segmentSuffix, infos, context);
   }
 }

Modified: lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankySegmentInfoFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankySegmentInfoFormat.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankySegmentInfoFormat.java (original)
+++ lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankySegmentInfoFormat.java Wed Oct 15 01:26:26 2014
@@ -21,9 +21,6 @@ import java.io.IOException;
 import java.util.Random;
 
 import org.apache.lucene.codecs.SegmentInfoFormat;
-import org.apache.lucene.codecs.SegmentInfoReader;
-import org.apache.lucene.codecs.SegmentInfoWriter;
-import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
@@ -38,30 +35,15 @@ class CrankySegmentInfoFormat extends Se
   }
   
   @Override
-  public SegmentInfoReader getSegmentInfoReader() {
-    return delegate.getSegmentInfoReader();
+  public SegmentInfo read(Directory directory, String segmentName, IOContext context) throws IOException {
+    return delegate.read(directory, segmentName, context);
   }
 
   @Override
-  public SegmentInfoWriter getSegmentInfoWriter() {
-    return new CrankySegmentInfoWriter(delegate.getSegmentInfoWriter(), random);
-  }
-  
-  static class CrankySegmentInfoWriter extends SegmentInfoWriter {
-    final SegmentInfoWriter delegate;
-    final Random random;
-    
-    CrankySegmentInfoWriter(SegmentInfoWriter delegate, Random random) {
-      this.delegate = delegate;
-      this.random = random;
-    }
-    
-    @Override
-    public void write(Directory dir, SegmentInfo info, FieldInfos fis, IOContext ioContext) throws IOException {
-      if (random.nextInt(100) == 0) {
-        throw new IOException("Fake IOException from SegmentInfoWriter.write()");
-      }
-      delegate.write(dir, info, fis, ioContext);
+  public void write(Directory dir, SegmentInfo info, IOContext ioContext) throws IOException {
+    if (random.nextInt(100) == 0) {
+      throw new IOException("Fake IOException from SegmentInfoFormat.write()");
     }
+    delegate.write(dir, info, ioContext);
   }
 }

Modified: lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java (original)
+++ lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java Wed Oct 15 01:26:26 2014
@@ -435,29 +435,8 @@ public abstract class BaseCompoundFormat
     expected.seek(0);
     assertSameStreams("basic clone two", expected, two);
     
-    // Now close the first stream
-    one.close();
-    
-    // The following should really fail since we couldn't expect to
-    // access a file once close has been called on it (regardless of
-    // buffering and/or clone magic)
-    expected.seek(0);
-    two.seek(0);
-    assertSameStreams("basic clone two/2", expected, two);
-    
     // Now close the compound reader
     cr.close();
-    
-    // The following may also fail since the compound stream is closed
-    expected.seek(0);
-    two.seek(0);
-    //assertSameStreams("basic clone two/3", expected, two);
-    
-    // Now close the second clone
-    two.close();
-    expected.seek(0);
-    //assertSameStreams("basic clone two/4", expected, two);
-    
     expected.close();
     dir.close();
   }

Modified: lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java (original)
+++ lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java Wed Oct 15 01:26:26 2014
@@ -125,7 +125,7 @@ public abstract class BaseStoredFieldsFo
       w.addDocument(doc);
       if (rand.nextInt(50) == 17) {
         // mixup binding of field name -> Number every so often
-        Collections.shuffle(fieldIDs);
+        Collections.shuffle(fieldIDs, random());
       }
       if (rand.nextInt(5) == 3 && i > 0) {
         final String delID = ""+rand.nextInt(i);

Modified: lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/util/BaseDocIdSetTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/util/BaseDocIdSetTestCase.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/util/BaseDocIdSetTestCase.java (original)
+++ lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/util/BaseDocIdSetTestCase.java Wed Oct 15 01:26:26 2014
@@ -57,8 +57,8 @@ public abstract class BaseDocIdSetTestCa
   /** Test length=0. */
   public void testNoBit() throws IOException {
     final BitSet bs = new BitSet(1);
-    final T copy = copyOf(bs, TestUtil.nextInt(random(), 1, 10000));
-    assertEquals(0, bs, copy);
+    final T copy = copyOf(bs, 1);
+    assertEquals(1, bs, copy);
   }
 
   /** Test length=1. */
@@ -67,7 +67,7 @@ public abstract class BaseDocIdSetTestCa
     if (random().nextBoolean()) {
       bs.set(0);
     }
-    final T copy = copyOf(bs, TestUtil.nextInt(random(), 1, 10000));
+    final T copy = copyOf(bs, 1);
     assertEquals(1, bs, copy);
   }
 
@@ -80,7 +80,7 @@ public abstract class BaseDocIdSetTestCa
     if (random().nextBoolean()) {
       bs.set(1);
     }
-    final T copy = copyOf(bs, TestUtil.nextInt(random(), 1, 10000));
+    final T copy = copyOf(bs, 2);
     assertEquals(2, bs, copy);
   }
 
@@ -88,7 +88,7 @@ public abstract class BaseDocIdSetTestCa
   public void testAgainstBitSet() throws IOException {
     final int numBits = TestUtil.nextInt(random(), 100, 1 << 20);
     // test various random sets with various load factors
-    for (float percentSet : new float[] {0f, 0.0001f, random().nextFloat() / 2, 0.9f, 1f}) {
+    for (float percentSet : new float[] {0f, 0.0001f, random().nextFloat(), 0.9f, 1f}) {
       final BitSet set = randomSet(numBits, percentSet);
       final T copy = copyOf(set, numBits);
       assertEquals(numBits, set, copy);

Modified: lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Wed Oct 15 01:26:26 2014
@@ -61,10 +61,51 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.*;
-import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.AlcoholicMergePolicy;
+import org.apache.lucene.index.AssertingDirectoryReader;
+import org.apache.lucene.index.AssertingLeafReader;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.CompositeReader;
+import org.apache.lucene.index.ConcurrentMergeScheduler;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.FieldFilterLeafReader;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.IndexReader.ReaderClosedListener;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.LiveIndexWriterConfig;
+import org.apache.lucene.index.LogByteSizeMergePolicy;
+import org.apache.lucene.index.LogDocMergePolicy;
+import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.MergeScheduler;
+import org.apache.lucene.index.MockRandomMergePolicy;
+import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.ParallelCompositeReader;
+import org.apache.lucene.index.ParallelLeafReader;
+import org.apache.lucene.index.SegmentReader;
+import org.apache.lucene.index.SerialMergeScheduler;
+import org.apache.lucene.index.SimpleMergedSegmentWarmer;
+import org.apache.lucene.index.SlowCompositeReaderWrapper;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.StorableField;
+import org.apache.lucene.index.StoredDocument;
+import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum.SeekStatus;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.TieredMergePolicy;
 import org.apache.lucene.search.AssertingIndexSearcher;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
@@ -95,7 +136,6 @@ import org.junit.Test;
 import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
 import org.junit.runner.RunWith;
-
 import com.carrotsearch.randomizedtesting.JUnit4MethodProvider;
 import com.carrotsearch.randomizedtesting.LifecycleScope;
 import com.carrotsearch.randomizedtesting.MixWithSuiteName;
@@ -707,8 +747,9 @@ public abstract class LuceneTestCase ext
    */
   public static SegmentReader getOnlySegmentReader(DirectoryReader reader) {
     List<LeafReaderContext> subReaders = reader.leaves();
-    if (subReaders.size() != 1)
+    if (subReaders.size() != 1) {
       throw new IllegalArgumentException(reader + " has " + subReaders.size() + " segments instead of exactly one");
+    }
     final LeafReader r = subReaders.get(0).reader();
     assertTrue(r instanceof SegmentReader);
     return (SegmentReader) r;
@@ -862,6 +903,8 @@ public abstract class LuceneTestCase ext
 
     c.setMergePolicy(newMergePolicy(r));
 
+    avoidPathologicalMerging(c);
+
     if (rarely(r)) {
       c.setMergedSegmentWarmer(new SimpleMergedSegmentWarmer(c.getInfoStream()));
     }
@@ -870,6 +913,53 @@ public abstract class LuceneTestCase ext
     return c;
   }
 
+  private static void avoidPathologicalMerging(IndexWriterConfig iwc) {
+    // Don't allow "tiny" flushed segments with "big" merge
+    // floor: this leads to pathological O(N^2) merge costs:
+    long estFlushSizeBytes = Long.MAX_VALUE;
+    if (iwc.getMaxBufferedDocs() != IndexWriterConfig.DISABLE_AUTO_FLUSH) {
+      // Gross estimation of 1 KB segment bytes for each doc indexed:
+      estFlushSizeBytes = Math.min(estFlushSizeBytes, iwc.getMaxBufferedDocs() * 1024);
+    }
+    if (iwc.getRAMBufferSizeMB() != IndexWriterConfig.DISABLE_AUTO_FLUSH) {
+      estFlushSizeBytes = Math.min(estFlushSizeBytes, (long) (iwc.getRAMBufferSizeMB() * 1024 * 1024));
+    }
+    assert estFlushSizeBytes > 0;
+
+    MergePolicy mp = iwc.getMergePolicy();
+    if (mp instanceof TieredMergePolicy) {
+      TieredMergePolicy tmp = (TieredMergePolicy) mp;
+      long floorSegBytes = (long) (tmp.getFloorSegmentMB() * 1024 * 1024);
+      if (floorSegBytes / estFlushSizeBytes > 10) {
+        double newValue = estFlushSizeBytes * 10.0 / 1024 / 1024;
+        if (VERBOSE) {
+          System.out.println("NOTE: LuceneTestCase: changing TieredMergePolicy.floorSegmentMB from " + tmp.getFloorSegmentMB() + " to " + newValue + " to avoid pathological merging");
+        }
+        tmp.setFloorSegmentMB(newValue);
+      }
+    } else if (mp instanceof LogByteSizeMergePolicy) {
+      LogByteSizeMergePolicy lmp = (LogByteSizeMergePolicy) mp;
+      if ((lmp.getMinMergeMB()*1024*1024) / estFlushSizeBytes > 10) {
+        double newValue = estFlushSizeBytes * 10.0 / 1024 / 1024;
+        if (VERBOSE) {
+          System.out.println("NOTE: LuceneTestCase: changing LogByteSizeMergePolicy.minMergeMB from " + lmp.getMinMergeMB() + " to " + newValue + " to avoid pathological merging");
+        }
+        lmp.setMinMergeMB(newValue);
+      }
+    } else if (mp instanceof LogDocMergePolicy) {
+      LogDocMergePolicy lmp = (LogDocMergePolicy) mp;
+      assert estFlushSizeBytes / 1024 < Integer.MAX_VALUE/10;
+      int estFlushDocs = Math.max(1, (int) (estFlushSizeBytes / 1024));
+      if (lmp.getMinMergeDocs() / estFlushDocs > 10) {
+        int newValue = estFlushDocs * 10;
+        if (VERBOSE) {
+          System.out.println("NOTE: LuceneTestCase: changing LogDocMergePolicy.minMergeDocs from " + lmp.getMinMergeDocs() + " to " + newValue + " to avoid pathological merging");
+        }
+        lmp.setMinMergeDocs(newValue);
+      }
+    }
+  }
+
   public static MergePolicy newMergePolicy(Random r) {
     if (rarely(r)) {
       return new MockRandomMergePolicy(r);

Modified: lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleLimitSysouts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleLimitSysouts.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleLimitSysouts.java (original)
+++ lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleLimitSysouts.java Wed Oct 15 01:26:26 2014
@@ -16,6 +16,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.apache.lucene.util.LuceneTestCase.Monster;
 import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
 
 import com.carrotsearch.randomizedtesting.RandomizedTest;
@@ -180,6 +181,7 @@ public class TestRuleLimitSysouts extend
 
     if (LuceneTestCase.VERBOSE || 
         LuceneTestCase.INFOSTREAM ||
+        target.isAnnotationPresent(Monster.class) ||
         target.isAnnotationPresent(SuppressSysoutChecks.class)) {
       return false;
     }

Modified: lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java (original)
+++ lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java Wed Oct 15 01:26:26 2014
@@ -1025,7 +1025,7 @@ public final class TestUtil {
       case 4:
         return new FilteredQuery.RandomAccessFilterStrategy() {
           @Override
-          protected boolean useRandomAccess(Bits bits, int firstFilterDoc) {
+          protected boolean useRandomAccess(Bits bits, long filterCost) {
             return LuceneTestCase.random().nextBoolean();
           }
         };

Modified: lucene/dev/branches/lucene5969/lucene/tools/forbiddenApis/base.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/tools/forbiddenApis/base.txt?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/tools/forbiddenApis/base.txt (original)
+++ lucene/dev/branches/lucene5969/lucene/tools/forbiddenApis/base.txt Wed Oct 15 01:26:26 2014
@@ -31,3 +31,6 @@ java.lang.Character#codePointBefore(char
 java.lang.Character#codePointAt(char[],int) @ Implicit end offset is error-prone when the char[] is a buffer and the last chars are random chars
 
 java.io.File#delete() @ use Files.delete for real exception, IOUtils.deleteFilesIgnoringExceptions if you dont care
+
+@defaultMessage Use shuffle(List, Random) instead so that it can be reproduced
+java.util.Collections#shuffle(java.util.List)

Modified: lucene/dev/branches/lucene5969/lucene/tools/junit4/cached-timehints.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/tools/junit4/cached-timehints.txt?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/tools/junit4/cached-timehints.txt (original)
+++ lucene/dev/branches/lucene5969/lucene/tools/junit4/cached-timehints.txt Wed Oct 15 01:26:26 2014
@@ -43,19 +43,19 @@ org.apache.lucene.analysis.compound.Test
 org.apache.lucene.analysis.core.TestAllAnalyzersHaveFactories=797,1216,957,1445,1085,479,1093
 org.apache.lucene.analysis.core.TestAnalyzers=1771,3282,4168,4511,6763,2480,2815
 org.apache.lucene.analysis.core.TestBugInSomething=36,38,807,7,23,13,46
-org.apache.lucene.analysis.core.TestClassicAnalyzer=993,3056,1978,1141,1280,3766,1754
+org.apache.lucene.analysis.standard.TestClassicAnalyzer=993,3056,1978,1141,1280,3766,1754
 org.apache.lucene.analysis.core.TestDuelingAnalyzers=1844,4104,2325,2041,2568,1984,5501
 org.apache.lucene.analysis.core.TestFactories=13995,17156,12500,15103,14760,22010,13206
 org.apache.lucene.analysis.core.TestKeywordAnalyzer=241,205,234,403,223,326,209
 org.apache.lucene.analysis.core.TestRandomChains=3048,4646,3050,5003,4274,4400,3679
-org.apache.lucene.analysis.core.TestStandardAnalyzer=3672,3118,2532,4071,2379,2271,2479
+org.apache.lucene.analysis.standard.TestStandardAnalyzer=3672,3118,2532,4071,2379,2271,2479
 org.apache.lucene.analysis.core.TestStopAnalyzer=31,39,16,22,66,10,19
 org.apache.lucene.analysis.core.TestStopFilter=76,83,64,12,148,19,19
 org.apache.lucene.analysis.core.TestStopFilterFactory=31,7,27,92,10,29,22
 org.apache.lucene.analysis.core.TestTypeTokenFilter=13,26,18,352,18,113,44
 org.apache.lucene.analysis.core.TestTypeTokenFilterFactory=23,36,85,60,23,16,112
-org.apache.lucene.analysis.core.TestUAX29URLEmailAnalyzer=315,243,502,493,183,240,564
-org.apache.lucene.analysis.core.TestUAX29URLEmailTokenizer=1656,837,1071,1402,2610,1318,1134
+org.apache.lucene.analysis.standard.TestUAX29URLEmailAnalyzer=315,243,502,493,183,240,564
+org.apache.lucene.analysis.standard.TestUAX29URLEmailTokenizer=1656,837,1071,1402,2610,1318,1134
 org.apache.lucene.analysis.cz.TestCzechAnalyzer=610,176,302,835,120,221,92
 org.apache.lucene.analysis.cz.TestCzechStemFilterFactory=8,10,8,18,23,35,53
 org.apache.lucene.analysis.cz.TestCzechStemmer=38,126,47,24,242,96,34

Modified: lucene/dev/branches/lucene5969/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/solr/CHANGES.txt?rev=1631928&r1=1631927&r2=1631928&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene5969/solr/CHANGES.txt Wed Oct 15 01:26:26 2014
@@ -89,6 +89,11 @@ Upgrading from Solr 4.x
   for "createcollection", "delete" for "removecollection" and "deleteshard" for
   "removeshard".
 
+* If you have been using the /update/json/docs to index documents, SOLR-6617 introduces
+  backward incompatible change. the key names created are fully qualified paths of keys .
+  If you need the old functionality back , please add an extra parameter f=/**
+  example: /update/json/docs?f=/**
+
 Detailed Change List
 ----------------------
 
@@ -152,6 +157,18 @@ New Features
   this property from all other replicas within a particular slice. 
   (Erick Erickson)
 
+* SOLR-6513: Add a collectionsAPI call BALANCESLICEUNIQUE. Allows the even
+  distribution of custom replica properties across nodes making up a 
+  collection, at most one node per slice will have the property.
+
+* SOLR-6605: Make ShardHandlerFactory maxConnections configurable.
+  (Christine Poerschke via shalin)
+
+* SOLR-6585: RequestHandlers can optionaly handle sub paths as well (Noble Paul)
+
+* SOLR-6617: /update/json/docs path will use fully qualified node names by default
+             (NOble Paul)
+
 Bug Fixes
 ----------------------
 
@@ -206,6 +223,19 @@ Bug Fixes
 * SOLR-6587: Misleading exception when creating collections in SolrCloud with bad configuration.
   (Tomás Fernández Löbbe)
 
+* SOLR-6545: Query field list with wild card on dynamic field fails.
+  (Burke Webster, Xu Zhang, shalin)
+
+* SOLR-6540 Fix NPE from strdist() func when doc value source does not exist in a doc (hossman)
+
+* SOLR-6624 Spelling mistakes in the Java source (Hrishikesh Gadre)
+
+Optimizations
+----------------------
+
+* SOLR-6603: LBHttpSolrServer - lazily allocate skipped-zombie-servers list.
+  (Christine Poerschke via shalin)
+
 Other Changes
 ----------------------
 
@@ -275,6 +305,21 @@ Other Changes
 * SOLR-6550: Provide simple mechanism for passing additional metadata / context about a server-side
    SolrException back to the client-side (Timothy Potter)
 
+* SOLR-6597: SolrIndexConfig parameter in one of the SolrIndexSearcher constructor has been removed.
+  It was just passed and never used via that constructor. (Anshum Gupta)
+
+* SOLR-6549: bin/solr script should support a -s option to set the -Dsolr.solr.home property.
+  (Timothy Potter)
+
+* SOLR-6529: Stop command in the start scripts should only stop the instance that it had started.
+  (Varun Thacker, Timothy Potter)
+
+* SOLR-5852: Add CloudSolrServer helper method to connect to a ZK ensemble. (Varun Thacker, Furkan KAMACI,
+  Shawn Heisey, Mark Miller, Erick Erickson via shalin)
+
+* SOLR-6592: Avoid waiting for the leader to see the down state if that leader is not live.
+  (Timothy Potter)
+
 ==================  4.10.1 ==================
 
 Bug Fixes
@@ -410,7 +455,6 @@ New Features
 * SOLR-6233: Provide basic command line tools for checking Solr status and health.
   (Timothy Potter)
 
-* SOLR-6585: RequestHandlers can optionaly handle sub paths as well (Noble Paul)
 
 Bug Fixes
 ----------------------



Mime
View raw message