lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rm...@apache.org
Subject [lucene-solr] branch master updated: LUCENE-9241: fix tests to pass with -Xmx128m
Date Fri, 06 Mar 2020 02:38:20 GMT
This is an automated email from the ASF dual-hosted git repository.

rmuir pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git


The following commit(s) were added to refs/heads/master by this push:
     new 9cfdf17  LUCENE-9241: fix tests to pass with -Xmx128m
9cfdf17 is described below

commit 9cfdf17b2895866877668002d443277a46cd04e8
Author: Robert Muir <rmuir@apache.org>
AuthorDate: Thu Mar 5 21:29:39 2020 -0500

    LUCENE-9241: fix tests to pass with -Xmx128m
---
 .../lucene/analysis/minhash/MinHashFilterTest.java | 13 +++---
 .../lucene/analysis/ja/dict/ConnectionCosts.java   | 26 ++++++------
 .../analysis/ja/util/BinaryDictionaryWriter.java   |  4 +-
 .../analysis/ja/util/ConnectionCostsWriter.java    | 21 +++++-----
 .../lucene/analysis/ja/TestJapaneseTokenizer.java  | 14 ++-----
 .../lucene/analysis/ko/dict/ConnectionCosts.java   |  2 -
 .../analysis/ko/util/BinaryDictionaryWriter.java   |  4 +-
 .../analysis/ko/util/ConnectionCostsWriter.java    | 21 +++++-----
 .../benchmark/byTask/TestPerfTasksLogic.java       |  2 +-
 .../lucene/util/automaton/ByteRunAutomaton.java    |  2 +-
 .../util/automaton/CharacterRunAutomaton.java      |  4 +-
 .../apache/lucene/util/automaton/RunAutomaton.java | 17 ++++----
 .../lucene/codecs/lucene80/TestIndexedDISI.java    | 46 +++++++++++-----------
 .../apache/lucene/search/TestAutomatonQuery.java   |  3 +-
 .../lucene/store/TestByteBuffersDataInput.java     |  2 +-
 .../lucene/store/TestByteBuffersDataOutput.java    |  8 +++-
 .../org/apache/lucene/util/TestOfflineSorter.java  |  9 +++--
 .../org/apache/lucene/util/TestPagedBytes.java     | 14 ++++++-
 .../util/automaton/TestDeterminizeLexicon.java     | 11 ++++--
 .../taxonomy/writercache/TestCharBlockArray.java   |  7 +++-
 .../lucene/monitor/TestConcurrentQueryLoader.java  |  6 ++-
 .../org/apache/lucene/monitor/TestMonitor.java     |  2 +
 22 files changed, 132 insertions(+), 106 deletions(-)

diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/minhash/MinHashFilterTest.java
b/lucene/analysis/common/src/test/org/apache/lucene/analysis/minhash/MinHashFilterTest.java
index 1bc6ed7..f1ff4cd 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/minhash/MinHashFilterTest.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/minhash/MinHashFilterTest.java
@@ -99,11 +99,14 @@ public class MinHashFilterTest extends BaseTokenStreamTestCase {
     }
     assertEquals(100, minSet.size());
     assertEquals(0, unadded.size());
+  }
 
+  public void testCollisions() {
     HashSet<LongPair> collisionDetection = new HashSet<LongPair>();
-    unadded = new HashSet<LongPair>();
-    minSet = new FixedSizeTreeSet<LongPair>(500);
-    for (int i = 0; i < 1000000; i++) {
+    HashSet<LongPair> unadded = new HashSet<LongPair>();
+    FixedSizeTreeSet<LongPair> minSet = new FixedSizeTreeSet<LongPair>(500);
+    int numElements = TEST_NIGHTLY ? 1000000 : 10000;
+    for (int i = 0; i < numElements; i++) {
       LongPair hash = new LongPair();
       MinHashFilter.murmurhash3_x64_128(MinHashFilter.getBytes(i), 0, 4, 0, hash);
       collisionDetection.add(hash);
@@ -122,9 +125,9 @@ public class MinHashFilterTest extends BaseTokenStreamTestCase {
         }
       }
     }
-    assertEquals(1000000, collisionDetection.size());
+    assertEquals(numElements, collisionDetection.size());
     assertEquals(500, minSet.size());
-    assertEquals(999500, unadded.size());
+    assertEquals(numElements - 500, unadded.size());
 
     LongPair last = null;
     LongPair current = null;
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/ConnectionCosts.java
b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/ConnectionCosts.java
index c886af2..c3ed8b6 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/ConnectionCosts.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/ConnectionCosts.java
@@ -20,6 +20,7 @@ package org.apache.lucene.analysis.ja.dict;
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.ByteBuffer;
 
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.store.DataInput;
@@ -35,7 +36,8 @@ public final class ConnectionCosts {
   public static final String HEADER = "kuromoji_cc";
   public static final int VERSION = 1;
   
-  private final short[][] costs; // array is backward IDs first since get is called using
the same backward ID consecutively. maybe doesn't matter.
+  private final ByteBuffer buffer;
+  private final int forwardSize;
   
   /**
    * @param scheme - scheme for loading resources (FILE or CLASSPATH).
@@ -43,24 +45,26 @@ public final class ConnectionCosts {
    */
   public ConnectionCosts(BinaryDictionary.ResourceScheme scheme, String path) throws IOException
{
     InputStream is = null;
-    short[][] costs = null;
     boolean success = false;
     try {
       is = BinaryDictionary.getResource(scheme, path.replace('.', '/') + FILENAME_SUFFIX);
       is = new BufferedInputStream(is);
       final DataInput in = new InputStreamDataInput(is);
       CodecUtil.checkHeader(in, HEADER, VERSION, VERSION);
-      int forwardSize = in.readVInt();
+      forwardSize = in.readVInt();
       int backwardSize = in.readVInt();
-      costs = new short[backwardSize][forwardSize];
+      int size = forwardSize * backwardSize;
+
+      // copy the matrix into a direct byte buffer
+      final ByteBuffer tmpBuffer = ByteBuffer.allocateDirect(size*2);
       int accum = 0;
-      for (int j = 0; j < costs.length; j++) {
-        final short[] a = costs[j];
-        for (int i = 0; i < a.length; i++) {
+      for (int j = 0; j < backwardSize; j++) {
+        for (int i = 0; i < forwardSize; i++) {
           accum += in.readZInt();
-          a[i] = (short)accum;
+          tmpBuffer.putShort((short) accum);
         }
       }
+      buffer = tmpBuffer.asReadOnlyBuffer();
       success = true;
     } finally {
       if (success) {
@@ -69,8 +73,6 @@ public final class ConnectionCosts {
         IOUtils.closeWhileHandlingException(is);
       }
     }
-    
-    this.costs = costs;
   }
 
   private ConnectionCosts() throws IOException {
@@ -78,7 +80,9 @@ public final class ConnectionCosts {
   }
 
   public int get(int forwardId, int backwardId) {
-    return costs[backwardId][forwardId];
+    // map 2d matrix into a single dimension short array
+    int offset = (backwardId * forwardSize + forwardId) * 2;
+    return buffer.getShort(offset);
   }
   
   public static ConnectionCosts getInstance() {
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/BinaryDictionaryWriter.java
b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/BinaryDictionaryWriter.java
index 2cd4cca..58bd64b 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/BinaryDictionaryWriter.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/BinaryDictionaryWriter.java
@@ -46,7 +46,7 @@ abstract class BinaryDictionaryWriter {
 
   BinaryDictionaryWriter(Class<? extends BinaryDictionary> implClazz, int size) {
     this.implClazz = implClazz;
-    buffer = ByteBuffer.allocate(size);
+    buffer = ByteBuffer.allocateDirect(size);
   }
   
   /**
@@ -97,7 +97,7 @@ abstract class BinaryDictionaryWriter {
     // worst case: two short, 3 bytes, and features (all as utf-16)
     int worstCase = 4 + 3 + 2*(baseForm.length() + reading.length() + pronunciation.length());
     if (worstCase > left) {
-      ByteBuffer newBuffer = ByteBuffer.allocate(ArrayUtil.oversize(buffer.limit() + worstCase
- left, 1));
+      ByteBuffer newBuffer = ByteBuffer.allocateDirect(ArrayUtil.oversize(buffer.limit()
+ worstCase - left, 1));
       buffer.flip();
       newBuffer.put(buffer);
       buffer = newBuffer;
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ConnectionCostsWriter.java
b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ConnectionCostsWriter.java
index 746a4f7..e2b8984 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ConnectionCostsWriter.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ConnectionCostsWriter.java
@@ -20,6 +20,7 @@ package org.apache.lucene.analysis.ja.util;
 import java.io.BufferedOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.nio.ByteBuffer;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
@@ -31,7 +32,7 @@ import org.apache.lucene.store.OutputStreamDataOutput;
 
 final class ConnectionCostsWriter {
   
-  private final short[][] costs; // array is backward IDs first since get is called using
the same backward ID consecutively. maybe doesn't matter.
+  private final ByteBuffer costs; // array is backward IDs first since get is called using
the same backward ID consecutively. maybe doesn't matter.
   private final int forwardSize;
   private final int backwardSize;
   /**
@@ -40,11 +41,12 @@ final class ConnectionCostsWriter {
   ConnectionCostsWriter(int forwardSize, int backwardSize) {
     this.forwardSize = forwardSize;
     this.backwardSize = backwardSize;
-    this.costs = new short[backwardSize][forwardSize];
+    this.costs = ByteBuffer.allocateDirect(2 * backwardSize * forwardSize);
   }
   
   public void add(int forwardId, int backwardId, int cost) {
-    this.costs[backwardId][forwardId] = (short)cost;
+    int offset = (backwardId * forwardSize + forwardId) * 2;
+    costs.putShort(offset, (short) cost);
   }
   
   public void write(Path baseDir) throws IOException {
@@ -57,14 +59,11 @@ final class ConnectionCostsWriter {
       out.writeVInt(forwardSize);
       out.writeVInt(backwardSize);
       int last = 0;
-      assert costs.length == backwardSize;
-      for (short[] a : costs) {
-        assert a.length == forwardSize;
-        for (short cost : a) {
-          int delta = (int) cost - last;
-          out.writeZInt(delta);
-          last = cost;
-        }
+      for (int i = 0; i < costs.limit() / 2; i++) {
+        short cost = costs.getShort(i * 2);
+        int delta = (int) cost - last;
+        out.writeZInt(delta);
+        last = cost;
       }
     }
   }
diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
index 447060f..d061106 100644
--- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
+++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
@@ -48,17 +48,9 @@ public class
     TestJapaneseTokenizer extends BaseTokenStreamTestCase {
 
   public static UserDictionary readDict() {
-    InputStream is = TestJapaneseTokenizer.class.getResourceAsStream("userdict.txt");
-    if (is == null) {
-      throw new RuntimeException("Cannot find userdict.txt in test classpath!");
-    }
-    try {
-      try {
-        Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8);
-        return UserDictionary.open(reader);
-      } finally {
-        is.close();
-      }
+    try (InputStream stream = TestJapaneseTokenizer.class.getResourceAsStream("userdict.txt");
+         Reader reader = new InputStreamReader(stream, StandardCharsets.UTF_8)) {
+      return UserDictionary.open(reader);
     } catch (IOException ioe) {
       throw new RuntimeException(ioe);
     }
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/ConnectionCosts.java
b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/ConnectionCosts.java
index 3d67897..fc7be7a 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/ConnectionCosts.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/ConnectionCosts.java
@@ -44,7 +44,6 @@ public final class ConnectionCosts {
    */
   public ConnectionCosts(BinaryDictionary.ResourceScheme scheme, String resourcePath) throws
IOException {
     InputStream is = null;
-    ByteBuffer buffer;
     boolean success = false;
     try {
       is = BinaryDictionary.getResource(scheme, resourcePath.replace('.', '/') + FILENAME_SUFFIX);
@@ -73,7 +72,6 @@ public final class ConnectionCosts {
         IOUtils.closeWhileHandlingException(is);
       }
     }
-    this.buffer = buffer;
   }
 
   private ConnectionCosts() throws IOException {
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/BinaryDictionaryWriter.java
b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/BinaryDictionaryWriter.java
index fec02db..c06c28f 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/BinaryDictionaryWriter.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/BinaryDictionaryWriter.java
@@ -49,7 +49,7 @@ abstract class BinaryDictionaryWriter {
 
   BinaryDictionaryWriter(Class<? extends BinaryDictionary> implClazz, int size) {
     this.implClazz = implClazz;
-    buffer = ByteBuffer.allocate(size);
+    buffer = ByteBuffer.allocateDirect(size);
   }
   
   /**
@@ -96,7 +96,7 @@ abstract class BinaryDictionaryWriter {
     // worst case, 3 short + 4 bytes and features (all as utf-16)
     int worstCase = 9 + 2*(expression.length() + reading.length());
     if (worstCase > left) {
-      ByteBuffer newBuffer = ByteBuffer.allocate(ArrayUtil.oversize(buffer.limit() + worstCase
- left, 1));
+      ByteBuffer newBuffer = ByteBuffer.allocateDirect(ArrayUtil.oversize(buffer.limit()
+ worstCase - left, 1));
       buffer.flip();
       newBuffer.put(buffer);
       buffer = newBuffer;
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/ConnectionCostsWriter.java
b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/ConnectionCostsWriter.java
index 586290d..4519bfc 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/ConnectionCostsWriter.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/ConnectionCostsWriter.java
@@ -19,6 +19,7 @@ package org.apache.lucene.analysis.ko.util;
 import java.io.BufferedOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.nio.ByteBuffer;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
@@ -30,7 +31,7 @@ import org.apache.lucene.store.OutputStreamDataOutput;
 
 final class ConnectionCostsWriter {
   
-  private final short[][] costs; // array is backward IDs first since get is called using
the same backward ID consecutively. maybe doesn't matter.
+  private final ByteBuffer costs; // array is backward IDs first since get is called using
the same backward ID consecutively. maybe doesn't matter.
   private final int forwardSize;
   private final int backwardSize;
   /**
@@ -39,11 +40,12 @@ final class ConnectionCostsWriter {
   ConnectionCostsWriter(int forwardSize, int backwardSize) {
     this.forwardSize = forwardSize;
     this.backwardSize = backwardSize;
-    this.costs = new short[backwardSize][forwardSize];
+    this.costs = ByteBuffer.allocateDirect(2 * backwardSize * forwardSize);
   }
   
   public void add(int forwardId, int backwardId, int cost) {
-    this.costs[backwardId][forwardId] = (short)cost;
+    int offset = (backwardId * forwardSize + forwardId) * 2;
+    costs.putShort(offset, (short) cost);
   }
   
   public void write(Path baseDir) throws IOException {
@@ -56,14 +58,11 @@ final class ConnectionCostsWriter {
       out.writeVInt(forwardSize);
       out.writeVInt(backwardSize);
       int last = 0;
-      assert costs.length == backwardSize;
-      for (short[] a : costs) {
-        assert a.length == forwardSize;
-        for (short cost : a) {
-          int delta = (int) cost - last;
-          out.writeZInt(delta);
-          last = cost;
-        }
+      for (int i = 0; i < costs.limit() / 2; i++) {
+        short cost = costs.getShort(i * 2);
+        int delta = (int) cost - last;
+        out.writeZInt(delta);
+        last = cost;
       }
     }
   }
diff --git a/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
b/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
index 8aa3aa8..24dc5a1 100644
--- a/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
+++ b/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
@@ -61,7 +61,7 @@ import org.apache.lucene.util.TestUtil;
 /**
  * Test very simply that perf tasks - simple algorithms - are doing what they should.
  */
-@LuceneTestCase.SuppressCodecs("SimpleText")
+@LuceneTestCase.SuppressCodecs({"SimpleText", "Direct"})
 public class TestPerfTasksLogic extends BenchmarkTestCase {
 
   @Override
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/ByteRunAutomaton.java b/lucene/core/src/java/org/apache/lucene/util/automaton/ByteRunAutomaton.java
index abe7560..3627e8e 100644
--- a/lucene/core/src/java/org/apache/lucene/util/automaton/ByteRunAutomaton.java
+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/ByteRunAutomaton.java
@@ -41,6 +41,6 @@ public class ByteRunAutomaton extends RunAutomaton {
       p = step(p, s[i] & 0xFF);
       if (p == -1) return false;
     }
-    return accept[p];
+    return accept.get(p);
   }
 }
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/CharacterRunAutomaton.java
b/lucene/core/src/java/org/apache/lucene/util/automaton/CharacterRunAutomaton.java
index c282e24..26de4fe 100644
--- a/lucene/core/src/java/org/apache/lucene/util/automaton/CharacterRunAutomaton.java
+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/CharacterRunAutomaton.java
@@ -49,7 +49,7 @@ public class CharacterRunAutomaton extends RunAutomaton {
       p = step(p, cp = s.codePointAt(i));
       if (p == -1) return false;
     }
-    return accept[p];
+    return accept.get(p);
   }
   
   /**
@@ -62,6 +62,6 @@ public class CharacterRunAutomaton extends RunAutomaton {
       p = step(p, cp = Character.codePointAt(s, i, l));
       if (p == -1) return false;
     }
-    return accept[p];
+    return accept.get(p);
   }
 }
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/RunAutomaton.java b/lucene/core/src/java/org/apache/lucene/util/automaton/RunAutomaton.java
index e105ac3..0e58e34 100644
--- a/lucene/core/src/java/org/apache/lucene/util/automaton/RunAutomaton.java
+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/RunAutomaton.java
@@ -32,6 +32,7 @@ package org.apache.lucene.util.automaton;
 import java.util.Arrays;
 
 import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.RamUsageEstimator;
 
 /**
@@ -45,7 +46,7 @@ public abstract class RunAutomaton implements Accountable {
   final Automaton automaton;
   final int alphabetSize;
   final int size;
-  final boolean[] accept;
+  final FixedBitSet accept;
   final int[] transitions; // delta(state,c) = transitions[state*points.length +
                      // getCharClass(c)]
   final int[] points; // char interval start points
@@ -75,12 +76,14 @@ public abstract class RunAutomaton implements Accountable {
     this.automaton = a;
     points = a.getStartPoints();
     size = Math.max(1,a.getNumStates());
-    accept = new boolean[size];
+    accept = new FixedBitSet(size);
     transitions = new int[size * points.length];
     Arrays.fill(transitions, -1);
     Transition transition = new Transition();
     for (int n=0;n<size;n++) {
-      accept[n] = a.isAccept(n);
+      if (a.isAccept(n)) {
+        accept.set(n);
+      }
       transition.source = n;
       transition.transitionUpto = -1;
       for (int c = 0; c < points.length; c++) {
@@ -112,7 +115,7 @@ public abstract class RunAutomaton implements Accountable {
     b.append("initial state: 0\n");
     for (int i = 0; i < size; i++) {
       b.append("state ").append(i);
-      if (accept[i]) b.append(" [accept]:\n");
+      if (accept.get(i)) b.append(" [accept]:\n");
       else b.append(" [reject]:\n");
       for (int j = 0; j < points.length; j++) {
         int k = transitions[i * points.length + j];
@@ -145,7 +148,7 @@ public abstract class RunAutomaton implements Accountable {
    * Returns acceptance status for given state.
    */
   public final boolean isAccept(int state) {
-    return accept[state];
+    return accept.get(state);
   }
   
   /**
@@ -208,7 +211,7 @@ public abstract class RunAutomaton implements Accountable {
     if (alphabetSize != other.alphabetSize) return false;
     if (size != other.size) return false;
     if (!Arrays.equals(points, other.points)) return false;
-    if (!Arrays.equals(accept, other.accept)) return false;
+    if (!accept.equals(other.accept)) return false;
     if (!Arrays.equals(transitions, other.transitions)) return false;
     return true;
   }
@@ -216,7 +219,7 @@ public abstract class RunAutomaton implements Accountable {
   @Override
   public long ramBytesUsed() {
     return BASE_RAM_BYTES +
-        RamUsageEstimator.sizeOfObject(accept) +
+        accept.ramBytesUsed() +
         RamUsageEstimator.sizeOfObject(automaton) +
         RamUsageEstimator.sizeOfObject(classmap) +
         RamUsageEstimator.sizeOfObject(points) +
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestIndexedDISI.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestIndexedDISI.java
index c91bbf1..28c618d 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestIndexedDISI.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestIndexedDISI.java
@@ -17,8 +17,6 @@
 package org.apache.lucene.codecs.lucene80;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Random;
 
 import org.apache.lucene.search.DocIdSetIterator;
@@ -27,9 +25,11 @@ import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.RandomAccessInput;
+import org.apache.lucene.util.BitSet;
 import org.apache.lucene.util.BitSetIterator;
 import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.SparseFixedBitSet;
 import org.apache.lucene.util.TestUtil;
 
 // Copied from the lucene70 package for separation of codec-code
@@ -37,7 +37,7 @@ public class TestIndexedDISI extends LuceneTestCase {
 
   public void testEmpty() throws IOException {
     int maxDoc = TestUtil.nextInt(random(), 1, 100000);
-    FixedBitSet set = new FixedBitSet(maxDoc);
+    BitSet set = new SparseFixedBitSet(maxDoc);
     try (Directory dir = newDirectory()) {
       doTest(set, dir);
     }
@@ -48,7 +48,7 @@ public class TestIndexedDISI extends LuceneTestCase {
   public void testEmptyBlocks() throws IOException {
     final int B = 65536;
     int maxDoc = B*11;
-    FixedBitSet set = new FixedBitSet(maxDoc);
+    BitSet set = new SparseFixedBitSet(maxDoc);
     // block 0: EMPTY
     set.set(B+5); // block 1: SPARSE
     // block 2: EMPTY
@@ -83,7 +83,7 @@ public class TestIndexedDISI extends LuceneTestCase {
   public void testLastEmptyBlocks() throws IOException {
     final int B = 65536;
     int maxDoc = B*3;
-    FixedBitSet set = new FixedBitSet(maxDoc);
+    BitSet set = new SparseFixedBitSet(maxDoc);
     for (int docID = 0 ; docID < B*2 ; docID++) { // first 2 blocks are ALL
       set.set(docID);
     }
@@ -96,7 +96,7 @@ public class TestIndexedDISI extends LuceneTestCase {
   }
 
   // Checks that advance after the end of the blocks has been reached has the correct behaviour
-  private void assertAdvanceBeyondEnd(FixedBitSet set, Directory dir) throws IOException
{
+  private void assertAdvanceBeyondEnd(BitSet set, Directory dir) throws IOException {
     final int cardinality = set.cardinality();
     final byte denseRankPower = 9; // Not tested here so fixed to isolate factors
     long length;
@@ -127,7 +127,7 @@ public class TestIndexedDISI extends LuceneTestCase {
   @Nightly
   public void testRandomBlocks() throws IOException {
     final int BLOCKS = 5;
-    FixedBitSet set = createSetWithRandomBlocks(BLOCKS);
+    BitSet set = createSetWithRandomBlocks(BLOCKS);
     try (Directory dir = newDirectory()) {
       doTestAllSingleJump(set, dir);
     }
@@ -138,7 +138,7 @@ public class TestIndexedDISI extends LuceneTestCase {
     final int BLOCKS = 10;
     final byte denseRankPower = rarely() ? -1 : (byte) (random().nextInt(7)+7); // sane +
chance of disable
 
-    FixedBitSet set = createSetWithRandomBlocks(BLOCKS);
+    BitSet set = createSetWithRandomBlocks(BLOCKS);
     try (Directory dir = newDirectory()) {
       final int cardinality = set.cardinality();
       int jumpTableEntryCount;
@@ -158,9 +158,9 @@ public class TestIndexedDISI extends LuceneTestCase {
     }
   }
 
-  private FixedBitSet createSetWithRandomBlocks(int blockCount) {
+  private BitSet createSetWithRandomBlocks(int blockCount) {
     final int B = 65536;
-    FixedBitSet set = new FixedBitSet(blockCount * B);
+    BitSet set = new SparseFixedBitSet(blockCount * B);
     for (int block = 0; block < blockCount; block++) {
       switch (random().nextInt(4)) {
         case 0: { // EMPTY
@@ -191,7 +191,7 @@ public class TestIndexedDISI extends LuceneTestCase {
   }
 
 
-  private void doTestAllSingleJump(FixedBitSet set, Directory dir) throws IOException {
+  private void doTestAllSingleJump(BitSet set, Directory dir) throws IOException {
     final int cardinality = set.cardinality();
     final byte denseRankPower = rarely() ? -1 : (byte) (random().nextInt(7)+7); // sane +
chance of disable
     long length;
@@ -222,7 +222,7 @@ public class TestIndexedDISI extends LuceneTestCase {
 
   public void testOneDoc() throws IOException {
     int maxDoc = TestUtil.nextInt(random(), 1, 100000);
-    FixedBitSet set = new FixedBitSet(maxDoc);
+    BitSet set = new SparseFixedBitSet(maxDoc);
     set.set(random().nextInt(maxDoc));
     try (Directory dir = newDirectory()) {
       doTest(set, dir);
@@ -231,7 +231,7 @@ public class TestIndexedDISI extends LuceneTestCase {
 
   public void testTwoDocs() throws IOException {
     int maxDoc = TestUtil.nextInt(random(), 1, 100000);
-    FixedBitSet set = new FixedBitSet(maxDoc);
+    BitSet set = new SparseFixedBitSet(maxDoc);
     set.set(random().nextInt(maxDoc));
     set.set(random().nextInt(maxDoc));
     try (Directory dir = newDirectory()) {
@@ -250,7 +250,7 @@ public class TestIndexedDISI extends LuceneTestCase {
 
   public void testHalfFull() throws IOException {
     int maxDoc = TestUtil.nextInt(random(), 1, 100000);
-    FixedBitSet set = new FixedBitSet(maxDoc);
+    BitSet set = new SparseFixedBitSet(maxDoc);
     for (int i = random().nextInt(2); i < maxDoc; i += TestUtil.nextInt(random(), 1, 3))
{
       set.set(i);
     }
@@ -371,7 +371,7 @@ public class TestIndexedDISI extends LuceneTestCase {
   }
 
   private void createAndOpenDISI(byte denseRankPowerWrite, byte denseRankPowerRead) throws
IOException {
-    FixedBitSet set = new FixedBitSet(10);
+    BitSet set = new FixedBitSet(10);
     set.set(set.length()-1);
     try (Directory dir = newDirectory()) {
       long length;
@@ -423,24 +423,22 @@ public class TestIndexedDISI extends LuceneTestCase {
 
   private void doTestRandom(Directory dir) throws IOException {
     Random random = random();
-    List<Integer> docs = new ArrayList<>();
     final int maxStep = TestUtil.nextInt(random, 1, 1 << TestUtil.nextInt(random, 2,
20));
     final int numDocs = TestUtil.nextInt(random, 1, Math.min(100000, Integer.MAX_VALUE /
maxStep));
+    BitSet docs = new SparseFixedBitSet(numDocs * (maxStep + 1));
+    int lastDoc = -1;
     for (int doc = -1, i = 0; i < numDocs; ++i) {
       doc += TestUtil.nextInt(random, 1, maxStep);
-      docs.add(doc);
-    }
-    final int maxDoc = docs.get(docs.size() - 1) + TestUtil.nextInt(random, 1, 100);
-
-    FixedBitSet set = new FixedBitSet(maxDoc);
-    for (int doc : docs) {
-      set.set(doc);
+      docs.set(doc);
+      lastDoc = doc;
     }
+    final int maxDoc = lastDoc + TestUtil.nextInt(random, 1, 100);
 
+    BitSet set = BitSet.of(new BitSetIterator(docs, docs.approximateCardinality()), maxDoc);
     doTest(set, dir);
   }
 
-  private void doTest(FixedBitSet set, Directory dir) throws IOException {
+  private void doTest(BitSet set, Directory dir) throws IOException {
     final int cardinality = set.cardinality();
     final byte denseRankPower = rarely() ? -1 : (byte) (random().nextInt(7)+7); // sane +
chance of disable
     long length;
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQuery.java
index 8fb8c65..d4d4a95 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQuery.java
@@ -241,8 +241,9 @@ public class TestAutomatonQuery extends LuceneTestCase {
   }
 
   public void testBiggishAutomaton() {
+    int numTerms = TEST_NIGHTLY ? 3000 : 500;
     List<BytesRef> terms = new ArrayList<>();
-    while (terms.size() < 3000) {
+    while (terms.size() < numTerms) {
       terms.add(new BytesRef(TestUtil.randomUnicodeString(random())));
     }
     Collections.sort(terms);
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataInput.java b/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataInput.java
index 00f329b..141192f 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataInput.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataInput.java
@@ -67,7 +67,7 @@ public final class TestByteBuffersDataInput extends RandomizedTest {
     ByteBuffersDataOutput dst = new ByteBuffersDataOutput();
 
     long seed = randomLong();
-    int max = 1_000_000;
+    int max = LuceneTestCase.TEST_NIGHTLY ? 1_000_000 : 100_000;
     List<IOConsumer<DataInput>> reply = 
         TestByteBuffersDataOutput.addRandomData(dst, new Xoroshiro128PlusRandom(seed), max);
 
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataOutput.java b/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataOutput.java
index 42e10b0..8a9012a 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataOutput.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataOutput.java
@@ -23,6 +23,7 @@ import java.nio.ByteBuffer;
 import java.util.Random;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.LuceneTestCase;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -122,7 +123,12 @@ public final class TestByteBuffersDataOutput extends BaseDataOutputTestCase<Byte
   public void testLargeArrayAdd() {
     ByteBuffersDataOutput o = new ByteBuffersDataOutput();
     int MB = 1024 * 1024;
-    byte [] bytes = randomBytesOfLength(5 * MB, 15 * MB);
+    final byte [] bytes;
+    if (LuceneTestCase.TEST_NIGHTLY) {
+      bytes = randomBytesOfLength(5 * MB, 15 * MB);
+    } else {
+      bytes = randomBytesOfLength(MB/2, MB);
+    }
     int offset = randomIntBetween(0, 100);
     int len = bytes.length - offset;
     o.writeBytes(bytes, offset, len);
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java b/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java
index d2961f2..927af14 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java
@@ -81,16 +81,17 @@ public class TestOfflineSorter extends LuceneTestCase {
     if (random().nextBoolean()) {
       return null;
     } else {
-      return new ThreadPoolExecutor(1, TestUtil.nextInt(random(), 2, 6), Long.MAX_VALUE,
TimeUnit.MILLISECONDS,
+      int maxThreads = TEST_NIGHTLY ? TestUtil.nextInt(random(), 2, 6) : 2;
+      return new ThreadPoolExecutor(1, maxThreads, Long.MAX_VALUE, TimeUnit.MILLISECONDS,
                                     new LinkedBlockingQueue<Runnable>(),
-                                    new NamedThreadFactory("TestIndexSearcher"));
+                                    new NamedThreadFactory("TestOfflineSorter"));
     }
   }
 
   @Slow
   public void testIntermediateMerges() throws Exception {
     // Sort 20 mb worth of data with 1mb buffer, binary merging.
-    try (Directory dir = newDirectory()) {
+    try (Directory dir = newFSDirectory(createTempDir())) {
       ExecutorService exec = randomExecutorServiceOrNull();
       SortInfo info = checkSort(dir, new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR,
BufferSize.megabytes(1), 2, -1, exec, TestUtil.nextInt(random(), 1, 4)),
                                 generateRandom((int)OfflineSorter.MB * 20));
@@ -104,7 +105,7 @@ public class TestOfflineSorter extends LuceneTestCase {
   @Slow
   public void testSmallRandom() throws Exception {
     // Sort 20 mb worth of data with 1mb buffer.
-    try (Directory dir = newDirectory()) {
+    try (Directory dir = newFSDirectory(createTempDir())) {
       ExecutorService exec = randomExecutorServiceOrNull();
       SortInfo sortInfo = checkSort(dir, new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR,
BufferSize.megabytes(1), OfflineSorter.MAX_TEMPFILES, -1, exec, TestUtil.nextInt(random(),
1, 4)),
                                     generateRandom((int)OfflineSorter.MB * 20));
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java b/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java
index a9971c6..f622058 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java
@@ -45,7 +45,12 @@ public class TestPagedBytes extends LuceneTestCase {
       final int blockSize = 1 << blockBits;
       final PagedBytes p = new PagedBytes(blockBits);
       final IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT);
-      final int numBytes = TestUtil.nextInt(random, 2, 10000000);
+      final int numBytes;
+      if (TEST_NIGHTLY) {
+        numBytes = TestUtil.nextInt(random(), 2, 10_000_000);
+      } else {
+        numBytes = TestUtil.nextInt(random(), 2, 1_000_000);
+      }
 
       final byte[] answer = new byte[numBytes];
       random.nextBytes(answer);
@@ -105,7 +110,12 @@ public class TestPagedBytes extends LuceneTestCase {
       final int blockSize = 1 << blockBits;
       final PagedBytes p = new PagedBytes(blockBits);
       final DataOutput out = p.getDataOutput();
-      final int numBytes = random().nextInt(10000000);
+      final int numBytes;
+      if (TEST_NIGHTLY) {
+        numBytes = random().nextInt(10_000_000);
+      } else {
+        numBytes = random().nextInt(1_000_000);
+      }
 
       final byte[] answer = new byte[numBytes];
       random().nextBytes(answer);
diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java
b/lucene/core/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java
index 1440b19..6e22525 100644
--- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java
+++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java
@@ -55,10 +55,13 @@ public class TestDeterminizeLexicon extends LuceneTestCase {
     for (String s : terms) {
       assertTrue(Operations.run(lex, s));
     }
-    final ByteRunAutomaton lexByte = new ByteRunAutomaton(lex, false, 1000000);
-    for (String s : terms) {
-      byte bytes[] = s.getBytes(StandardCharsets.UTF_8);
-      assertTrue(lexByte.run(bytes, 0, bytes.length));
+    if (TEST_NIGHTLY) {
+      // TODO: very wasteful of RAM to do this without minimizing first.
+      final ByteRunAutomaton lexByte = new ByteRunAutomaton(lex, false, 1000000);
+      for (String s : terms) {
+        byte bytes[] = s.getBytes(StandardCharsets.UTF_8);
+        assertTrue(lexByte.run(bytes, 0, bytes.length));
+      }
     }
   }
 }
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/TestCharBlockArray.java
b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/TestCharBlockArray.java
index f03a54a6..177a9c7 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/TestCharBlockArray.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/TestCharBlockArray.java
@@ -33,7 +33,12 @@ public class TestCharBlockArray extends FacetTestCase {
     CharBlockArray array = new CharBlockArray();
     StringBuilder builder = new StringBuilder();
 
-    final int n = 100 * 1000;
+    final int n;
+    if (TEST_NIGHTLY) {
+      n = 100 * 1000;
+    } else {
+      n = 1000;
+    }
 
     byte[] buffer = new byte[50];
 
diff --git a/lucene/monitor/src/test/org/apache/lucene/monitor/TestConcurrentQueryLoader.java
b/lucene/monitor/src/test/org/apache/lucene/monitor/TestConcurrentQueryLoader.java
index 44b9fd1..20da4c7 100644
--- a/lucene/monitor/src/test/org/apache/lucene/monitor/TestConcurrentQueryLoader.java
+++ b/lucene/monitor/src/test/org/apache/lucene/monitor/TestConcurrentQueryLoader.java
@@ -20,13 +20,15 @@ package org.apache.lucene.monitor;
 public class TestConcurrentQueryLoader extends MonitorTestBase {
 
   public void testLoading() throws Exception {
+    // TODO: there are problems if this thing hits OOM (it hides it)
+    int numQueries = TEST_NIGHTLY ? 2000 : 200;
     try (Monitor monitor = newMonitor()) {
       try (ConcurrentQueryLoader loader = new ConcurrentQueryLoader(monitor)) {
-        for (int i = 0; i < 2000; i++) {
+        for (int i = 0; i < numQueries; i++) {
           loader.add(new MonitorQuery(Integer.toString(i), parse("\"test " + i + "\"")));
         }
       }
-      assertEquals(2000, monitor.getQueryCount());
+      assertEquals(numQueries, monitor.getQueryCount());
     }
   }
 
diff --git a/lucene/monitor/src/test/org/apache/lucene/monitor/TestMonitor.java b/lucene/monitor/src/test/org/apache/lucene/monitor/TestMonitor.java
index 73443e1..ca4da71 100644
--- a/lucene/monitor/src/test/org/apache/lucene/monitor/TestMonitor.java
+++ b/lucene/monitor/src/test/org/apache/lucene/monitor/TestMonitor.java
@@ -130,6 +130,8 @@ public class TestMonitor extends MonitorTestBase {
 
   }
 
+  // takes huge amounts of ram. TODO: what is this test doing?
+  @Nightly
   public void testUpdateReporting() throws IOException {
 
     List<MonitorQuery> queries = new ArrayList<>(10400);


Mime
View raw message