Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id F3154200B7E for ; Tue, 2 Aug 2016 03:07:27 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id F1D56160AAF; Tue, 2 Aug 2016 01:07:27 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 91A1B160AAB for ; Tue, 2 Aug 2016 03:07:26 +0200 (CEST) Received: (qmail 68152 invoked by uid 500); 2 Aug 2016 01:07:22 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 67533 invoked by uid 99); 2 Aug 2016 01:07:22 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 02 Aug 2016 01:07:22 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 0D686EE696; Tue, 2 Aug 2016 01:07:22 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sarowe@apache.org To: commits@lucene.apache.org Date: Tue, 02 Aug 2016 01:07:35 -0000 Message-Id: <46a0797e5b58484a88fea52bb6778ca9@git.apache.org> In-Reply-To: <9e8dc7f6eba045719a28c7abe9fbc036@git.apache.org> References: <9e8dc7f6eba045719a28c7abe9fbc036@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [15/40] lucene-solr:apiv2: LUCENE-7390: improve points indexing performance by letting the codec use transient heap in proportion to IndexWriter's indexing buffer, by default archived-at: Tue, 02 Aug 2016 01:07:28 -0000 LUCENE-7390: improve points indexing performance by letting the codec use transient heap in proportion to IndexWriter's indexing buffer, by default Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3a0a9fd2 Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3a0a9fd2 Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3a0a9fd2 Branch: refs/heads/apiv2 Commit: 3a0a9fd2c25965db4bde20975d9e6df4b540588a Parents: 4ed68bc Author: Mike McCandless Authored: Mon Jul 25 11:33:34 2016 -0400 Committer: Mike McCandless Committed: Mon Jul 25 11:33:34 2016 -0400 ---------------------------------------------------------------------- lucene/CHANGES.txt | 5 +++++ .../simpletext/SimpleTextPointsWriter.java | 4 ++-- .../org/apache/lucene/codecs/PointsWriter.java | 11 ++++++++--- .../codecs/lucene60/Lucene60PointsWriter.java | 15 ++++++++------- .../lucene/index/DocumentsWriterPerThread.java | 2 +- .../org/apache/lucene/index/IndexWriter.java | 2 +- .../lucene/index/LiveIndexWriterConfig.java | 19 ++++++++----------- .../apache/lucene/index/PointValuesWriter.java | 6 +++++- .../org/apache/lucene/util/bkd/BKDWriter.java | 2 +- .../lucene60/TestLucene60PointsFormat.java | 5 ++--- .../apache/lucene/search/TestPointQueries.java | 5 ++--- .../apache/lucene/spatial3d/TestGeo3DPoint.java | 5 ++--- .../codecs/asserting/AssertingPointsFormat.java | 4 ++-- .../lucene/codecs/cranky/CrankyPointsFormat.java | 4 ++-- .../apache/lucene/geo/BaseGeoPointTestCase.java | 3 +-- .../org/apache/lucene/index/RandomCodec.java | 9 +++------ 16 files changed, 53 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a0a9fd2/lucene/CHANGES.txt ---------------------------------------------------------------------- diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 7b53017..917dfa2 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -135,6 +135,11 @@ Improvements * LUCENE-7385: Improve/fix assert messages in SpanScorer. (David Smiley) +* LUCENE-7390: Improve performance of indexing points by allowing the + codec to use transient heap in proportion to IndexWriter's RAM + buffer, instead of a fixed 16.0 MB. A custom codec can still + override the buffer size itself. (Mike McCandless) + Optimizations * LUCENE-7330, LUCENE-7339: Speed up conjunction queries. (Adrien Grand) http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a0a9fd2/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java ---------------------------------------------------------------------- diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java index 8d5c034..67289b6 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java @@ -68,7 +68,7 @@ class SimpleTextPointsWriter extends PointsWriter { } @Override - public void writeField(FieldInfo fieldInfo, PointsReader values) throws IOException { + public void writeField(FieldInfo fieldInfo, PointsReader values, double maxMBSortInHeap) throws IOException { boolean singleValuePerDoc = values.size(fieldInfo.name) == values.getDocCount(fieldInfo.name); @@ -79,7 +79,7 @@ class SimpleTextPointsWriter extends PointsWriter { fieldInfo.getPointDimensionCount(), fieldInfo.getPointNumBytes(), BKDWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE, - BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP, + maxMBSortInHeap, values.size(fieldInfo.name), singleValuePerDoc) { http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a0a9fd2/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java ---------------------------------------------------------------------- diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java index 05084db..8458497 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java @@ -22,6 +22,7 @@ import java.io.IOException; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.MergeState; +import org.apache.lucene.util.bkd.BKDWriter; /** Abstract API to write points * @@ -34,8 +35,9 @@ public abstract class PointsWriter implements Closeable { protected PointsWriter() { } - /** Write all values contained in the provided reader */ - public abstract void writeField(FieldInfo fieldInfo, PointsReader values) throws IOException; + /** Write all values contained in the provided reader. {@code maxMBSortInHeap} is the maximum + * transient heap that can be used to sort values, before spilling to disk for offline sorting */ + public abstract void writeField(FieldInfo fieldInfo, PointsReader values, double maxMBSortInHeap) throws IOException; /** Default naive merge implementation for one field: it just re-indexes all the values * from the incoming segment. The default codec overrides this for 1D fields and uses @@ -145,7 +147,10 @@ public abstract class PointsWriter implements Closeable { public int getDocCount(String fieldName) { return finalDocCount; } - }); + }, + // TODO: also let merging of > 1D fields tap into IW's indexing buffer size, somehow (1D fields do an optimized merge sort + // and don't need heap) + BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP); } /** Default merge implementation to merge incoming points readers by visiting all their points and http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a0a9fd2/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java ---------------------------------------------------------------------- diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java index 63308c4..3acfac3 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java @@ -39,7 +39,9 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.bkd.BKDReader; import org.apache.lucene.util.bkd.BKDWriter; -/** Writes dimensional values */ +/** Writes dimensional values + * + * @lucene.experimental */ public class Lucene60PointsWriter extends PointsWriter implements Closeable { /** Output used to write the BKD tree data file */ @@ -50,15 +52,13 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable { final SegmentWriteState writeState; final int maxPointsInLeafNode; - final double maxMBSortInHeap; private boolean finished; /** Full constructor */ - public Lucene60PointsWriter(SegmentWriteState writeState, int maxPointsInLeafNode, double maxMBSortInHeap) throws IOException { + public Lucene60PointsWriter(SegmentWriteState writeState, int maxPointsInLeafNode) throws IOException { assert writeState.fieldInfos.hasPointValues(); this.writeState = writeState; this.maxPointsInLeafNode = maxPointsInLeafNode; - this.maxMBSortInHeap = maxMBSortInHeap; String dataFileName = IndexFileNames.segmentFileName(writeState.segmentInfo.name, writeState.segmentSuffix, Lucene60PointsFormat.DATA_EXTENSION); @@ -80,11 +80,11 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable { /** Uses the defaults values for {@code maxPointsInLeafNode} (1024) and {@code maxMBSortInHeap} (16.0) */ public Lucene60PointsWriter(SegmentWriteState writeState) throws IOException { - this(writeState, BKDWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE, BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP); + this(writeState, BKDWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE); } @Override - public void writeField(FieldInfo fieldInfo, PointsReader values) throws IOException { + public void writeField(FieldInfo fieldInfo, PointsReader values, double maxMBSortInHeap) throws IOException { boolean singleValuePerDoc = values.size(fieldInfo.name) == values.getDocCount(fieldInfo.name); @@ -173,7 +173,8 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable { fieldInfo.getPointDimensionCount(), fieldInfo.getPointNumBytes(), maxPointsInLeafNode, - maxMBSortInHeap, + // NOTE: not used, since BKDWriter.merge does a merge sort: + BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP, totMaxSize, singleValuePerDoc)) { List bkdReaders = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a0a9fd2/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java ---------------------------------------------------------------------- diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java index e72145c..351235e 100644 --- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java +++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java @@ -153,7 +153,7 @@ class DocumentsWriterPerThread { final Allocator byteBlockAllocator; final IntBlockPool.Allocator intBlockAllocator; private final AtomicLong pendingNumDocs; - private final LiveIndexWriterConfig indexWriterConfig; + final LiveIndexWriterConfig indexWriterConfig; private final boolean enableTestPoints; private final IndexWriter indexWriter; http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a0a9fd2/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java ---------------------------------------------------------------------- diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java index 0fb23d9..bdd9552 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java @@ -762,7 +762,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable { * {@link #getConfig()}. * *

- * NOTE: after ths writer is created, the given configuration instance + * NOTE: after this writer is created, the given configuration instance * cannot be passed to another writer. * * @param d http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a0a9fd2/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java ---------------------------------------------------------------------- diff --git a/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java b/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java index cec70c0..fe4924d 100644 --- a/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java +++ b/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java @@ -168,9 +168,14 @@ public class LiveIndexWriterConfig { /** * Determines the amount of RAM that may be used for buffering added documents - * and deletions before they are flushed to the Directory. Generally for - * faster indexing performance it's best to flush by RAM usage instead of - * document count and use as large a RAM buffer as you can. + * and deletions before beginning to flush them to the Directory. For + * faster indexing performance it's best to use as large a RAM buffer as you can. + *

+ * Note that this setting is not a hard limit on memory usage during indexing, as + * transient and non-trivial memory well beyond this buffer size may be used, + * for example due to segment merges or writing points to new segments. + * For application stability the available memory in the JVM + * should be significantly larger than the RAM buffer used for indexing. *

* When this is set, the writer will flush whenever buffered documents and * deletions use this much RAM. Pass in @@ -178,14 +183,6 @@ public class LiveIndexWriterConfig { * due to RAM usage. Note that if flushing by document count is also enabled, * then the flush will be triggered by whichever comes first. *

- * The maximum RAM limit is inherently determined by the JVMs available - * memory. Yet, an {@link IndexWriter} session can consume a significantly - * larger amount of memory than the given RAM limit since this limit is just - * an indicator when to flush memory resident documents to the Directory. - * Flushes are likely happen concurrently while other threads adding documents - * to the writer. For application stability the available memory in the JVM - * should be significantly larger than the RAM buffer used for indexing. - *

* NOTE: the account of RAM usage for pending deletions is only * approximate. Specifically, if you delete by Query, Lucene currently has no * way to measure the RAM usage of individual Queries so the accounting will http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a0a9fd2/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java ---------------------------------------------------------------------- diff --git a/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java index e3d1c1d..511635c 100644 --- a/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java @@ -24,6 +24,7 @@ import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Counter; +import org.apache.lucene.util.bkd.BKDWriter; /** Buffers up pending byte[][] value(s) per doc, then flushes when segment flushes. */ class PointValuesWriter { @@ -35,6 +36,7 @@ class PointValuesWriter { private int numDocs; private int lastDocID = -1; private final byte[] packedValue; + private final LiveIndexWriterConfig indexWriterConfig; public PointValuesWriter(DocumentsWriterPerThread docWriter, FieldInfo fieldInfo) { this.fieldInfo = fieldInfo; @@ -43,6 +45,7 @@ class PointValuesWriter { docIDs = new int[16]; iwBytesUsed.addAndGet(16 * Integer.BYTES); packedValue = new byte[fieldInfo.getPointDimensionCount() * fieldInfo.getPointNumBytes()]; + indexWriterConfig = docWriter.indexWriterConfig; } // TODO: if exactly the same value is added to exactly the same doc, should we dedup? @@ -124,6 +127,7 @@ class PointValuesWriter { public int getDocCount(String fieldName) { return numDocs; } - }); + }, + Math.max(indexWriterConfig.getRAMBufferSizeMB()/8.0, BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP)); } } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a0a9fd2/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java ---------------------------------------------------------------------- diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java index 09e6412..97651e4 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java @@ -204,7 +204,7 @@ public class BKDWriter implements Closeable { // all recursive halves (i.e. 16 + 8 + 4 + 2) so the memory usage is 2X // what that level would consume, so we multiply by 0.5 to convert from // bytes to points here. Each dimension has its own sorted partition, so - // we must divide by numDims as wel. + // we must divide by numDims as well. maxPointsSortInHeap = (int) (0.5 * (maxMBSortInHeap * 1024 * 1024) / (bytesPerDoc * numDims)); http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a0a9fd2/lucene/core/src/test/org/apache/lucene/codecs/lucene60/TestLucene60PointsFormat.java ---------------------------------------------------------------------- diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene60/TestLucene60PointsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene60/TestLucene60PointsFormat.java index afa8ec4..4b898c3 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/lucene60/TestLucene60PointsFormat.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene60/TestLucene60PointsFormat.java @@ -41,9 +41,8 @@ public class TestLucene60PointsFormat extends BasePointsFormatTestCase { if (random().nextBoolean()) { // randomize parameters int maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 500); - double maxMBSortInHeap = 3.0 + (3*random().nextDouble()); if (VERBOSE) { - System.out.println("TEST: using Lucene60PointsFormat with maxPointsInLeafNode=" + maxPointsInLeafNode + " and maxMBSortInHeap=" + maxMBSortInHeap); + System.out.println("TEST: using Lucene60PointsFormat with maxPointsInLeafNode=" + maxPointsInLeafNode); } // sneaky impersonation! @@ -53,7 +52,7 @@ public class TestLucene60PointsFormat extends BasePointsFormatTestCase { return new PointsFormat() { @Override public PointsWriter fieldsWriter(SegmentWriteState writeState) throws IOException { - return new Lucene60PointsWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap); + return new Lucene60PointsWriter(writeState, maxPointsInLeafNode); } @Override http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a0a9fd2/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java ---------------------------------------------------------------------- diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java index cf8372d..c139b64 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java @@ -1156,9 +1156,8 @@ public class TestPointQueries extends LuceneTestCase { private static Codec getCodec() { if (Codec.getDefault().getName().equals("Lucene62")) { int maxPointsInLeafNode = TestUtil.nextInt(random(), 16, 2048); - double maxMBSortInHeap = 5.0 + (3*random().nextDouble()); if (VERBOSE) { - System.out.println("TEST: using Lucene60PointsFormat with maxPointsInLeafNode=" + maxPointsInLeafNode + " and maxMBSortInHeap=" + maxMBSortInHeap); + System.out.println("TEST: using Lucene60PointsFormat with maxPointsInLeafNode=" + maxPointsInLeafNode); } return new FilterCodec("Lucene62", Codec.getDefault()) { @@ -1167,7 +1166,7 @@ public class TestPointQueries extends LuceneTestCase { return new PointsFormat() { @Override public PointsWriter fieldsWriter(SegmentWriteState writeState) throws IOException { - return new Lucene60PointsWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap); + return new Lucene60PointsWriter(writeState, maxPointsInLeafNode); } @Override http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a0a9fd2/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java ---------------------------------------------------------------------- diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index d9baf61..bb98145 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -87,9 +87,8 @@ public class TestGeo3DPoint extends LuceneTestCase { private static Codec getCodec() { if (Codec.getDefault().getName().equals("Lucene62")) { int maxPointsInLeafNode = TestUtil.nextInt(random(), 16, 2048); - double maxMBSortInHeap = 3.0 + (3*random().nextDouble()); if (VERBOSE) { - System.out.println("TEST: using Lucene60PointsFormat with maxPointsInLeafNode=" + maxPointsInLeafNode + " and maxMBSortInHeap=" + maxMBSortInHeap); + System.out.println("TEST: using Lucene60PointsFormat with maxPointsInLeafNode=" + maxPointsInLeafNode); } return new FilterCodec("Lucene62", Codec.getDefault()) { @@ -98,7 +97,7 @@ public class TestGeo3DPoint extends LuceneTestCase { return new PointsFormat() { @Override public PointsWriter fieldsWriter(SegmentWriteState writeState) throws IOException { - return new Lucene60PointsWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap); + return new Lucene60PointsWriter(writeState, maxPointsInLeafNode); } @Override http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a0a9fd2/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java ---------------------------------------------------------------------- diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java index 731aaec..c3c672b 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java @@ -254,11 +254,11 @@ public final class AssertingPointsFormat extends PointsFormat { } @Override - public void writeField(FieldInfo fieldInfo, PointsReader values) throws IOException { + public void writeField(FieldInfo fieldInfo, PointsReader values, double maxMBSortInHeap) throws IOException { if (fieldInfo.getPointDimensionCount() == 0) { throw new IllegalArgumentException("writing field=\"" + fieldInfo.name + "\" but pointDimensionalCount is 0"); } - in.writeField(fieldInfo, values); + in.writeField(fieldInfo, values, maxMBSortInHeap); } @Override http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a0a9fd2/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointsFormat.java ---------------------------------------------------------------------- diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointsFormat.java index fd2260b..ffd9a8c 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointsFormat.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointsFormat.java @@ -56,11 +56,11 @@ class CrankyPointsFormat extends PointsFormat { } @Override - public void writeField(FieldInfo fieldInfo, PointsReader values) throws IOException { + public void writeField(FieldInfo fieldInfo, PointsReader values, double maxMBSortInHeap) throws IOException { if (random.nextInt(100) == 0) { throw new IOException("Fake IOException"); } - delegate.writeField(fieldInfo, values); + delegate.writeField(fieldInfo, values, maxMBSortInHeap); } @Override http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a0a9fd2/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java ---------------------------------------------------------------------- diff --git a/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java index 275c186..926132f 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java @@ -67,7 +67,6 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.SloppyMath; import org.apache.lucene.util.TestUtil; -import org.apache.lucene.util.bkd.BKDWriter; /** * Abstract class to do basic tests for a geospatial impl (high level @@ -1248,7 +1247,7 @@ public abstract class BaseGeoPointTestCase extends LuceneTestCase { return new PointsFormat() { @Override public PointsWriter fieldsWriter(SegmentWriteState writeState) throws IOException { - return new Lucene60PointsWriter(writeState, pointsInLeaf, BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP); + return new Lucene60PointsWriter(writeState, pointsInLeaf); } @Override http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a0a9fd2/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java ---------------------------------------------------------------------- diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java b/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java index 127549f..6e8f7bc 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java @@ -92,7 +92,6 @@ public class RandomCodec extends AssertingCodec { // which is less effective for testing. // TODO: improve how we randomize this... private final int maxPointsInLeafNode; - private final double maxMBSortInHeap; private final int bkdSplitRandomSeed; @Override @@ -103,9 +102,9 @@ public class RandomCodec extends AssertingCodec { // Randomize how BKDWriter chooses its splis: - return new Lucene60PointsWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap) { + return new Lucene60PointsWriter(writeState, maxPointsInLeafNode) { @Override - public void writeField(FieldInfo fieldInfo, PointsReader values) throws IOException { + public void writeField(FieldInfo fieldInfo, PointsReader values, double maxMBSortInHeap) throws IOException { boolean singleValuePerDoc = values.size(fieldInfo.name) == values.getDocCount(fieldInfo.name); @@ -185,7 +184,6 @@ public class RandomCodec extends AssertingCodec { int lowFreqCutoff = TestUtil.nextInt(random, 2, 100); maxPointsInLeafNode = TestUtil.nextInt(random, 16, 2048); - maxMBSortInHeap = 5.0 + (3*random.nextDouble()); bkdSplitRandomSeed = random.nextInt(); add(avoidCodecs, @@ -253,8 +251,7 @@ public class RandomCodec extends AssertingCodec { public String toString() { return super.toString() + ": " + previousMappings.toString() + ", docValues:" + previousDVMappings.toString() + - ", maxPointsInLeafNode=" + maxPointsInLeafNode + - ", maxMBSortInHeap=" + maxMBSortInHeap; + ", maxPointsInLeafNode=" + maxPointsInLeafNode; } /** Just like {@link BKDWriter} except it evilly picks random ways to split cells on