From commits-return-98916-archive-asf-public=cust-asf.ponee.io@lucene.apache.org Thu Feb 15 15:23:53 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id EC63818064A for ; Thu, 15 Feb 2018 15:23:51 +0100 (CET) Received: (qmail 4865 invoked by uid 500); 15 Feb 2018 14:23:50 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 4856 invoked by uid 99); 15 Feb 2018 14:23:50 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 15 Feb 2018 14:23:50 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C244FE0904; Thu, 15 Feb 2018 14:23:50 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jpountz@apache.org To: commits@lucene.apache.org Date: Thu, 15 Feb 2018 14:23:50 -0000 Message-Id: <64659c8cd0a142af97348151a5afd277@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/4] lucene-solr:master: LUCENE-8135: Implement block-max WAND. Repository: lucene-solr Updated Branches: refs/heads/master a07493d50 -> 4fb7e3d02 http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java ---------------------------------------------------------------------- diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java index 405e481..a0f86af 100644 --- a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java +++ b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java @@ -543,7 +543,7 @@ public class TestJoinUtil extends LuceneTestCase { return (float) price.longValue(); } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { return Float.POSITIVE_INFINITY; } }; http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java ---------------------------------------------------------------------- diff --git a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java index 5e98349f..db17a90 100644 --- a/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java +++ b/lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java @@ -505,7 +505,7 @@ public class TestDiversifiedTopDocsCollector extends LuceneTestCase { } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { return Float.POSITIVE_INFINITY; } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java ---------------------------------------------------------------------- diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java index da41246..f996306 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java @@ -125,7 +125,7 @@ public class FunctionQuery extends Query { } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { return Float.POSITIVE_INFINITY; } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java ---------------------------------------------------------------------- diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java index 09a592b..0d39e8b 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java @@ -209,7 +209,7 @@ public final class FunctionScoreQuery extends Query { return 0; } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { return Float.POSITIVE_INFINITY; } }; http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java ---------------------------------------------------------------------- diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java index 4b8111e..5290dcc 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java @@ -107,7 +107,7 @@ public abstract class ValueSource { } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { return Float.POSITIVE_INFINITY; } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java ---------------------------------------------------------------------- diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java index 9e8534b..509f454 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java @@ -90,7 +90,7 @@ public abstract class ValueSourceScorer extends Scorer { } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { return Float.POSITIVE_INFINITY; } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/lucene/sandbox/src/java/org/apache/lucene/search/CoveringScorer.java ---------------------------------------------------------------------- diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/CoveringScorer.java b/lucene/sandbox/src/java/org/apache/lucene/search/CoveringScorer.java index eb3255e..a9743be 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/CoveringScorer.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/CoveringScorer.java @@ -212,7 +212,7 @@ final class CoveringScorer extends Scorer { } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { // TODO: implement but beware of floating-point errors return Float.POSITIVE_INFINITY; } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonScorer.java ---------------------------------------------------------------------- diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonScorer.java b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonScorer.java index 6094c01..f1ab32f 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonScorer.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonScorer.java @@ -360,7 +360,7 @@ class TermAutomatonScorer extends Scorer { } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { return docScorer.maxScore(); } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java ---------------------------------------------------------------------- diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java index b6062a2..80cd4da 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java @@ -24,7 +24,7 @@ import java.util.Random; /** Wraps a Scorer with additional checks */ public class AssertingScorer extends Scorer { - static enum IteratorState { START, APPROXIMATING, ITERATING, FINISHED }; + static enum IteratorState { APPROXIMATING, ITERATING, SHALLOW_ADVANCING, FINISHED }; public static Scorer wrap(Random random, Scorer other, ScoreMode scoreMode) { if (other == null) { @@ -37,9 +37,10 @@ public class AssertingScorer extends Scorer { final Scorer in; final ScoreMode scoreMode; - IteratorState state = IteratorState.START; + IteratorState state = IteratorState.ITERATING; int doc; float minCompetitiveScore = 0; + int lastShallowTarget; private AssertingScorer(Random random, Scorer in, ScoreMode scoreMode) { super(in.weight); @@ -60,7 +61,7 @@ public class AssertingScorer extends Scorer { case DocIdSetIterator.NO_MORE_DOCS: return false; default: - return state != IteratorState.APPROXIMATING; // Matches must be confirmed before calling freq() or score() + return state == IteratorState.ITERATING; } } @@ -74,19 +75,30 @@ public class AssertingScorer extends Scorer { } @Override - public float maxScore() { - float maxScore = in.maxScore(); - assert Float.isNaN(maxScore) == false; + public int advanceShallow(int target) throws IOException { + assert target >= lastShallowTarget : "called on decreasing targets: target = " + target + " < last target = " + lastShallowTarget; + assert target >= docID() : "target = " + target + " < docID = " + docID(); + int upTo = in.advanceShallow(target); + assert upTo >= target : "upTo = " + upTo + " < target = " + target; + lastShallowTarget = target; + state = target != doc ? IteratorState.SHALLOW_ADVANCING : state; + return upTo; + } + + @Override + public float getMaxScore(int upTo) throws IOException { + assert upTo >= lastShallowTarget : "uTo = " + upTo + " < last target = " + lastShallowTarget; + float maxScore = in.getMaxScore(upTo); return maxScore; } @Override public float score() throws IOException { assert scoreMode.needsScores(); - assert iterating(); + assert iterating() : state; final float score = in.score(); assert !Float.isNaN(score) : "NaN score for in="+in; - assert score <= maxScore(); + assert score <= getMaxScore(DocIdSetIterator.NO_MORE_DOCS); assert Float.compare(score, 0f) >= 0 : score; return score; } @@ -125,6 +137,7 @@ public class AssertingScorer extends Scorer { @Override public int nextDoc() throws IOException { assert state != IteratorState.FINISHED : "nextDoc() called after NO_MORE_DOCS"; + assert docID() + 1 >= lastShallowTarget; int nextDoc = in.nextDoc(); assert nextDoc > doc : "backwards nextDoc from " + doc + " to " + nextDoc + " " + in; if (nextDoc == DocIdSetIterator.NO_MORE_DOCS) { @@ -141,6 +154,7 @@ public class AssertingScorer extends Scorer { public int advance(int target) throws IOException { assert state != IteratorState.FINISHED : "advance() called after NO_MORE_DOCS"; assert target > doc : "target must be > docID(), got " + target + " <= " + doc; + assert target >= lastShallowTarget; int advanced = in.advance(target); assert advanced >= target : "backwards advance from: " + target + " to: " + advanced; if (advanced == DocIdSetIterator.NO_MORE_DOCS) { @@ -178,6 +192,7 @@ public class AssertingScorer extends Scorer { @Override public int nextDoc() throws IOException { assert state != IteratorState.FINISHED : "advance() called after NO_MORE_DOCS"; + assert docID() + 1 >= lastShallowTarget; final int nextDoc = inApproximation.nextDoc(); assert nextDoc > doc : "backwards advance from: " + doc + " to: " + nextDoc; if (nextDoc == NO_MORE_DOCS) { @@ -193,6 +208,7 @@ public class AssertingScorer extends Scorer { public int advance(int target) throws IOException { assert state != IteratorState.FINISHED : "advance() called after NO_MORE_DOCS"; assert target > doc : "target must be > docID(), got " + target + " <= " + doc; + assert target >= lastShallowTarget; final int advanced = inApproximation.advance(target); assert advanced >= target : "backwards advance from: " + target + " to: " + advanced; if (advanced == NO_MORE_DOCS) { @@ -213,7 +229,7 @@ public class AssertingScorer extends Scorer { return new TwoPhaseIterator(assertingApproximation) { @Override public boolean matches() throws IOException { - assert state == IteratorState.APPROXIMATING; + assert state == IteratorState.APPROXIMATING : state; final boolean matches = in.matches(); if (matches) { assert AssertingScorer.this.in.iterator().docID() == inApproximation.docID() : "Approximation and scorer don't advance synchronously"; http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/lucene/test-framework/src/java/org/apache/lucene/search/BlockScoreQueryWrapper.java ---------------------------------------------------------------------- diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/BlockScoreQueryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/search/BlockScoreQueryWrapper.java new file mode 100644 index 0000000..3b9a740 --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/search/BlockScoreQueryWrapper.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.search; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; +import java.util.Set; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Term; +import org.apache.lucene.util.ArrayUtil; + +/** + * Query wrapper that reduces the size of max-score blocks to more easily detect + * problems with the max-score logic. + */ +public final class BlockScoreQueryWrapper extends Query { + + private final Query query; + private final int blockLength; + + /** Sole constructor. */ + public BlockScoreQueryWrapper(Query query, int blockLength) { + this.query = Objects.requireNonNull(query); + this.blockLength = blockLength; + } + + @Override + public String toString(String field) { + return query.toString(field); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || getClass() != obj.getClass()) { + return false; + } + BlockScoreQueryWrapper that = (BlockScoreQueryWrapper) obj; + return Objects.equals(query, that.query) && blockLength == that.blockLength; + } + + @Override + public int hashCode() { + int h = classHash(); + h = 31 * h + query.hashCode(); + h = 31 * h + blockLength; + return h; + } + + @Override + public Query rewrite(IndexReader reader) throws IOException { + final Query rewritten = query.rewrite(reader); + if (rewritten != query) { + return new BlockScoreQueryWrapper(rewritten, blockLength); + } + return super.rewrite(reader); + } + + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + final Weight inWeight = query.createWeight(searcher, scoreMode, boost); + if (scoreMode.needsScores() == false) { + return inWeight; + } + return new Weight(this) { + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return inWeight.isCacheable(ctx); + } + + @Override + public Scorer scorer(LeafReaderContext context) throws IOException { + Scorer inScorer = inWeight.scorer(context); + if (inScorer == null) { + return null; + } + + int[] tmpDocs = new int[2]; + float[] tmpScores = new float[2]; + tmpDocs[0] = -1; + DocIdSetIterator it = inScorer.iterator(); + int i = 1; + for (int doc = it.nextDoc(); ; doc = it.nextDoc()) { + if (i == tmpDocs.length) { + tmpDocs = ArrayUtil.grow(tmpDocs); + tmpScores = Arrays.copyOf(tmpScores, tmpDocs.length); + } + tmpDocs[i] = doc; + if (doc == DocIdSetIterator.NO_MORE_DOCS) { + i++; + break; + } + tmpScores[i] = inScorer.score(); + i++; + } + final int[] docs = Arrays.copyOf(tmpDocs, i); + final float[] scores = Arrays.copyOf(tmpScores, i); + + return new Scorer(inWeight) { + + int i = 0; + + @Override + public int docID() { + return docs[i]; + } + + @Override + public float score() throws IOException { + return scores[i]; + } + + @Override + public DocIdSetIterator iterator() { + return new DocIdSetIterator() { + + @Override + public int nextDoc() throws IOException { + assert docs[i] != NO_MORE_DOCS; + return docs[++i]; + } + + @Override + public int docID() { + return docs[i]; + } + + @Override + public long cost() { + return docs.length - 2; + } + + @Override + public int advance(int target) throws IOException { + i = Arrays.binarySearch(docs, target); + if (i < 0) { + i = -1 - i; + } + assert docs[i] >= target; + return docs[i]; + } + }; + } + + private int startOfBlock(int target) { + int i = Arrays.binarySearch(docs, target); + if (i < 0) { + i = -1 - i; + } + return i - i % blockLength; + } + + private int endOfBlock(int target) { + return Math.min(startOfBlock(target) + blockLength, docs.length - 1); + } + + int lastShallowTarget = -1; + + @Override + public int advanceShallow(int target) throws IOException { + lastShallowTarget = target; + if (target == DocIdSetIterator.NO_MORE_DOCS) { + return DocIdSetIterator.NO_MORE_DOCS; + } + return docs[endOfBlock(target)] - 1; + } + + @Override + public float getMaxScore(int upTo) throws IOException { + float max = 0; + for (int j = startOfBlock(Math.max(docs[i], lastShallowTarget)); ; ++j) { + if (docs[j] > upTo) { + break; + } + max = Math.max(max, scores[j]); + if (j == docs.length - 1) { + break; + } + } + return max; + } + + }; + } + + @Override + public void extractTerms(Set terms) { + inWeight.extractTerms(terms); + } + + @Override + public Explanation explain(LeafReaderContext context, int doc) throws IOException { + return inWeight.explain(context, doc); + } + }; + } +} http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/lucene/test-framework/src/java/org/apache/lucene/search/BulkScorerWrapperScorer.java ---------------------------------------------------------------------- diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/BulkScorerWrapperScorer.java b/lucene/test-framework/src/java/org/apache/lucene/search/BulkScorerWrapperScorer.java index db1de52..9206b04 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/BulkScorerWrapperScorer.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/BulkScorerWrapperScorer.java @@ -70,7 +70,7 @@ public class BulkScorerWrapperScorer extends Scorer { } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { return Float.POSITIVE_INFINITY; } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java ---------------------------------------------------------------------- diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java b/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java index 9c8943d..8d8b60b 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java @@ -16,6 +16,9 @@ */ package org.apache.lucene.search; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + import java.io.IOException; import java.util.Locale; import java.util.Set; @@ -520,6 +523,150 @@ public class CheckHits { } } -} + public static void checkTopScores(Random random, Query query, IndexSearcher searcher) throws IOException { + // Check it computed the top hits correctly + doCheckTopScores(query, searcher, 1); + doCheckTopScores(query, searcher, 10); + + // Now check that the exposed max scores and block boundaries are valid + doCheckMaxScores(random, query, searcher); + } + + private static void doCheckTopScores(Query query, IndexSearcher searcher, int numHits) throws IOException { + TopScoreDocCollector collector1 = TopScoreDocCollector.create(numHits, null, true); // COMPLETE + TopScoreDocCollector collector2 = TopScoreDocCollector.create(numHits, null, false); // TOP_SCORES + searcher.search(query, collector1); + searcher.search(query, collector2); + checkEqual(query, collector1.topDocs().scoreDocs, collector2.topDocs().scoreDocs); + } + + private static void doCheckMaxScores(Random random, Query query, IndexSearcher searcher) throws IOException { + Weight w1 = searcher.createNormalizedWeight(query, ScoreMode.COMPLETE); + Weight w2 = searcher.createNormalizedWeight(query, ScoreMode.TOP_SCORES); + + // Check boundaries and max scores when iterating all matches + for (LeafReaderContext ctx : searcher.getIndexReader().leaves()) { + Scorer s1 = w1.scorer(ctx); + Scorer s2 = w2.scorer(ctx); + if (s1 == null) { + Assert.assertTrue(s2 == null || s2.iterator().nextDoc() == DocIdSetIterator.NO_MORE_DOCS); + continue; + } + TwoPhaseIterator twoPhase1 = s1.twoPhaseIterator(); + TwoPhaseIterator twoPhase2 = s2.twoPhaseIterator(); + DocIdSetIterator approx1 = twoPhase1 == null ? s1.iterator() : twoPhase1.approximation; + DocIdSetIterator approx2 = twoPhase2 == null ? s2.iterator() : twoPhase2.approximation; + int upTo = -1; + float maxScore = 0; + float minScore = 0; + for (int doc2 = approx2.nextDoc(); ; doc2 = approx2.nextDoc()) { + int doc1; + for (doc1 = approx1.nextDoc(); doc1 < doc2; doc1 = approx1.nextDoc()) { + if (twoPhase1 == null || twoPhase1.matches()) { + Assert.assertTrue(s1.score() < minScore); + } + } + Assert.assertEquals(doc1, doc2); + if (doc2 == DocIdSetIterator.NO_MORE_DOCS) { + break; + } + + if (doc2 > upTo) { + upTo = s2.advanceShallow(doc2); + Assert.assertTrue(upTo >= doc2); + maxScore = s2.getMaxScore(upTo); + } + + if (twoPhase2 == null || twoPhase2.matches()) { + Assert.assertTrue(twoPhase1 == null || twoPhase1.matches()); + float score = s2.score(); + Assert.assertEquals(s1.score(), score); + Assert.assertTrue(score <= maxScore); + + if (score >= minScore && random.nextInt(10) == 0) { + // On some scorers, changing the min score changes the way that docs are iterated + minScore = score; + s2.setMinCompetitiveScore(minScore); + } + } + } + } + + // Now check advancing + for (LeafReaderContext ctx : searcher.getIndexReader().leaves()) { + Scorer s1 = w1.scorer(ctx); + Scorer s2 = w2.scorer(ctx); + if (s1 == null) { + Assert.assertTrue(s2 == null || s2.iterator().nextDoc() == DocIdSetIterator.NO_MORE_DOCS); + continue; + } + TwoPhaseIterator twoPhase1 = s1.twoPhaseIterator(); + TwoPhaseIterator twoPhase2 = s2.twoPhaseIterator(); + DocIdSetIterator approx1 = twoPhase1 == null ? s1.iterator() : twoPhase1.approximation; + DocIdSetIterator approx2 = twoPhase2 == null ? s2.iterator() : twoPhase2.approximation; + + int upTo = -1; + float minScore = 0; + float maxScore = 0; + while (true) { + int doc2 = s2.docID(); + boolean advance; + int target; + if (random.nextBoolean()) { + advance = false; + target = doc2 + 1; + } else { + advance = true; + int delta = Math.min(1 + random.nextInt(512), DocIdSetIterator.NO_MORE_DOCS - doc2); + target = s2.docID() + delta; + } + + if (target > upTo && random.nextBoolean()) { + int delta = Math.min(random.nextInt(512), DocIdSetIterator.NO_MORE_DOCS - target); + upTo = target + delta; + int m = s2.advanceShallow(target); + assertTrue(m >= target); + maxScore = s2.getMaxScore(upTo); + } + if (advance) { + doc2 = approx2.advance(target); + } else { + doc2 = approx2.nextDoc(); + } + + int doc1; + for (doc1 = approx1.advance(target); doc1 < doc2; doc1 = approx1.nextDoc()) { + if (twoPhase1 == null || twoPhase1.matches()) { + Assert.assertTrue(s1.score() < minScore); + } + } + assertEquals(doc1, doc2); + + if (doc2 == DocIdSetIterator.NO_MORE_DOCS) { + break; + } + if (twoPhase2 == null || twoPhase2.matches()) { + Assert.assertTrue(twoPhase1 == null || twoPhase1.matches()); + float score = s2.score(); + Assert.assertEquals(s1.score(), score); + + if (doc2 > upTo) { + upTo = s2.advanceShallow(doc2); + Assert.assertTrue(upTo >= doc2); + maxScore = s2.getMaxScore(upTo); + } + + Assert.assertTrue(score <= maxScore); + + if (score >= minScore && random.nextInt(10) == 0) { + // On some scorers, changing the min score changes the way that docs are iterated + minScore = score; + s2.setMinCompetitiveScore(minScore); + } + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/lucene/test-framework/src/java/org/apache/lucene/search/RandomApproximationQuery.java ---------------------------------------------------------------------- diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/RandomApproximationQuery.java b/lucene/test-framework/src/java/org/apache/lucene/search/RandomApproximationQuery.java index e88afa7..a050b50 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/RandomApproximationQuery.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/RandomApproximationQuery.java @@ -109,8 +109,13 @@ public class RandomApproximationQuery extends Query { } @Override - public float maxScore() { - return scorer.maxScore(); + public int advanceShallow(int target) throws IOException { + return scorer.advanceShallow(target); + } + + @Override + public float getMaxScore(int upTo) throws IOException { + return scorer.getMaxScore(upTo); } @Override http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java ---------------------------------------------------------------------- diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java index cf564bb..c143d81 100644 --- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java +++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/LTRScoringQuery.java @@ -523,7 +523,7 @@ public class LTRScoringQuery extends Query { } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { return Float.POSITIVE_INFINITY; } @@ -582,7 +582,7 @@ public class LTRScoringQuery extends Query { } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { return Float.POSITIVE_INFINITY; } @@ -669,7 +669,7 @@ public class LTRScoringQuery extends Query { } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { return Float.POSITIVE_INFINITY; } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java ---------------------------------------------------------------------- diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java index 026f760..066d281 100644 --- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java +++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/Feature.java @@ -328,7 +328,7 @@ public abstract class Feature extends Query { } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { return constScore; } } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/FieldLengthFeature.java ---------------------------------------------------------------------- diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/FieldLengthFeature.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/FieldLengthFeature.java index f8b544f..b2fc154 100644 --- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/FieldLengthFeature.java +++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/FieldLengthFeature.java @@ -147,7 +147,7 @@ public class FieldLengthFeature extends Feature { } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { return Float.POSITIVE_INFINITY; } } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/FieldValueFeature.java ---------------------------------------------------------------------- diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/FieldValueFeature.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/FieldValueFeature.java index d9e7f02..78cdbaa 100644 --- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/FieldValueFeature.java +++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/FieldValueFeature.java @@ -142,7 +142,7 @@ public class FieldValueFeature extends Feature { } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { return Float.POSITIVE_INFINITY; } } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/OriginalScoreFeature.java ---------------------------------------------------------------------- diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/OriginalScoreFeature.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/OriginalScoreFeature.java index d351a04..b538b86 100644 --- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/OriginalScoreFeature.java +++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/OriginalScoreFeature.java @@ -109,7 +109,7 @@ public class OriginalScoreFeature extends Feature { } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { return Float.POSITIVE_INFINITY; } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/SolrFeature.java ---------------------------------------------------------------------- diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/SolrFeature.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/SolrFeature.java index b616bd5..612085d 100644 --- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/SolrFeature.java +++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/SolrFeature.java @@ -288,7 +288,7 @@ public class SolrFeature extends Feature { } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { return Float.POSITIVE_INFINITY; } } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java ---------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java index 2385533..1a34b58 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java @@ -1481,7 +1481,7 @@ public class QueryComponent extends SearchComponent } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { return Float.POSITIVE_INFINITY; } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/solr/core/src/java/org/apache/solr/schema/LatLonType.java ---------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/schema/LatLonType.java b/solr/core/src/java/org/apache/solr/schema/LatLonType.java index 8cf86b8..9f9dcd1 100644 --- a/solr/core/src/java/org/apache/solr/schema/LatLonType.java +++ b/solr/core/src/java/org/apache/solr/schema/LatLonType.java @@ -486,7 +486,7 @@ class SpatialDistanceQuery extends ExtendedQueryBase implements PostFilter { } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { return Float.POSITIVE_INFINITY; } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java ---------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index 9af1c89..be8741c 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -452,7 +452,7 @@ public class CollapsingQParserPlugin extends QParserPlugin { } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { return Float.POSITIVE_INFINITY; } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java ---------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java index 5842e92..4402a26 100644 --- a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java +++ b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java @@ -307,7 +307,7 @@ public class GraphQuery extends Query { } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { return score; } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fb7e3d0/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java ---------------------------------------------------------------------- diff --git a/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java b/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java index fa74fb4..6d2143a 100644 --- a/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java +++ b/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java @@ -458,7 +458,7 @@ public class TestRankQueryPlugin extends QParserPlugin { } @Override - public float maxScore() { + public float getMaxScore(int upTo) throws IOException { return score; }