Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 95B32200C5D for ; Thu, 23 Mar 2017 15:40:47 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 947DE160B9D; Thu, 23 Mar 2017 14:40:47 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 40C06160B92 for ; Thu, 23 Mar 2017 15:40:46 +0100 (CET) Received: (qmail 49243 invoked by uid 500); 23 Mar 2017 14:40:44 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 49056 invoked by uid 99); 23 Mar 2017 14:40:44 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 23 Mar 2017 14:40:44 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 55A0FE0896; Thu, 23 Mar 2017 14:40:44 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: shalin@apache.org To: commits@lucene.apache.org Date: Thu, 23 Mar 2017 14:40:48 -0000 Message-Id: <811df5b1ca9649f1bcf59357275add62@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [05/36] lucene-solr:feature/autoscaling: SOLR-10283: Learning to Rank (LTR) SolrFeature to reject searches with missing efi (External Feature Information) used by fq. archived-at: Thu, 23 Mar 2017 14:40:47 -0000 SOLR-10283: Learning to Rank (LTR) SolrFeature to reject searches with missing efi (External Feature Information) used by fq. Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9f935735 Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9f935735 Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9f935735 Branch: refs/heads/feature/autoscaling Commit: 9f935735b0d9cdbb688d24358a7840503c037d9f Parents: d836ff1 Author: Christine Poerschke Authored: Thu Mar 16 14:10:28 2017 +0000 Committer: Shalin Shekhar Mangar Committed: Thu Mar 23 20:10:09 2017 +0530 ---------------------------------------------------------------------- solr/CHANGES.txt | 3 +++ .../apache/solr/ltr/feature/SolrFeature.java | 3 +++ .../featureExamples/external_features.json | 6 +++++ .../solr/ltr/TestSelectiveWeightCreation.java | 8 +++++-- .../solr/ltr/feature/TestExternalFeatures.java | 25 ++++++++++++++++---- 5 files changed, 39 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f935735/solr/CHANGES.txt ---------------------------------------------------------------------- diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 5e2a518..2d4ce6c 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -260,6 +260,9 @@ Bug Fixes * SOLR-10302: Solr's zkcli scripts now able to find the metrics libraries, which it couldn't earlier (kiran, Ishan Chattopadhyaya) +* SOLR-10283: Learning to Rank (LTR) SolrFeature to reject searches with missing efi (External Feature Information) used by fq. + (Christine Poerschke) + Optimizations ---------------------- http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f935735/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/SolrFeature.java ---------------------------------------------------------------------- diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/SolrFeature.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/SolrFeature.java index 13eb96f..4aa872d 100644 --- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/SolrFeature.java +++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/SolrFeature.java @@ -157,6 +157,9 @@ public class SolrFeature extends Feature { for (String fq : fqs) { if ((fq != null) && (fq.trim().length() != 0)) { fq = macroExpander.expand(fq); + if (fq == null) { + throw new FeatureException(this.getClass().getSimpleName()+" requires efi parameter that was not passed in request."); + } final QParser fqp = QParser.getParser(fq, req); final Query filterQuery = fqp.getQuery(); if (filterQuery != null) { http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f935735/solr/contrib/ltr/src/test-files/featureExamples/external_features.json ---------------------------------------------------------------------- diff --git a/solr/contrib/ltr/src/test-files/featureExamples/external_features.json b/solr/contrib/ltr/src/test-files/featureExamples/external_features.json index d8a9eca..5c4f12d 100644 --- a/solr/contrib/ltr/src/test-files/featureExamples/external_features.json +++ b/solr/contrib/ltr/src/test-files/featureExamples/external_features.json @@ -48,4 +48,10 @@ "params" : { "q" : "{!field f=title}${user_query}" } +}, { + "name" : "titlePhrasesMatch", + "class" : "org.apache.solr.ltr.feature.SolrFeature", + "params" : { + "fq" : [ "{!field f=title}${userTitlePhrase1}", "{!field f=title}${userTitlePhrase2}"] + } } ] http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f935735/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java ---------------------------------------------------------------------- diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java index 5cfd999..7bf8373 100644 --- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java +++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java @@ -210,18 +210,22 @@ public class TestSelectiveWeightCreation extends TestRerankBase { @Test public void testSelectiveWeightsRequestFeaturesFromDifferentStore() throws Exception { - final String docs0fv = FeatureLoggerTestUtils.toFeatureVector( + final String docs0fv_sparse = FeatureLoggerTestUtils.toFeatureVector( "matchedTitle","1.0", "titlePhraseMatch","0.6103343"); + final String docs0fv_dense = FeatureLoggerTestUtils.toFeatureVector( + "matchedTitle","1.0", "titlePhraseMatch","0.6103343", "titlePhrasesMatch","0.0"); final String docs0fv_fstore4= FeatureLoggerTestUtils.toFeatureVector( "popularity","3.0", "originalScore","1.0"); + final String docs0fv = chooseDefaultFeatureVector(docs0fv_dense, docs0fv_sparse); + // extract all features in externalmodel's store (default store) // rerank using externalmodel (default store) final SolrQuery query = new SolrQuery(); query.setQuery("*:*"); query.add("fl", "*,score,fv:[fv]"); query.add("rows", "5"); - query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}"); + query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3 efi.userTitlePhrase1=w2 efi.userTitlePhrase2=w1}"); assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'"); assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='4'"); http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f935735/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java ---------------------------------------------------------------------- diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java index 4010ee1..c6ae30f 100644 --- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java +++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java @@ -67,7 +67,7 @@ public class TestExternalFeatures extends TestRerankBase { query.remove("fl"); query.add("fl", "*,score,[fv]"); - query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}"); + query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3 efi.userTitlePhrase1=w4 efi.userTitlePhrase2=w5}"); assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'"); assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.7693934"); @@ -77,7 +77,7 @@ public class TestExternalFeatures extends TestRerankBase { // Adding an efi in the transformer should not affect the rq ranking with a // different value for efi of the same parameter query.remove("fl"); - query.add("fl", "*,score,[fv efi.user_query=w2]"); + query.add("fl", "*,score,[fv efi.user_query=w2 efi.userTitlePhrase1=w4 efi.userTitlePhrase2=w5]"); assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'"); assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.7693934"); @@ -92,11 +92,12 @@ public class TestExternalFeatures extends TestRerankBase { query.add("fl", "*,score,fv:[fv]"); query.add("rows", "1"); // Stopword only query passed in - query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query='a'}"); + query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query='a' efi.userTitlePhrase1='b' efi.userTitlePhrase2='c'}"); final String docs0fv_dense_csv = FeatureLoggerTestUtils.toFeatureVector( "matchedTitle","0.0", - "titlePhraseMatch","0.0"); + "titlePhraseMatch","0.0", + "titlePhrasesMatch","0.0"); final String docs0fv_sparse_csv = FeatureLoggerTestUtils.toFeatureVector(); final String docs0fv_default_csv = chooseDefaultFeatureVector(docs0fv_dense_csv, docs0fv_sparse_csv); @@ -181,4 +182,20 @@ public class TestExternalFeatures extends TestRerankBase { query.add("fl", "fvalias:[fv store=fstore4]"); assertJQ("/query" + query.toQueryString(), "/error/msg=='Exception from createWeight for ValueFeature [name=popularity, params={value=${myPop}, required=true}] ValueFeatureWeight requires efi parameter that was not passed in request.'"); } + + @Test + public void featureExtraction_valueFeatureRequiredInFq_shouldThrowException() throws Exception { + final String userTitlePhrase1 = "userTitlePhrase1"; + final String userTitlePhrase2 = "userTitlePhrase2"; + final String userTitlePhrasePresent = (random().nextBoolean() ? userTitlePhrase1 : userTitlePhrase2); + + final SolrQuery query = new SolrQuery(); + query.setQuery("*:*"); + query.add("rows", "1"); + query.add("fl", "score,features:[fv efi.user_query=uq "+userTitlePhrasePresent+"=utpp]"); + assertJQ("/query" + query.toQueryString(), "/error/msg=='Exception from createWeight for " + + "SolrFeature [name=titlePhrasesMatch, params={fq=[{!field f=title}${"+userTitlePhrase1+"}, {!field f=title}${"+userTitlePhrase2+"}]}] " + + "SolrFeatureWeight requires efi parameter that was not passed in request.'"); + } + }