lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yo...@apache.org
Subject svn commit: r1061065 - in /lucene/dev/trunk/solr/src: java/org/apache/solr/handler/component/ java/org/apache/solr/search/ java/org/apache/solr/search/function/ test/org/apache/solr/ test/org/apache/solr/search/function/
Date Wed, 19 Jan 2011 23:48:28 GMT
Author: yonik
Date: Wed Jan 19 23:48:28 2011
New Revision: 1061065

URL: http://svn.apache.org/viewvc?rev=1061065&view=rev
Log:
SOLR-1297: fix weighting when sorting by function

Added:
    lucene/dev/trunk/solr/src/java/org/apache/solr/search/SolrSortField.java   (with props)
Modified:
    lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/QueryComponent.java
    lucene/dev/trunk/solr/src/java/org/apache/solr/search/Grouping.java
    lucene/dev/trunk/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java
    lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/ValueSource.java
    lucene/dev/trunk/solr/src/test/org/apache/solr/TestDistributedSearch.java
    lucene/dev/trunk/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java

Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=1061065&r1=1061064&r2=1061065&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/QueryComponent.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/QueryComponent.java Wed
Jan 19 23:48:28 2011
@@ -443,7 +443,7 @@ public class QueryComponent extends Sear
     // take the documents given and re-derive the sort values.
     boolean fsv = req.getParams().getBool(ResponseBuilder.FIELD_SORT_VALUES,false);
     if(fsv){
-      Sort sort = rb.getSortSpec().getSort();
+      Sort sort = searcher.weightSort(rb.getSortSpec().getSort());
       SortField[] sortFields = sort==null ? new SortField[]{SortField.FIELD_SCORE} : sort.getSort();
       NamedList sortVals = new NamedList(); // order is important for the sort fields
       Field field = new Field("dummy", "", Field.Store.YES, Field.Index.NO); // a dummy Field

Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/search/Grouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/search/Grouping.java?rev=1061065&r1=1061064&r2=1061065&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/Grouping.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/search/Grouping.java Wed Jan 19 23:48:28
2011
@@ -162,7 +162,7 @@ public class Grouping {
       // if we aren't going to return any groups, disregard the offset 
       if (numGroups == 0) maxGroupToFind = 0;
 
-      collector = new TopGroupCollector(groupBy, context, normalizeSort(sort), maxGroupToFind);
+      collector = new TopGroupCollector(groupBy, context, searcher.weightSort(normalizeSort(sort)),
maxGroupToFind);
 
       /*** if we need a different algorithm when sort != group.sort
       if (compareSorts(sort, groupSort)) {
@@ -185,9 +185,9 @@ public class Grouping {
       int collectorOffset = format==Format.Simple ? 0 : offset;
 
       if (groupBy instanceof StrFieldSource) {
-        collector2 = new Phase2StringGroupCollector(collector, groupBy, context, groupSort,
docsToCollect, needScores, collectorOffset);
+        collector2 = new Phase2StringGroupCollector(collector, groupBy, context, searcher.weightSort(groupSort),
docsToCollect, needScores, collectorOffset);
       } else {
-        collector2 = new Phase2GroupCollector(collector, groupBy, context, groupSort, docsToCollect,
needScores, collectorOffset);
+        collector2 = new Phase2GroupCollector(collector, groupBy, context, searcher.weightSort(groupSort),
docsToCollect, needScores, collectorOffset);
       }
       return collector2;
     }
@@ -306,11 +306,11 @@ public class Grouping {
     return v;
   }
 
-  static TopDocsCollector newCollector(Sort sort, int numHits, boolean fillFields, boolean
needScores) throws IOException {
+  TopDocsCollector newCollector(Sort sort, int numHits, boolean fillFields, boolean needScores)
throws IOException {
     if (sort==null || sort==byScoreDesc) {
       return TopScoreDocCollector.create(numHits, true);
     } else {
-      return TopFieldCollector.create(sort, numHits, false, needScores, needScores, true);
+      return TopFieldCollector.create(searcher.weightSort(sort), numHits, false, needScores,
needScores, true);
     }
   }
 
@@ -505,12 +505,12 @@ class TopGroupCollector extends GroupCol
 
   int matches;
 
-  public TopGroupCollector(ValueSource groupByVS, Map vsContext, Sort sort, int nGroups)
throws IOException {
+  public TopGroupCollector(ValueSource groupByVS, Map vsContext, Sort weightedSort, int nGroups)
throws IOException {
     this.vs = groupByVS;
     this.context = vsContext;
     this.nGroups = nGroups = Math.max(1,nGroups);  // we need a minimum of 1 for this collector
 
-    SortField[] sortFields = sort.getSort();
+    SortField[] sortFields = weightedSort.getSort();
     this.comparators = new FieldComparator[sortFields.length];
     this.reversed = new int[sortFields.length];
     for (int i = 0; i < sortFields.length; i++) {
@@ -719,7 +719,7 @@ class Phase2GroupCollector extends Colle
   int docBase;
 
   // TODO: may want to decouple from the phase1 collector
-  public Phase2GroupCollector(TopGroupCollector topGroups, ValueSource groupByVS, Map vsContext,
Sort sort, int docsPerGroup, boolean getScores, int offset) throws IOException {
+  public Phase2GroupCollector(TopGroupCollector topGroups, ValueSource groupByVS, Map vsContext,
Sort weightedSort, int docsPerGroup, boolean getScores, int offset) throws IOException {
     boolean getSortFields = false;
 
     if (topGroups.orderedGroups == null)
@@ -733,10 +733,10 @@ class Phase2GroupCollector extends Colle
       }
       SearchGroupDocs groupDocs = new SearchGroupDocs();
       groupDocs.groupValue = group.groupValue;
-      if (sort==null)
+      if (weightedSort==null)
         groupDocs.collector = TopScoreDocCollector.create(docsPerGroup, true);        
       else
-        groupDocs.collector = TopFieldCollector.create(sort, docsPerGroup, getSortFields,
getScores, getScores, true);
+        groupDocs.collector = TopFieldCollector.create(weightedSort, docsPerGroup, getSortFields,
getScores, getScores, true);
       groupMap.put(groupDocs.groupValue, groupDocs);
     }
 
@@ -791,8 +791,8 @@ class Phase2StringGroupCollector extends
   final SearchGroupDocs[] groups;
   final BytesRef spare = new BytesRef();
 
-  public Phase2StringGroupCollector(TopGroupCollector topGroups, ValueSource groupByVS, Map
vsContext, Sort sort, int docsPerGroup, boolean getScores, int offset) throws IOException
{
-    super(topGroups, groupByVS, vsContext,sort,docsPerGroup,getScores,offset);
+  public Phase2StringGroupCollector(TopGroupCollector topGroups, ValueSource groupByVS, Map
vsContext, Sort weightedSort, int docsPerGroup, boolean getScores, int offset) throws IOException
{
+    super(topGroups, groupByVS, vsContext,weightedSort,docsPerGroup,getScores,offset);
     ordSet = new SentinelIntSet(groupMap.size(), -1);
     groups = new SearchGroupDocs[ordSet.keys.length];
   }

Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1061065&r1=1061064&r2=1061065&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java Wed Jan 19
23:48:28 2011
@@ -460,6 +460,30 @@ public class SolrIndexSearcher extends I
     return fieldValueCache;
   }
 
+  /** Returns a weighted sort according to this searcher */
+  public Sort weightSort(Sort sort) throws IOException {
+    if (sort == null) return null;
+    SortField[] sorts = sort.getSort();
+
+    boolean needsWeighting = false;
+    for (SortField sf : sorts) {
+      if (sf instanceof SolrSortField) {
+        needsWeighting = true;
+        break;
+      }
+    }
+    if (!needsWeighting) return sort;
+
+    SortField[] newSorts = Arrays.copyOf(sorts, sorts.length);
+    for (int i=0; i<newSorts.length; i++) {
+      if (newSorts[i] instanceof SolrSortField) {
+        newSorts[i] = ((SolrSortField)newSorts[i]).weight(this);
+      }
+    }
+
+    return new Sort(newSorts);
+  }
+
 
   /**
    * Returns the first document number containing the term <code>t</code>
@@ -1156,7 +1180,7 @@ public class SolrIndexSearcher extends I
       if (cmd.getSort() == null) {
         topCollector = TopScoreDocCollector.create(len, true);
       } else {
-        topCollector = TopFieldCollector.create(cmd.getSort(), len, false, needScores, needScores,
true);
+        topCollector = TopFieldCollector.create(weightSort(cmd.getSort()), len, false, needScores,
needScores, true);
       }
       Collector collector = topCollector;
       if( timeAllowed > 0 ) {
@@ -1266,7 +1290,7 @@ public class SolrIndexSearcher extends I
       if (cmd.getSort() == null) {
         topCollector = TopScoreDocCollector.create(len, true);
       } else {
-        topCollector = TopFieldCollector.create(cmd.getSort(), len, false, needScores, needScores,
true);
+        topCollector = TopFieldCollector.create(weightSort(cmd.getSort()), len, false, needScores,
needScores, true);
       }
 
       DocSetCollector setCollector = new DocSetDelegateCollector(maxDoc>>6, maxDoc,
topCollector);
@@ -1548,7 +1572,7 @@ public class SolrIndexSearcher extends I
     // bit of a hack to tell if a set is sorted - do it better in the futute.
     boolean inOrder = set instanceof BitDocSet || set instanceof SortedIntDocSet;
 
-    TopDocsCollector topCollector = TopFieldCollector.create(sort, nDocs, false, false, false,
inOrder);
+    TopDocsCollector topCollector = TopFieldCollector.create(weightSort(sort), nDocs, false,
false, false, inOrder);
 
     DocIterator iter = set.iterator();
     int base=0;

Added: lucene/dev/trunk/solr/src/java/org/apache/solr/search/SolrSortField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/search/SolrSortField.java?rev=1061065&view=auto
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/SolrSortField.java (added)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/search/SolrSortField.java Wed Jan 19 23:48:28
2011
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.solr.search;
+
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.SortField;
+
+import java.io.IOException;
+
+/**@lucene.internal
+ *
+ */
+public interface SolrSortField {
+  public SortField weight(IndexSearcher searcher) throws IOException;
+}

Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/ValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/ValueSource.java?rev=1061065&r1=1061064&r2=1061065&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/ValueSource.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/ValueSource.java Wed Jan
19 23:48:28 2011
@@ -26,12 +26,13 @@ import org.apache.lucene.search.IndexSea
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.index.MultiFields;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.search.SolrSortField;
 
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.IdentityHashMap;
 import java.util.Map;
-import java.util.Collections;
 
 /**
  * Instantiates {@link org.apache.solr.search.function.DocValues} for a particular reader.
@@ -61,24 +62,6 @@ public abstract class ValueSource implem
     return description();
   }
 
-  /**
-   * EXPERIMENTAL: This method is subject to change.
-   * <br>WARNING: Sorted function queries are not currently weighted.
-   * <p>
-   * Get the SortField for this ValueSource.  Uses the {@link #getValues(java.util.Map, AtomicReaderContext)}
-   * to populate the SortField.
-   * 
-   * @param reverse true if this is a reverse sort.
-   * @return The {@link org.apache.lucene.search.SortField} for the ValueSource
-   * @throws IOException if there was a problem reading the values.
-   */
-  public SortField getSortField(boolean reverse) throws IOException {
-    //should we pass in the description for the field name?
-    //Hmm, Lucene is going to intern whatever we pass in, not sure I like that
-    //and we can't pass in null, either, as that throws an illegal arg. exception
-    return new SortField(description(), new ValueSourceComparatorSource(), reverse);
-  }
-
 
   /**
    * Implementations should propagate createWeight to sub-ValueSources which can optionally
store
@@ -97,16 +80,56 @@ public abstract class ValueSource implem
     return context;
   }
 
-  class ValueSourceComparatorSource extends FieldComparatorSource {
 
+  //
+  // Sorting by function
+  //
 
-    public ValueSourceComparatorSource() {
+  /**
+   * EXPERIMENTAL: This method is subject to change.
+   * <br>WARNING: Sorted function queries are not currently weighted.
+   * <p>
+   * Get the SortField for this ValueSource.  Uses the {@link #getValues(java.util.Map, AtomicReaderContext)}
+   * to populate the SortField.
+   *
+   * @param reverse true if this is a reverse sort.
+   * @return The {@link org.apache.lucene.search.SortField} for the ValueSource
+   * @throws IOException if there was a problem reading the values.
+   */
+  public SortField getSortField(boolean reverse) throws IOException {
+    return new ValueSourceSortField(reverse);
+  }
+
+  private static FieldComparatorSource dummyComparator = new FieldComparatorSource() {
+    @Override
+    public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean
reversed) throws IOException {
+      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unweighted use of sort
" + fieldname);
+    }
+  };
+
+  class ValueSourceSortField extends SortField implements SolrSortField {
+    public ValueSourceSortField(boolean reverse) {
+      super(description(), dummyComparator, reverse);
+    }
+
+    @Override
+    public SortField weight(IndexSearcher searcher) throws IOException {
+      Map context = newContext(searcher);
+      createWeight(context, searcher);
+      return new SortField(getField(), new ValueSourceComparatorSource(context), getReverse());
+    }
+  }
+
+  class ValueSourceComparatorSource extends FieldComparatorSource {
+    private final Map context;
 
+    public ValueSourceComparatorSource(Map context) {
+      this.context = context;
     }
 
     public FieldComparator newComparator(String fieldname, int numHits,
                                          int sortPos, boolean reversed) throws IOException
{
-      return new ValueSourceComparator(numHits);
+      return new ValueSourceComparator(context, numHits);
     }
   }
 
@@ -119,8 +142,10 @@ public abstract class ValueSource implem
     private final double[] values;
     private DocValues docVals;
     private double bottom;
+    private Map fcontext;
 
-    ValueSourceComparator(int numHits) {
+    ValueSourceComparator(Map fcontext, int numHits) {
+      this.fcontext = fcontext;
       values = new double[numHits];
     }
 
@@ -153,7 +178,7 @@ public abstract class ValueSource implem
     }
 
     public FieldComparator setNextReader(AtomicReaderContext context) throws IOException
{
-      docVals = getValues(Collections.emptyMap(), context);
+      docVals = getValues(fcontext, context);
       return this;
     }
 
@@ -162,7 +187,7 @@ public abstract class ValueSource implem
     }
 
     public Comparable value(int slot) {
-      return Double.valueOf(values[slot]);
+      return values[slot];
     }
   }
 }

Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/TestDistributedSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/TestDistributedSearch.java?rev=1061065&r1=1061064&r2=1061065&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/TestDistributedSearch.java (original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/TestDistributedSearch.java Wed Jan 19 23:48:28
2011
@@ -95,6 +95,7 @@ public class TestDistributedSearch exten
 
     // these queries should be exactly ordered and scores should exactly match
     query("q","*:*", "sort",i1+" desc");
+    query("q","*:*", "sort","{!func}add("+i1+",5)"+" desc");
     query("q","*:*", "sort",i1+" asc");
     query("q","*:*", "sort",i1+" desc", "fl","*,score");
     query("q","*:*", "sort",tlong+" asc", "fl","score");  // test legacy behavior - "score"=="*,score"

Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java?rev=1061065&r1=1061064&r2=1061065&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java
(original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java
Wed Jan 19 23:48:28 2011
@@ -326,17 +326,18 @@ public class TestFunctionQuery extends S
       assertU(adoc("id",""+i, "text","batman"));
     }
     assertU(commit());
-    assertU(adoc("id","120", "text","batman superman"));   // in a segment by itself
+    assertU(adoc("id","120", "text","batman superman"));   // in a smaller segment
+    assertU(adoc("id","121", "text","superman"));
     assertU(commit());
 
-    // batman and superman have the same idf in single-doc segment, but very different in
the complete index.
+    // superman has a higher df (thus lower idf) in one segment, but reversed in the complete
index
     String q ="{!func}query($qq)";
     String fq="id:120"; 
     assertQ(req("fl","*,score","q", q, "qq","text:batman", "fq",fq), "//float[@name='score']<'1.0'");
     assertQ(req("fl","*,score","q", q, "qq","text:superman", "fq",fq), "//float[@name='score']>'1.0'");
 
     // test weighting through a function range query
-    assertQ(req("fl","*,score", "q", "{!frange l=1 u=10}query($qq)", "qq","text:superman"),
"//*[@numFound='1']");
+    assertQ(req("fl","*,score", "fq",fq,  "q", "{!frange l=1 u=10}query($qq)", "qq","text:superman"),
"//*[@numFound='1']");
 
     // test weighting through a complex function
     q ="{!func}sub(div(sum(0.0,product(1,query($qq))),1),0)";
@@ -360,6 +361,14 @@ public class TestFunctionQuery extends S
       // OK
     }
 
+    // test that sorting by function weights correctly.  superman should sort higher than
batman due to idf of the whole index
+
+    assertQ(req("q", "*:*", "fq","id:120 OR id:121", "sort","{!func v=$sortfunc} desc", "sortfunc","query($qq)",
"qq","text:(batman OR superman)")
+           ,"*//doc[1]/float[.='120.0']"
+           ,"*//doc[2]/float[.='121.0']"
+    );
+
+
     purgeFieldCache(FieldCache.DEFAULT);   // avoid FC insanity
   }
 



Mime
View raw message