lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sar...@apache.org
Subject svn commit: r1546461 - in /lucene/dev/branches/branch_4x: ./ solr/ solr/contrib/ solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ solr/core/ solr/core/src/java/org/apache/solr/handler/component/ solr/core/src/java/org/apache/solr/schema/ s...
Date Thu, 28 Nov 2013 21:29:10 GMT
Author: sarowe
Date: Thu Nov 28 21:29:09 2013
New Revision: 1546461

URL: http://svn.apache.org/r1546461
Log:
SOLR-5354: Distributed sort is broken with CUSTOM FieldType (merged trunk r1546457)

Added:
    lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml
      - copied unchanged from r1546457, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml
    lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml
      - copied unchanged from r1546457, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/TestDistributedMissingSort.java
      - copied unchanged from r1546457, lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestDistributedMissingSort.java
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentCustomSortTest.java
      - copied unchanged from r1546457, lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentCustomSortTest.java
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java
      - copied, changed from r1546457, lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestCustomSort.java
      - copied unchanged from r1546457, lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestCustomSort.java
Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/solr/   (props changed)
    lucene/dev/branches/branch_4x/solr/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/branch_4x/solr/contrib/   (props changed)
    lucene/dev/branches/branch_4x/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
    lucene/dev/branches/branch_4x/solr/core/   (props changed)
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/CollationField.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/FieldType.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/SortableIntField.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/SortableLongField.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/StrField.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/TextField.java
    lucene/dev/branches/branch_4x/solr/test-framework/   (props changed)
    lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
    lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java

Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1546461&r1=1546460&r2=1546461&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Thu Nov 28 21:29:09 2013
@@ -127,6 +127,9 @@ Bug Fixes
 
 * SOLR-5494: CoreContainer#remove throws NPE rather than returning null when
   a SolrCore does not exist in core discovery mode. (Mark Miller)
+  
+* SOLR-5354: Distributed sort is broken with CUSTOM FieldType. 
+  (Steve Rowe, hossman, Jessica Cheng)  
 
 Optimizations
 ----------------------

Modified: lucene/dev/branches/branch_4x/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java?rev=1546461&r1=1546460&r2=1546461&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
(original)
+++ lucene/dev/branches/branch_4x/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
Thu Nov 28 21:29:09 2013
@@ -43,6 +43,7 @@ import org.apache.lucene.util.Version;
 import org.apache.lucene.analysis.util.ResourceLoader;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.util.Base64;
 import org.apache.solr.response.TextResponseWriter;
 import org.apache.solr.search.QParser;
 
@@ -304,4 +305,23 @@ public class ICUCollationField extends F
       return Collections.singletonList(createField(field, value, boost));
     }
   }
+
+  @Override
+  public Object marshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    final BytesRef val = (BytesRef)value;
+    return Base64.byteArrayToBase64(val.bytes, val.offset, val.length);
+  }
+
+  @Override
+  public Object unmarshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    final String val = (String)value;
+    final byte[] bytes = Base64.base64ToByteArray(val);
+    return new BytesRef(bytes);
+  }
 }

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=1546461&r1=1546460&r2=1546461&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
Thu Nov 28 21:29:09 2013
@@ -34,7 +34,6 @@ import org.apache.lucene.search.grouping
 import org.apache.lucene.search.grouping.TopGroups;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CharsRef;
-import org.apache.lucene.util.UnicodeUtil;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
@@ -47,6 +46,7 @@ import org.apache.solr.request.SolrQuery
 import org.apache.solr.response.ResultContext;
 import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.IndexSchema;
 import org.apache.solr.schema.SchemaField;
 import org.apache.solr.search.DocIterator;
 import org.apache.solr.search.DocList;
@@ -449,7 +449,6 @@ public class QueryComponent extends Sear
   {
     SolrQueryRequest req = rb.req;
     SolrQueryResponse rsp = rb.rsp;
-    final CharsRef spare = new CharsRef();
     // The query cache doesn't currently store sort field values, and SolrIndexSearcher doesn't
     // currently have an option to return sort field values.  Because of this, we
     // take the documents given and re-derive the sort values.
@@ -458,7 +457,6 @@ public class QueryComponent extends Sear
       Sort sort = searcher.weightSort(rb.getSortSpec().getSort());
       SortField[] sortFields = sort==null ? new SortField[]{SortField.FIELD_SCORE} : sort.getSort();
       NamedList<Object[]> sortVals = new NamedList<Object[]>(); // order is important
for the sort fields
-      Field field = new StringField("dummy", "", Field.Store.NO); // a dummy Field
       IndexReaderContext topReaderContext = searcher.getTopReaderContext();
       List<AtomicReaderContext> leaves = topReaderContext.leaves();
       AtomicReaderContext currentLeaf = null;
@@ -516,27 +514,7 @@ public class QueryComponent extends Sear
           doc -= currentLeaf.docBase;  // adjust for what segment this is in
           comparator.copy(0, doc);
           Object val = comparator.value(0);
-
-          // Sortable float, double, int, long types all just use a string
-          // comparator. For these, we need to put the type into a readable
-          // format.  One reason for this is that XML can't represent all
-          // string values (or even all unicode code points).
-          // indexedToReadable() should be a no-op and should
-          // thus be harmless anyway (for all current ways anyway)
-          if (val instanceof String) {
-            field.setStringValue((String)val);
-            val = ft.toObject(field);
-          }
-
-          // Must do the same conversion when sorting by a
-          // String field in Lucene, which returns the terms
-          // data as BytesRef:
-          if (val instanceof BytesRef) {
-            UnicodeUtil.UTF8toUTF16((BytesRef)val, spare);
-            field.setStringValue(spare.toString());
-            val = ft.toObject(field);
-          }
-
+          if (null != ft) val = ft.marshalSortValue(val); 
           vals[position] = val;
         }
 
@@ -778,7 +756,8 @@ public class QueryComponent extends Sear
         sortFields = new SortField[]{SortField.FIELD_SCORE};
       }
  
-      SchemaField uniqueKeyField = rb.req.getSchema().getUniqueKeyField();
+      IndexSchema schema = rb.req.getSchema();
+      SchemaField uniqueKeyField = schema.getUniqueKeyField();
 
 
       // id to shard mapping, to eliminate any accidental dups
@@ -787,7 +766,7 @@ public class QueryComponent extends Sear
       // Merge the docs via a priority queue so we don't have to sort *all* of the
       // documents... we only need to order the top (rows+start)
       ShardFieldSortedHitQueue queue;
-      queue = new ShardFieldSortedHitQueue(sortFields, ss.getOffset() + ss.getCount());
+      queue = new ShardFieldSortedHitQueue(sortFields, ss.getOffset() + ss.getCount(), rb.req.getSearcher());
 
       NamedList<Object> shardInfo = null;
       if(rb.req.getParams().getBool(ShardParams.SHARDS_INFO, false)) {
@@ -886,7 +865,7 @@ public class QueryComponent extends Sear
             }
           }
 
-          shardDoc.sortFieldValues = sortFieldValues;
+          shardDoc.sortFieldValues = unmarshalSortValues(sortFieldValues, schema);
 
           queue.insertWithOverflow(shardDoc);
         } // end for-each-doc-in-response
@@ -928,6 +907,26 @@ public class QueryComponent extends Sear
       }
   }
 
+  private NamedList unmarshalSortValues(NamedList sortFieldValues, IndexSchema schema) {
+    NamedList unmarshalledSortValsPerField = new NamedList();
+    for (int fieldNum = 0 ; fieldNum < sortFieldValues.size() ; ++fieldNum) {
+      String fieldName = sortFieldValues.getName(fieldNum);
+      SchemaField field = schema.getFieldOrNull(fieldName);
+      List sortVals = (List)sortFieldValues.getVal(fieldNum);
+      if (null == field) {
+        unmarshalledSortValsPerField.add(fieldName, sortVals);
+      } else {
+        FieldType fieldType = field.getType();
+        List unmarshalledSortVals = new ArrayList();
+        for (Object sortVal : sortVals) {
+          unmarshalledSortVals.add(fieldType.unmarshalSortValue(sortVal));
+        }
+        unmarshalledSortValsPerField.add(fieldName, unmarshalledSortVals);
+      }
+    }
+    return unmarshalledSortValsPerField;
+  }
+
   private void createRetrieveDocs(ResponseBuilder rb) {
 
     // TODO: in a system with nTiers > 2, we could be passed "ids" here

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java?rev=1546461&r1=1546460&r2=1546461&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java
Thu Nov 28 21:29:09 2013
@@ -16,18 +16,21 @@
  */
 package org.apache.solr.handler.component;
 
-import org.apache.lucene.search.FieldComparatorSource;
+import org.apache.lucene.search.FieldComparator;
 import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.PriorityQueue;
+import org.apache.solr.common.SolrException;
 import org.apache.solr.common.util.NamedList;
-import org.apache.solr.search.MissingStringLastComparatorSource;
+import org.apache.solr.search.SolrIndexSearcher;
 
-import java.text.Collator;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
-import java.util.Locale;
+
+import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
 
 public class ShardDoc extends FieldDoc {
   public String shard;
@@ -101,7 +104,7 @@ public class ShardDoc extends FieldDoc {
 class ShardFieldSortedHitQueue extends PriorityQueue<ShardDoc> {
 
   /** Stores a comparator corresponding to each field being sorted by */
-  protected Comparator[] comparators;
+  protected Comparator<ShardDoc>[] comparators;
 
   /** Stores the sort criteria being used. */
   protected SortField[] fields;
@@ -109,9 +112,10 @@ class ShardFieldSortedHitQueue extends P
   /** The order of these fieldNames should correspond to the order of sort field values retrieved
from the shard */
   protected List<String> fieldNames = new ArrayList<String>();
 
-  public ShardFieldSortedHitQueue(SortField[] fields, int size) {
+  public ShardFieldSortedHitQueue(SortField[] fields, int size, IndexSearcher searcher) {
     super(size);
     final int n = fields.length;
+    //noinspection unchecked
     comparators = new Comparator[n];
     this.fields = new SortField[n];
     for (int i = 0; i < n; ++i) {
@@ -123,8 +127,7 @@ class ShardFieldSortedHitQueue extends P
       }
 
       String fieldname = fields[i].getField();
-      comparators[i] = getCachedComparator(fieldname, fields[i]
-          .getType(), fields[i].getComparatorSource());
+      comparators[i] = getCachedComparator(fields[i], searcher);
 
      if (fields[i].getType() == SortField.Type.STRING) {
         this.fields[i] = new SortField(fieldname, SortField.Type.STRING,
@@ -169,47 +172,36 @@ class ShardFieldSortedHitQueue extends P
     return c < 0;
   }
 
-  Comparator getCachedComparator(String fieldname, SortField.Type type, FieldComparatorSource
factory) {
-    Comparator comparator = null;
-    switch (type) {
-    case SCORE:
-      comparator = comparatorScore(fieldname);
-      break;
-    case STRING:
-      comparator = comparatorNatural(fieldname);
-      break;
-    case CUSTOM:
-      if (factory instanceof MissingStringLastComparatorSource){
-        comparator = comparatorMissingStringLast(fieldname);
-      } else {
-        // TODO: support other types such as random... is there a way to
-        // support generically?  Perhaps just comparing Object
-        comparator = comparatorNatural(fieldname);
-        // throw new RuntimeException("Custom sort not supported factory is "+factory.getClass());
-      }
-      break;
-    case DOC:
-      // TODO: we can support this!
-      throw new RuntimeException("Doc sort not supported");
-    default:
-      comparator = comparatorNatural(fieldname);
-      break;
-    }
-    return comparator;
-  }
-
-  class ShardComparator implements Comparator {
-    String fieldName;
-    int fieldNum;
-    public ShardComparator(String fieldName) {
-      this.fieldName = fieldName;
-      this.fieldNum=0;
+  Comparator<ShardDoc> getCachedComparator(SortField sortField, IndexSearcher searcher)
{
+    SortField.Type type = sortField.getType();
+    if (type == SortField.Type.SCORE) {
+      return comparatorScore();
+    } else if (type == SortField.Type.REWRITEABLE) {
+      try {
+        sortField = sortField.rewrite(searcher);
+      } catch (IOException e) {
+        throw new SolrException(SERVER_ERROR, "Exception rewriting sort field " + sortField,
e);
+      }
+    }
+    return comparatorFieldComparator(sortField);
+  }
+
+  abstract class ShardComparator implements Comparator<ShardDoc> {
+    final SortField sortField;
+    final String fieldName;
+    final int fieldNum;
+
+    public ShardComparator(SortField sortField) {
+      this.sortField = sortField;
+      this.fieldName = sortField.getField();
+      int fieldNum = 0;
       for (int i=0; i<fieldNames.size(); i++) {
         if (fieldNames.get(i).equals(fieldName)) {
-          this.fieldNum = i;
+          fieldNum = i;
           break;
         }
       }
+      this.fieldNum = fieldNum;
     }
 
     Object sortVal(ShardDoc shardDoc) {
@@ -217,22 +209,14 @@ class ShardFieldSortedHitQueue extends P
       List lst = (List)shardDoc.sortFieldValues.getVal(fieldNum);
       return lst.get(shardDoc.orderInShard);
     }
-
-    @Override
-    public int compare(Object o1, Object o2) {
-      return 0;
-    }
   }
 
-  static Comparator comparatorScore(final String fieldName) {
-    return new Comparator() {
+  static Comparator<ShardDoc> comparatorScore() {
+    return new Comparator<ShardDoc>() {
       @Override
-      public final int compare(final Object o1, final Object o2) {
-        ShardDoc e1 = (ShardDoc) o1;
-        ShardDoc e2 = (ShardDoc) o2;
-
-        final float f1 = e1.score;
-        final float f2 = e2.score;
+      public final int compare(final ShardDoc o1, final ShardDoc o2) {
+        final float f1 = o1.score;
+        final float f2 = o2.score;
         if (f1 < f2)
           return -1;
         if (f1 > f2)
@@ -242,71 +226,24 @@ class ShardFieldSortedHitQueue extends P
     };
   }
 
-  // The lucene natural sort ordering corresponds to numeric
-  // and string natural sort orderings (ascending).  Since
-  // the PriorityQueue keeps the biggest elements by default,
-  // we need to reverse the natural compare ordering so that the
-  // smallest elements are kept instead of the largest... hence
-  // the negative sign on the final compareTo().
-  Comparator comparatorNatural(String fieldName) {
-    return new ShardComparator(fieldName) {
-      @Override
-      public final int compare(final Object o1, final Object o2) {
-        ShardDoc sd1 = (ShardDoc) o1;
-        ShardDoc sd2 = (ShardDoc) o2;
-        Comparable v1 = (Comparable)sortVal(sd1);
-        Comparable v2 = (Comparable)sortVal(sd2);
-        if (v1==v2)
-          return 0;
-        if (v1==null)
-          return 1;
-        if(v2==null)
-          return -1;
-        return -v1.compareTo(v2);
-      }
-    };
-  }
-
-
-  Comparator comparatorStringLocale(final String fieldName,
-      Locale locale) {
-    final Collator collator = Collator.getInstance(locale);
-    return new ShardComparator(fieldName) {
-      @Override
-      public final int compare(final Object o1, final Object o2) {
-        ShardDoc sd1 = (ShardDoc) o1;
-        ShardDoc sd2 = (ShardDoc) o2;
-        Comparable v1 = (Comparable)sortVal(sd1);
-        Comparable v2 = (Comparable)sortVal(sd2);
-        if (v1==v2)
-          return 0;
-        if (v1==null)
-          return 1;
-        if(v2==null)
-          return -1;
-        return -collator.compare(v1,v2);
-      }
-    };
-  }
-
+  Comparator<ShardDoc> comparatorFieldComparator(SortField sortField) {
+    final FieldComparator fieldComparator;
+    try {
+      fieldComparator = sortField.getComparator(0, 0);
+    } catch (IOException e) {
+      throw new RuntimeException("Unable to get FieldComparator for sortField " + sortField);
+    }
 
-  Comparator comparatorMissingStringLast(final String fieldName) {
-     return new ShardComparator(fieldName) {
+    return new ShardComparator(sortField) {
+      // Since the PriorityQueue keeps the biggest elements by default,
+      // we need to reverse the field compare ordering so that the
+      // smallest elements are kept instead of the largest... hence
+      // the negative sign.
       @Override
-      public final int compare(final Object o1, final Object o2) {
-        ShardDoc sd1 = (ShardDoc) o1;
-        ShardDoc sd2 = (ShardDoc) o2;
-        Comparable v1 = (Comparable)sortVal(sd1);
-        Comparable v2 = (Comparable)sortVal(sd2);
-        if (v1==v2)
-          return 0;
-        if (v1==null)
-          return -1;
-        if(v2==null)
-          return 1;
-        return -v1.compareTo(v2);
+      public int compare(final ShardDoc o1, final ShardDoc o2) {
+        //noinspection unchecked
+        return -fieldComparator.compareValues(sortVal(o1), sortVal(o2));
       }
     };
   }
-
 }

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/CollationField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/CollationField.java?rev=1546461&r1=1546460&r2=1546461&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/CollationField.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/CollationField.java
Thu Nov 28 21:29:09 2013
@@ -47,6 +47,7 @@ import org.apache.lucene.util.Version;
 import org.apache.lucene.analysis.util.ResourceLoader;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.util.Base64;
 import org.apache.solr.response.TextResponseWriter;
 import org.apache.solr.search.QParser;
 
@@ -279,4 +280,23 @@ public class CollationField extends Fiel
       return Collections.singletonList(createField(field, value, boost));
     }
   }
+
+  @Override
+  public Object marshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    final BytesRef val = (BytesRef)value;
+    return Base64.byteArrayToBase64(val.bytes, val.offset, val.length);
+  }
+
+  @Override
+  public Object unmarshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    final String val = (String)value;
+    final byte[] bytes = Base64.base64ToByteArray(val);
+    return new BytesRef(bytes);
+  }
 }

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/FieldType.java?rev=1546461&r1=1546460&r2=1546461&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/FieldType.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/FieldType.java
Thu Nov 28 21:29:09 2013
@@ -932,4 +932,20 @@ public abstract class FieldType extends 
     }
     return analyzerProps;
   }
+  
+  /** 
+   * Convert a value used by the FieldComparator for this FieldType's SortField
+   * into a marshalable value for distributed sorting.
+   */
+  public Object marshalSortValue(Object value) {
+    return value;
+  }
+  
+  /**
+   * Convert a value marshaled via {@link #marshalSortValue} back 
+   * into a value usable by the FieldComparator for this FieldType's SortField
+   */
+  public Object unmarshalSortValue(Object value) {
+    return value;
+  }
 }

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java?rev=1546461&r1=1546460&r2=1546461&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java
Thu Nov 28 21:29:09 2013
@@ -98,6 +98,27 @@ public class SortableDoubleField extends
     String sval = f.stringValue();
     writer.writeDouble(name, NumberUtils.SortableStr2double(sval));
   }
+
+  @Override
+  public Object marshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    CharsRef chars = new CharsRef();
+    UnicodeUtil.UTF8toUTF16((BytesRef)value, chars);
+    return NumberUtils.SortableStr2double(chars.toString());
+  }
+
+  @Override
+  public Object unmarshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    String sortableString = NumberUtils.double2sortableStr(value.toString());
+    BytesRef bytes = new BytesRef();
+    UnicodeUtil.UTF16toUTF8(sortableString, 0, sortableString.length(), bytes);
+    return bytes;
+  }
 }
 
 class SortableDoubleFieldSource extends FieldCacheSource {

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java?rev=1546461&r1=1546460&r2=1546461&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java
Thu Nov 28 21:29:09 2013
@@ -99,6 +99,27 @@ public class SortableFloatField extends 
     String sval = f.stringValue();
     writer.writeFloat(name, NumberUtils.SortableStr2float(sval));
   }
+
+  @Override
+  public Object marshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    CharsRef chars = new CharsRef();
+    UnicodeUtil.UTF8toUTF16((BytesRef)value, chars);
+    return NumberUtils.SortableStr2float(chars.toString());
+  }
+
+  @Override
+  public Object unmarshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    String sortableString = NumberUtils.float2sortableStr(value.toString());
+    BytesRef bytes = new BytesRef();
+    UnicodeUtil.UTF16toUTF8(sortableString, 0, sortableString.length(), bytes);
+    return bytes;
+  }
 }
 
 

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/SortableIntField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/SortableIntField.java?rev=1546461&r1=1546460&r2=1546461&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/SortableIntField.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/SortableIntField.java
Thu Nov 28 21:29:09 2013
@@ -102,6 +102,27 @@ public class SortableIntField extends Pr
     String sval = f.stringValue();
     writer.writeInt(name, NumberUtils.SortableStr2int(sval,0,sval.length()));
   }
+
+  @Override
+  public Object marshalSortValue(Object value) {
+    if (null == value) { 
+      return null;
+    }
+    CharsRef chars = new CharsRef();
+    UnicodeUtil.UTF8toUTF16((BytesRef)value, chars);
+    return NumberUtils.SortableStr2int(chars.toString());
+  }
+
+  @Override
+  public Object unmarshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    String sortableString = NumberUtils.int2sortableStr(value.toString());
+    BytesRef bytes = new BytesRef();
+    UnicodeUtil.UTF16toUTF8(sortableString, 0, sortableString.length(), bytes);
+    return bytes;
+  }
 }
 
 

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/SortableLongField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/SortableLongField.java?rev=1546461&r1=1546460&r2=1546461&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/SortableLongField.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/SortableLongField.java
Thu Nov 28 21:29:09 2013
@@ -98,6 +98,27 @@ public class SortableLongField extends P
     String sval = f.stringValue();
     writer.writeLong(name, NumberUtils.SortableStr2long(sval,0,sval.length()));
   }
+
+  @Override
+  public Object marshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    CharsRef chars = new CharsRef();
+    UnicodeUtil.UTF8toUTF16((BytesRef)value, chars);
+    return NumberUtils.SortableStr2long(chars.toString());
+  }
+
+  @Override
+  public Object unmarshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    String sortableString = NumberUtils.long2sortableStr(value.toString());
+    BytesRef bytes = new BytesRef();
+    UnicodeUtil.UTF16toUTF8(sortableString, 0, sortableString.length(), bytes);
+    return bytes;
+  }
 }
 
 

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/StrField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/StrField.java?rev=1546461&r1=1546460&r2=1546461&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/StrField.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/StrField.java
Thu Nov 28 21:29:09 2013
@@ -29,6 +29,8 @@ import org.apache.lucene.document.Sorted
 import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.UnicodeUtil;
 import org.apache.solr.response.TextResponseWriter;
 import org.apache.solr.search.QParser;
 
@@ -81,6 +83,27 @@ public class StrField extends PrimitiveF
   @Override
   public void checkSchemaField(SchemaField field) {
   }
+
+  @Override
+  public Object marshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    CharsRef spare = new CharsRef();
+    UnicodeUtil.UTF8toUTF16((BytesRef)value, spare);
+    return spare.toString();
+  }
+
+  @Override
+  public Object unmarshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    BytesRef spare = new BytesRef();
+    String stringVal = (String)value;
+    UnicodeUtil.UTF16toUTF8(stringVal, 0, stringVal.length(), spare);
+    return spare;
+  }
 }
 
 

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/TextField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/TextField.java?rev=1546461&r1=1546460&r2=1546461&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/TextField.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/TextField.java
Thu Nov 28 21:29:09 2013
@@ -23,8 +23,10 @@ import org.apache.lucene.index.Indexable
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.QueryBuilder;
+import org.apache.lucene.util.UnicodeUtil;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.response.TextResponseWriter;
 import org.apache.solr.search.QParser;
@@ -170,4 +172,25 @@ public class TextField extends FieldType
   public boolean isExplicitMultiTermAnalyzer() {
     return isExplicitMultiTermAnalyzer;
   }
+
+  @Override
+  public Object marshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    CharsRef spare = new CharsRef();
+    UnicodeUtil.UTF8toUTF16((BytesRef)value, spare);
+    return spare.toString();
+  }
+
+  @Override
+  public Object unmarshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    BytesRef spare = new BytesRef();
+    String stringVal = (String)value;
+    UnicodeUtil.UTF16toUTF8(stringVal, 0, stringVal.length(), spare);
+    return spare;
+  }
 }

Copied: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java
(from r1546457, lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java?p2=lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java&p1=lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java&r1=1546457&r2=1546461&rev=1546461&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java
Thu Nov 28 21:29:09 2013
@@ -19,7 +19,8 @@ package org.apache.solr.schema;
 
 import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.SortedSetDocValuesField;
-import org.apache.lucene.index.StorableField;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.FieldComparator;
 import org.apache.lucene.search.FieldComparatorSource;
 import org.apache.lucene.search.SortField;
@@ -46,10 +47,10 @@ public class SortableBinaryField extends
   }
 
   @Override
-  public List<StorableField> createFields(SchemaField field, Object value, float boost)
{
+  public List<IndexableField> createFields(SchemaField field, Object value, float boost)
{
     if (field.hasDocValues()) {
-      List<StorableField> fields = new ArrayList<StorableField>();
-      StorableField storedField = createField(field, value, boost);
+      List<IndexableField> fields = new ArrayList<IndexableField>();
+      IndexableField storedField = createField(field, value, boost);
       fields.add(storedField);
       ByteBuffer byteBuffer = toObject(storedField);
       BytesRef bytes = new BytesRef

Modified: lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java?rev=1546461&r1=1546460&r2=1546461&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
(original)
+++ lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
Thu Nov 28 21:29:09 2013
@@ -507,11 +507,18 @@ public abstract class BaseDistributedSea
     return rsp;
   }
 
-  protected void query(Object... q) throws Exception {
-    query(true, q);
+  /**
+   * Sets distributed params.
+   * Returns the QueryResponse from {@link #queryServer},
+   */
+  protected QueryResponse query(Object... q) throws Exception {
+    return query(true, q);
   }
-  
-  protected void query(boolean setDistribParams, Object[] q) throws Exception {
+
+  /**
+   * Returns the QueryResponse from {@link #queryServer}  
+   */
+  protected QueryResponse query(boolean setDistribParams, Object[] q) throws Exception {
     
     final ModifiableSolrParams params = new ModifiableSolrParams();
 
@@ -558,6 +565,7 @@ public abstract class BaseDistributedSea
         thread.join();
       }
     }
+    return rsp;
   }
   
   public QueryResponse queryAndCompare(SolrParams params, SolrServer... servers) throws SolrServerException
{

Modified: lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java?rev=1546461&r1=1546460&r2=1546461&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
(original)
+++ lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
Thu Nov 28 21:29:09 2013
@@ -47,6 +47,8 @@ import org.apache.lucene.util.LuceneTest
 import org.apache.lucene.util.QuickPatchThreadsFilter;
 import org.apache.lucene.util._TestUtil;
 import org.apache.solr.client.solrj.util.ClientUtils;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.SolrInputField;
@@ -1627,6 +1629,30 @@ public abstract class SolrTestCaseJ4 ext
       throw new RuntimeException("XPath is invalid", e2);
     }
   }
+                                                         
+  /**
+   * Fails if the number of documents in the given SolrDocumentList differs
+   * from the given number of expected values, or if any of the values in the
+   * given field don't match the expected values in the same order.
+   */
+  public static void assertFieldValues(SolrDocumentList documents, String fieldName, Object...
expectedValues) {
+    if (documents.size() != expectedValues.length) {
+      fail("Number of documents (" + documents.size()
+          + ") is different from number of expected values (" + expectedValues.length);
+    }
+    for (int docNum = 1 ; docNum <= documents.size() ; ++docNum) {
+      SolrDocument doc = documents.get(docNum - 1);
+      Object expected = expectedValues[docNum - 1];
+      Object actual = doc.get(fieldName);
+      if (null != expected && null != actual) {
+        if ( ! expected.equals(actual)) {
+          fail( "Unexpected " + fieldName + " field value in document #" + docNum
+              + ": expected=[" + expected + "], actual=[" + actual + "]");
+        }
+      }
+    }
+  }
+
   public static void copyMinConf(File dstRoot) throws IOException {
     copyMinConf(dstRoot, null);
   }



Mime
View raw message