lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rm...@apache.org
Subject svn commit: r1101016 [9/10] - in /lucene/dev/branches/bulkpostings: ./ dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/ dev-tools/idea/solr/ dev-tools/maven/lucene/contrib/ant/ dev-tools/maven/lucene/contrib/db/bdb-je/ dev-tools/maven/luce...
Date Mon, 09 May 2011 13:19:39 GMT
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/SolrQueryResponse.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/SolrQueryResponse.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/SolrQueryResponse.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/SolrQueryResponse.java Mon May  9 13:19:28 2011
@@ -19,6 +19,7 @@ package org.apache.solr.response;
 
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.search.ReturnFields;
 
 import java.util.*;
 
@@ -67,13 +68,14 @@ public class SolrQueryResponse {
    * @see <a href="#returnable_data">Note on Returnable Data</a>
    */
   protected NamedList<Object> values = new SimpleOrderedMap<Object>();
-
-  /**
+  
+  
+/**
    * Container for storing information that should be logged by Solr before returning.
    */
   protected NamedList<Object> toLog = new SimpleOrderedMap<Object>();
 
-  protected Set<String> defaultReturnFields;
+  protected ReturnFields returnFields;
 
   // error if this is set...
   protected Exception err;
@@ -111,18 +113,19 @@ public class SolrQueryResponse {
    * Sets the document field names of fields to return by default when
    * returning DocLists
    */
-  public void setReturnFields(Set<String> fields) {
-    defaultReturnFields=fields;
+  public void setReturnFields(ReturnFields fields) {
+    returnFields=fields;
   }
-  // TODO: should this be represented as a String[] such
-  // that order can be maintained if needed?
 
   /**
    * Gets the document field names of fields to return by default when
    * returning DocLists
    */
-  public Set<String> getReturnFields() {
-    return defaultReturnFields;
+  public ReturnFields getReturnFields() {
+    if( returnFields == null ) {
+      returnFields = new ReturnFields(); // by default return everything
+    }
+    return returnFields;
   }
 
 

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/TextResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/TextResponseWriter.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/TextResponseWriter.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/TextResponseWriter.java Mon May  9 13:19:28 2011
@@ -17,17 +17,23 @@
 
 package org.apache.solr.response;
 
+import java.io.IOException;
+import java.io.Writer;
+import java.util.*;
+
 import org.apache.lucene.document.Document;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.FastWriter;
+import org.apache.lucene.document.Fieldable;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.util.FastWriter;
+import org.apache.solr.common.util.NamedList;
 import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.transform.DocTransformer;
+import org.apache.solr.response.transform.TransformContext;
 import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
 import org.apache.solr.search.DocList;
-import java.io.IOException;
-import java.io.Writer;
-import java.util.*;
+import org.apache.solr.search.ReturnFields;
 
 /** Base class for text-oriented response writers.
  *
@@ -49,7 +55,7 @@ public abstract class TextResponseWriter
   protected final SolrQueryResponse rsp;
 
   // the default set of fields to return for each document
-  protected Set<String> returnFields;
+  protected ReturnFields returnFields;
 
   protected int level;
   protected boolean doIndent;
@@ -114,6 +120,15 @@ public abstract class TextResponseWriter
     } else if (val instanceof String) {
       writeStr(name, val.toString(), true);
       // micro-optimization... using toString() avoids a cast first
+    } else if (val instanceof Fieldable) {
+      Fieldable f = (Fieldable)val;
+      SchemaField sf = schema.getFieldOrNull( f.name() );
+      if( sf != null ) {
+        sf.getType().write(this, name, f);
+      }
+      else {
+        writeStr(name, f.stringValue(), true);
+      }
     } else if (val instanceof Integer) {
       writeInt(name, val.toString());
     } else if (val instanceof Boolean) {
@@ -129,19 +144,25 @@ public abstract class TextResponseWriter
     } else if (val instanceof Double) {
       writeDouble(name, ((Double)val).doubleValue());
     } else if (val instanceof Document) {
-      writeDoc(name, (Document)val, returnFields, 0.0f, false);
+      SolrDocument doc = toSolrDocument( (Document)val );
+      writeSolrDocument(name, doc, returnFields, 0 );
     } else if (val instanceof SolrDocument) {
-      writeSolrDocument(name, (SolrDocument)val, returnFields, null);
-    } else if (val instanceof DocList) {
+      writeSolrDocument(name, (SolrDocument)val, returnFields, 0);
+    } else if (val instanceof ResultContext) {
       // requires access to IndexReader
-      writeDocList(name, (DocList)val, returnFields,null);
+      writeDocuments(name, (ResultContext)val, returnFields);
+    } else if (val instanceof DocList) {
+      // Should not happen normally
+      ResultContext ctx = new ResultContext();
+      ctx.docs = (DocList)val;
+      writeDocuments(name, ctx, returnFields);
     // }
     // else if (val instanceof DocSet) {
     // how do we know what fields to read?
     // todo: have a DocList/DocSet wrapper that
     // restricts the fields to write...?
     } else if (val instanceof SolrDocumentList) {
-      writeSolrDocumentList(name, (SolrDocumentList)val, returnFields, null);
+      writeSolrDocumentList(name, (SolrDocumentList)val, returnFields);
     } else if (val instanceof Map) {
       writeMap(name, (Map)val, false, true);
     } else if (val instanceof NamedList) {
@@ -162,20 +183,82 @@ public abstract class TextResponseWriter
   // types of formats, including those where the name may come after the value (like
   // some XML formats).
 
-  public abstract void writeDoc(String name, Document doc, Set<String> returnFields, float score, boolean includeScore) throws IOException;
+  public abstract void writeStartDocumentList(String name, long start, int size, long numFound, Float maxScore) throws IOException;  
 
-  /**
-   * @since solr 1.3
-   */
-  public abstract void writeSolrDocument(String name, SolrDocument doc, Set<String> returnFields, Map pseudoFields) throws IOException;  
-
-  public abstract void writeDocList(String name, DocList ids, Set<String> fields, Map otherFields) throws IOException;
-
-  /**
-   * @since solr 1.3
-   */
-  public abstract void writeSolrDocumentList(String name, SolrDocumentList docs, Set<String> fields, Map otherFields) throws IOException;  
+  public abstract void writeSolrDocument(String name, SolrDocument doc, ReturnFields returnFields, int idx) throws IOException;  
+  
+  public abstract void writeEndDocumentList() throws IOException;
+  
+  // Assume each SolrDocument is already transformed
+  public final void writeSolrDocumentList(String name, SolrDocumentList docs, ReturnFields returnFields) throws IOException
+  {
+    writeStartDocumentList(name, docs.getStart(), docs.size(), docs.getNumFound(), docs.getMaxScore() );
+    for( int i=0; i<docs.size(); i++ ) {
+      writeSolrDocument( null, docs.get(i), returnFields, i );
+    }
+    writeEndDocumentList();
+  }
 
+  public final SolrDocument toSolrDocument( Document doc )
+  {
+    SolrDocument out = new SolrDocument();
+    for( Fieldable f : doc.getFields() ) {
+      if( "gack_i".equals( f.name() ) ) {
+        System.out.println( f );
+      }
+      
+      // Make sure multivalued fields are represented as lists
+      Object existing = out.get(f.name());
+      if (existing == null) {
+        SchemaField sf = schema.getFieldOrNull(f.name());
+        if (sf != null && sf.multiValued()) {
+          List<Object> vals = new ArrayList<Object>();
+          vals.add( f );
+          out.setField( f.name(), vals );
+        } 
+        else{
+          out.setField( f.name(), f );
+        }
+      }
+      else {
+        out.addField( f.name(), f );
+      }
+    }
+    return out;
+  }
+  
+  public final void writeDocuments(String name, ResultContext res, ReturnFields fields ) throws IOException {
+    DocList ids = res.docs;
+    TransformContext context = new TransformContext();
+    context.query = res.query;
+    context.wantsScores = fields.wantsScore() && ids.hasScores();
+    writeStartDocumentList(name, ids.offset(), ids.size(), ids.matches(), 
+        context.wantsScores ? new Float(ids.maxScore()) : null );
+    
+    DocTransformer transformer = fields.getTransformer();
+    context.searcher = req.getSearcher();
+    context.iterator = ids.iterator();
+    if( transformer != null ) {
+      transformer.setContext( context );
+    }
+    int sz = ids.size();
+    Set<String> fnames = fields.getLuceneFieldNames();
+    for (int i=0; i<sz; i++) {
+      int id = context.iterator.nextDoc();
+      Document doc = context.searcher.doc(id, fnames);
+      SolrDocument sdoc = toSolrDocument( doc );
+      if( transformer != null ) {
+        transformer.transform( sdoc, id );
+      }
+      writeSolrDocument( null, sdoc, returnFields, i );
+    }
+    if( transformer != null ) {
+      transformer.setContext( null );
+    }
+    writeEndDocumentList();
+  }
+  
+  
   public abstract void writeStr(String name, String val, boolean needsEscaping) throws IOException;
 
   public abstract void writeMap(String name, Map val, boolean excludeOuter, boolean isFirstVal) throws IOException;

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/XMLWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/XMLWriter.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/XMLWriter.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/XMLWriter.java Mon May  9 13:19:28 2011
@@ -17,26 +17,23 @@
 
 package org.apache.solr.response;
 
+import java.io.IOException;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.XML;
 import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.search.SolrIndexSearcher;
-import org.apache.solr.search.DocList;
-import org.apache.solr.search.DocIterator;
-import org.apache.solr.search.DocSet;
 import org.apache.solr.schema.SchemaField;
-import org.apache.solr.schema.TextField;
-
-import java.io.Writer;
-import java.io.IOException;
-import java.util.*;
-
-import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.document.Document;
+import org.apache.solr.search.ReturnFields;
 
 
 public final class XMLWriter extends TextResponseWriter {
@@ -166,257 +163,59 @@ public final class XMLWriter extends Tex
     }
   }
 
-  private static final Comparator fieldnameComparator = new Comparator() {
-    public int compare(Object o, Object o1) {
-      Fieldable f1 = (Fieldable)o; Fieldable f2 = (Fieldable)o1;
-      int cmp = f1.name().compareTo(f2.name());
-      return cmp;
-      // note - the sort is stable, so this should not have affected the ordering
-      // of fields with the same name w.r.t eachother.
-    }
-  };
 
   @Override
-  public final void writeDoc(String name, Document doc, Set<String> returnFields, float score, boolean includeScore) throws IOException {
-    startTag("doc", name, false);
-    incLevel();
-
-    if (includeScore) {
-      writeFloat("score", score);
-    }
-
-
-    // Lucene Documents have multivalued types as multiple fields
-    // with the same name.
-    // The XML needs to represent these as
-    // an array.  The fastest way to detect multiple fields
-    // with the same name is to sort them first.
-
-
-    // using global tlst here, so we shouldn't call any other
-    // function that uses it until we are done.
-    tlst.clear();
-    for (Object obj : doc.getFields()) {
-      Fieldable ff = (Fieldable)obj;
-      // skip this field if it is not a field to be returned.
-      if (returnFields!=null && !returnFields.contains(ff.name())) {
-        continue;
-      }
-      tlst.add(ff);
-    }
-    Collections.sort(tlst, fieldnameComparator);
-
-    int sz = tlst.size();
-    int fidx1 = 0, fidx2 = 0;
-    while (fidx1 < sz) {
-      Fieldable f1 = (Fieldable)tlst.get(fidx1);
-      String fname = f1.name();
-
-      // find the end of fields with this name
-      fidx2 = fidx1+1;
-      while (fidx2 < sz && fname.equals(((Fieldable)tlst.get(fidx2)).name()) ) {
-        fidx2++;
-      }
-
-      /***
-      // more efficient to use getFieldType instead of
-      // getField since that way dynamic fields won't have
-      // to create a SchemaField on the fly.
-      FieldType ft = schema.getFieldType(fname);
-      ***/
-
-      SchemaField sf = schema.getFieldOrNull(fname);
-      if( sf == null ) {
-        sf = new SchemaField( fname, new TextField() );
-      }
-      if (fidx1+1 == fidx2) {
-        // single field value
-        if (sf.multiValued()) {
-          startTag("arr",fname,false);
-          doIndent=false;
-          sf.write(this, null, f1);
-          writer.write("</arr>");
-          doIndent=defaultIndent;
-        } else {
-          sf.write(this, f1.name(), f1);
-        }
-      } else {
-        // multiple fields with same name detected
+  public void writeStartDocumentList(String name, 
+      long start, int size, long numFound, Float maxScore) throws IOException
+  {
+    if (doIndent) indent();
 
-        startTag("arr",fname,false);
-        incLevel();
-        doIndent=false;
-        int cnt=0;
-        for (int i=fidx1; i<fidx2; i++) {
-          if (defaultIndent && ++cnt==4) { // only indent every 4th item
-            indent();
-            cnt=0;
-          }
-          sf.write(this, null, (Fieldable)tlst.get(i));
-        }
-        decLevel();
-        // if (doIndent) indent();
-        writer.write("</arr>");
-        // doIndent=true;
-        doIndent=defaultIndent;
-      }
-      fidx1 = fidx2;
+    writer.write("<result");
+    writeAttr("name",name);
+    writeAttr("numFound",Long.toString(numFound));
+    writeAttr("start",Long.toString(start));
+    if(maxScore!=null) {
+      writeAttr("maxScore",Float.toString(maxScore));
     }
-
-    decLevel();
-    if (doIndent) indent();
-    writer.write("</doc>");
+    writer.write(">");
+    
+    incLevel();
   }
 
+
+  /**
+   * The SolrDocument should already have multivalued fields implemented as
+   * Collections -- this will not rewrite to <arr>
+   */ 
   @Override
-  public void writeSolrDocument(String name, SolrDocument doc, Set<String> returnFields, Map pseudoFields) throws IOException {
+  public void writeSolrDocument(String name, SolrDocument doc, ReturnFields returnFields, int idx ) throws IOException {
     startTag("doc", name, false);
     incLevel();
 
     for (String fname : doc.getFieldNames()) {
-      if (returnFields!=null && !returnFields.contains(fname)) {
+      if (!returnFields.wantsField(fname)) {
         continue;
       }
+      
       Object val = doc.getFieldValue(fname);
-
-      if (val instanceof Collection) {
-        writeVal(fname, val);
-      } else {
-        // single valued... figure out if we should put <arr> tags around it anyway
-        SchemaField sf = schema.getFieldOrNull(fname);
-        if (sf!=null && sf.multiValued()) {
-          startTag("arr",fname,false);
-          doIndent=false;
-          writeVal(fname, val);
-          writer.write("</arr>");
-          doIndent=defaultIndent;
-        } else {
-          writeVal(fname, val);
-        }
-      }
-    }
-
-    if (pseudoFields != null) {
-      for (Object fname : pseudoFields.keySet()) {
-        writeVal(fname.toString(), pseudoFields.get(fname));
+      if( "_explain_".equals( fname ) ) {
+        System.out.println( val );
       }
+      writeVal(fname, val);
     }
-
+    
     decLevel();
-    if (doIndent) indent();
     writer.write("</doc>");
   }
-
-
-  private static interface DocumentListInfo {
-    Float getMaxScore();
-    int getCount();
-    long getNumFound();
-    long getStart();
-    void writeDocs( boolean includeScore, Set<String> fields ) throws IOException;
-  }
-
-  private final void writeDocuments(
-      String name,
-      DocumentListInfo docs,
-      Set<String> fields) throws IOException
+  
+  @Override
+  public void writeEndDocumentList() throws IOException
   {
-    boolean includeScore=false;
-    if (fields!=null) {
-      includeScore = fields.contains("score");
-      if (fields.size()==0 || (fields.size()==1 && includeScore) || fields.contains("*")) {
-        fields=null;  // null means return all stored fields
-      }
-    }
-
-    int sz=docs.getCount();
-    if (doIndent) indent();
-
-    writer.write("<result");
-    writeAttr("name",name);
-    writeAttr("numFound",Long.toString(docs.getNumFound()));
-    writeAttr("start",Long.toString(docs.getStart()));
-    if (includeScore && docs.getMaxScore()!=null) {
-      writeAttr("maxScore",Float.toString(docs.getMaxScore()));
-    }
-    if (sz==0) {
-      writer.write("/>");
-      return;
-    } else {
-      writer.write('>');
-    }
-
-    incLevel();
-    docs.writeDocs(includeScore, fields);
     decLevel();
-
     if (doIndent) indent();
     writer.write("</result>");
   }
 
-  @Override
-  public final void writeSolrDocumentList(String name, final SolrDocumentList docs, Set<String> fields, Map otherFields) throws IOException
-  {
-    this.writeDocuments( name, new DocumentListInfo()
-    {
-      public int getCount() {
-        return docs.size();
-      }
-
-      public Float getMaxScore() {
-        return docs.getMaxScore();
-      }
-
-      public long getNumFound() {
-        return docs.getNumFound();
-      }
-
-      public long getStart() {
-        return docs.getStart();
-      }
-
-      public void writeDocs(boolean includeScore, Set<String> fields) throws IOException {
-        for( SolrDocument doc : docs ) {
-          writeSolrDocument(null, doc, fields, null);
-        }
-      }
-    }, fields );
-  }
-
-  @Override
-  public void writeDocList(String name, final DocList ids, Set<String> fields, Map otherFields) throws IOException
-  {
-    this.writeDocuments( name, new DocumentListInfo()
-    {
-      public int getCount() {
-        return ids.size();
-      }
-
-      public Float getMaxScore() {
-        return ids.maxScore();
-      }
-
-      public long getNumFound() {
-        return ids.matches();
-      }
-
-      public long getStart() {
-        return ids.offset();
-      }
-
-      public void writeDocs(boolean includeScore, Set<String> fields) throws IOException {
-        SolrIndexSearcher searcher = req.getSearcher();
-        DocIterator iterator = ids.iterator();
-        int sz = ids.size();
-        includeScore = includeScore && ids.hasScores();
-        for (int i=0; i<sz; i++) {
-          int id = iterator.nextDoc();
-          Document doc = searcher.doc(id, fields);
-          writeDoc(null, doc, fields, (includeScore ? iterator.score() : 0.0f), includeScore);
-        }
-      }
-    }, fields );
-  }
 
 
   //

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/BinaryField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/BinaryField.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/BinaryField.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/BinaryField.java Mon May  9 13:19:28 2011
@@ -54,6 +54,7 @@ public class BinaryField extends FieldTy
     return  ByteBuffer.wrap(f.getBinaryValue(), f.getBinaryOffset(), f.getBinaryLength() ) ;
   }
 
+  @Override
   public Fieldable createField(SchemaField field, Object val, float boost) {
     if (val == null) return null;
     if (!field.stored()) {

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/DateField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/DateField.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/DateField.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/DateField.java Mon May  9 13:19:28 2011
@@ -180,6 +180,14 @@ public class DateField extends FieldType
                                "Invalid Date Math String:'" +val+'\'',e);
     }
   }
+
+  public Fieldable createField(SchemaField field, Object value, float boost) {
+    // Convert to a string before indexing
+    if(value instanceof Date) {
+      value = toInternal( (Date)value ) + 'Z';
+    }
+    return super.createField(field, value, boost);
+  }
   
   public String toInternal(Date val) {
     return formatDate(val);

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/FieldProperties.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/FieldProperties.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/FieldProperties.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/FieldProperties.java Mon May  9 13:19:28 2011
@@ -22,29 +22,31 @@ import java.util.HashMap;
 
 /**
  * @version $Id$
+ * 
+ * @lucene.internal
  */
-abstract class FieldProperties {
+public abstract class FieldProperties {
 
   // use a bitfield instead of many different boolean variables since
   // many of the variables are independent or semi-independent.
 
   // bit values for boolean field properties.
-  final static int INDEXED             = 0x00000001;
-  final static int TOKENIZED           = 0x00000002;
-  final static int STORED              = 0x00000004;
-  final static int BINARY              = 0x00000008;
-  final static int OMIT_NORMS          = 0x00000010;
-  final static int OMIT_TF_POSITIONS   = 0x00000020;
-  final static int STORE_TERMVECTORS   = 0x00000040;
-  final static int STORE_TERMPOSITIONS = 0x00000080;
-  final static int STORE_TERMOFFSETS   = 0x00000100;
+  protected final static int INDEXED             = 0x00000001;
+  protected final static int TOKENIZED           = 0x00000002;
+  protected final static int STORED              = 0x00000004;
+  protected final static int BINARY              = 0x00000008;
+  protected final static int OMIT_NORMS          = 0x00000010;
+  protected final static int OMIT_TF_POSITIONS   = 0x00000020;
+  protected final static int STORE_TERMVECTORS   = 0x00000040;
+  protected final static int STORE_TERMPOSITIONS = 0x00000080;
+  protected final static int STORE_TERMOFFSETS   = 0x00000100;
 
 
-  final static int MULTIVALUED         = 0x00000200;
-  final static int SORT_MISSING_FIRST  = 0x00000400;
-  final static int SORT_MISSING_LAST   = 0x00000800;
+  protected final static int MULTIVALUED         = 0x00000200;
+  protected final static int SORT_MISSING_FIRST  = 0x00000400;
+  protected final static int SORT_MISSING_LAST   = 0x00000800;
   
-  final static int REQUIRED            = 0x00001000;
+  protected final static int REQUIRED            = 0x00001000;
   
   static final String[] propertyNames = {
           "indexed", "tokenized", "stored",

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/FieldType.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/FieldType.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/FieldType.java Mon May  9 13:19:28 2011
@@ -25,6 +25,7 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Similarity;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TermRangeQuery;
@@ -58,7 +59,7 @@ public abstract class FieldType extends 
   /**
    * The default poly field separator.
    *
-   * @see #createFields(SchemaField, String, float)
+   * @see #createFields(SchemaField, Object, float)
    * @see #isPolyField()
    */
   public static final String POLY_FIELD_SEPARATOR = "___";
@@ -83,11 +84,16 @@ public abstract class FieldType extends 
   public boolean isMultiValued() {
     return (properties & MULTIVALUED) != 0;
   }
+  
+  /** Check if a property is set */
+  protected boolean hasProperty( int p ) {
+    return (properties & p) != 0;
+  }
 
   /**
-   * A "polyField" is a FieldType that can produce more than one Fieldable instance for a single value, via the {@link #createFields(org.apache.solr.schema.SchemaField, String, float)} method.  This is useful
+   * A "polyField" is a FieldType that can produce more than one Fieldable instance for a single value, via the {@link #createFields(org.apache.solr.schema.SchemaField, Object, float)} method.  This is useful
    * when hiding the implementation details of a field from the Solr end user.  For instance, a spatial point may be represented by multiple different fields.
-   * @return true if the {@link #createFields(org.apache.solr.schema.SchemaField, String, float)} method may return more than one field
+   * @return true if the {@link #createFields(org.apache.solr.schema.SchemaField, Object, float)} method may return more than one field
    */
   public boolean isPolyField(){
     return false;
@@ -223,17 +229,18 @@ public abstract class FieldType extends 
    *
    *
    */
-  public Fieldable createField(SchemaField field, String externalVal, float boost) {
+  public Fieldable createField(SchemaField field, Object value, float boost) {
     if (!field.indexed() && !field.stored()) {
       if (log.isTraceEnabled())
         log.trace("Ignoring unindexed/unstored field: " + field);
       return null;
     }
+    
     String val;
     try {
-      val = toInternal(externalVal);
+      val = toInternal(value.toString());
     } catch (RuntimeException e) {
-      throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Error while creating field '" + field + "' from value '" + externalVal + "'", e, false);
+      throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Error while creating field '" + field + "' from value '" + value + "'", e, false);
     }
     if (val==null) return null;
 
@@ -272,15 +279,15 @@ public abstract class FieldType extends 
   /**
    * Given a {@link org.apache.solr.schema.SchemaField}, create one or more {@link org.apache.lucene.document.Fieldable} instances
    * @param field the {@link org.apache.solr.schema.SchemaField}
-   * @param externalVal The value to add to the field
+   * @param value The value to add to the field
    * @param boost The boost to apply
    * @return An array of {@link org.apache.lucene.document.Fieldable}
    *
-   * @see #createField(SchemaField, String, float)
+   * @see #createField(SchemaField, Object, float)
    * @see #isPolyField()
    */
-  public Fieldable[] createFields(SchemaField field, String externalVal, float boost) {
-    Fieldable f = createField( field, externalVal, boost);
+  public Fieldable[] createFields(SchemaField field, Object value, float boost) {
+    Fieldable f = createField( field, value, boost);
     return f==null ? new Fieldable[]{} : new Fieldable[]{f};
   }
 
@@ -502,6 +509,34 @@ public abstract class FieldType extends 
     throw e;
   }
 
+  /** @lucene.internal */
+  protected Similarity similarity;
+  
+  /**
+   * Gets the Similarity used when scoring fields of this type
+   * 
+   * <p>
+   * The default implementation returns null, which means this type
+   * has no custom similarity associated with it.
+   * </p>
+   * 
+   * This method exists to internally support SolrSimilarityProvider. 
+   * Custom application code interested in a field's Similarity should
+   * instead query via the searcher's SimilarityProvider.
+   * @lucene.internal
+   */
+  public Similarity getSimilarity() {
+    return similarity;
+  }
+  
+  /**
+   * Sets the Similarity used when scoring fields of this type
+   * @lucene.internal
+   */
+  public void setSimilarity(Similarity similarity) {
+    this.similarity = similarity;
+  }
+  
   /**
    * calls back to TextResponseWriter to write the field value
    */

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/IndexSchema.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/IndexSchema.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/IndexSchema.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/IndexSchema.java Mon May  9 13:19:28 2011
@@ -20,7 +20,9 @@ package org.apache.solr.schema;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.search.DefaultSimilarity;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Similarity;
 import org.apache.lucene.search.SimilarityProvider;
 import org.apache.lucene.queryParser.QueryParser;
 import org.apache.lucene.util.Version;
@@ -28,6 +30,7 @@ import org.apache.solr.common.ResourceLo
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.DOMUtil;
+import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.SystemIdResolver;
 import org.apache.solr.core.SolrConfig;
 import org.apache.solr.core.Config;
@@ -37,6 +40,7 @@ import org.apache.solr.analysis.TokenFil
 import org.apache.solr.analysis.TokenizerChain;
 import org.apache.solr.analysis.TokenizerFactory;
 import org.apache.solr.search.SolrQueryParser;
+import org.apache.solr.search.SolrSimilarityProvider;
 import org.apache.solr.util.plugin.AbstractPluginLoader;
 import org.apache.solr.util.plugin.SolrCoreAware;
 import org.w3c.dom.*;
@@ -185,19 +189,22 @@ public final class IndexSchema {
    */
   public Collection<SchemaField> getRequiredFields() { return requiredFields; }
 
-  private SimilarityFactory similarityFactory;
+  private SimilarityProviderFactory similarityProviderFactory;
 
   /**
-   * Returns the Similarity used for this index
+   * Returns the SimilarityProvider used for this index
    */
-  public SimilarityProvider getSimilarityProvider() { return similarityFactory.getSimilarityProvider(); }
+  public SimilarityProvider getSimilarityProvider() { return similarityProviderFactory.getSimilarityProvider(this); }
 
   /**
-   * Returns the SimilarityFactory used for this index
+   * Returns the SimilarityProviderFactory used for this index
    */
-  public SimilarityFactory getSimilarityFactory() { return similarityFactory; }
-
+  public SimilarityProviderFactory getSimilarityProviderFactory() { return similarityProviderFactory; }
 
+  private Similarity fallbackSimilarity;
+  
+  /** fallback similarity, in the case a field doesnt specify */
+  public Similarity getFallbackSimilarity() { return fallbackSimilarity; }
 
   /**
    * Returns the Analyzer used when indexing documents for this index
@@ -387,6 +394,11 @@ public final class IndexSchema {
           expression = "./analyzer[not(@type)] | ./analyzer[@type='index']";
           anode = (Node)xpath.evaluate(expression, node, XPathConstants.NODE);
           Analyzer analyzer = readAnalyzer(anode);
+          
+          // a custom similarity[Factory]
+          expression = "./similarity";
+          anode = (Node)xpath.evaluate(expression, node, XPathConstants.NODE);
+          Similarity similarity = readSimilarity(anode);
 
           if (queryAnalyzer==null) queryAnalyzer=analyzer;
           if (analyzer==null) analyzer=queryAnalyzer;
@@ -394,6 +406,9 @@ public final class IndexSchema {
             ft.setAnalyzer(analyzer);
             ft.setQueryAnalyzer(queryAnalyzer);
           }
+          if (similarity!=null) {
+            ft.setSimilarity(similarity);
+          }
           if (ft instanceof SchemaAware){
             schemaAware.add((SchemaAware) ft);
           }
@@ -491,36 +506,31 @@ public final class IndexSchema {
     // stuff it in a normal array for faster access
     dynamicFields = dFields.toArray(new DynamicField[dFields.size()]);
 
-
     Node node = (Node) xpath.evaluate("/schema/similarity", document, XPathConstants.NODE);
+    Similarity similarity = readSimilarity(node);
+    fallbackSimilarity = similarity == null ? new DefaultSimilarity() : similarity;
+
+    node = (Node) xpath.evaluate("/schema/similarityProvider", document, XPathConstants.NODE);
     if (node==null) {
-      similarityFactory = new SimilarityFactory() {
+      final SolrSimilarityProvider provider = new SolrSimilarityProvider(this);
+      similarityProviderFactory = new SimilarityProviderFactory() {
         @Override
-        public SimilarityProvider getSimilarityProvider() {
-          return IndexSearcher.getDefaultSimilarityProvider();
+        public SolrSimilarityProvider getSimilarityProvider(IndexSchema schema) {
+          return provider;
         }
       };
-      log.debug("using default similarity");
+      log.debug("using default similarityProvider");
     } else {
       final Object obj = loader.newInstance(((Element) node).getAttribute("class"));
-      if (obj instanceof SimilarityFactory) {
-        // configure a factory, get a similarity back
-        SolrParams params = SolrParams.toSolrParams(DOMUtil.childNodesToNamedList(node));
-        similarityFactory = (SimilarityFactory)obj;
-        similarityFactory.init(params);
-      } else {
-        // just like always, assume it's a SimilarityProvider and get a ClassCastException - reasonable error handling
-        similarityFactory = new SimilarityFactory() {
-          @Override
-          public SimilarityProvider getSimilarityProvider() {
-            return (SimilarityProvider) obj;
-          }
-        };
-      }
-      if (similarityFactory instanceof SchemaAware){
-        schemaAware.add((SchemaAware) similarityFactory);
+      // just like always, assume it's a SimilarityProviderFactory and get a ClassCastException - reasonable error handling
+      // configure a factory, get a similarity back
+      NamedList<?> args = DOMUtil.childNodesToNamedList(node);
+      similarityProviderFactory = (SimilarityProviderFactory)obj;
+      similarityProviderFactory.init(args);
+      if (similarityProviderFactory instanceof SchemaAware){
+        schemaAware.add((SchemaAware) similarityProviderFactory);
       }
-      log.debug("using similarity factory" + similarityFactory.getClass().getName());
+      log.debug("using similarityProvider factory" + similarityProviderFactory.getClass().getName());
     }
 
     node = (Node) xpath.evaluate("/schema/defaultSearchField/text()", document, XPathConstants.NODE);
@@ -750,6 +760,30 @@ public final class IndexSchema {
     return newArr;
   }
 
+  private Similarity readSimilarity(Node node) throws XPathExpressionException {
+    if (node==null) {
+      return null;
+    } else {
+      SimilarityFactory similarityFactory;
+      final Object obj = loader.newInstance(((Element) node).getAttribute("class"));
+      if (obj instanceof SimilarityFactory) {
+        // configure a factory, get a similarity back
+        SolrParams params = SolrParams.toSolrParams(DOMUtil.childNodesToNamedList(node));
+        similarityFactory = (SimilarityFactory)obj;
+        similarityFactory.init(params);
+      } else {
+        // just like always, assume it's a Similarity and get a ClassCastException - reasonable error handling
+        similarityFactory = new SimilarityFactory() {
+          @Override
+          public Similarity getSimilarity() {
+            return (Similarity) obj;
+          }
+        };
+      }
+      return similarityFactory.getSimilarity();
+    }
+  }
+
   //
   // <analyzer><tokenizer class="...."/><tokenizer class="...." arg="....">
   //

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/LatLonType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/LatLonType.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/LatLonType.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/LatLonType.java Mon May  9 13:19:28 2011
@@ -54,7 +54,8 @@ public class LatLonType extends Abstract
   }
 
   @Override
-  public Fieldable[] createFields(SchemaField field, String externalVal, float boost) {
+  public Fieldable[] createFields(SchemaField field, Object value, float boost) {
+    String externalVal = value.toString();
     //we could have tileDiff + 3 fields (two for the lat/lon, one for storage)
     Fieldable[] f = new Fieldable[(field.indexed() ? 2 : 0) + (field.stored() ? 1 : 0)];
     if (field.indexed()) {
@@ -280,7 +281,7 @@ public class LatLonType extends Abstract
   //It never makes sense to create a single field, so make it impossible to happen
 
   @Override
-  public Fieldable createField(SchemaField field, String externalVal, float boost) {
+  public Fieldable createField(SchemaField field, Object value, float boost) {
     throw new UnsupportedOperationException("LatLonType uses multiple fields.  field=" + field.getName());
   }
 

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/PointType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/PointType.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/PointType.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/PointType.java Mon May  9 13:19:28 2011
@@ -68,7 +68,8 @@ public class PointType extends Coordinat
   }
 
   @Override
-  public Fieldable[] createFields(SchemaField field, String externalVal, float boost) {
+  public Fieldable[] createFields(SchemaField field, Object value, float boost) {
+    String externalVal = value.toString();
     String[] point = new String[0];
     try {
       point = DistanceUtils.parsePoint(null, externalVal, dimension);
@@ -112,7 +113,7 @@ public class PointType extends Coordinat
    *
    */
   @Override
-  public Fieldable createField(SchemaField field, String externalVal, float boost) {
+  public Fieldable createField(SchemaField field, Object value, float boost) {
     throw new UnsupportedOperationException("PointType uses multiple fields.  field=" + field.getName());
   }
 

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SchemaField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SchemaField.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SchemaField.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SchemaField.java Mon May  9 13:19:28 2011
@@ -93,16 +93,16 @@ public final class SchemaField extends F
   boolean isBinary() { return (properties & BINARY)!=0; }
 
 
-  public Fieldable createField(String val, float boost) {
+  public Fieldable createField(Object val, float boost) {
     return type.createField(this,val,boost);
   }
   
-  public Fieldable[] createFields(String val, float boost) {
+  public Fieldable[] createFields(Object val, float boost) {
     return type.createFields(this,val,boost);
   }
 
   /**
-   * If true, then use {@link #createFields(String, float)}, else use {@link #createField} to save an extra allocation
+   * If true, then use {@link #createFields(Object, float)}, else use {@link #createField} to save an extra allocation
    * @return true if this field is a poly field
    */
   public boolean isPolyField(){

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SimilarityFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SimilarityFactory.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SimilarityFactory.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SimilarityFactory.java Mon May  9 13:19:28 2011
@@ -16,7 +16,7 @@ package org.apache.solr.schema;
  * limitations under the License.
  */
 
-import org.apache.lucene.search.SimilarityProvider;
+import org.apache.lucene.search.Similarity;
 import org.apache.solr.common.params.SolrParams;
 
 public abstract class SimilarityFactory {
@@ -25,5 +25,5 @@ public abstract class SimilarityFactory 
   public void init(SolrParams params) { this.params = params; }
   public SolrParams getParams() { return params; }
 
-  public abstract SimilarityProvider getSimilarityProvider();
+  public abstract Similarity getSimilarity();
 }

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/TrieDateField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/TrieDateField.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/TrieDateField.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/TrieDateField.java Mon May  9 13:19:28 2011
@@ -68,7 +68,7 @@ public class TrieDateField extends DateF
   public Date toObject(Fieldable f) {
     byte[] arr = f.getBinaryValue();
     if (arr==null) throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,TrieField.badFieldString(f));
-    return new Date(TrieField.toLong(arr));
+    return new Date(TrieFieldHelper.toLong(arr));
   }
 
   @Override
@@ -107,7 +107,7 @@ public class TrieDateField extends DateF
       return;
     }
 
-    writer.writeDate(name,new Date(TrieField.toLong(arr)));
+    writer.writeDate(name,new Date(TrieFieldHelper.toLong(arr)));
   }
 
   @Override
@@ -146,7 +146,7 @@ public class TrieDateField extends DateF
   public String toExternal(Fieldable f) {
     byte[] arr = f.getBinaryValue();
     if (arr==null) return TrieField.badFieldString(f);
-     return super.toExternal(new Date(TrieField.toLong(arr)));
+     return super.toExternal(new Date(TrieFieldHelper.toLong(arr)));
   }
 
   @Override
@@ -168,7 +168,7 @@ public class TrieDateField extends DateF
   }
 
   @Override
-  public Fieldable createField(SchemaField field, String externalVal, float boost) {
+  public Fieldable createField(SchemaField field, Object value, float boost) {
     boolean indexed = field.indexed();
     boolean stored = field.stored();
 
@@ -183,8 +183,11 @@ public class TrieDateField extends DateF
     byte[] arr=null;
     TokenStream ts=null;
 
-    long time = super.parseMath(null, externalVal).getTime();
-    if (stored) arr = TrieField.toArr(time);
+    long time = (value instanceof Date) 
+      ? ((Date)value).getTime() 
+      : super.parseMath(null, value.toString()).getTime();
+      
+    if (stored) arr = TrieFieldHelper.toArr(time);
     if (indexed) ts = new NumericTokenStream(ps).setLongValue(time);
 
     Field f;

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/TrieField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/TrieField.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/TrieField.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/TrieField.java Mon May  9 13:19:28 2011
@@ -17,7 +17,6 @@
 package org.apache.solr.schema;
 
 import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.search.*;
 import org.apache.lucene.search.cache.CachedArrayCreator;
 import org.apache.lucene.search.cache.DoubleValuesCreator;
@@ -26,8 +25,6 @@ import org.apache.lucene.search.cache.In
 import org.apache.lucene.search.cache.LongValuesCreator;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.NumericUtils;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.NumericTokenStream;
 import org.apache.noggit.CharArr;
 import org.apache.solr.analysis.*;
 import org.apache.solr.common.SolrException;
@@ -68,6 +65,7 @@ public class TrieField extends FieldType
   protected TrieTypes type;
   protected Object missingValue;
 
+  
   /**
    * Used for handling date types following the same semantics as DateField
    */
@@ -107,15 +105,15 @@ public class TrieField extends FieldType
     if (arr==null) return badFieldString(f);
     switch (type) {
       case INTEGER:
-        return toInt(arr);
+        return TrieFieldHelper.toInt(arr);
       case FLOAT:
-        return toFloat(arr);
+        return TrieFieldHelper.toFloat(arr);
       case LONG:
-        return toLong(arr);
+        return TrieFieldHelper.toLong(arr);
       case DOUBLE:
-        return toDouble(arr);
+        return TrieFieldHelper.toDouble(arr);
       case DATE:
-        return new Date(toLong(arr));
+        return new Date(TrieFieldHelper.toLong(arr));
       default:
         throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
     }
@@ -207,19 +205,19 @@ public class TrieField extends FieldType
     }
     switch (type) {
       case INTEGER:
-        writer.writeInt(name,toInt(arr));
+        writer.writeInt(name,TrieFieldHelper.toInt(arr));
         break;
       case FLOAT:
-        writer.writeFloat(name,toFloat(arr));
+        writer.writeFloat(name,TrieFieldHelper.toFloat(arr));
         break;
       case LONG:
-        writer.writeLong(name,toLong(arr));
+        writer.writeLong(name,TrieFieldHelper.toLong(arr));
         break;
       case DOUBLE:
-        writer.writeDouble(name,toDouble(arr));
+        writer.writeDouble(name,TrieFieldHelper.toDouble(arr));
         break;
       case DATE:
-        writer.writeDate(name,new Date(toLong(arr)));
+        writer.writeDate(name,new Date(TrieFieldHelper.toLong(arr)));
         break;
       default:
         throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
@@ -293,55 +291,6 @@ public class TrieField extends FieldType
   }
 
 
-  static int toInt(byte[] arr) {
-    return (arr[0]<<24) | ((arr[1]&0xff)<<16) | ((arr[2]&0xff)<<8) | (arr[3]&0xff);
-  }
-  
-  static long toLong(byte[] arr) {
-    int high = (arr[0]<<24) | ((arr[1]&0xff)<<16) | ((arr[2]&0xff)<<8) | (arr[3]&0xff);
-    int low = (arr[4]<<24) | ((arr[5]&0xff)<<16) | ((arr[6]&0xff)<<8) | (arr[7]&0xff);
-    return (((long)high)<<32) | (low&0x0ffffffffL);
-  }
-
-  static float toFloat(byte[] arr) {
-    return Float.intBitsToFloat(toInt(arr));
-  }
-
-  static double toDouble(byte[] arr) {
-    return Double.longBitsToDouble(toLong(arr));
-  }
-
-  static byte[] toArr(int val) {
-    byte[] arr = new byte[4];
-    arr[0] = (byte)(val>>>24);
-    arr[1] = (byte)(val>>>16);
-    arr[2] = (byte)(val>>>8);
-    arr[3] = (byte)(val);
-    return arr;
-  }
-
-  static byte[] toArr(long val) {
-    byte[] arr = new byte[8];
-    arr[0] = (byte)(val>>>56);
-    arr[1] = (byte)(val>>>48);
-    arr[2] = (byte)(val>>>40);
-    arr[3] = (byte)(val>>>32);
-    arr[4] = (byte)(val>>>24);
-    arr[5] = (byte)(val>>>16);
-    arr[6] = (byte)(val>>>8);
-    arr[7] = (byte)(val);
-    return arr;
-  }
-
-  static byte[] toArr(float val) {
-    return toArr(Float.floatToRawIntBits(val));
-  }
-
-  static byte[] toArr(double val) {
-    return toArr(Double.doubleToRawLongBits(val));
-  }
-
-
   @Override
   public String storedToReadable(Fieldable f) {
     return toExternal(f);
@@ -396,15 +345,15 @@ public class TrieField extends FieldType
     if (arr==null) return badFieldString(f);
     switch (type) {
       case INTEGER:
-        return Integer.toString(toInt(arr));
+        return Integer.toString(TrieFieldHelper.toInt(arr));
       case FLOAT:
-        return Float.toString(toFloat(arr));
+        return Float.toString(TrieFieldHelper.toFloat(arr));
       case LONG:
-        return Long.toString(toLong(arr));
+        return Long.toString(TrieFieldHelper.toLong(arr));
       case DOUBLE:
-        return Double.toString(toDouble(arr));
+        return Double.toString(TrieFieldHelper.toDouble(arr));
       case DATE:
-        return dateField.formatDate(new Date(toLong(arr)));
+        return dateField.formatDate(new Date(TrieFieldHelper.toLong(arr)));
       default:
         throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
     }
@@ -482,66 +431,54 @@ public class TrieField extends FieldType
   }
 
   @Override
-  public Fieldable createField(SchemaField field, String externalVal, float boost) {
-    boolean indexed = field.indexed();
-    boolean stored = field.stored();
-
-    if (!indexed && !stored) {
+  public Fieldable createField(SchemaField field, Object value, float boost) {
+    TrieFieldHelper.FieldInfo info = new TrieFieldHelper.FieldInfo();
+    info.index = field.indexed();
+    info.store = field.stored();
+    info.precisionStep = precisionStep;
+    info.omitNorms = field.omitNorms();
+    info.omitTF = field.omitTf();
+    
+    if (!info.index && !info.store) {
       if (log.isTraceEnabled())
         log.trace("Ignoring unindexed/unstored field: " + field);
       return null;
     }
 
-    int ps = precisionStep;
-
-    byte[] arr=null;
-    TokenStream ts=null;
-    // String indexedVal = indexed && precisionStep==0 ? readableToIndexed(externalVal) : null;
-
     switch (type) {
       case INTEGER:
-        int i = Integer.parseInt(externalVal);
-        if (stored) arr = toArr(i);
-        if (indexed) ts = new NumericTokenStream(ps).setIntValue(i);
-        break;
+        int i = (value instanceof Number)
+          ? ((Number)value).intValue()
+          : Integer.parseInt(value.toString());
+        return TrieFieldHelper.createIntField(field.getName(), i, info, boost);
+
       case FLOAT:
-        float f = Float.parseFloat(externalVal);
-        if (stored) arr = toArr(f);
-        if (indexed) ts = new NumericTokenStream(ps).setFloatValue(f);
-        break;
+        float f = (value instanceof Number)
+          ? ((Number)value).floatValue()
+          : Float.parseFloat(value.toString());
+        return TrieFieldHelper.createFloatField(field.getName(), f, info, boost);
+        
       case LONG:
-        long l = Long.parseLong(externalVal);
-        if (stored) arr = toArr(l);
-        if (indexed) ts = new NumericTokenStream(ps).setLongValue(l);
-        break;
-      case DOUBLE:
-        double d = Double.parseDouble(externalVal);
-        if (stored) arr = toArr(d);
-        if (indexed) ts = new NumericTokenStream(ps).setDoubleValue(d);
-        break;
+        long l = (value instanceof Number)
+          ? ((Number)value).longValue()
+          : Long.parseLong(value.toString());
+        return TrieFieldHelper.createLongField(field.getName(), l, info, boost);
+          
+      case DOUBLE:
+        double d = (value instanceof Number)
+          ? ((Number)value).doubleValue()
+          : Double.parseDouble(value.toString());
+        return TrieFieldHelper.createDoubleField(field.getName(), d, info, boost);
+        
       case DATE:
-        long time = dateField.parseMath(null, externalVal).getTime();
-        if (stored) arr = toArr(time);
-        if (indexed) ts = new NumericTokenStream(ps).setLongValue(time);
-        break;
+        Date date = (value instanceof Date)
+          ? ((Date)value)
+          : dateField.parseMath(null, value.toString());
+        return TrieFieldHelper.createDateField(field.getName(), date, info, boost);
+        
       default:
         throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type);
     }
-
-    Field f;
-    if (stored) {
-      f = new Field(field.getName(), arr);
-      if (indexed) f.setTokenStream(ts);
-    } else {
-      f = new Field(field.getName(), ts);
-    }
-
-    // term vectors aren't supported
-
-    f.setOmitNorms(field.omitNorms());
-    f.setOmitTermFreqAndPositions(field.omitTf());
-    f.setBoost(boost);
-    return f;
   }
 
   public enum TrieTypes {

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/QueryParsing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/QueryParsing.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/QueryParsing.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/QueryParsing.java Mon May  9 13:19:28 2011
@@ -582,7 +582,7 @@ public class QueryParsing {
 
     boolean opt(char ch) {
       eatws();
-      if (val.charAt(pos) == ch) {
+      if (pos < end && val.charAt(pos) == ch) {
         pos++;
         return true;
       }
@@ -693,7 +693,29 @@ public class QueryParsing {
         pos++;
         while (pos < end) {
           ch = val.charAt(pos);
-          if (!Character.isJavaIdentifierPart(ch) && ch != '.') {
+          if (!Character.isJavaIdentifierPart(ch) && ch != '.' && ch != ':') {
+            break;
+          }
+          pos++;
+        }
+        return val.substring(id_start, pos);
+      }
+
+      if (errMessage != null) {
+        throw new ParseException(errMessage + " at pos " + pos + " str='" + val + "'");
+      }
+      return null;
+    }
+
+    public String getGlobbedId(String errMessage) throws ParseException {
+      eatws();
+      int id_start = pos;
+      char ch;
+      if (pos < end && (ch = val.charAt(pos)) != '$' && (Character.isJavaIdentifierStart(ch) || ch=='?' || ch=='*')) {
+        pos++;
+        while (pos < end) {
+          ch = val.charAt(pos);
+          if (!(Character.isJavaIdentifierPart(ch) || ch=='?' || ch=='*') && ch != '.') {
             break;
           }
           pos++;

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/SolrQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/SolrQueryParser.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/SolrQueryParser.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/SolrQueryParser.java Mon May  9 13:19:28 2011
@@ -122,9 +122,9 @@ public class SolrQueryParser extends Que
     SchemaField sf = schema.getFieldOrNull(field);
     if (sf != null) {
       FieldType ft = sf.getType();
-      // delegate to type for everything except TextField
-      if (ft instanceof TextField) {
-        return super.getFieldQuery(field, queryText, quoted || ((TextField)ft).getAutoGeneratePhraseQueries());
+      // delegate to type for everything except tokenized fields
+      if (ft.isTokenized()) {
+        return super.getFieldQuery(field, queryText, quoted || (ft instanceof TextField && ((TextField)ft).getAutoGeneratePhraseQueries()));
       } else {
         return sf.getType().getFieldQuery(parser, sf, queryText);
       }

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/ValueSourceParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/ValueSourceParser.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/ValueSourceParser.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/ValueSourceParser.java Mon May  9 13:19:28 2011
@@ -583,7 +583,7 @@ public abstract class ValueSourceParser 
     if (ft == null) ft = new StrField();
 
     if (ft instanceof TextField) {
-      // need to do analyisis on the term
+      // need to do analysis on the term
       String indexedVal = tinfo.val;
       Query q = ft.getFieldQuery(fp, fp.getReq().getSchema().getFieldOrNull(tinfo.field), tinfo.val);
       if (q instanceof TermQuery) {

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellTernarySearchTrie.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellTernarySearchTrie.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellTernarySearchTrie.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellTernarySearchTrie.java Mon May  9 13:19:28 2011
@@ -1,5 +1,34 @@
 package org.apache.solr.spelling.suggest.jaspell;
 
+/** 
+ * Copyright (c) 2005 Bruno Martins
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without 
+ * modification, are permitted provided that the following conditions 
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright 
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the organization nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/update/DocumentBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/update/DocumentBuilder.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/update/DocumentBuilder.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/update/DocumentBuilder.java Mon May  9 13:19:28 2011
@@ -18,12 +18,10 @@
 package org.apache.solr.update;
 
 import java.util.ArrayList;
-import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrException;
@@ -194,7 +192,7 @@ public class DocumentBuilder {
   }
 
 
-  private static void addField(Document doc, SchemaField field, String val, float boost) {
+  private static void addField(Document doc, SchemaField field, Object val, float boost) {
     if (field.isPolyField()) {
       Fieldable[] farr = field.getType().createFields(field, val, boost);
       for (Fieldable f : farr) {
@@ -257,30 +255,10 @@ public class DocumentBuilder {
           if( v == null ) {
             continue;
           }
-          String val = null;
           hasField = true;
-          boolean isBinaryField = false;
-          if (sfield != null && sfield.getType() instanceof BinaryField) {
-            isBinaryField = true;
-            BinaryField binaryField = (BinaryField) sfield.getType();
-            Fieldable f = binaryField.createField(sfield,v,boost);
-            if(f != null){
-              out.add(f);
-            }
+          if (sfield != null) {
             used = true;
-          } else {
-            // TODO!!! HACK -- date conversion
-            if (sfield != null && v instanceof Date && sfield.getType() instanceof DateField) {
-              DateField df = (DateField) sfield.getType();
-              val = df.toInternal((Date) v) + 'Z';
-            } else if (v != null) {
-              val = v.toString();
-            }
-  
-            if (sfield != null) {
-              used = true;
-              addField(out, sfield, val, boost);
-            }
+            addField(out, sfield, v, boost);
           }
   
           // Check if we should copy this field to any other fields.
@@ -289,27 +267,24 @@ public class DocumentBuilder {
           for (CopyField cf : copyFields) {
             SchemaField destinationField = cf.getDestination();
             // check if the copy field is a multivalued or not
-            if (!destinationField.multiValued() && out.get(destinationField.getName()) != null) {
+            if (!destinationField.multiValued() && out.getFieldable(destinationField.getName()) != null) {
               throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
                       "ERROR: "+getID(doc, schema)+"multiple values encountered for non multiValued copy field " +
-                              destinationField.getName() + ": " + val);
+                              destinationField.getName() + ": " + v);
             }
   
             used = true;
-            //Don't worry about poly fields here
-            Fieldable [] fields = null;
-            if (isBinaryField) {
-              if (destinationField.getType() instanceof BinaryField) {
-                BinaryField binaryField = (BinaryField) destinationField.getType();
-                //TODO: safe to assume that binary fields only create one?
-                fields = new Fieldable[]{binaryField.createField(destinationField, v, boost)};
-              }
-            } else {
-              fields = destinationField.createFields(cf.getLimitedValue(val), boost);
+            
+            // Perhaps trim the length of a copy field
+            Object val = v;
+            if( val instanceof String && cf.getMaxChars() > 0 ) {
+              val = cf.getLimitedValue((String)val);
             }
+            
+            Fieldable [] fields = destinationField.createFields(val, boost);
             if (fields != null) { // null fields are not added
               for (Fieldable f : fields) {
-                out.add(f);
+                if(f != null) out.add(f);
               }
             }
           }

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/util/SolrPluginUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/util/SolrPluginUtils.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/util/SolrPluginUtils.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/util/SolrPluginUtils.java Mon May  9 13:19:28 2011
@@ -115,41 +115,6 @@ public class SolrPluginUtils {
      return splitList.split(value.trim(), 0);
   }
 
-  /**
-   * Assumes the standard query param of "fl" to specify the return fields
-   * @see #setReturnFields(String,SolrQueryResponse)
-   */
-  public static int setReturnFields(SolrQueryRequest req,
-                                    SolrQueryResponse res) {
-
-    return setReturnFields(req.getParams().get(CommonParams.FL), res);
-  }
-
-  /**
-   * Given a space seperated list of field names, sets the field list on the
-   * SolrQueryResponse.
-   *
-   * @return bitfield of SolrIndexSearcher flags that need to be set
-   */
-  public static int setReturnFields(String fl,
-                                    SolrQueryResponse res) {
-    int flags = 0;
-    if (fl != null) {
-      // TODO - this could become more efficient if widely used.
-      // TODO - should field order be maintained?
-      String[] flst = split(fl);
-      if (flst.length > 0 && !(flst.length==1 && flst[0].length()==0)) {
-        Set<String> set = new LinkedHashSet<String>();
-        for (String fname : flst) {
-          if("score".equalsIgnoreCase(fname))
-            flags |= SolrIndexSearcher.GET_SCORES;
-          set.add(fname);
-        }
-        res.setReturnFields(set);
-      }
-    }
-    return flags;
-  }
 
   /**
    * Pre-fetch documents into the index searcher's document cache.
@@ -180,14 +145,13 @@ public class SolrPluginUtils {
       return;
     }
 
-    Set<String> returnFields = res.getReturnFields();
-    Set<String> fieldFilter = returnFields;
-
-    if(returnFields != null) {
+    ReturnFields returnFields = res.getReturnFields();
+    if(returnFields.getLuceneFieldNames() != null) {
+      Set<String> fieldFilter = returnFields.getLuceneFieldNames();
 
       if (rb.doHighlights) {
         // copy return fields list
-        fieldFilter = new HashSet<String>(returnFields);
+        fieldFilter = new HashSet<String>(fieldFilter);
         // add highlight fields
 
         SolrHighlighter highlighter = HighlightComponent.getHighlighter(req.getCore());

Modified: lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/content/xdocs/features.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/content/xdocs/features.xml?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/content/xdocs/features.xml (original)
+++ lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/content/xdocs/features.xml Mon May  9 13:19:28 2011
@@ -28,7 +28,7 @@
 <section>
 <title>Solr in a Nutshell</title>
   <p>
-  Solr is a standalone enterprise search server with a web-services like API. You put documents in it (called "indexing") via XML over HTTP. You query it via HTTP GET and receive XML results.
+  Solr is a standalone enterprise search server with a REST-like API. You put documents in it (called "indexing") via XML, JSON or binary over HTTP. You query it via HTTP GET and receive XML, JSON, or binary results.
   </p>
   <ul>
     <li> Advanced Full-Text Search Capabilities </li>
@@ -48,6 +48,7 @@
     <li> A Real Data Schema, with Numeric Types, Dynamic Fields, Unique Keys </li>
     <li> Powerful Extensions to the Lucene Query Language </li>
     <li> Faceted Search and Filtering </li>
+    <li> Geospatial Search </li>
     <li> Advanced, Configurable Text Analysis </li>
     <li> Highly Configurable and User Extensible Caching </li>
     <li> Performance Optimizations </li>
@@ -56,9 +57,10 @@
     <li> Monitorable Logging </li>
     <li> Fast Incremental Updates and Index Replication </li>
     <li> Highly Scalable Distributed search with sharded index across multiple hosts </li>
-    <li> XML, CSV/delimited-text, and binary update formats </li>
+    <li> JSON, XML, CSV/delimited-text, and binary update formats </li>
     <li> Easy ways to pull in data from databases and XML files from local disk and HTTP sources </li>
     <li> Rich Document Parsing and Indexing (PDF, Word, HTML, etc) using Apache Tika </li>
+    <li> Apache UIMA integration for configurable metadata extraction </li>
     <li> Multiple search indices </li>
   </ul>
 </section>
@@ -82,10 +84,10 @@
   <section><title>Query</title>
     <ul>
       <li>HTTP interface with configurable response formats (XML/XSLT, JSON, Python, Ruby, PHP, Velocity, binary)</li>
-      <li>Sort by any number of fields</li>
+      <li>Sort by any number of fields, and by complex functions of numeric fields</li>
       <li>Advanced DisMax query parser for high relevancy results from user-entered queries</li> 
       <li>Highlighted context snippets</li>
-      <li>Faceted Searching based on unique field values, explicit queries, or date ranges</li>
+      <li>Faceted Searching based on unique field values, explicit queries, date ranges, and numeric ranges</li>
       <li>Multi-Select Faceting by tagging and selectively excluding filters</li>
       <li>Spelling suggestions for user queries</li>
       <li>More Like This suggestions for given document</li>
@@ -96,7 +98,7 @@
       <li>Dynamic search results clustering using Carrot2</li>
       <li>Numeric field statistics such as min, max, average, standard deviation </li>
       <li>Combine queries derived from different syntaxes</li>
-      <li>Auto-suggest functionality</li>
+      <li>Auto-suggest functionality for completing user queries</li>
       <li>Allow configuration of top results for a query, overriding normal scoring and sorting</li>
       <li>Performance Optimizations</li>
     </ul>

Modified: lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/content/xdocs/index.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/content/xdocs/index.xml?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/content/xdocs/index.xml (original)
+++ lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/content/xdocs/index.xml Mon May  9 13:19:28 2011
@@ -28,8 +28,8 @@
 Solr is the popular, blazing fast open source enterprise search
 platform from the Apache Lucene project.  Its major features include
 powerful full-text search, hit highlighting, faceted search, dynamic
-clustering, database integration, and rich document (e.g., Word, PDF)
-handling.  Solr is highly scalable, providing distributed search and
+clustering, database integration, rich document (e.g., Word, PDF)
+handling, and geospatial search.  Solr is highly scalable, providing distributed search and
 index replication, and it powers the search and navigation features of
 many of the world's largest internet sites.
 </p>
@@ -66,6 +66,37 @@ customization is required.
     </section>
     <section id="news">
       <title>News</title>
+            <section>
+   <title>March 2011 - Solr 3.1 Released</title>
+  <p>The Lucene PMC is pleased to announce the release of <a href="http://www.apache.org/dyn/closer.cgi/lucene/solr">Apache Solr 3.1</a>!
+  </p> 
+   <p>
+    Solr's version number was synced with Lucene following the Lucene/Solr merge, so Solr 3.1 contains Lucene 3.1.
+    Solr 3.1 is the first release after Solr 1.4.1.
+   </p>
+  <p>
+   Solr 3.1 release highlights include
+ </p>
+  <ul>
+    <li><a href="http://wiki.apache.org/solr/SimpleFacetParameters#Facet_by_Range">Numeric range facets</a> (similar to date faceting).</li>
+    <li>New <a href="http://wiki.apache.org/solr/SpatialSearch">spatial search</a>, including spatial filtering, boosting and sorting capabilities.</li>
+    <li>Example Velocity driven search UI at <a href="http://localhost:8983/solr/browse">http://localhost:8983/solr/browse</a></li>
+    <li>A new faster <a href="http://wiki.apache.org/solr/HighlightingParameters">termvector-based highlighter</a>.</li>
+    <li>Extended dismax (edismax) query parser with support for fielded queries, enhanced relevancy, and full lucene syntax support.</li>
+    <li>Distributed search support for the <a href="http://wiki.apache.org/solr/SpellCheckComponent">Spell check</a>
+ and <a href="http://wiki.apache.org/solr/TermsComponent">Terms</a> components.</li>
+    <li><a href="http://wiki.apache.org/solr/Suggester">Suggester</a>, a fast trie-based autocomplete component.</li>
+    <li>Sort results by any <a href="http://wiki.apache.org/solr/FunctionQuery#Sort_By_Function any function">function query</a>.</li>
+    <li><a href="http://wiki.apache.org/solr/UpdateJSON">JSON document indexing</a>.</li>
+    <li><a href="http://wiki.apache.org/solr/CSVResponseWriter">CSV response format</a></li>
+    <li><a href="http://wiki.apache.org/solr/SolrUIMA">Apache UIMA integration</a> for metadata extraction.</li>
+    <li>Tons of optimizations, bugfixes, and new analysis capabilities via <a href="http://lucene.apache.org/java/docs/index.html">Apache Lucene 3.1</a>.</li>
+  </ul>
+     
+  <p>See the <a href="http://svn.apache.org/repos/asf/lucene/dev/tags/lucene_solr_3_1/solr/CHANGES.txt">release notes</a> for a more complete list of all the new features, improvements, and bugfixes.
+ </p>
+
+</section>
         <section>
           <title>25 June 2010 - Solr 1.4.1 Released</title>
           <p>

Modified: lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/content/xdocs/site.xml?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/content/xdocs/site.xml (original)
+++ lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/content/xdocs/site.xml Mon May  9 13:19:28 2011
@@ -42,12 +42,13 @@ See http://forrest.apache.org/docs/linki
   </about>
 
   <docs label="Documentation">
-    <features    label="Features"         href="features.html" />
-    <tutorial    label="Tutorial"         href="tutorial.html" />
-    <wiki        label="Docs (Wiki)" href="ext:wiki" />
-    <faq         label="FAQ"              href="ext:faq" />
-    <apidocs     label="All Javadoc"     href="api/index.html" />
-    <apidocs     label="SolrJ Javadoc"     href="api/solrj/index.html" />
+    <features    label="Features"                href="features.html" />
+    <tutorial    label="Tutorial"                href="tutorial.html" />
+    <wiki        label="Docs (Wiki)"             href="ext:wiki" />
+    <faq         label="FAQ"                     href="ext:faq" />
+    <apidocs     label="All Javadoc"             href="api/index.html" />
+    <apidocs     label="SolrJ Javadoc"           href="api/solrj/index.html" />
+    <apidocs     label="Test Framework Javadoc"  href="api/test-framework/index.html" />
   </docs>
 
   <resources label="Resources">

Modified: lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/content/xdocs/tutorial.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/content/xdocs/tutorial.xml?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/content/xdocs/tutorial.xml (original)
+++ lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/content/xdocs/tutorial.xml Mon May  9 13:19:28 2011
@@ -46,10 +46,9 @@ To follow along with this tutorial, you 
 
 <ol>
   <li>Java 1.6 or greater.  Some places you can get it are from
-  <a href="http://java.sun.com/j2se/downloads.html">OpenJDK</a>,
-  <a href="http://java.sun.com/j2se/downloads.html">Sun</a>,
+  <a href="http://www.oracle.com/technetwork/java/javase/downloads/index.html">Oracle</a>,
+  <a href="http://openjdk.java.net/">Open JDK</a>,
   <a href="http://www.ibm.com/developerworks/java/jdk/">IBM</a>, or
-  <a href="http://www.oracle.com/technology/products/jrockit/index.html">Oracle</a>.
   <br/>
   Running <code>java -version</code> at the command line should indicate a version
   number starting with 1.6.  Gnu's GCJ is not supported and does not work with Solr.
@@ -86,7 +85,7 @@ To launch Jetty with the Solr WAR, and t
 <source>
 user:~/solr/example$ <strong>java -jar start.jar</strong>
 2009-10-23 16:42:53.816::INFO:  Logging to STDERR via org.mortbay.log.StdErrLog
-2009-10-23 16:42:53.907::INFO:  jetty-6.1.3
+2009-10-23 16:42:53.907::INFO:  jetty-6.1.26
 
 ...
 
@@ -141,11 +140,11 @@ You can now search for "solr" using the 
 Clicking the "Search" button should take you to the following URL...
 </p>
 <p>
-<a href="http://localhost:8983/solr/select/?q=solr&amp;version=2.2&amp;start=0&amp;rows=10&amp;indent=on">http://localhost:8983/solr/select/?q=solr&amp;version=2.2&amp;start=0&amp;rows=10&amp;indent=on</a>
+<a href="http://localhost:8983/solr/select/?q=solr&amp;start=0&amp;rows=10&amp;indent=on">http://localhost:8983/solr/select/?q=solr&amp;start=0&amp;rows=10&amp;indent=on</a>
 </p>
 
 <p>
-You can index all of the sample data, using the following command (assuming your shell supports the *.xml notation):
+You can index all of the sample data, using the following command (assuming your command line shell supports the *.xml notation):
 </p>
 
 <source>
@@ -189,6 +188,8 @@ SimplePostTool: COMMITting Solr index ch
   <li><a href="http://wiki.apache.org/solr/UpdateCSV">Load a CSV file</a> (comma separated values),
    including those exported by Excel or MySQL.
   </li> 
+  <li><a href="http://wiki.apache.org/solr/UpdateJSON">POST JSON documents</a>
+  </li> 
   <li>Index binary documents such as Word and PDF with 
     <a href="http://wiki.apache.org/solr/ExtractingRequestHandler">Solr Cell</a> (ExtractingRequestHandler).
   </li>
@@ -313,6 +314,14 @@ in subsequent searches.
     </ul>
 
     <p>
+      Complex functions may also be used to sort results...
+    </p>
+    <ul>
+      <li><a href="http://localhost:8983/solr/select/?indent=on&amp;q=*:*&amp;sort=div(popularity,add(price,1))+desc">q=video&amp;sort=div(popularity,add(price,1)) desc</a></li>
+    </ul>
+
+
+    <p>
       If no sort is specified, the default is <code>score desc</code> to return the matches having the highest relevancy.
     </p>
 
@@ -402,6 +411,21 @@ in subsequent searches.
 
 
 <section>
+  <title>Search UI</title>
+  <p>
+    Solr includes an example search interface built with velocity templating
+    that demonstrates many features, including searching, faceting, highlighting,
+    autocomplete, and geospatial searching. 
+  </p>
+  <p>
+    Try it out at
+    <a href="http://localhost:8983/solr/browse">http://localhost:8983/solr/browse</a>
+  </p>
+</section> <!-- Search UI -->
+
+
+
+<section>
   <title>Text Analysis</title>
 
   <p>

Modified: lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/resources/schema/catalog.xcat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/resources/schema/catalog.xcat?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/resources/schema/catalog.xcat (original)
+++ lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/resources/schema/catalog.xcat Mon May  9 13:19:28 2011
@@ -21,6 +21,5 @@
 <catalog xmlns="urn:oasis:names:tc:entity:xmlns:xml:catalog"
          prefer="public">
   <!-- Sets of symbols. e.g. for string replacements -->
-  <public publicId="-//Apache Solr//ENTITIES Symbols Project v1.0//EN"
-        uri="../../../../../../build/solr-specific-forrest-variables.ent"/>
+
 </catalog>

Modified: lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/skinconf.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/skinconf.xml?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/skinconf.xml (original)
+++ lucene/dev/branches/bulkpostings/solr/src/site/src/documentation/skinconf.xml Mon May  9 13:19:28 2011
@@ -21,12 +21,7 @@ Skin configuration file. This file conta
 which will be used to configure the chosen Forrest skin.
 -->
 
-<!DOCTYPE skinconfig PUBLIC "-//APACHE//DTD Skin Configuration V0.7-1//EN" "http://forrest.apache.org/dtd/skinconfig-v07-1.dtd" [
-<!ENTITY % symbols-project
-    PUBLIC "-//Apache Solr//ENTITIES Symbols Project v1.0//EN"
-    "solr-specific-forrest-variables.ent"> 
-  %symbols-project;
-]>
+<!DOCTYPE skinconfig PUBLIC "-//APACHE//DTD Skin Configuration V0.7-1//EN" "http://forrest.apache.org/dtd/skinconfig-v07-1.dtd">
 <skinconfig>
   <!-- To enable lucene search add provider="lucene" (default is google).
     Add box-location="alt" to move the search box to an alternate location
@@ -156,16 +151,6 @@ which will be used to configure the chos
       both : both
     -->
 
-  <motd>
-    <motd-option pattern="tutorial">
-      <motd-title>version &solr.specversion;</motd-title>
-      <motd-page location="both">
-      This document is for Apache Solr version &solr.specversion;.  If you are using a different version of Solr, please consult the documentation that was distributed with the version you are using.
-        </motd-page>
-    </motd-option>
-  </motd>
-
-
   <!--
     extra-css - here you can define custom css-elements that are 
     A) overriding the fallback elements or 

Modified: lucene/dev/branches/bulkpostings/solr/src/solrj/org/apache/solr/client/solrj/impl/XMLResponseParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/solrj/org/apache/solr/client/solrj/impl/XMLResponseParser.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/solrj/org/apache/solr/client/solrj/impl/XMLResponseParser.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/solrj/org/apache/solr/client/solrj/impl/XMLResponseParser.java Mon May  9 13:19:28 2011
@@ -37,6 +37,7 @@ import org.apache.solr.client.solrj.util
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.DateUtil;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.SimpleOrderedMap;
 import org.apache.solr.common.util.XMLErrorLogger;
@@ -165,7 +166,7 @@ public class XMLResponseParser extends R
       @Override 
       public Date read( String txt ) { 
         try {
-          return ClientUtils.parseDate(txt);      
+          return DateUtil.parseDate(txt);      
         }
         catch( Exception ex ) {
           ex.printStackTrace();

Modified: lucene/dev/branches/bulkpostings/solr/src/test-files/solr/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/test-files/solr/conf/schema.xml?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/test-files/solr/conf/schema.xml (original)
+++ lucene/dev/branches/bulkpostings/solr/src/test-files/solr/conf/schema.xml Mon May  9 13:19:28 2011
@@ -391,6 +391,32 @@
 
   <fieldType name="latLon" class="solr.LatLonType" subFieldType="double"/>
 
+  <!--  some per-field similarity examples -->
+  
+  <!--  specify a Similarity classname directly -->
+  <fieldType name="sim1" class="solr.TextField">
+    <analyzer>
+      <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+    </analyzer>
+    <similarity class="org.apache.lucene.misc.SweetSpotSimilarity"/>
+  </fieldType>
+
+  <!--  specify a Similarity factory -->  
+  <fieldType name="sim2" class="solr.TextField">
+    <analyzer>
+      <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+    </analyzer>
+    <similarity class="org.apache.solr.schema.CustomSimilarityFactory">
+      <str name="echo">is there an echo?</str>
+    </similarity>
+  </fieldType>
+  
+  <!-- don't specify any sim at all: get the default  -->
+  <fieldType name="sim3" class="solr.TextField">
+    <analyzer>
+      <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+    </analyzer>
+  </fieldType>
  </types>
 
 
@@ -496,6 +522,9 @@
    <field name="multiDefault" type="string" indexed="true" stored="true" default="muLti-Default" multiValued="true"/>
    <field name="intDefault" type="int" indexed="true" stored="true" default="42" multiValued="false"/>
 
+   <field name="sim1text" type="sim1" indexed="true" stored="true"/>
+   <field name="sim2text" type="sim2" indexed="true" stored="true"/>
+   <field name="sim3text" type="sim3" indexed="true" stored="true"/>
 
    <field name="tlong" type="tlong" indexed="true" stored="true" />
 
@@ -574,6 +603,10 @@
 
    <dynamicField name="*_mfacet" type="string" indexed="true" stored="false" multiValued="true" />
 
+   <!-- make sure custom sims work with dynamic fields -->
+   <dynamicField name="*_sim1" type="sim1" indexed="true" stored="true"/>
+   <dynamicField name="*_sim2" type="sim2" indexed="true" stored="true"/>
+   <dynamicField name="*_sim3" type="sim3" indexed="true" stored="true"/>
  </fields>
 
  <defaultSearchField>text</defaultSearchField>
@@ -607,12 +640,18 @@
    <!-- dynamic destination -->
    <copyField source="*_dynamic" dest="dynamic_*"/>
 
- <!-- Similarity is the scoring routine for each document vs a query.
-      A custom similarity may be specified here, but the default is fine
+ <!-- expert: SimilarityProvider contains scoring routines that are not field-specific,
+      such as coord() and queryNorm(). most scoring customization happens in the fieldtype.
+      A custom similarity provider may be specified here, but the default is fine
       for most applications.
  -->
- <similarity class="org.apache.solr.schema.CustomSimilarityFactory">
+ <similarityProvider class="org.apache.solr.schema.CustomSimilarityProviderFactory">
    <str name="echo">is there an echo?</str>
- </similarity>
+ </similarityProvider>
 
+ <!-- default similarity, unless otherwise specified by the fieldType
+  -->
+ <similarity class="org.apache.solr.schema.CustomSimilarityFactory">
+   <str name="echo">I am your default sim</str>
+ </similarity>
 </schema>



Mime
View raw message