lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject cvs commit: jakarta-lucene/src/test/org/apache/lucene/search TestDocBoost.java
Date Mon, 29 Jul 2002 19:11:15 GMT
cutting     2002/07/29 12:11:15

  Modified:    .        CHANGES.txt
               src/java/org/apache/lucene/document Document.java Field.java
               src/java/org/apache/lucene/index DocumentWriter.java
                        IndexReader.java
               src/java/org/apache/lucene/search PhrasePrefixQuery.java
                        PhraseScorer.java Similarity.java TermScorer.java
  Added:       src/test/org/apache/lucene/search TestDocBoost.java
  Log:
  msg.txt
  
  Revision  Changes    Path
  1.28      +12 -1     jakarta-lucene/CHANGES.txt
  
  Index: CHANGES.txt
  ===================================================================
  RCS file: /home/cvs/jakarta-lucene/CHANGES.txt,v
  retrieving revision 1.27
  retrieving revision 1.28
  diff -u -r1.27 -r1.28
  --- CHANGES.txt	26 Jul 2002 17:32:54 -0000	1.27
  +++ CHANGES.txt	29 Jul 2002 19:11:14 -0000	1.28
  @@ -48,6 +48,17 @@
        stems from nouns and verbs derived from the same word.
        (gschwarz)
   
  + 12. Added support for boosting the score of documents and fields via
  +     the new methods Document.setBoost(float) and Field.setBoost(float).
  +
  +     Note: This changes the encoding of an indexed value.  Indexes
  +     should be re-created from scratch in order for search scores to
  +     be correct.  With the new code and an old index, searches will
  +     yield very large scores for shorter fields, and very small scores
  +     for longer fields.  Once the index is re-created, scores will be
  +     as before. (cutting)
  +
  +
   1.2 RC6
   
    1. Changed QueryParser.jj to have "?" be a special character which
  
  
  
  1.3       +32 -0     jakarta-lucene/src/java/org/apache/lucene/document/Document.java
  
  Index: Document.java
  ===================================================================
  RCS file: /home/cvs/jakarta-lucene/src/java/org/apache/lucene/document/Document.java,v
  retrieving revision 1.2
  retrieving revision 1.3
  diff -u -r1.2 -r1.3
  --- Document.java	17 Jul 2002 21:54:38 -0000	1.2
  +++ Document.java	29 Jul 2002 19:11:14 -0000	1.3
  @@ -55,6 +55,8 @@
    */
   
   import java.util.Enumeration;
  +import org.apache.lucene.index.IndexReader;
  +import org.apache.lucene.search.Hits;
   
   /** Documents are the unit of indexing and search.
    *
  @@ -66,9 +68,39 @@
   
   public final class Document implements java.io.Serializable {
     DocumentFieldList fieldList = null;
  +  private float boost = 1.0f;
   
     /** Constructs a new document with no fields. */
     public Document() {}
  +
  +
  +  /** Sets a boost factor for hits on any field of this document.  This value
  +   * will be multiplied into the score of all hits on this document.
  +   *
  +   * <p>Values are multiplied into the value of {@link Field#getBoost()} of
  +   * each field in this document.  Thus, this method in effect sets a default
  +   * boost for the fields of this document.
  +   *
  +   * @see Field#setBoost(float)
  +   */
  +  public void setBoost(float boost) {
  +    this.boost = boost;
  +  }
  +
  +  /** Returns the boost factor for hits on any field of this document.
  +   *
  +   * <p>The default value is 1.0.
  +   *
  +   * <p>Note: This value is not stored directly with the document in the index.
  +   * Documents returned from {@link IndexReader#document(int)} and {@link
  +   * Hits#doc(int)} may thus not have the same value present as when this
  +   * document was indexed.
  +   *
  +   * @see #setBoost(float)
  +   */
  +  public float getBoost() {
  +    return boost;
  +  }
   
     /** Adds a field to a document.  Several fields may be added with
      * the same name.  In this case, if the fields are indexed, their text is
  
  
  
  1.7       +40 -0     jakarta-lucene/src/java/org/apache/lucene/document/Field.java
  
  Index: Field.java
  ===================================================================
  RCS file: /home/cvs/jakarta-lucene/src/java/org/apache/lucene/document/Field.java,v
  retrieving revision 1.6
  retrieving revision 1.7
  diff -u -r1.6 -r1.7
  --- Field.java	17 Jul 2002 21:54:38 -0000	1.6
  +++ Field.java	29 Jul 2002 19:11:14 -0000	1.7
  @@ -56,6 +56,9 @@
   
   import java.io.Reader;
   import java.util.Date;
  +import org.apache.lucene.index.IndexReader;
  +import org.apache.lucene.search.Similarity;
  +import org.apache.lucene.search.Hits;
   
   /**
     A field is a section of a Document.  Each field has two parts, a name and a
  @@ -72,6 +75,43 @@
     private boolean isStored = false;
     private boolean isIndexed = true;
     private boolean isTokenized = true;
  +
  +  private float boost = 1.0f;
  +
  +  /** Sets the boost factor hits on this field.  This value will be
  +   * multiplied into the score of all hits on this this field of this
  +   * document.
  +   *
  +   * <p>The boost is multiplied by {@link Document#getBoost()} of the document
  +   * containing this field.  If a document has multiple fields with the same
  +   * name, all such values are multiplied together.  This product is then
  +   * multipled by the value {@link Similarity#normalizeLength(int)}, and
  +   * rounded by {@link Similarity#encodeNorm(float)} before it is stored in the
  +   * index.  One should attempt to ensure that this product does not overflow
  +   * the range of that encoding.
  +   *
  +   * @see Document#setBoost(float)
  +   * @see Similarity#normalizeLength(int)
  +   * @see Similarity#encodeNorm(float)
  +   */
  +  public void setBoost(float boost) {
  +    this.boost = boost;
  +  }
  +
  +  /** Returns the boost factor for hits on any field of this document.
  +   *
  +   * <p>The default value is 1.0.
  +   *
  +   * <p>Note: this value is not stored directly with the document in the index.
  +   * Documents returned from {@link IndexReader#document(int)} and {@link
  +   * Hits#doc(int)} may thus not have the same value present as when this field
  +   * was indexed.
  +   *
  +   * @see #setBoost(float)
  +   */
  +  public float getBoost() {
  +    return boost;
  +  }
   
     /** Constructs a String-valued Field that is not tokenized, but is indexed
       and stored.  Useful for non-text fields, e.g. date or url.  */
  
  
  
  1.2       +13 -4     jakarta-lucene/src/java/org/apache/lucene/index/DocumentWriter.java
  
  Index: DocumentWriter.java
  ===================================================================
  RCS file: /home/cvs/jakarta-lucene/src/java/org/apache/lucene/index/DocumentWriter.java,v
  retrieving revision 1.1
  retrieving revision 1.2
  diff -u -r1.1 -r1.2
  --- DocumentWriter.java	18 Sep 2001 16:29:52 -0000	1.1
  +++ DocumentWriter.java	29 Jul 2002 19:11:15 -0000	1.2
  @@ -59,6 +59,7 @@
   import java.io.StringReader;
   import java.util.Hashtable;
   import java.util.Enumeration;
  +import java.util.Arrays;
   
   import org.apache.lucene.document.Document;
   import org.apache.lucene.document.Field;
  @@ -100,6 +101,10 @@
       // invert doc into postingTable
       postingTable.clear();			  // clear postingTable
       fieldLengths = new int[fieldInfos.size()];	  // init fieldLengths
  +
  +    fieldBoosts = new float[fieldInfos.size()];	  // init fieldBoosts
  +    Arrays.fill(fieldBoosts, doc.getBoost());
  +
       invertDocument(doc);
   
       // sort postingTable into an array
  @@ -130,6 +135,7 @@
     // Used to buffer a document before it is written to the index.
     private final Hashtable postingTable = new Hashtable();
     private int[] fieldLengths;
  +  private float[] fieldBoosts;
   
     // Tokenizes the fields of a document into Postings.
     private final void invertDocument(Document doc)
  @@ -168,6 +174,7 @@
   	}
   
   	fieldLengths[fieldNumber] = position;	  // save field length
  +        fieldBoosts[fieldNumber] *= field.getBoost();
         }
       }
     }
  @@ -310,12 +317,14 @@
       while (fields.hasMoreElements()) {
         Field field = (Field)fields.nextElement();
         if (field.isIndexed()) {
  -	int fieldNumber = fieldInfos.fieldNumber(field.name());
  -	OutputStream norm = directory.createFile(segment + ".f" + fieldNumber);
  +	int n = fieldInfos.fieldNumber(field.name());
  +        float norm =
  +          fieldBoosts[n] * Similarity.normalizeLength(fieldLengths[n]);
  +	OutputStream norms = directory.createFile(segment + ".f" + n);
   	try {
  -	  norm.writeByte(Similarity.norm(fieldLengths[fieldNumber]));
  +	  norms.writeByte(Similarity.encodeNorm(norm));
   	} finally {
  -	  norm.close();
  +	  norms.close();
   	}
         }
       }
  
  
  
  1.10      +5 -3      jakarta-lucene/src/java/org/apache/lucene/index/IndexReader.java
  
  Index: IndexReader.java
  ===================================================================
  RCS file: /home/cvs/jakarta-lucene/src/java/org/apache/lucene/index/IndexReader.java,v
  retrieving revision 1.9
  retrieving revision 1.10
  diff -u -r1.9 -r1.10
  --- IndexReader.java	15 Feb 2002 18:59:42 -0000	1.9
  +++ IndexReader.java	29 Jul 2002 19:11:15 -0000	1.10
  @@ -60,6 +60,7 @@
   import org.apache.lucene.store.FSDirectory;
   import org.apache.lucene.store.Lock;
   import org.apache.lucene.document.Document;
  +import org.apache.lucene.document.Field;
   
   /** IndexReader is an abstract class, providing an interface for accessing an
     index.  Search of an index is done entirely through this abstract interface,
  @@ -177,9 +178,10 @@
     abstract public boolean isDeleted(int n);
   
     /** Returns the byte-encoded normalization factor for the named field of
  -    every document.  This is used by the search code to score documents.
  -    @see org.apache.lucene.search.Similarity#norm
  -    */
  +   * every document.  This is used by the search code to score documents.
  +   *
  +   * @see Field#setBoost(float)
  +   */
     abstract public byte[] norms(String field) throws IOException;
   
     /** Returns an enumeration of all the terms in the index.
  
  
  
  1.2       +1 -1      jakarta-lucene/src/java/org/apache/lucene/search/PhrasePrefixQuery.java
  
  Index: PhrasePrefixQuery.java
  ===================================================================
  RCS file: /home/cvs/jakarta-lucene/src/java/org/apache/lucene/search/PhrasePrefixQuery.java,v
  retrieving revision 1.1
  retrieving revision 1.2
  diff -u -r1.1 -r1.2
  --- PhrasePrefixQuery.java	18 Jul 2002 14:39:58 -0000	1.1
  +++ PhrasePrefixQuery.java	29 Jul 2002 19:11:15 -0000	1.2
  @@ -66,7 +66,7 @@
   
   /**
    * PhrasePrefixQuery is a generalized version of PhraseQuery, with an added
  - * method {@link add(Term[])}.
  + * method {@link #add(Term[])}.
    * To use this class, to search for the phrase "Microsoft app*" first use
    * add(Term) on the term "Microsoft", then find all terms that has "app" as
    * prefix using IndexReader.terms(Term), and use PhrasePrefixQuery.add(Term[]
  
  
  
  1.2       +1 -1      jakarta-lucene/src/java/org/apache/lucene/search/PhraseScorer.java
  
  Index: PhraseScorer.java
  ===================================================================
  RCS file: /home/cvs/jakarta-lucene/src/java/org/apache/lucene/search/PhraseScorer.java,v
  retrieving revision 1.1
  retrieving revision 1.2
  diff -u -r1.1 -r1.2
  --- PhraseScorer.java	18 Sep 2001 16:29:57 -0000	1.1
  +++ PhraseScorer.java	29 Jul 2002 19:11:15 -0000	1.2
  @@ -93,7 +93,7 @@
   
         if (freq > 0.0) {
   	float score = Similarity.tf(freq)*weight; // compute score
  -	score *= Similarity.norm(norms[first.doc]); // normalize
  +	score *= Similarity.decodeNorm(norms[first.doc]); // normalize
   	results.collect(first.doc, score);	  // add to results
         }
         last.next();				  // resume scanning
  
  
  
  1.2       +69 -20    jakarta-lucene/src/java/org/apache/lucene/search/Similarity.java
  
  Index: Similarity.java
  ===================================================================
  RCS file: /home/cvs/jakarta-lucene/src/java/org/apache/lucene/search/Similarity.java,v
  retrieving revision 1.1
  retrieving revision 1.2
  diff -u -r1.1 -r1.2
  --- Similarity.java	18 Sep 2001 16:29:58 -0000	1.1
  +++ Similarity.java	29 Jul 2002 19:11:15 -0000	1.2
  @@ -56,6 +56,7 @@
   
   import java.io.IOException;
   import org.apache.lucene.index.Term;
  +import org.apache.lucene.document.Field;
   
   /** Internal class used for scoring.
    * <p>Public only so that the indexing code can compute and store the
  @@ -63,32 +64,80 @@
   public final class Similarity {
     private Similarity() {}			  // no public constructor
   
  -  /** Computes the normalization byte for a document given the total number of
  -   * terms contained in the document.  These values are stored in an index and
  -   * used by the search code. */
  -  public static final byte norm(int numTerms) {
  -    // Scales 1/sqrt(numTerms) into a byte, i.e. 256/sqrt(numTerms).
  -    // Math.ceil is used to ensure that even very long documents don't get a
  -    // zero norm byte, as that is reserved for zero-lengthed documents and
  -    // deleted documents.
  -    return (byte) Math.ceil(255.0 / Math.sqrt(numTerms));
  +  static final float[] NORM_TABLE = new float[256];
  +
  +  static {
  +    for (int i = 0; i < 256; i++)
  +      NORM_TABLE[i] = byteToFloat((byte)i);
     }
   
  +  /** Computes the normalization value for a document given the total number of
  +   * terms contained in a field.  These values are stored in an index and used
  +   * by the search code.
  +   *
  +   * <p>The formula used is: <code>1.0f / Math.sqrt(numTerms)</code>
  +   *
  +   * @see Field#setBoost(float)
  +   */
  +  public static float normalizeLength(int numTerms) {
  +    return (float)(1.0 / Math.sqrt(numTerms));
  +  }
  +  
  +  /** Decodes a normalization factor stored in an index.
  +   * @see #encodeNorm(float)
  +   */
  +  public static float decodeNorm(byte b) {
  +    return NORM_TABLE[b & 0xFF];
  +  }
   
  -  private static final float[] makeNormTable() {
  -    float[] result = new float[256];
  -    for (int i = 0; i < 256; i++)
  -      result[i] = i / 255.0F;
  -    return result;
  +  /** Encodes a normalization factor for storage in an index.  
  +   *
  +   * <p>The encoding uses a five-bit exponent and three-bit mantissa, thus
  +   * representing values from around 7x10^9 to 2x10^-9 with about one
  +   * significant decimal digit of accuracy.  Zero is also represented.
  +   * Negative numbers are rounded up to zero.  Values too large to represent
  +   * are rounded down to the largest representable value.  Positive values too
  +   * small to represent are rounded up to the smallest positive representable
  +   * value.
  +   *
  +   * @see Field#setBoost(float)
  +   */
  +  public static byte encodeNorm(float f) {
  +    return floatToByte(f);
     }
   
  -  static final float[] NORM_TABLE = makeNormTable();
  -    
  -  static final float norm(byte normByte) {
  -    // Un-scales from the byte encoding of a norm into a float, i.e.,
  -    // approximately 1/sqrt(numTerms).
  -    return NORM_TABLE[normByte & 0xFF];
  +  private static float byteToFloat(byte b) {
  +    if (b == 0)                                   // zero is a special case
  +      return 0.0f;
  +    int mantissa = b & 7;
  +    int exponent = (b >> 3) & 31;
  +    int bits = ((exponent+(63-15)) << 24) | (mantissa << 21);
  +    return Float.intBitsToFloat(bits);
     }
  +   
  +  private static byte floatToByte(float f) {
  +    if (f < 0.0f)                                 // round negatives up to zero
  +      f = 0.0f;
  +
  +    if (f == 0.0f)                                // zero is a special case
  +      return 0;
  +
  +    int bits = Float.floatToIntBits(f);           // parse float into parts
  +    int mantissa = (bits & 0xffffff) >> 21;
  +    int exponent = (((bits >> 24) & 0x7f) - 63) + 15;
  +
  +    if (exponent > 31) {                          // overflow: use max value
  +      exponent = 31;
  +      mantissa = 7;
  +    }
  +
  +    if (exponent < 1) {                           // underflow: use min value
  +      exponent = 1;
  +      mantissa = 0;
  +    }
  +
  +    return (byte)((exponent << 3) | mantissa);    // pack into a byte
  +   }
   
     static final float tf(int freq) {
       return (float)Math.sqrt(freq);
  
  
  
  1.2       +1 -1      jakarta-lucene/src/java/org/apache/lucene/search/TermScorer.java
  
  Index: TermScorer.java
  ===================================================================
  RCS file: /home/cvs/jakarta-lucene/src/java/org/apache/lucene/search/TermScorer.java,v
  retrieving revision 1.1
  retrieving revision 1.2
  diff -u -r1.1 -r1.2
  --- TermScorer.java	18 Sep 2001 16:29:58 -0000	1.1
  +++ TermScorer.java	29 Jul 2002 19:11:15 -0000	1.2
  @@ -98,7 +98,7 @@
   	 ? scoreCache[f]			  // cache hit
   	 : Similarity.tf(f)*weight;		  // cache miss
   
  -      score *= Similarity.norm(norms[d]);	  // normalize for field
  +      score *= Similarity.decodeNorm(norms[d]);	  // normalize for field
   
         c.collect(d, score);			  // collect score
   
  
  
  
  1.1                  jakarta-lucene/src/test/org/apache/lucene/search/TestDocBoost.java
  
  Index: TestDocBoost.java
  ===================================================================
  package org.apache.lucene.search;
  
  /* ====================================================================
   * The Apache Software License, Version 1.1
   *
   * Copyright (c) 2001 The Apache Software Foundation.  All rights
   * reserved.
   *
   * Redistribution and use in source and binary forms, with or without
   * modification, are permitted provided that the following conditions
   * are met:
   *
   * 1. Redistributions of source code must retain the above copyright
   *    notice, this list of conditions and the following disclaimer.
   *
   * 2. Redistributions in binary form must reproduce the above copyright
   *    notice, this list of conditions and the following disclaimer in
   *    the documentation and/or other materials provided with the
   *    distribution.
   *
   * 3. The end-user documentation included with the redistribution,
   *    if any, must include the following acknowledgment:
   *       "This product includes software developed by the
   *        Apache Software Foundation (http://www.apache.org/)."
   *    Alternately, this acknowledgment may appear in the software itself,
   *    if and wherever such third-party acknowledgments normally appear.
   *
   * 4. The names "Apache" and "Apache Software Foundation" and
   *    "Apache Lucene" must not be used to endorse or promote products
   *    derived from this software without prior written permission. For
   *    written permission, please contact apache@apache.org.
   *
   * 5. Products derived from this software may not be called "Apache",
   *    "Apache Lucene", nor may "Apache" appear in their name, without
   *    prior written permission of the Apache Software Foundation.
   *
   * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
   * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
   * DISCLAIMED.  IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
   * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
   * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
   * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
   * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
   * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
   * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
   * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   * SUCH DAMAGE.
   * ====================================================================
   *
   * This software consists of voluntary contributions made by many
   * individuals on behalf of the Apache Software Foundation.  For more
   * information on the Apache Software Foundation, please see
   * <http://www.apache.org/>.
   */
  
  import org.apache.lucene.index.Term;
  import org.apache.lucene.index.IndexWriter;
  import org.apache.lucene.search.Query;
  import org.apache.lucene.search.Hits;
  import org.apache.lucene.search.IndexSearcher;
  import org.apache.lucene.store.RAMDirectory;
  import org.apache.lucene.analysis.SimpleAnalyzer;
  import org.apache.lucene.document.Document;
  import org.apache.lucene.document.Field;
  
  import junit.framework.TestCase;
  
   /** Document boost unit test.
    *
    * @author Doug Cutting
    * @version $Revision: 1.1 $
    */
  public class TestDocBoost extends TestCase {
    public TestDocBoost(String name) {
      super(name);
    }
    
    public static void test() throws Exception {
      RAMDirectory store = new RAMDirectory();
      IndexWriter writer = new IndexWriter(store, new SimpleAnalyzer(), true);
      
      Field f1 = Field.Text("field", "word");
      Field f2 = Field.Text("field", "word");
      f2.setBoost(2.0f);
      
      Document d1 = new Document();
      Document d2 = new Document();
      Document d3 = new Document();
      Document d4 = new Document();
      d3.setBoost(3.0f);
      d4.setBoost(2.0f);
      
      d1.add(f1);                                 // boost = 1
      d2.add(f2);                                 // boost = 2
      d3.add(f1);                                 // boost = 3
      d4.add(f2);                                 // boost = 4
      
      writer.addDocument(d1);
      writer.addDocument(d2);
      writer.addDocument(d3);
      writer.addDocument(d4);
      writer.optimize();
      writer.close();
  
      final float[] scores = new float[4];
  
      new IndexSearcher(store).search
        (new TermQuery(new Term("field", "word")),
         new HitCollector() {
           public final void collect(int doc, float score) {
             scores[doc] = score;
           }
         });
      
      float lastScore = 0.0f;
  
      for (int i = 0; i < 4; i++) {
        assertTrue(scores[i] > lastScore);
        lastScore = scores[i];
      }
    }
  }
  
  
  

--
To unsubscribe, e-mail:   <mailto:lucene-dev-unsubscribe@jakarta.apache.org>
For additional commands, e-mail: <mailto:lucene-dev-help@jakarta.apache.org>


Mime
View raw message