lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From uschind...@apache.org
Subject svn commit: r931278 [8/10] - in /lucene/dev/trunk: lucene/ lucene/backwards/src/ lucene/backwards/src/java/org/apache/lucene/index/ lucene/backwards/src/java/org/apache/lucene/index/codecs/ lucene/backwards/src/java/org/apache/lucene/search/ lucene/bac...
Date Tue, 06 Apr 2010 19:19:36 GMT
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/TermRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/TermRangeQuery.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/TermRangeQuery.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/TermRangeQuery.java Tue Apr  6 19:19:27 2010
@@ -21,6 +21,9 @@ import java.io.IOException;
 import java.text.Collator;
 
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.util.ToStringUtils;
 
 /**
@@ -41,7 +44,6 @@ public class TermRangeQuery extends Mult
   private String lowerTerm;
   private String upperTerm;
   private Collator collator;
-  private String field;
   private boolean includeLower;
   private boolean includeUpper;
 
@@ -104,7 +106,7 @@ public class TermRangeQuery extends Mult
    */
   public TermRangeQuery(String field, String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper,
                     Collator collator) {
-    this.field = field;
+    super(field);
     this.lowerTerm = lowerTerm;
     this.upperTerm = upperTerm;
     this.includeLower = includeLower;
@@ -112,9 +114,6 @@ public class TermRangeQuery extends Mult
     this.collator = collator;
   }
 
-  /** Returns the field name for this query */
-  public String getField() { return field; }
-  
   /** Returns the lower value of this range query */
   public String getLowerTerm() { return lowerTerm; }
 
@@ -130,12 +129,33 @@ public class TermRangeQuery extends Mult
   /** Returns the collator used to determine range inclusion, if any. */
   public Collator getCollator() { return collator; }
   
-  @Override
+  @Override @Deprecated
   protected FilteredTermEnum getEnum(IndexReader reader) throws IOException {
     return new TermRangeTermEnum(reader, field, lowerTerm,
         upperTerm, includeLower, includeUpper, collator);
   }
 
+  @Override
+  protected TermsEnum getTermsEnum(IndexReader reader) throws IOException {
+    if (collator == null && lowerTerm != null && upperTerm != null && lowerTerm.compareTo(upperTerm) > 0) {
+      return TermsEnum.EMPTY;
+    }
+    if ((lowerTerm == null || (collator == null && includeLower && "".equals(lowerTerm))) && upperTerm == null) {
+      // NOTE: debateably, the caller should never pass in a
+      // multi reader...
+      final Terms terms = MultiFields.getTerms(reader, field);
+      return (terms != null) ? terms.iterator() : null;
+    }
+    return new TermRangeTermsEnum(reader, field,
+        lowerTerm, upperTerm, includeLower, includeUpper, collator);
+  }
+
+  /** @deprecated */
+  @Deprecated
+  public String field() {
+    return getField();
+  }
+
   /** Prints a user-readable version of this query. */
   @Override
   public String toString(String field) {
@@ -158,7 +178,6 @@ public class TermRangeQuery extends Mult
     final int prime = 31;
     int result = super.hashCode();
     result = prime * result + ((collator == null) ? 0 : collator.hashCode());
-    result = prime * result + ((field == null) ? 0 : field.hashCode());
     result = prime * result + (includeLower ? 1231 : 1237);
     result = prime * result + (includeUpper ? 1231 : 1237);
     result = prime * result + ((lowerTerm == null) ? 0 : lowerTerm.hashCode());
@@ -180,11 +199,6 @@ public class TermRangeQuery extends Mult
         return false;
     } else if (!collator.equals(other.collator))
       return false;
-    if (field == null) {
-      if (other.field != null)
-        return false;
-    } else if (!field.equals(other.field))
-      return false;
     if (includeLower != other.includeLower)
       return false;
     if (includeUpper != other.includeUpper)

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/TermRangeTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/TermRangeTermEnum.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/TermRangeTermEnum.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/TermRangeTermEnum.java Tue Apr  6 19:19:27 2010
@@ -31,7 +31,9 @@ import org.apache.lucene.util.StringHelp
  * Term enumerations are always ordered by Term.compareTo().  Each term in
  * the enumeration is greater than all that precede it.
  * @since 2.9
+ * @deprecated Please switch to {@link TermRangeTermsEnum}
  */
+@Deprecated
 public class TermRangeTermEnum extends FilteredTermEnum {
 
   private Collator collator = null;

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/TermScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/TermScorer.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/TermScorer.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/TermScorer.java Tue Apr  6 19:19:27 2010
@@ -19,25 +19,26 @@ package org.apache.lucene.search;
 
 import java.io.IOException;
 
-import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.DocsEnum;
 
 /** Expert: A <code>Scorer</code> for documents matching a <code>Term</code>.
  */
 final class TermScorer extends Scorer {
-  
   private Weight weight;
-  private TermDocs termDocs;
+  private DocsEnum docsEnum;
   private byte[] norms;
   private float weightValue;
   private int doc = -1;
+  private int freq;
 
-  private final int[] docs = new int[32];         // buffered doc numbers
-  private final int[] freqs = new int[32];        // buffered term freqs
   private int pointer;
   private int pointerMax;
 
   private static final int SCORE_CACHE_SIZE = 32;
   private float[] scoreCache = new float[SCORE_CACHE_SIZE];
+  private int[] docs;
+  private int[] freqs;
+  private final DocsEnum.BulkReadResult bulkResult;
 
   /**
    * Construct a <code>TermScorer</code>.
@@ -52,13 +53,14 @@ final class TermScorer extends Scorer {
    * @param norms
    *          The field norms of the document fields for the <code>Term</code>.
    */
-  TermScorer(Weight weight, TermDocs td, Similarity similarity, byte[] norms) {
+  TermScorer(Weight weight, DocsEnum td, Similarity similarity, byte[] norms) {
     super(similarity);
     
     this.weight = weight;
-    this.termDocs = td;
+    this.docsEnum = td;
     this.norms = norms;
     this.weightValue = weight.getValue();
+    bulkResult = td.getBulkResult();
 
     for (int i = 0; i < SCORE_CACHE_SIZE; i++)
       scoreCache[i] = getSimilarity().tf(i) * weightValue;
@@ -69,62 +71,69 @@ final class TermScorer extends Scorer {
     score(c, Integer.MAX_VALUE, nextDoc());
   }
 
+  private final void refillBuffer() throws IOException {
+    pointerMax = docsEnum.read();  // refill
+    docs = bulkResult.docs.ints;
+    freqs = bulkResult.freqs.ints;
+  }
+
   // firstDocID is ignored since nextDoc() sets 'doc'
   @Override
   protected boolean score(Collector c, int end, int firstDocID) throws IOException {
     c.setScorer(this);
     while (doc < end) {                           // for docs in window
       c.collect(doc);                      // collect score
-        
       if (++pointer >= pointerMax) {
-        pointerMax = termDocs.read(docs, freqs);  // refill buffers
+        refillBuffer();
         if (pointerMax != 0) {
           pointer = 0;
         } else {
-          termDocs.close();                       // close stream
-          doc = Integer.MAX_VALUE;                // set to sentinel value
+          doc = NO_MORE_DOCS;                // set to sentinel value
           return false;
         }
       } 
       doc = docs[pointer];
+      freq = freqs[pointer];
     }
     return true;
   }
 
   @Override
-  public int docID() { return doc; }
+  public int docID() {
+    return doc;
+  }
 
   /**
    * Advances to the next document matching the query. <br>
    * The iterator over the matching documents is buffered using
    * {@link TermDocs#read(int[],int[])}.
    * 
-   * @return the document matching the query or -1 if there are no more documents.
+   * @return the document matching the query or NO_MORE_DOCS if there are no more documents.
    */
   @Override
   public int nextDoc() throws IOException {
     pointer++;
     if (pointer >= pointerMax) {
-      pointerMax = termDocs.read(docs, freqs);    // refill buffer
+      refillBuffer();
       if (pointerMax != 0) {
         pointer = 0;
       } else {
-        termDocs.close();                         // close stream
         return doc = NO_MORE_DOCS;
       }
     } 
     doc = docs[pointer];
+    freq = freqs[pointer];
+    assert doc != NO_MORE_DOCS;
     return doc;
   }
   
   @Override
   public float score() {
-    assert doc != -1;
-    int f = freqs[pointer];
+    assert doc != NO_MORE_DOCS;
     float raw =                                   // compute tf(f)*weight
-      f < SCORE_CACHE_SIZE                        // check cache
-      ? scoreCache[f]                             // cache hit
-      : getSimilarity().tf(f)*weightValue;        // cache miss
+      freq < SCORE_CACHE_SIZE                        // check cache
+      ? scoreCache[freq]                             // cache hit
+      : getSimilarity().tf(freq)*weightValue;        // cache miss
 
     return norms == null ? raw : raw * getSimilarity().decodeNormValue(norms[doc]); // normalize for field
   }
@@ -132,34 +141,34 @@ final class TermScorer extends Scorer {
   /**
    * Advances to the first match beyond the current whose document number is
    * greater than or equal to a given target. <br>
-   * The implementation uses {@link TermDocs#skipTo(int)}.
+   * The implementation uses {@link DocsEnum#advance(int)}.
    * 
    * @param target
    *          The target document number.
-   * @return the matching document or -1 if none exist.
+   * @return the matching document or NO_MORE_DOCS if none exist.
    */
   @Override
   public int advance(int target) throws IOException {
     // first scan in cache
     for (pointer++; pointer < pointerMax; pointer++) {
       if (docs[pointer] >= target) {
+        freq = freqs[pointer];
         return doc = docs[pointer];
       }
     }
 
-    // not found in cache, seek underlying stream
-    boolean result = termDocs.skipTo(target);
-    if (result) {
-      pointerMax = 1;
-      pointer = 0;
-      docs[pointer] = doc = termDocs.doc();
-      freqs[pointer] = termDocs.freq();
+    // not found in readahead cache, seek underlying stream
+    int newDoc = docsEnum.advance(target);
+    //System.out.println("ts.advance docsEnum=" + docsEnum);
+    if (newDoc != DocsEnum.NO_MORE_DOCS) {
+      doc = newDoc;
+      freq = docsEnum.freq();
     } else {
       doc = NO_MORE_DOCS;
     }
     return doc;
   }
-  
+
   /** Returns a string representation of this <code>TermScorer</code>. */
   @Override
   public String toString() { return "scorer(" + weight + ")"; }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/WildcardQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/WildcardQuery.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/WildcardQuery.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/WildcardQuery.java Tue Apr  6 19:19:27 2010
@@ -19,101 +19,95 @@ package org.apache.lucene.search;
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.util.ToStringUtils;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.BasicAutomata;
+import org.apache.lucene.util.automaton.BasicOperations;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 /** Implements the wildcard search query. Supported wildcards are <code>*</code>, which
  * matches any character sequence (including the empty one), and <code>?</code>,
  * which matches any single character. Note this query can be slow, as it
  * needs to iterate over many terms. In order to prevent extremely slow WildcardQueries,
- * a Wildcard term should not start with one of the wildcards <code>*</code> or
- * <code>?</code>.
+ * a Wildcard term should not start with the wildcard <code>*</code>
  * 
  * <p>This query uses the {@link
  * MultiTermQuery#CONSTANT_SCORE_AUTO_REWRITE_DEFAULT}
  * rewrite method.
  *
- * @see WildcardTermEnum */
-public class WildcardQuery extends MultiTermQuery {
-  private boolean termContainsWildcard;
-  private boolean termIsPrefix;
-  protected Term term;
-    
+ * @see AutomatonQuery
+ */
+public class WildcardQuery extends AutomatonQuery {
+  /** String equality with support for wildcards */
+  public static final char WILDCARD_STRING = '*';
+
+  /** Char equality with support for wildcards */
+  public static final char WILDCARD_CHAR = '?';
+
+  /**
+   * Constructs a query for terms matching <code>term</code>. 
+   */
   public WildcardQuery(Term term) {
-    this.term = term;
-    String text = term.text();
-    this.termContainsWildcard = (text.indexOf('*') != -1)
-        || (text.indexOf('?') != -1);
-    this.termIsPrefix = termContainsWildcard 
-        && (text.indexOf('?') == -1) 
-        && (text.indexOf('*') == text.length() - 1);
+    super(term, toAutomaton(term));
   }
-
-  @Override
+  
+  /**
+   * Convert Lucene wildcard syntax into an automaton.
+   */
+  static Automaton toAutomaton(Term wildcardquery) {
+    List<Automaton> automata = new ArrayList<Automaton>();
+    
+    String wildcardText = wildcardquery.text();
+    
+    for (int i = 0; i < wildcardText.length(); i++) {
+      final char c = wildcardText.charAt(i);
+      switch(c) {
+        case WILDCARD_STRING: 
+          automata.add(BasicAutomata.makeAnyString());
+          break;
+        case WILDCARD_CHAR:
+          automata.add(BasicAutomata.makeAnyChar());
+          break;
+        default:
+          automata.add(BasicAutomata.makeChar(c));
+      }
+    }
+    
+    return BasicOperations.concatenate(automata);
+  }
+  
+  @Override @Deprecated
   protected FilteredTermEnum getEnum(IndexReader reader) throws IOException {
-    if (termContainsWildcard)
-      return new WildcardTermEnum(reader, getTerm());
-    else
-      return new SingleTermEnum(reader, getTerm());
+    return new WildcardTermEnum(reader, term);
   }
   
+  // we override this method, else backwards layer in MTQ will prefer getEnum!
+  @Override
+  protected TermsEnum getTermsEnum(IndexReader reader) throws IOException {
+    return super.getTermsEnum(reader);
+  }
+
   /**
    * Returns the pattern term.
    */
   public Term getTerm() {
     return term;
   }
-
-  @Override
-  public Query rewrite(IndexReader reader) throws IOException {
-    if (termIsPrefix) {
-      MultiTermQuery rewritten = new PrefixQuery(term.createTerm(term.text()
-          .substring(0, term.text().indexOf('*'))));
-      rewritten.setBoost(getBoost());
-      rewritten.setRewriteMethod(getRewriteMethod());
-      return rewritten;
-    } else {
-      return super.rewrite(reader);
-    }
-  }
   
   /** Prints a user-readable version of this query. */
   @Override
   public String toString(String field) {
     StringBuilder buffer = new StringBuilder();
-    if (!term.field().equals(field)) {
-      buffer.append(term.field());
+    if (!getField().equals(field)) {
+      buffer.append(getField());
       buffer.append(":");
     }
     buffer.append(term.text());
     buffer.append(ToStringUtils.boost(getBoost()));
     return buffer.toString();
   }
-
-  @Override
-  public int hashCode() {
-    final int prime = 31;
-    int result = super.hashCode();
-    result = prime * result + ((term == null) ? 0 : term.hashCode());
-    return result;
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (this == obj)
-      return true;
-    if (!super.equals(obj))
-      return false;
-    if (getClass() != obj.getClass())
-      return false;
-    WildcardQuery other = (WildcardQuery) obj;
-    if (term == null) {
-      if (other.term != null)
-        return false;
-    } else if (!term.equals(other.term))
-      return false;
-    return true;
-  }
-
 }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/WildcardTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/WildcardTermEnum.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/WildcardTermEnum.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/WildcardTermEnum.java Tue Apr  6 19:19:27 2010
@@ -28,7 +28,9 @@ import org.apache.lucene.index.Term;
  * <p>
  * Term enumerations are always ordered by Term.compareTo().  Each term in
  * the enumeration is greater than all that precede it.
+ * @deprecated Please use {@link AutomatonTermsEnum} instead.
  */
+@Deprecated
 public class WildcardTermEnum extends FilteredTermEnum {
   final Term searchTerm;
   final String field;
@@ -91,8 +93,8 @@ public class WildcardTermEnum extends Fi
    * String equality with support for wildcards
    ********************************************/
 
-  public static final char WILDCARD_STRING = '*';
-  public static final char WILDCARD_CHAR = '?';
+  public static final char WILDCARD_STRING = WildcardQuery.WILDCARD_STRING;
+  public static final char WILDCARD_CHAR = WildcardQuery.WILDCARD_CHAR;
 
   /**
    * Determines if a word matches a wildcard pattern.

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java Tue Apr  6 19:19:27 2010
@@ -18,10 +18,11 @@ package org.apache.lucene.search.functio
  */
 
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
 import org.apache.lucene.search.*;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.util.ToStringUtils;
+import org.apache.lucene.util.Bits;
 
 import java.io.IOException;
 import java.util.Set;
@@ -56,7 +57,7 @@ public class ValueSourceQuery extends Qu
     return this;
   }
 
-  /*(non-Javadoc) @see org.apache.lucene.search.Query#extractTerms(java.util.Set) */
+  /*(non-Javadoc) @see org.apache.lucene.search.Query#extractTerms(Set) */
   @Override
   public void extractTerms(Set<Term> terms) {
     // no terms involved here
@@ -127,7 +128,8 @@ public class ValueSourceQuery extends Qu
   private class ValueSourceScorer extends Scorer {
     private final float qWeight;
     private final DocValues vals;
-    private final TermDocs termDocs;
+    private final Bits delDocs;
+    private final int maxDoc;
     private int doc = -1;
 
     // constructor
@@ -136,28 +138,37 @@ public class ValueSourceQuery extends Qu
       qWeight = w.getValue();
       // this is when/where the values are first created.
       vals = valSrc.getValues(reader);
-      termDocs = reader.termDocs(null);
+      delDocs = MultiFields.getDeletedDocs(reader);
+      maxDoc = reader.maxDoc();
     }
 
     @Override
     public int nextDoc() throws IOException {
-      return doc = termDocs.next() ? termDocs.doc() : NO_MORE_DOCS;
+      doc++;
+      while (delDocs != null && doc < maxDoc && delDocs.get(doc)) {
+        doc++;
+      }
+      if (doc == maxDoc) {
+        doc = NO_MORE_DOCS;
+      }
+      return doc;
     }
-    
+
     @Override
     public int docID() {
       return doc;
     }
-    
+
     @Override
     public int advance(int target) throws IOException {
-      return doc = termDocs.skipTo(target) ? termDocs.doc() : NO_MORE_DOCS;
+      doc = target - 1;
+      return nextDoc();
     }
     
     /*(non-Javadoc) @see org.apache.lucene.search.Scorer#score() */
     @Override
     public float score() throws IOException {
-      return qWeight * vals.floatVal(termDocs.doc());
+      return qWeight * vals.floatVal(doc);
     }
   }
 

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java Tue Apr  6 19:19:27 2010
@@ -18,8 +18,8 @@ package org.apache.lucene.search.payload
  */
 
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.DocsAndPositionsEnum;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.TermPositions;
 import org.apache.lucene.search.Searcher;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.Weight;
@@ -30,6 +30,7 @@ import org.apache.lucene.search.spans.Te
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.search.spans.SpanWeight;
 import org.apache.lucene.search.spans.SpanScorer;
+import org.apache.lucene.util.BytesRef;
 
 import java.io.IOException;
 
@@ -80,16 +81,15 @@ public class PayloadTermQuery extends Sp
     }
 
     protected class PayloadTermSpanScorer extends SpanScorer {
-      // TODO: is this the best way to allocate this?
-      protected byte[] payload = new byte[256];
-      protected TermPositions positions;
+      protected BytesRef payload;
       protected float payloadScore;
       protected int payloadsSeen;
+      private final TermSpans termSpans;
 
       public PayloadTermSpanScorer(TermSpans spans, Weight weight,
           Similarity similarity, byte[] norms) throws IOException {
         super(spans, weight, similarity, norms);
-        positions = spans.getPositions();
+        termSpans = spans;
       }
 
       @Override
@@ -115,12 +115,24 @@ public class PayloadTermQuery extends Sp
       }
 
       protected void processPayload(Similarity similarity) throws IOException {
-        if (positions.isPayloadAvailable()) {
-          payload = positions.getPayload(payload, 0);
-          payloadScore = function.currentScore(doc, term.field(),
-              spans.start(), spans.end(), payloadsSeen, payloadScore,
-              similarity.scorePayload(doc, term.field(), spans.start(), spans
-                  .end(), payload, 0, positions.getPayloadLength()));
+        final DocsAndPositionsEnum postings = termSpans.getPostings();
+        if (postings.hasPayload()) {
+          payload = postings.getPayload();
+          if (payload != null) {
+            payloadScore = function.currentScore(doc, term.field(),
+                                                 spans.start(), spans.end(), payloadsSeen, payloadScore,
+                                                 similarity.scorePayload(doc, term.field(), spans.start(),
+                                                                         spans.end(), payload.bytes,
+                                                                         payload.offset,
+                                                                         payload.length));
+          } else {
+            payloadScore = function.currentScore(doc, term.field(),
+                                                 spans.start(), spans.end(), payloadsSeen, payloadScore,
+                                                 similarity.scorePayload(doc, term.field(), spans.start(),
+                                                                         spans.end(), null,
+                                                                         0,
+                                                                         0));
+          }
           payloadsSeen++;
 
         } else {

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java Tue Apr  6 19:19:27 2010
@@ -19,6 +19,9 @@ package org.apache.lucene.search.spans;
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.ToStringUtils;
 
 import java.io.IOException;
@@ -39,7 +42,7 @@ public class SpanTermQuery extends SpanQ
   
   @Override
   public void extractTerms(Set<Term> terms) {
-	  terms.add(term);
+    terms.add(term);
   }
 
   @Override
@@ -80,7 +83,24 @@ public class SpanTermQuery extends SpanQ
 
   @Override
   public Spans getSpans(final IndexReader reader) throws IOException {
-    return new TermSpans(reader.termPositions(term), term);
+    // NOTE: debateably, the caller should never pass in a
+    // multi reader...
+    final BytesRef textBytes = new BytesRef(term.text());
+    final DocsAndPositionsEnum postings = MultiFields.getTermPositionsEnum(reader,
+                                                                           MultiFields.getDeletedDocs(reader),
+                                                                           term.field(),
+                                                                           textBytes);
+
+    if (postings != null) {
+      return new TermSpans(postings, term);
+    } else {
+      if (MultiFields.getTermDocsEnum(reader, MultiFields.getDeletedDocs(reader), term.field(), textBytes) != null) {
+        // term does exist, but has no positions
+        throw new IllegalStateException("field \"" + term.field() + "\" was indexed with Field.omitTermFreqAndPositions=true; cannot run SpanTermQuery (term=" + term.text() + ")");
+      } else {
+        // term does not exist
+        return TermSpans.EMPTY_TERM_SPANS;
+      }
+    }
   }
-
 }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/spans/Spans.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/spans/Spans.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/spans/Spans.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/spans/Spans.java Tue Apr  6 19:19:27 2010
@@ -83,5 +83,4 @@ public abstract class Spans {
    * @return true if there is a payload available at this position that can be loaded
    */
   public abstract boolean isPayloadAvailable();
-
 }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/spans/TermSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/spans/TermSpans.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/spans/TermSpans.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/spans/TermSpans.java Tue Apr  6 19:19:27 2010
@@ -17,7 +17,9 @@ package org.apache.lucene.search.spans;
 
 
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.BytesRef;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -28,49 +30,53 @@ import java.util.Collection;
  * Public for extension only
  */
 public class TermSpans extends Spans {
-  protected TermPositions positions;
-  protected Term term;
+  protected final DocsAndPositionsEnum postings;
+  protected final Term term;
   protected int doc;
   protected int freq;
   protected int count;
   protected int position;
 
-
-  public TermSpans(TermPositions positions, Term term) throws IOException {
-
-    this.positions = positions;
+  public TermSpans(DocsAndPositionsEnum postings, Term term) throws IOException {
+    this.postings = postings;
     this.term = term;
     doc = -1;
   }
 
+  // only for EmptyTermSpans (below)
+  TermSpans() {
+    term = null;
+    postings = null;
+  }
+
   @Override
   public boolean next() throws IOException {
     if (count == freq) {
-      if (!positions.next()) {
-        doc = Integer.MAX_VALUE;
+      if (postings == null) {
         return false;
       }
-      doc = positions.doc();
-      freq = positions.freq();
+      doc = postings.nextDoc();
+      if (doc == DocsAndPositionsEnum.NO_MORE_DOCS) {
+        return false;
+      }
+      freq = postings.freq();
       count = 0;
     }
-    position = positions.nextPosition();
+    position = postings.nextPosition();
     count++;
     return true;
   }
 
   @Override
   public boolean skipTo(int target) throws IOException {
-    if (!positions.skipTo(target)) {
-      doc = Integer.MAX_VALUE;
+    doc = postings.advance(target);
+    if (doc == DocsAndPositionsEnum.NO_MORE_DOCS) {
       return false;
     }
 
-    doc = positions.doc();
-    freq = positions.freq();
+    freq = postings.freq();
     count = 0;
-
-    position = positions.nextPosition();
+    position = postings.nextPosition();
     count++;
 
     return true;
@@ -94,15 +100,21 @@ public class TermSpans extends Spans {
   // TODO: Remove warning after API has been finalized
   @Override
   public Collection<byte[]> getPayload() throws IOException {
-    byte [] bytes = new byte[positions.getPayloadLength()]; 
-    bytes = positions.getPayload(bytes, 0);
+    final BytesRef payload = postings.getPayload();
+    final byte[] bytes;
+    if (payload != null) {
+      bytes = new byte[payload.length];
+      System.arraycopy(payload.bytes, payload.offset, bytes, 0, payload.length);
+    } else {
+      bytes = null;
+    }
     return Collections.singletonList(bytes);
   }
 
   // TODO: Remove warning after API has been finalized
   @Override
   public boolean isPayloadAvailable() {
-    return positions.isPayloadAvailable();
+    return postings.hasPayload();
   }
 
   @Override
@@ -111,8 +123,47 @@ public class TermSpans extends Spans {
             (doc == -1 ? "START" : (doc == Integer.MAX_VALUE) ? "END" : doc + "-" + position);
   }
 
+  public DocsAndPositionsEnum getPostings() {
+    return postings;
+  }
+
+  private static final class EmptyTermSpans extends TermSpans {
+
+    @Override
+    public boolean next() {
+      return false;
+    }
+
+    @Override
+    public boolean skipTo(int target) {
+      return false;
+    }
 
-  public TermPositions getPositions() {
-    return positions;
+    @Override
+    public int doc() {
+      return DocIdSetIterator.NO_MORE_DOCS;
+    }
+    
+    @Override
+    public int start() {
+      return -1;
+    }
+
+    @Override
+    public int end() {
+      return -1;
+    }
+
+    @Override
+    public Collection<byte[]> getPayload() {
+      return null;
+    }
+
+    @Override
+    public boolean isPayloadAvailable() {
+      return false;
+    }
   }
+
+  public static final TermSpans EMPTY_TERM_SPANS = new EmptyTermSpans();
 }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/Directory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/Directory.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/Directory.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/Directory.java Tue Apr  6 19:19:27 2010
@@ -19,15 +19,9 @@ package org.apache.lucene.store;
 
 import java.io.IOException;
 import java.io.Closeable;
-import java.util.Collection;
-import java.util.Collections;
+import java.util.Collection; // for javadocs
 
-import java.util.ArrayList;
-import static java.util.Arrays.asList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import org.apache.lucene.index.IndexFileNameFilter;
+import java.util.Arrays;
 import org.apache.lucene.util.IOUtils;
 
 /** A Directory is a flat list of files.  Files may be written once, when they
@@ -200,14 +194,7 @@ public abstract class Directory implemen
    * @param to        destination directory
    */
   public final void copyTo(Directory to) throws IOException {
-    List<String> filenames = new ArrayList<String>();
-    IndexFileNameFilter filter = IndexFileNameFilter.getFilter();
-
-    for (String name : listAll())
-      if (filter.accept(null, name))
-        filenames.add(name);
-
-    copyTo(to, filenames);
+    copyTo(to, Arrays.asList(listAll()));
   }
 
   /**

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/IndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/IndexInput.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/IndexInput.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/IndexInput.java Tue Apr  6 19:19:27 2010
@@ -17,187 +17,14 @@ package org.apache.lucene.store;
  * limitations under the License.
  */
 
-import java.io.IOException;
 import java.io.Closeable;
-import java.util.Map;
-import java.util.HashMap;
-
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.RamUsageEstimator;
+import java.io.IOException;
 
 /** Abstract base class for input from a file in a {@link Directory}.  A
  * random-access input stream.  Used for all Lucene index input operations.
  * @see Directory
  */
-public abstract class IndexInput implements Cloneable,Closeable {
-  private byte[] bytes;                           // used by readString()
-  private char[] chars;                           // used by readModifiedUTF8String()
-  private boolean preUTF8Strings;                 // true if we are reading old (modified UTF8) string format
-
-  /** Reads and returns a single byte.
-   * @see IndexOutput#writeByte(byte)
-   */
-  public abstract byte readByte() throws IOException;
-
-  /** Reads a specified number of bytes into an array at the specified offset.
-   * @param b the array to read bytes into
-   * @param offset the offset in the array to start storing bytes
-   * @param len the number of bytes to read
-   * @see IndexOutput#writeBytes(byte[],int)
-   */
-  public abstract void readBytes(byte[] b, int offset, int len)
-    throws IOException;
-
-  /** Reads a specified number of bytes into an array at the
-   * specified offset with control over whether the read
-   * should be buffered (callers who have their own buffer
-   * should pass in "false" for useBuffer).  Currently only
-   * {@link BufferedIndexInput} respects this parameter.
-   * @param b the array to read bytes into
-   * @param offset the offset in the array to start storing bytes
-   * @param len the number of bytes to read
-   * @param useBuffer set to false if the caller will handle
-   * buffering.
-   * @see IndexOutput#writeBytes(byte[],int)
-   */
-  public void readBytes(byte[] b, int offset, int len, boolean useBuffer)
-    throws IOException
-  {
-    // Default to ignoring useBuffer entirely
-    readBytes(b, offset, len);
-  }
-
-  /** Reads four bytes and returns an int.
-   * @see IndexOutput#writeInt(int)
-   */
-  public int readInt() throws IOException {
-    return ((readByte() & 0xFF) << 24) | ((readByte() & 0xFF) << 16)
-         | ((readByte() & 0xFF) <<  8) |  (readByte() & 0xFF);
-  }
-
-  /** Reads an int stored in variable-length format.  Reads between one and
-   * five bytes.  Smaller values take fewer bytes.  Negative numbers are not
-   * supported.
-   * @see IndexOutput#writeVInt(int)
-   */
-  public int readVInt() throws IOException {
-    byte b = readByte();
-    int i = b & 0x7F;
-    for (int shift = 7; (b & 0x80) != 0; shift += 7) {
-      b = readByte();
-      i |= (b & 0x7F) << shift;
-    }
-    return i;
-  }
-
-  /** Reads eight bytes and returns a long.
-   * @see IndexOutput#writeLong(long)
-   */
-  public long readLong() throws IOException {
-    return (((long)readInt()) << 32) | (readInt() & 0xFFFFFFFFL);
-  }
-
-  /** Reads a long stored in variable-length format.  Reads between one and
-   * nine bytes.  Smaller values take fewer bytes.  Negative numbers are not
-   * supported. */
-  public long readVLong() throws IOException {
-    byte b = readByte();
-    long i = b & 0x7F;
-    for (int shift = 7; (b & 0x80) != 0; shift += 7) {
-      b = readByte();
-      i |= (b & 0x7FL) << shift;
-    }
-    return i;
-  }
-
-  /** Call this if readString should read characters stored
-   *  in the old modified UTF8 format (length in java chars
-   *  and java's modified UTF8 encoding).  This is used for
-   *  indices written pre-2.4 See LUCENE-510 for details. */
-  public void setModifiedUTF8StringsMode() {
-    preUTF8Strings = true;
-  }
-
-  /** Reads a string.
-   * @see IndexOutput#writeString(String)
-   */
-  public String readString() throws IOException {
-    if (preUTF8Strings)
-      return readModifiedUTF8String();
-    int length = readVInt();
-    if (bytes == null || length > bytes.length) {
-      bytes = new byte[ArrayUtil.oversize(length, 1)];
-    }
-    readBytes(bytes, 0, length);
-    return new String(bytes, 0, length, "UTF-8");
-  }
-
-  private String readModifiedUTF8String() throws IOException {
-    int length = readVInt();
-    if (chars == null || length > chars.length) {
-      chars = new char[ArrayUtil.oversize(length, RamUsageEstimator.NUM_BYTES_CHAR)];
-    }
-    readChars(chars, 0, length);
-    return new String(chars, 0, length);
-  }
-
-  /** Reads Lucene's old "modified UTF-8" encoded
-   *  characters into an array.
-   * @param buffer the array to read characters into
-   * @param start the offset in the array to start storing characters
-   * @param length the number of characters to read
-   * @see IndexOutput#writeChars(String,int,int)
-   * @deprecated -- please use readString or readBytes
-   *                instead, and construct the string
-   *                from those utf8 bytes
-   */
-  @Deprecated
-  public void readChars(char[] buffer, int start, int length)
-       throws IOException {
-    final int end = start + length;
-    for (int i = start; i < end; i++) {
-      byte b = readByte();
-      if ((b & 0x80) == 0)
-	buffer[i] = (char)(b & 0x7F);
-      else if ((b & 0xE0) != 0xE0) {
-	buffer[i] = (char)(((b & 0x1F) << 6)
-		 | (readByte() & 0x3F));
-      } else {
-	buffer[i] = (char)(((b & 0x0F) << 12)
-		| ((readByte() & 0x3F) << 6)
-	        |  (readByte() & 0x3F));
-      }
-    }
-  }
-
-  /**
-   * Expert
-   * 
-   * Similar to {@link #readChars(char[], int, int)} but does not do any conversion operations on the bytes it is reading in.  It still
-   * has to invoke {@link #readByte()} just as {@link #readChars(char[], int, int)} does, but it does not need a buffer to store anything
-   * and it does not have to do any of the bitwise operations, since we don't actually care what is in the byte except to determine
-   * how many more bytes to read
-   * @param length The number of chars to read
-   * @deprecated this method operates on old "modified utf8" encoded
-   *             strings
-   */
-  @Deprecated
-  public void skipChars(int length) throws IOException{
-    for (int i = 0; i < length; i++) {
-      byte b = readByte();
-      if ((b & 0x80) == 0){
-        //do nothing, we only need one byte
-      } else if ((b & 0xE0) != 0xE0) {
-        readByte();//read an additional byte
-      } else {      
-        //read two additional bytes.
-        readByte();
-        readByte();
-      }
-    }
-  }
-  
-
+public abstract class IndexInput extends DataInput implements Cloneable,Closeable {
   /** Closes the stream to further operations. */
   public abstract void close() throws IOException;
 
@@ -214,38 +41,4 @@ public abstract class IndexInput impleme
 
   /** The number of bytes in the file. */
   public abstract long length();
-
-  /** Returns a clone of this stream.
-   *
-   * <p>Clones of a stream access the same data, and are positioned at the same
-   * point as the stream they were cloned from.
-   *
-   * <p>Expert: Subclasses must ensure that clones may be positioned at
-   * different points in the input from each other and from the stream they
-   * were cloned from.
-   */
-  @Override
-  public Object clone() {
-    IndexInput clone = null;
-    try {
-      clone = (IndexInput)super.clone();
-    } catch (CloneNotSupportedException e) {}
-
-    clone.bytes = null;
-    clone.chars = null;
-
-    return clone;
-  }
-
-  public Map<String,String> readStringStringMap() throws IOException {
-    final Map<String,String> map = new HashMap<String,String>();
-    final int count = readInt();
-    for(int i=0;i<count;i++) {
-      final String key = readString();
-      final String val = readString();
-      map.put(key, val);
-    }
-
-    return map;
-  }
-}
+}
\ No newline at end of file

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/IndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/IndexOutput.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/IndexOutput.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/IndexOutput.java Tue Apr  6 19:19:27 2010
@@ -17,168 +17,15 @@ package org.apache.lucene.store;
  * limitations under the License.
  */
 
-import java.io.IOException;
 import java.io.Closeable;
-import java.util.Map;
-import org.apache.lucene.util.UnicodeUtil;
+import java.io.IOException;
 
 /** Abstract base class for output to a file in a Directory.  A random-access
  * output stream.  Used for all Lucene index output operations.
  * @see Directory
  * @see IndexInput
  */
-public abstract class IndexOutput implements Closeable {
-
-  private UnicodeUtil.UTF8Result utf8Result = new UnicodeUtil.UTF8Result();
-
-  /** Writes a single byte.
-   * @see IndexInput#readByte()
-   */
-  public abstract void writeByte(byte b) throws IOException;
-
-  /** Writes an array of bytes.
-   * @param b the bytes to write
-   * @param length the number of bytes to write
-   * @see IndexInput#readBytes(byte[],int,int)
-   */
-  public void writeBytes(byte[] b, int length) throws IOException {
-    writeBytes(b, 0, length);
-  }
-
-  /** Writes an array of bytes.
-   * @param b the bytes to write
-   * @param offset the offset in the byte array
-   * @param length the number of bytes to write
-   * @see IndexInput#readBytes(byte[],int,int)
-   */
-  public abstract void writeBytes(byte[] b, int offset, int length) throws IOException;
-
-  /** Writes an int as four bytes.
-   * @see IndexInput#readInt()
-   */
-  public void writeInt(int i) throws IOException {
-    writeByte((byte)(i >> 24));
-    writeByte((byte)(i >> 16));
-    writeByte((byte)(i >>  8));
-    writeByte((byte) i);
-  }
-
-  /** Writes an int in a variable-length format.  Writes between one and
-   * five bytes.  Smaller values take fewer bytes.  Negative numbers are not
-   * supported.
-   * @see IndexInput#readVInt()
-   */
-  public void writeVInt(int i) throws IOException {
-    while ((i & ~0x7F) != 0) {
-      writeByte((byte)((i & 0x7f) | 0x80));
-      i >>>= 7;
-    }
-    writeByte((byte)i);
-  }
-
-  /** Writes a long as eight bytes.
-   * @see IndexInput#readLong()
-   */
-  public void writeLong(long i) throws IOException {
-    writeInt((int) (i >> 32));
-    writeInt((int) i);
-  }
-
-  /** Writes an long in a variable-length format.  Writes between one and five
-   * bytes.  Smaller values take fewer bytes.  Negative numbers are not
-   * supported.
-   * @see IndexInput#readVLong()
-   */
-  public void writeVLong(long i) throws IOException {
-    while ((i & ~0x7F) != 0) {
-      writeByte((byte)((i & 0x7f) | 0x80));
-      i >>>= 7;
-    }
-    writeByte((byte)i);
-  }
-
-  /** Writes a string.
-   * @see IndexInput#readString()
-   */
-  public void writeString(String s) throws IOException {
-    UnicodeUtil.UTF16toUTF8(s, 0, s.length(), utf8Result);
-    writeVInt(utf8Result.length);
-    writeBytes(utf8Result.result, 0, utf8Result.length);
-  }
-
-  /** Writes a sub sequence of characters from s as the old
-   *  format (modified UTF-8 encoded bytes).
-   * @param s the source of the characters
-   * @param start the first character in the sequence
-   * @param length the number of characters in the sequence
-   * @deprecated -- please pre-convert to utf8 bytes
-   * instead or use {@link #writeString}
-   */
-  @Deprecated
-  public void writeChars(String s, int start, int length)
-       throws IOException {
-    final int end = start + length;
-    for (int i = start; i < end; i++) {
-      final int code = s.charAt(i);
-      if (code >= 0x01 && code <= 0x7F)
-	writeByte((byte)code);
-      else if (((code >= 0x80) && (code <= 0x7FF)) || code == 0) {
-	writeByte((byte)(0xC0 | (code >> 6)));
-	writeByte((byte)(0x80 | (code & 0x3F)));
-      } else {
-	writeByte((byte)(0xE0 | (code >>> 12)));
-	writeByte((byte)(0x80 | ((code >> 6) & 0x3F)));
-	writeByte((byte)(0x80 | (code & 0x3F)));
-      }
-    }
-  }
-
-  /** Writes a sub sequence of characters from char[] as
-   *  the old format (modified UTF-8 encoded bytes).
-   * @param s the source of the characters
-   * @param start the first character in the sequence
-   * @param length the number of characters in the sequence
-   * @deprecated -- please pre-convert to utf8 bytes instead or use {@link #writeString}
-   */
-  @Deprecated
-  public void writeChars(char[] s, int start, int length)
-    throws IOException {
-    final int end = start + length;
-    for (int i = start; i < end; i++) {
-      final int code = s[i];
-      if (code >= 0x01 && code <= 0x7F)
-	writeByte((byte)code);
-      else if (((code >= 0x80) && (code <= 0x7FF)) || code == 0) {
-	writeByte((byte)(0xC0 | (code >> 6)));
-	writeByte((byte)(0x80 | (code & 0x3F)));
-      } else {
-	writeByte((byte)(0xE0 | (code >>> 12)));
-	writeByte((byte)(0x80 | ((code >> 6) & 0x3F)));
-	writeByte((byte)(0x80 | (code & 0x3F)));
-      }
-    }
-  }
-
-  private static int COPY_BUFFER_SIZE = 16384;
-  private byte[] copyBuffer;
-
-  /** Copy numBytes bytes from input to ourself. */
-  public void copyBytes(IndexInput input, long numBytes) throws IOException {
-    assert numBytes >= 0: "numBytes=" + numBytes;
-    long left = numBytes;
-    if (copyBuffer == null)
-      copyBuffer = new byte[COPY_BUFFER_SIZE];
-    while(left > 0) {
-      final int toCopy;
-      if (left > COPY_BUFFER_SIZE)
-        toCopy = COPY_BUFFER_SIZE;
-      else
-        toCopy = (int) left;
-      input.readBytes(copyBuffer, 0, toCopy);
-      writeBytes(copyBuffer, 0, toCopy);
-      left -= toCopy;
-    }
-  }
+public abstract class IndexOutput extends DataOutput implements Closeable {
 
   /** Forces any buffered output to be written. */
   public abstract void flush() throws IOException;
@@ -211,16 +58,4 @@ public abstract class IndexOutput implem
    * @param length file length
    */
   public void setLength(long length) throws IOException {}
-
-  public void writeStringStringMap(Map<String,String> map) throws IOException {
-    if (map == null) {
-      writeInt(0);
-    } else {
-      writeInt(map.size());
-      for(final Map.Entry<String, String> entry: map.entrySet()) {
-        writeString(entry.getKey());
-        writeString(entry.getValue());
-      }
-    }
-  }
 }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/ArrayUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/ArrayUtil.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/ArrayUtil.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/ArrayUtil.java Tue Apr  6 19:19:27 2010
@@ -232,6 +232,29 @@ public final class ArrayUtil {
       return currentSize;
   }
 
+  public static short[] grow(short[] array, int minSize) {
+    if (array.length < minSize) {
+      short[] newArray = new short[oversize(minSize, RamUsageEstimator.NUM_BYTES_SHORT)];
+      System.arraycopy(array, 0, newArray, 0, array.length);
+      return newArray;
+    } else
+      return array;
+  }
+
+  public static short[] grow(short[] array) {
+    return grow(array, 1 + array.length);
+  }
+
+  public static short[] shrink(short[] array, int targetSize) {
+    final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_SHORT);
+    if (newSize != array.length) {
+      short[] newArray = new short[newSize];
+      System.arraycopy(array, 0, newArray, 0, newSize);
+      return newArray;
+    } else
+      return array;
+  }
+
   public static int[] grow(int[] array, int minSize) {
     if (array.length < minSize) {
       int[] newArray = new int[oversize(minSize, RamUsageEstimator.NUM_BYTES_INT)];

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/BitVector.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/BitVector.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/BitVector.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/BitVector.java Tue Apr  6 19:19:27 2010
@@ -32,7 +32,7 @@ import org.apache.lucene.store.IndexOutp
   <li>store and load, as bit set or d-gaps, depending on sparseness;</li> 
   </ul>
   */
-public final class BitVector implements Cloneable {
+public final class BitVector implements Cloneable, Bits {
 
   private byte[] bits;
   private int size;
@@ -110,6 +110,11 @@ public final class BitVector implements 
     return size;
   }
 
+  // @Override -- not until Java 1.6
+  public int length() {
+    return size;
+  }
+
   /** Returns the total number of one bits in this vector.  This is efficiently
     computed and cached, so that, if the vector is not changed, no
     recomputation is done for repeated calls. */

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/NumericUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/NumericUtils.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/NumericUtils.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/NumericUtils.java Tue Apr  6 19:19:27 2010
@@ -22,6 +22,8 @@ import org.apache.lucene.document.Numeri
 import org.apache.lucene.search.NumericRangeQuery; // for javadocs
 import org.apache.lucene.search.NumericRangeFilter; // for javadocs
 
+// TODO: Remove the commented out methods before release!
+
 /**
  * This is a helper class to generate prefix-encoded representations for numerical values
  * and supplies converters to represent float/double values as sortable integers/longs.
@@ -32,10 +34,10 @@ import org.apache.lucene.search.NumericR
  * more exactly. This reduces the number of terms dramatically.
  *
  * <p>This class generates terms to achieve this: First the numerical integer values need to
- * be converted to strings. For that integer values (32 bit or 64 bit) are made unsigned
- * and the bits are converted to ASCII chars with each 7 bit. The resulting string is
- * sortable like the original integer value. Each value is also prefixed
- * (in the first char) by the <code>shift</code> value (number of bits removed) used
+ * be converted to bytes. For that integer values (32 bit or 64 bit) are made unsigned
+ * and the bits are converted to ASCII chars with each 7 bit. The resulting byte[] is
+ * sortable like the original integer value (even using UTF-8 sort order). Each value is also
+ * prefixed (in the first char) by the <code>shift</code> value (number of bits removed) used
  * during encoding.
  *
  * <p>To also index floating point numbers, this class supplies two methods to convert them
@@ -51,13 +53,12 @@ import org.apache.lucene.search.NumericR
  * {@link NumericRangeQuery} and {@link NumericRangeFilter} implement the query part
  * for the same data types.
  *
- * <p>This class can also be used, to generate lexicographically sortable (according
- * {@link String#compareTo(String)}) representations of numeric data types for other
- * usages (e.g. sorting).
- *
- * @lucene.experimental
+ * <p>This class can also be used, to generate lexicographically sortable (according to
+ * {@link BytesRef#getUTF8SortedAsUTF16Comparator()}) representations of numeric data
+ * types for other usages (e.g. sorting).
  *
- * @since 2.9
+ * @lucene.internal
+ * @since 2.9, API changed non backwards-compliant in 3.1
  */
 public final class NumericUtils {
 
@@ -70,126 +71,150 @@ public final class NumericUtils {
   public static final int PRECISION_STEP_DEFAULT = 4;
   
   /**
-   * Expert: Longs are stored at lower precision by shifting off lower bits. The shift count is
-   * stored as <code>SHIFT_START_LONG+shift</code> in the first character
+   * Longs are stored at lower precision by shifting off lower bits. The shift count is
+   * stored as <code>SHIFT_START_LONG+shift</code> in the first byte
    */
-  public static final char SHIFT_START_LONG = (char)0x20;
+  public static final byte SHIFT_START_LONG = 0x20;
 
   /**
-   * Expert: The maximum term length (used for <code>char[]</code> buffer size)
+   * The maximum term length (used for <code>byte[]</code> buffer size)
    * for encoding <code>long</code> values.
-   * @see #longToPrefixCoded(long,int,char[])
+   * @see #longToPrefixCoded(long,int,BytesRef)
    */
   public static final int BUF_SIZE_LONG = 63/7 + 2;
 
   /**
-   * Expert: Integers are stored at lower precision by shifting off lower bits. The shift count is
-   * stored as <code>SHIFT_START_INT+shift</code> in the first character
+   * Integers are stored at lower precision by shifting off lower bits. The shift count is
+   * stored as <code>SHIFT_START_INT+shift</code> in the first byte
    */
-  public static final char SHIFT_START_INT  = (char)0x60;
+  public static final byte SHIFT_START_INT  = 0x60;
 
   /**
-   * Expert: The maximum term length (used for <code>char[]</code> buffer size)
+   * The maximum term length (used for <code>byte[]</code> buffer size)
    * for encoding <code>int</code> values.
-   * @see #intToPrefixCoded(int,int,char[])
+   * @see #intToPrefixCoded(int,int,BytesRef)
    */
   public static final int BUF_SIZE_INT = 31/7 + 2;
 
   /**
-   * Expert: Returns prefix coded bits after reducing the precision by <code>shift</code> bits.
+   * Returns prefix coded bits after reducing the precision by <code>shift</code> bits.
    * This is method is used by {@link NumericTokenStream}.
    * @param val the numeric value
    * @param shift how many bits to strip from the right
-   * @param buffer that will contain the encoded chars, must be at least of {@link #BUF_SIZE_LONG}
-   * length
-   * @return number of chars written to buffer
+   * @param bytes will contain the encoded value
+   * @return the hash code for indexing (TermsHash)
    */
-  public static int longToPrefixCoded(final long val, final int shift, final char[] buffer) {
+  public static int longToPrefixCoded(final long val, final int shift, final BytesRef bytes) {
     if (shift>63 || shift<0)
       throw new IllegalArgumentException("Illegal shift value, must be 0..63");
-    int nChars = (63-shift)/7 + 1, len = nChars+1;
-    buffer[0] = (char)(SHIFT_START_LONG + shift);
+    if (bytes.bytes == null) {
+      bytes.bytes = new byte[NumericUtils.BUF_SIZE_LONG];
+    } else if (bytes.bytes.length < NumericUtils.BUF_SIZE_LONG) {
+      bytes.grow(NumericUtils.BUF_SIZE_LONG);
+    }
+    int hash, nChars = (63-shift)/7 + 1;
+    bytes.length = nChars+1;
+    bytes.bytes[0] = (byte) (hash = (SHIFT_START_LONG + shift));
     long sortableBits = val ^ 0x8000000000000000L;
     sortableBits >>>= shift;
-    while (nChars>=1) {
-      // Store 7 bits per character for good efficiency when UTF-8 encoding.
-      // The whole number is right-justified so that lucene can prefix-encode
-      // the terms more efficiently.
-      buffer[nChars--] = (char)(sortableBits & 0x7f);
+    while (nChars > 0) {
+      // Store 7 bits per byte for compatibility
+      // with UTF-8 encoding of terms
+      bytes.bytes[nChars--] = (byte)(sortableBits & 0x7f);
       sortableBits >>>= 7;
     }
-    return len;
+    // calculate hash
+    for (int i = 1; i < bytes.length; i++) {
+      hash = 31*hash + bytes.bytes[i];
+    }
+    return hash;
   }
 
   /**
-   * Expert: Returns prefix coded bits after reducing the precision by <code>shift</code> bits.
+   * Returns prefix coded bits after reducing the precision by <code>shift</code> bits.
    * This is method is used by {@link LongRangeBuilder}.
    * @param val the numeric value
    * @param shift how many bits to strip from the right
-   */
+   * @deprecated This method is no longer needed!
+   *
+  @Deprecated
   public static String longToPrefixCoded(final long val, final int shift) {
-    final char[] buffer = new char[BUF_SIZE_LONG];
-    final int len = longToPrefixCoded(val, shift, buffer);
-    return new String(buffer, 0, len);
-  }
+    final BytesRef buffer = new BytesRef(BUF_SIZE_LONG);
+    longToPrefixCoded(val, shift, buffer);
+    return buffer.utf8ToString();
+  }*/
 
   /**
    * This is a convenience method, that returns prefix coded bits of a long without
    * reducing the precision. It can be used to store the full precision value as a
    * stored field in index.
    * <p>To decode, use {@link #prefixCodedToLong}.
-   */
+   * @deprecated This method is no longer needed!
+   *
+  @Deprecated
   public static String longToPrefixCoded(final long val) {
     return longToPrefixCoded(val, 0);
-  }
+  }*/
   
   /**
-   * Expert: Returns prefix coded bits after reducing the precision by <code>shift</code> bits.
+   * Returns prefix coded bits after reducing the precision by <code>shift</code> bits.
    * This is method is used by {@link NumericTokenStream}.
    * @param val the numeric value
    * @param shift how many bits to strip from the right
-   * @param buffer that will contain the encoded chars, must be at least of {@link #BUF_SIZE_INT}
-   * length
-   * @return number of chars written to buffer
+   * @param bytes will contain the encoded value
+   * @return the hash code for indexing (TermsHash)
    */
-  public static int intToPrefixCoded(final int val, final int shift, final char[] buffer) {
+  public static int intToPrefixCoded(final int val, final int shift, final BytesRef bytes) {
     if (shift>31 || shift<0)
       throw new IllegalArgumentException("Illegal shift value, must be 0..31");
-    int nChars = (31-shift)/7 + 1, len = nChars+1;
-    buffer[0] = (char)(SHIFT_START_INT + shift);
+    if (bytes.bytes == null) {
+      bytes.bytes = new byte[NumericUtils.BUF_SIZE_INT];
+    } else if (bytes.bytes.length < NumericUtils.BUF_SIZE_INT) {
+      bytes.grow(NumericUtils.BUF_SIZE_INT);
+    }
+    int hash, nChars = (31-shift)/7 + 1;
+    bytes.length = nChars+1;
+    bytes.bytes[0] = (byte) (hash = (SHIFT_START_INT + shift));
     int sortableBits = val ^ 0x80000000;
     sortableBits >>>= shift;
-    while (nChars>=1) {
-      // Store 7 bits per character for good efficiency when UTF-8 encoding.
-      // The whole number is right-justified so that lucene can prefix-encode
-      // the terms more efficiently.
-      buffer[nChars--] = (char)(sortableBits & 0x7f);
+    while (nChars > 0) {
+      // Store 7 bits per byte for compatibility
+      // with UTF-8 encoding of terms
+      bytes.bytes[nChars--] = (byte)(sortableBits & 0x7f);
       sortableBits >>>= 7;
     }
-    return len;
+    // calculate hash
+    for (int i = 1; i < bytes.length; i++) {
+      hash = 31*hash + bytes.bytes[i];
+    }
+    return hash;
   }
 
   /**
-   * Expert: Returns prefix coded bits after reducing the precision by <code>shift</code> bits.
+   * Returns prefix coded bits after reducing the precision by <code>shift</code> bits.
    * This is method is used by {@link IntRangeBuilder}.
    * @param val the numeric value
    * @param shift how many bits to strip from the right
-   */
+   * @deprecated This method is no longer needed!
+   *
+  @Deprecated
   public static String intToPrefixCoded(final int val, final int shift) {
-    final char[] buffer = new char[BUF_SIZE_INT];
-    final int len = intToPrefixCoded(val, shift, buffer);
-    return new String(buffer, 0, len);
-  }
+    final BytesRef buffer = new BytesRef(BUF_SIZE_INT);
+    intToPrefixCoded(val, shift, buffer);
+    return buffer.utf8ToString();
+  }*/
 
   /**
    * This is a convenience method, that returns prefix coded bits of an int without
    * reducing the precision. It can be used to store the full precision value as a
    * stored field in index.
    * <p>To decode, use {@link #prefixCodedToInt}.
-   */
+   * @deprecated This method is no longer needed!
+   *
+  @Deprecated
   public static String intToPrefixCoded(final int val) {
     return intToPrefixCoded(val, 0);
-  }
+  }*/
 
   /**
    * Returns a long from prefixCoded characters.
@@ -198,51 +223,97 @@ public final class NumericUtils {
    * @throws NumberFormatException if the supplied string is
    * not correctly prefix encoded.
    * @see #longToPrefixCoded(long)
-   */
+   * @deprecated This method is no longer needed!
+   *
+  @Deprecated
   public static long prefixCodedToLong(final String prefixCoded) {
-    final int shift = prefixCoded.charAt(0)-SHIFT_START_LONG;
-    if (shift>63 || shift<0)
-      throw new NumberFormatException("Invalid shift value in prefixCoded string (is encoded value really a LONG?)");
+    return prefixCodedToLong(new BytesRef(prefixCoded));
+  }*/
+
+  /**
+   * Returns the shift value from a prefix encoded {@code long}.
+   * @throws NumberFormatException if the supplied {@link BytesRef} is
+   * not correctly prefix encoded.
+   */
+  public static int getPrefixCodedLongShift(final BytesRef val) {
+    final int shift = val.bytes[val.offset] - SHIFT_START_LONG;
+    if (shift > 63 || shift < 0)
+      throw new NumberFormatException("Invalid shift value in prefixCoded bytes (is encoded value really an INT?)");
+    return shift;
+  }
+
+  /**
+   * Returns the shift value from a prefix encoded {@code int}.
+   * @throws NumberFormatException if the supplied {@link BytesRef} is
+   * not correctly prefix encoded.
+   */
+  public static int getPrefixCodedIntShift(final BytesRef val) {
+    final int shift = val.bytes[val.offset] - SHIFT_START_INT;
+    if (shift > 31 || shift < 0)
+      throw new NumberFormatException("Invalid shift value in prefixCoded bytes (is encoded value really an INT?)");
+    return shift;
+  }
+
+  /**
+   * Returns a long from prefixCoded bytes.
+   * Rightmost bits will be zero for lower precision codes.
+   * This method can be used to decode a term's value.
+   * @throws NumberFormatException if the supplied {@link BytesRef} is
+   * not correctly prefix encoded.
+   * @see #longToPrefixCoded(long,int,BytesRef)
+   */
+  public static long prefixCodedToLong(final BytesRef val) {
     long sortableBits = 0L;
-    for (int i=1, len=prefixCoded.length(); i<len; i++) {
+    for (int i=val.offset+1, limit=val.offset+val.length; i<limit; i++) {
       sortableBits <<= 7;
-      final char ch = prefixCoded.charAt(i);
-      if (ch>0x7f) {
+      final byte b = val.bytes[i];
+      if (b < 0) {
         throw new NumberFormatException(
-          "Invalid prefixCoded numerical value representation (char "+
-          Integer.toHexString(ch)+" at position "+i+" is invalid)"
+          "Invalid prefixCoded numerical value representation (byte "+
+          Integer.toHexString(b&0xff)+" at position "+(i-val.offset)+" is invalid)"
         );
       }
-      sortableBits |= ch;
+      sortableBits |= b;
     }
-    return (sortableBits << shift) ^ 0x8000000000000000L;
+    return (sortableBits << getPrefixCodedLongShift(val)) ^ 0x8000000000000000L;
   }
 
   /**
    * Returns an int from prefixCoded characters.
    * Rightmost bits will be zero for lower precision codes.
-   * This method can be used to decode e.g. a stored field.
+   * This method can be used to decode a term's value.
    * @throws NumberFormatException if the supplied string is
    * not correctly prefix encoded.
    * @see #intToPrefixCoded(int)
-   */
+   * @deprecated This method is no longer needed!
+   *
+  @Deprecated
   public static int prefixCodedToInt(final String prefixCoded) {
-    final int shift = prefixCoded.charAt(0)-SHIFT_START_INT;
-    if (shift>31 || shift<0)
-      throw new NumberFormatException("Invalid shift value in prefixCoded string (is encoded value really an INT?)");
+    return prefixCodedToInt(new BytesRef(prefixCoded));
+  }*/
+
+  /**
+   * Returns an int from prefixCoded bytes.
+   * Rightmost bits will be zero for lower precision codes.
+   * This method can be used to decode a term's value.
+   * @throws NumberFormatException if the supplied {@link BytesRef} is
+   * not correctly prefix encoded.
+   * @see #intToPrefixCoded(int,int,BytesRef)
+   */
+  public static int prefixCodedToInt(final BytesRef val) {
     int sortableBits = 0;
-    for (int i=1, len=prefixCoded.length(); i<len; i++) {
+    for (int i=val.offset+1, limit=val.offset+val.length; i<limit; i++) {
       sortableBits <<= 7;
-      final char ch = prefixCoded.charAt(i);
-      if (ch>0x7f) {
+      final byte b = val.bytes[i];
+      if (b < 0) {
         throw new NumberFormatException(
-          "Invalid prefixCoded numerical value representation (char "+
-          Integer.toHexString(ch)+" at position "+i+" is invalid)"
+          "Invalid prefixCoded numerical value representation (byte "+
+          Integer.toHexString(b&0xff)+" at position "+(i-val.offset)+" is invalid)"
         );
       }
-      sortableBits |= ch;
+      sortableBits |= b;
     }
-    return (sortableBits << shift) ^ 0x80000000;
+    return (sortableBits << getPrefixCodedIntShift(val)) ^ 0x80000000;
   }
 
   /**
@@ -261,10 +332,12 @@ public final class NumericUtils {
   /**
    * Convenience method: this just returns:
    *   longToPrefixCoded(doubleToSortableLong(val))
-   */
+   * @deprecated This method is no longer needed!
+   *
+  @Deprecated
   public static String doubleToPrefixCoded(double val) {
     return longToPrefixCoded(doubleToSortableLong(val));
-  }
+  }*/
 
   /**
    * Converts a sortable <code>long</code> back to a <code>double</code>.
@@ -278,10 +351,12 @@ public final class NumericUtils {
   /**
    * Convenience method: this just returns:
    *    sortableLongToDouble(prefixCodedToLong(val))
-   */
+   * @deprecated This method is no longer needed!
+   *
+  @Deprecated
   public static double prefixCodedToDouble(String val) {
     return sortableLongToDouble(prefixCodedToLong(val));
-  }
+  }*/
 
   /**
    * Converts a <code>float</code> value to a sortable signed <code>int</code>.
@@ -299,10 +374,12 @@ public final class NumericUtils {
   /**
    * Convenience method: this just returns:
    *   intToPrefixCoded(floatToSortableInt(val))
-   */
+   * @deprecated This method is no longer needed!
+   *
+  @Deprecated
   public static String floatToPrefixCoded(float val) {
     return intToPrefixCoded(floatToSortableInt(val));
-  }
+  }*/
 
   /**
    * Converts a sortable <code>int</code> back to a <code>float</code>.
@@ -316,16 +393,18 @@ public final class NumericUtils {
   /**
    * Convenience method: this just returns:
    *    sortableIntToFloat(prefixCodedToInt(val))
-   */
+   * @deprecated This method is no longer needed!
+   *
+  @Deprecated
   public static float prefixCodedToFloat(String val) {
     return sortableIntToFloat(prefixCodedToInt(val));
-  }
+  }*/
 
   /**
-   * Expert: Splits a long range recursively.
+   * Splits a long range recursively.
    * You may implement a builder that adds clauses to a
    * {@link org.apache.lucene.search.BooleanQuery} for each call to its
-   * {@link LongRangeBuilder#addRange(String,String)}
+   * {@link LongRangeBuilder#addRange(BytesRef,BytesRef)}
    * method.
    * <p>This method is used by {@link NumericRangeQuery}.
    */
@@ -336,10 +415,10 @@ public final class NumericUtils {
   }
   
   /**
-   * Expert: Splits an int range recursively.
+   * Splits an int range recursively.
    * You may implement a builder that adds clauses to a
    * {@link org.apache.lucene.search.BooleanQuery} for each call to its
-   * {@link IntRangeBuilder#addRange(String,String)}
+   * {@link IntRangeBuilder#addRange(BytesRef,BytesRef)}
    * method.
    * <p>This method is used by {@link NumericRangeQuery}.
    */
@@ -412,10 +491,10 @@ public final class NumericUtils {
   }
 
   /**
-   * Expert: Callback for {@link #splitLongRange}.
+   * Callback for {@link #splitLongRange}.
    * You need to overwrite only one of the methods.
-   * <p><font color="red"><b>NOTE:</b> This is a very low-level interface,
-   * the method signatures may change in later versions.</font>
+   * @lucene.internal
+   * @since 2.9, API changed non backwards-compliant in 3.1
    */
   public static abstract class LongRangeBuilder {
     
@@ -423,7 +502,7 @@ public final class NumericUtils {
      * Overwrite this method, if you like to receive the already prefix encoded range bounds.
      * You can directly build classical (inclusive) range queries from them.
      */
-    public void addRange(String minPrefixCoded, String maxPrefixCoded) {
+    public void addRange(BytesRef minPrefixCoded, BytesRef maxPrefixCoded) {
       throw new UnsupportedOperationException();
     }
     
@@ -432,16 +511,19 @@ public final class NumericUtils {
      * You can use this for e.g. debugging purposes (print out range bounds).
      */
     public void addRange(final long min, final long max, final int shift) {
-      addRange(longToPrefixCoded(min, shift), longToPrefixCoded(max, shift));
+      final BytesRef minBytes = new BytesRef(BUF_SIZE_LONG), maxBytes = new BytesRef(BUF_SIZE_LONG);
+      longToPrefixCoded(min, shift, minBytes);
+      longToPrefixCoded(max, shift, maxBytes);
+      addRange(minBytes, maxBytes);
     }
   
   }
   
   /**
-   * Expert: Callback for {@link #splitIntRange}.
+   * Callback for {@link #splitIntRange}.
    * You need to overwrite only one of the methods.
-   * <p><font color="red"><b>NOTE:</b> This is a very low-level interface,
-   * the method signatures may change in later versions.</font>
+   * @lucene.internal
+   * @since 2.9, API changed non backwards-compliant in 3.1
    */
   public static abstract class IntRangeBuilder {
     
@@ -449,7 +531,7 @@ public final class NumericUtils {
      * Overwrite this method, if you like to receive the already prefix encoded range bounds.
      * You can directly build classical range (inclusive) queries from them.
      */
-    public void addRange(String minPrefixCoded, String maxPrefixCoded) {
+    public void addRange(BytesRef minPrefixCoded, BytesRef maxPrefixCoded) {
       throw new UnsupportedOperationException();
     }
     
@@ -458,7 +540,10 @@ public final class NumericUtils {
      * You can use this for e.g. debugging purposes (print out range bounds).
      */
     public void addRange(final int min, final int max, final int shift) {
-      addRange(intToPrefixCoded(min, shift), intToPrefixCoded(max, shift));
+      final BytesRef minBytes = new BytesRef(BUF_SIZE_INT), maxBytes = new BytesRef(BUF_SIZE_INT);
+      intToPrefixCoded(min, shift, minBytes);
+      intToPrefixCoded(max, shift, maxBytes);
+      addRange(minBytes, maxBytes);
     }
   
   }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/OpenBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/OpenBitSet.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/OpenBitSet.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/OpenBitSet.java Tue Apr  6 19:19:27 2010
@@ -75,7 +75,7 @@ Test system: AMD Opteron, 64 bit linux, 
 </table>
  */
 
-public class OpenBitSet extends DocIdSet implements Cloneable, Serializable {
+public class OpenBitSet extends DocIdSet implements Bits, Cloneable, Serializable {
   protected long[] bits;
   protected int wlen;   // number of words (elements) used in the array
 
@@ -132,6 +132,11 @@ public class OpenBitSet extends DocIdSet
       return capacity();
   }
 
+  // @Override -- not until Java 1.6
+  public int length() {
+    return bits.length << 6;
+  }
+
   /** Returns true if there are no set bits */
   public boolean isEmpty() { return cardinality()==0; }
 

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/RamUsageEstimator.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/RamUsageEstimator.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/RamUsageEstimator.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/RamUsageEstimator.java Tue Apr  6 19:19:27 2010
@@ -37,6 +37,16 @@ import java.util.*;
  * @lucene.internal
  */
 public final class RamUsageEstimator {
+
+  public final static int NUM_BYTES_SHORT = 2;
+  public final static int NUM_BYTES_INT = 4;
+  public final static int NUM_BYTES_LONG = 8;
+  public final static int NUM_BYTES_FLOAT = 4;
+  public final static int NUM_BYTES_DOUBLE = 8;
+  public final static int NUM_BYTES_OBJ_HEADER = 8;
+  public final static int NUM_BYTES_OBJ_REF = Constants.JRE_IS_64BIT ? 8 : 4;
+  public final static int NUM_BYTES_ARRAY_HEADER = NUM_BYTES_OBJ_HEADER + NUM_BYTES_INT + NUM_BYTES_OBJ_REF;
+
   private MemoryModel memoryModel;
 
   private final Map<Object,Object> seen;
@@ -47,11 +57,6 @@ public final class RamUsageEstimator {
 
   public final static int NUM_BYTES_OBJECT_REF = Constants.JRE_IS_64BIT ? 8 : 4;
   public final static int NUM_BYTES_CHAR = 2;
-  public final static int NUM_BYTES_SHORT = 2;
-  public final static int NUM_BYTES_INT = 4;
-  public final static int NUM_BYTES_LONG = 8;
-  public final static int NUM_BYTES_FLOAT = 4;
-  public final static int NUM_BYTES_DOUBLE = 8;
 
   private boolean checkInterned;
 

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/ReaderUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/ReaderUtil.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/ReaderUtil.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/ReaderUtil.java Tue Apr  6 19:19:27 2010
@@ -19,6 +19,7 @@ package org.apache.lucene.util;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.io.IOException;
 
 import org.apache.lucene.index.IndexReader;
 
@@ -29,22 +30,85 @@ import org.apache.lucene.index.IndexRead
  */
 public class ReaderUtil {
 
+  public static class Slice {
+    public static final Slice[] EMPTY_ARRAY = new Slice[0];
+    public final int start;
+    public final int length;
+    public final int readerIndex;
+
+    public Slice(int start, int length, int readerIndex) {
+      this.start = start;
+      this.length = length;
+      this.readerIndex = readerIndex;
+    }
+
+    public String toString() {
+      return "slice start=" + start + " length=" + length;
+    }
+  }
+
   /**
-   * Gathers sub-readers from reader into a List.
+   * Gathers sub-readers from reader into a List.  See
+   * {@link Gather} for are more general way to gather
+   * whatever you need to, per reader.
+   *
+   * @lucene.experimental
    * 
    * @param allSubReaders
    * @param reader
    */
-  public static void gatherSubReaders(List<IndexReader> allSubReaders, IndexReader reader) {
-    IndexReader[] subReaders = reader.getSequentialSubReaders();
-    if (subReaders == null) {
-      // Add the reader itself, and do not recurse
-      allSubReaders.add(reader);
-    } else {
-      for (int i = 0; i < subReaders.length; i++) {
-        gatherSubReaders(allSubReaders, subReaders[i]);
+
+  public static void gatherSubReaders(final List<IndexReader> allSubReaders, IndexReader reader) {
+    try {
+      new Gather(reader) {
+        @Override
+          protected void add(int base, IndexReader r) {
+          allSubReaders.add(r);
+        }
+      }.run();
+    } catch (IOException ioe) {
+      // won't happen
+      throw new RuntimeException(ioe);
+    }
+  }
+
+  /** Recursively visits all sub-readers of a reader.  You
+   *  should subclass this and override the add method to
+   *  gather what you need.
+   *
+   * @lucene.experimental */
+  public static abstract class Gather {
+    private final IndexReader topReader;
+
+    public Gather(IndexReader r) {
+      topReader = r;
+    }
+
+    public int run() throws IOException {
+      return run(0, topReader);
+    }
+
+    public int run(int docBase) throws IOException {
+      return run(docBase, topReader);
+    }
+
+    private int run(int base, IndexReader reader) throws IOException {
+      IndexReader[] subReaders = reader.getSequentialSubReaders();
+      if (subReaders == null) {
+        // atomic reader
+        add(base, reader);
+        base += reader.maxDoc();
+      } else {
+        // composite reader
+        for (int i = 0; i < subReaders.length; i++) {
+          base = run(base, subReaders[i]);
+        }
       }
+
+      return base;
     }
+
+    protected abstract void add(int base, IndexReader r) throws IOException;
   }
 
   /**



Mime
View raw message