lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From busc...@apache.org
Subject svn commit: r1058718 [7/18] - in /lucene/dev/branches/realtime_search: ./ lucene/ lucene/contrib/ lucene/contrib/ant/src/java/org/apache/lucene/ant/ lucene/contrib/ant/src/test/org/apache/lucene/ant/ lucene/contrib/benchmark/ lucene/contrib/demo/src/ja...
Date Thu, 13 Jan 2011 19:53:39 GMT
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java Thu Jan 13 19:53:21 2011
@@ -19,12 +19,12 @@ package org.apache.lucene.search;
 
 import java.io.IOException;
 
-import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.util.OpenBitSet;
 import org.apache.lucene.util.Bits;
 
@@ -105,8 +105,9 @@ public class MultiTermQueryWrapperFilter
    * results.
    */
   @Override
-  public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
-    final Fields fields = MultiFields.getFields(reader);
+  public DocIdSet getDocIdSet(AtomicReaderContext context) throws IOException {
+    final IndexReader reader = context.reader;
+    final Fields fields = reader.fields();
     if (fields == null) {
       // reader has no fields
       return DocIdSet.EMPTY_DOCIDSET;
@@ -122,9 +123,9 @@ public class MultiTermQueryWrapperFilter
     assert termsEnum != null;
     if (termsEnum.next() != null) {
       // fill into a OpenBitSet
-      final OpenBitSet bitSet = new OpenBitSet(reader.maxDoc());
+      final OpenBitSet bitSet = new OpenBitSet(context.reader.maxDoc());
       int termCount = 0;
-      final Bits delDocs = MultiFields.getDeletedDocs(reader);
+      final Bits delDocs = reader.getDeletedDocs();
       DocsEnum docsEnum = null;
       do {
         termCount++;

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/PhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/PhraseQuery.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/PhraseQuery.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/PhraseQuery.java Thu Jan 13 19:53:21 2011
@@ -21,10 +21,12 @@ import java.io.IOException;
 import java.util.Set;
 import java.util.ArrayList;
 
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.DocsAndPositionsEnum;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.Explanation.IDFExplanation;
+import org.apache.lucene.search.Weight.ScorerContext;
 import org.apache.lucene.util.ToStringUtils;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.Bits;
@@ -143,7 +145,7 @@ public class PhraseQuery extends Query {
     private float queryWeight;
     private IDFExplanation idfExp;
 
-    public PhraseWeight(Searcher searcher)
+    public PhraseWeight(IndexSearcher searcher)
       throws IOException {
       this.similarity = getSimilarity(searcher);
 
@@ -174,10 +176,10 @@ public class PhraseQuery extends Query {
     }
 
     @Override
-    public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException {
+    public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
       if (terms.size() == 0)			  // optimize zero-term case
         return null;
-
+      final IndexReader reader = context.reader;
       PostingsAndFreq[] postingsFreqs = new PostingsAndFreq[terms.size()];
       final Bits delDocs = reader.getDeletedDocs();
       for (int i = 0; i < terms.size(); i++) {
@@ -206,7 +208,7 @@ public class PhraseQuery extends Query {
 
       if (slop == 0) {				  // optimize exact case
         ExactPhraseScorer s = new ExactPhraseScorer(this, postingsFreqs, similarity,
-                                                    reader.norms(field));
+            reader.norms(field));
         if (s.noDocs) {
           return null;
         } else {
@@ -215,12 +217,12 @@ public class PhraseQuery extends Query {
       } else {
         return
           new SloppyPhraseScorer(this, postingsFreqs, similarity, slop,
-                                 reader.norms(field));
+              reader.norms(field));
       }
     }
 
     @Override
-    public Explanation explain(IndexReader reader, int doc)
+    public Explanation explain(AtomicReaderContext context, int doc)
       throws IOException {
 
       Explanation result = new Explanation();
@@ -267,7 +269,7 @@ public class PhraseQuery extends Query {
       fieldExpl.setDescription("fieldWeight("+field+":"+query+" in "+doc+
                                "), product of:");
 
-      Scorer scorer = scorer(reader, true, false);
+      Scorer scorer = scorer(context, ScorerContext.def());
       if (scorer == null) {
         return new Explanation(0.0f, "no matching docs");
       }
@@ -287,7 +289,7 @@ public class PhraseQuery extends Query {
       fieldExpl.addDetail(idfExpl);
 
       Explanation fieldNormExpl = new Explanation();
-      byte[] fieldNorms = reader.norms(field);
+      byte[] fieldNorms = context.reader.norms(field);
       float fieldNorm =
         fieldNorms!=null ? similarity.decodeNormValue(fieldNorms[doc]) : 1.0f;
       fieldNormExpl.setValue(fieldNorm);
@@ -311,7 +313,7 @@ public class PhraseQuery extends Query {
   }
 
   @Override
-  public Weight createWeight(Searcher searcher) throws IOException {
+  public Weight createWeight(IndexSearcher searcher) throws IOException {
     if (terms.size() == 1) {			  // optimize one-term case
       Term term = terms.get(0);
       Query termQuery = new TermQuery(term);

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/PositiveScoresOnlyCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/PositiveScoresOnlyCollector.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/PositiveScoresOnlyCollector.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/PositiveScoresOnlyCollector.java Thu Jan 13 19:53:21 2011
@@ -19,7 +19,7 @@ package org.apache.lucene.search;
 
 import java.io.IOException;
 
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
 
 /**
  * A {@link Collector} implementation which wraps another
@@ -43,8 +43,8 @@ public class PositiveScoresOnlyCollector
   }
 
   @Override
-  public void setNextReader(IndexReader reader, int docBase) throws IOException {
-    c.setNextReader(reader, docBase);
+  public void setNextReader(AtomicReaderContext context) throws IOException {
+    c.setNextReader(context);
   }
 
   @Override

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/Query.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/Query.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/Query.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/Query.java Thu Jan 13 19:53:21 2011
@@ -19,8 +19,6 @@ package org.apache.lucene.search;
 
 import java.io.IOException;
 
-import java.util.HashSet;
-
 import java.util.Set;
 
 import org.apache.lucene.index.IndexReader;
@@ -89,14 +87,14 @@ public abstract class Query implements j
    * <p>
    * Only implemented by primitive queries, which re-write to themselves.
    */
-  public Weight createWeight(Searcher searcher) throws IOException {
+  public Weight createWeight(IndexSearcher searcher) throws IOException {
     throw new UnsupportedOperationException();
   }
 
   /**
    * Expert: Constructs and initializes a Weight for a top-level query.
    */
-  public Weight weight(Searcher searcher) throws IOException {
+  public Weight weight(IndexSearcher searcher) throws IOException {
     Query query = searcher.rewrite(this);
     Weight weight = query.createWeight(searcher);
     float sum = weight.sumOfSquaredWeights();
@@ -116,52 +114,6 @@ public abstract class Query implements j
     return this;
   }
   
-
-  /** Expert: called when re-writing queries under MultiSearcher.
-   *
-   * Create a single query suitable for use by all subsearchers (in 1-1
-   * correspondence with queries). This is an optimization of the OR of
-   * all queries. We handle the common optimization cases of equal
-   * queries and overlapping clauses of boolean OR queries (as generated
-   * by MultiTermQuery.rewrite()).
-   * Be careful overriding this method as queries[0] determines which
-   * method will be called and is not necessarily of the same type as
-   * the other queries.
-  */
-  public Query combine(Query[] queries) {
-    HashSet<Query> uniques = new HashSet<Query>();
-    for (int i = 0; i < queries.length; i++) {
-      Query query = queries[i];
-      BooleanClause[] clauses = null;
-      // check if we can split the query into clauses
-      boolean splittable = (query instanceof BooleanQuery);
-      if(splittable){
-        BooleanQuery bq = (BooleanQuery) query;
-        splittable = bq.isCoordDisabled();
-        clauses = bq.getClauses();
-        for (int j = 0; splittable && j < clauses.length; j++) {
-          splittable = (clauses[j].getOccur() == BooleanClause.Occur.SHOULD);
-        }
-      }
-      if(splittable){
-        for (int j = 0; j < clauses.length; j++) {
-          uniques.add(clauses[j].getQuery());
-        }
-      } else {
-        uniques.add(query);
-      }
-    }
-    // optimization: if we have just one query, just return it
-    if(uniques.size() == 1){
-        return uniques.iterator().next();
-    }
-    BooleanQuery result = new BooleanQuery(true);
-    for (final Query query : uniques)
-      result.add(query, BooleanClause.Occur.SHOULD);
-    return result;
-  }
-  
-
   /**
    * Expert: adds all terms occurring in this query to the terms set. Only
    * works if this query is in its {@link #rewrite rewritten} form.
@@ -174,35 +126,11 @@ public abstract class Query implements j
   }
   
 
-
-  /** Expert: merges the clauses of a set of BooleanQuery's into a single
-   * BooleanQuery.
-   *
-   *<p>A utility for use by {@link #combine(Query[])} implementations.
-   */
-  public static Query mergeBooleanQueries(BooleanQuery... queries) {
-    HashSet<BooleanClause> allClauses = new HashSet<BooleanClause>();
-    for (BooleanQuery booleanQuery : queries) {
-      for (BooleanClause clause : booleanQuery) {
-        allClauses.add(clause);
-      }
-    }
-
-    boolean coordDisabled =
-      queries.length==0? false : queries[0].isCoordDisabled();
-    BooleanQuery result = new BooleanQuery(coordDisabled);
-    for(BooleanClause clause2 : allClauses) {
-      result.add(clause2);
-    }
-    return result;
-  }
-  
-
   /** Expert: Returns the Similarity implementation to be used for this query.
    * Subclasses may override this method to specify their own Similarity
    * implementation, perhaps one that delegates through that of the Searcher.
    * By default the Searcher's Similarity implementation is returned.*/
-  public Similarity getSimilarity(Searcher searcher) {
+  public Similarity getSimilarity(IndexSearcher searcher) {
     return searcher.getSimilarity();
   }
 

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java Thu Jan 13 19:53:21 2011
@@ -19,8 +19,8 @@ package org.apache.lucene.search;
 
 import java.io.IOException;
 
-
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.search.Weight.ScorerContext;
 
 /** 
  * Constrains search results to only match those which also match a provided
@@ -48,12 +48,15 @@ public class QueryWrapperFilter extends 
   }
 
   @Override
-  public DocIdSet getDocIdSet(final IndexReader reader) throws IOException {
-    final Weight weight = query.weight(new IndexSearcher(reader));
+  public DocIdSet getDocIdSet(final AtomicReaderContext context) throws IOException {
+    // get a private context that is used to rewrite, createWeight and score eventually
+    assert context.reader.getTopReaderContext().isAtomic;
+    final AtomicReaderContext privateContext = (AtomicReaderContext) context.reader.getTopReaderContext();
+    final Weight weight = query.weight(new IndexSearcher(privateContext));
     return new DocIdSet() {
       @Override
       public DocIdSetIterator iterator() throws IOException {
-        return weight.scorer(reader, true, false);
+        return weight.scorer(privateContext, ScorerContext.def());
       }
       @Override
       public boolean isCacheable() { return false; }

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java Thu Jan 13 19:53:21 2011
@@ -43,7 +43,7 @@ public class ScoreCachingWrappingScorer 
   }
 
   @Override
-  protected boolean score(Collector collector, int max, int firstDocID) throws IOException {
+  public boolean score(Collector collector, int max, int firstDocID) throws IOException {
     return scorer.score(collector, max, firstDocID);
   }
 

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/ScoreDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/ScoreDoc.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/ScoreDoc.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/ScoreDoc.java Thu Jan 13 19:53:21 2011
@@ -24,7 +24,7 @@ public class ScoreDoc implements java.io
   public float score;
 
   /** Expert: A hit document's number.
-   * @see Searcher#doc(int)
+   * @see IndexSearcher#doc(int)
    */
   public int doc;
 

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/Scorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/Scorer.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/Scorer.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/Scorer.java Thu Jan 13 19:53:21 2011
@@ -90,7 +90,7 @@ public abstract class Scorer extends Doc
    *          this method.
    * @return true if more matching documents may remain.
    */
-  protected boolean score(Collector collector, int max, int firstDocID) throws IOException {
+  public boolean score(Collector collector, int max, int firstDocID) throws IOException {
     collector.setScorer(this);
     int doc = firstDocID;
     while (doc < max) {

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/ScoringRewrite.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/ScoringRewrite.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/ScoringRewrite.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/ScoringRewrite.java Thu Jan 13 19:53:21 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.search;
 import java.io.IOException;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermState;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.MultiTermQuery.RewriteMethod;
 
@@ -27,6 +28,7 @@ import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.ByteBlockPool;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefHash;
+import org.apache.lucene.util.PerReaderTermState;
 import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util.BytesRefHash.DirectBytesStartArray;
 
@@ -53,8 +55,9 @@ public abstract class ScoringRewrite<Q e
     }
     
     @Override
-    protected void addClause(BooleanQuery topLevel, Term term, int docCount, float boost) {
-      final TermQuery tq = new TermQuery(term, docCount);
+    protected void addClause(BooleanQuery topLevel, Term term, int docCount,
+        float boost, PerReaderTermState states) {
+      final TermQuery tq = new TermQuery(term, states);
       tq.setBoost(boost);
       topLevel.add(tq, BooleanClause.Occur.SHOULD);
     }
@@ -114,13 +117,13 @@ public abstract class ScoringRewrite<Q e
     final int size = col.terms.size();
     if (size > 0) {
       final int sort[] = col.terms.sort(col.termsEnum.getComparator());
-      final int[] docFreq = col.array.docFreq;
       final float[] boost = col.array.boost;
+      final PerReaderTermState[] termStates = col.array.termState;
       for (int i = 0; i < size; i++) {
         final int pos = sort[i];
         final Term term = placeholderTerm.createTerm(col.terms.get(pos, new BytesRef()));
-        assert reader.docFreq(term) == docFreq[pos];
-        addClause(result, term, docFreq[pos], query.getBoost() * boost[pos]);
+        assert reader.docFreq(term) == termStates[pos].docFreq();
+        addClause(result, term, termStates[pos].docFreq(), query.getBoost() * boost[pos], termStates[pos]);
       }
     }
     query.incTotalNumberOfTerms(size);
@@ -143,15 +146,17 @@ public abstract class ScoringRewrite<Q e
     @Override
     public boolean collect(BytesRef bytes) throws IOException {
       final int e = terms.add(bytes);
+      final TermState state = termsEnum.termState();
+      assert state != null; 
       if (e < 0 ) {
         // duplicate term: update docFreq
         final int pos = (-e)-1;
-        array.docFreq[pos] += termsEnum.docFreq();
+        array.termState[pos].register(state, readerContext.ord, termsEnum.docFreq());
         assert array.boost[pos] == boostAtt.getBoost() : "boost should be equal in all segment TermsEnums";
       } else {
         // new entry: we populate the entry initially
-        array.docFreq[e] = termsEnum.docFreq();
         array.boost[e] = boostAtt.getBoost();
+        array.termState[e] = new PerReaderTermState(topReaderContext, state, readerContext.ord, termsEnum.docFreq());
         ScoringRewrite.this.checkMaxClauseCount(terms.size());
       }
       return true;
@@ -160,8 +165,8 @@ public abstract class ScoringRewrite<Q e
   
   /** Special implementation of BytesStartArray that keeps parallel arrays for boost and docFreq */
   static final class TermFreqBoostByteStart extends DirectBytesStartArray  {
-    int[] docFreq;
     float[] boost;
+    PerReaderTermState[] termState;
     
     public TermFreqBoostByteStart(int initSize) {
       super(initSize);
@@ -171,24 +176,28 @@ public abstract class ScoringRewrite<Q e
     public int[] init() {
       final int[] ord = super.init();
       boost = new float[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_FLOAT)];
-      docFreq = new int[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_INT)];
-      assert boost.length >= ord.length && docFreq.length >= ord.length;
+      termState = new PerReaderTermState[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+      assert termState.length >= ord.length && boost.length >= ord.length;
       return ord;
     }
 
     @Override
     public int[] grow() {
       final int[] ord = super.grow();
-      docFreq = ArrayUtil.grow(docFreq, ord.length);
       boost = ArrayUtil.grow(boost, ord.length);
-      assert boost.length >= ord.length && docFreq.length >= ord.length;
+      if (termState.length < ord.length) {
+        PerReaderTermState[] tmpTermState = new PerReaderTermState[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+        System.arraycopy(termState, 0, tmpTermState, 0, termState.length);
+        termState = tmpTermState;
+      }     
+      assert termState.length >= ord.length && boost.length >= ord.length;
       return ord;
     }
 
     @Override
     public int[] clear() {
      boost = null;
-     docFreq = null;
+     termState = null;
      return super.clear();
     }
     

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/Similarity.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/Similarity.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/Similarity.java Thu Jan 13 19:53:21 2011
@@ -462,12 +462,14 @@ import org.apache.lucene.util.SmallFloat
  *        {@link org.apache.lucene.document.Fieldable#setBoost(float) field.setBoost()}
  *        before adding the field to a document.
  *        </li>
- *        <li>{@link #lengthNorm(String, int) <b>lengthNorm</b>(field)} - computed
+ *        <li><b>lengthNorm</b> - computed
  *        when the document is added to the index in accordance with the number of tokens
  *        of this field in the document, so that shorter fields contribute more to the score.
  *        LengthNorm is computed by the Similarity class in effect at indexing.
  *        </li>
  *      </ul>
+ *      The {@link #computeNorm} method is responsible for
+ *      combining all of these factors into a single float.
  *
  *      <p>
  *      When a document is added to the index, all the above factors are multiplied.
@@ -480,7 +482,7 @@ import org.apache.lucene.util.SmallFloat
  *            norm(t,d) &nbsp; = &nbsp;
  *            {@link org.apache.lucene.document.Document#getBoost() doc.getBoost()}
  *            &nbsp;&middot;&nbsp;
- *            {@link #lengthNorm(String, int) lengthNorm(field)}
+ *            lengthNorm
  *            &nbsp;&middot;&nbsp;
  *          </td>
  *          <td valign="bottom" align="center" rowspan="1">
@@ -521,8 +523,8 @@ import org.apache.lucene.util.SmallFloat
  * </ol>
  *
  * @see #setDefault(Similarity)
- * @see org.apache.lucene.index.IndexWriter#setSimilarity(Similarity)
- * @see Searcher#setSimilarity(Similarity)
+ * @see org.apache.lucene.index.IndexWriterConfig#setSimilarity(Similarity)
+ * @see IndexSearcher#setSimilarity(Similarity)
  */
 public abstract class Similarity implements Serializable {
   
@@ -535,8 +537,8 @@ public abstract class Similarity impleme
   /** Set the default Similarity implementation used by indexing and search
    * code.
    *
-   * @see Searcher#setSimilarity(Similarity)
-   * @see org.apache.lucene.index.IndexWriter#setSimilarity(Similarity)
+   * @see IndexSearcher#setSimilarity(Similarity)
+   * @see org.apache.lucene.index.IndexWriterConfig#setSimilarity(Similarity)
    */
   public static void setDefault(Similarity similarity) {
     Similarity.defaultImpl = similarity;
@@ -547,8 +549,8 @@ public abstract class Similarity impleme
    *
    * <p>This is initially an instance of {@link DefaultSimilarity}.
    *
-   * @see Searcher#setSimilarity(Similarity)
-   * @see org.apache.lucene.index.IndexWriter#setSimilarity(Similarity)
+   * @see IndexSearcher#setSimilarity(Similarity)
+   * @see org.apache.lucene.index.IndexWriterConfig#setSimilarity(Similarity)
    */
   public static Similarity getDefault() {
     return Similarity.defaultImpl;
@@ -570,12 +572,23 @@ public abstract class Similarity impleme
   }
 
   /**
-   * Compute the normalization value for a field, given the accumulated
+   * Computes the normalization value for a field, given the accumulated
    * state of term processing for this field (see {@link FieldInvertState}).
    * 
    * <p>Implementations should calculate a float value based on the field
    * state and then return that value.
    *
+   * <p>Matches in longer fields are less precise, so implementations of this
+   * method usually return smaller values when <code>state.getLength()</code> is large,
+   * and larger values when <code>state.getLength()</code> is small.
+   * 
+   * <p>Note that the return values are computed under 
+   * {@link org.apache.lucene.index.IndexWriter#addDocument(org.apache.lucene.document.Document)} 
+   * and then stored using
+   * {@link #encodeNormValue(float)}.  
+   * Thus they have limited precision, and documents
+   * must be re-indexed if this method is altered.
+   *
    * <p>For backward compatibility this method by default calls
    * {@link #lengthNorm(String, int)} passing
    * {@link FieldInvertState#getLength()} as the second argument, and
@@ -587,9 +600,7 @@ public abstract class Similarity impleme
    * @param state current processing state for this field
    * @return the calculated float norm
    */
-  public float computeNorm(String field, FieldInvertState state) {
-    return (state.getBoost() * lengthNorm(field, state.getLength()));
-  }
+  public abstract float computeNorm(String field, FieldInvertState state);
   
   /** Computes the normalization value for a field given the total number of
    * terms contained in a field.  These values, together with field boosts, are
@@ -613,8 +624,13 @@ public abstract class Similarity impleme
    * @return a normalization factor for hits on this field of this document
    *
    * @see org.apache.lucene.document.Field#setBoost(float)
+   *
+   * @deprecated Please override computeNorm instead
    */
-  public abstract float lengthNorm(String fieldName, int numTokens);
+  @Deprecated
+  public final float lengthNorm(String fieldName, int numTokens) {
+    throw new UnsupportedOperationException("please use computeNorm instead");
+  }
 
   /** Computes the normalization value for a query given the sum of the squared
    * weights of each of the query terms.  This value is multiplied into the
@@ -641,7 +657,6 @@ public abstract class Similarity impleme
    * are rounded down to the largest representable value.  Positive values too
    * small to represent are rounded up to the smallest positive representable
    * value.
-   *
    * @see org.apache.lucene.document.Field#setBoost(float)
    * @see org.apache.lucene.util.SmallFloat
    */
@@ -709,11 +724,11 @@ public abstract class Similarity impleme
    * idf(docFreq, searcher.maxDoc());
    * </pre>
    * 
-   * Note that {@link Searcher#maxDoc()} is used instead of
+   * Note that {@link IndexSearcher#maxDoc()} is used instead of
    * {@link org.apache.lucene.index.IndexReader#numDocs() IndexReader#numDocs()} because also 
-   * {@link Searcher#docFreq(Term)} is used, and when the latter 
-   * is inaccurate, so is {@link Searcher#maxDoc()}, and in the same direction.
-   * In addition, {@link Searcher#maxDoc()} is more efficient to compute
+   * {@link IndexSearcher#docFreq(Term)} is used, and when the latter 
+   * is inaccurate, so is {@link IndexSearcher#maxDoc()}, and in the same direction.
+   * In addition, {@link IndexSearcher#maxDoc()} is more efficient to compute
    *   
    * @param term the term in question
    * @param searcher the document collection being searched
@@ -722,7 +737,7 @@ public abstract class Similarity impleme
              and an explanation for the term.
    * @throws IOException
    */
-  public IDFExplanation idfExplain(final Term term, final Searcher searcher, int docFreq) throws IOException {
+  public IDFExplanation idfExplain(final Term term, final IndexSearcher searcher, int docFreq) throws IOException {
     final int df = docFreq;
     final int max = searcher.maxDoc();
     final float idf = idf(df, max);
@@ -736,16 +751,16 @@ public abstract class Similarity impleme
         public float getIdf() {
           return idf;
         }};
-   }
+  }
 
   /**
    * This method forwards to {@link
-   * #idfExplain(Term,Searcher,int)} by passing
+   * #idfExplain(Term,IndexSearcher,int)} by passing
    * <code>searcher.docFreq(term)</code> as the docFreq.
    */
-  public IDFExplanation idfExplain(final Term term, final Searcher searcher) throws IOException {
+  public IDFExplanation idfExplain(final Term term, final IndexSearcher searcher) throws IOException {
     return idfExplain(term, searcher, searcher.docFreq(term));
-   }
+  }
 
   /**
    * Computes a score factor for a phrase.
@@ -761,7 +776,7 @@ public abstract class Similarity impleme
    *         for each term.
    * @throws IOException
    */
-  public IDFExplanation idfExplain(Collection<Term> terms, Searcher searcher) throws IOException {
+  public IDFExplanation idfExplain(Collection<Term> terms, IndexSearcher searcher) throws IOException {
     final int max = searcher.maxDoc();
     float idf = 0.0f;
     final StringBuilder exp = new StringBuilder();

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/Sort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/Sort.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/Sort.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/Sort.java Thu Jan 13 19:53:21 2011
@@ -103,7 +103,7 @@ implements Serializable {
   /**
    * Represents sorting by computed relevance. Using this sort criteria returns
    * the same results as calling
-   * {@link Searcher#search(Query,int) Searcher#search()}without a sort criteria,
+   * {@link IndexSearcher#search(Query,int) IndexSearcher#search()}without a sort criteria,
    * only with slightly more overhead.
    */
   public static final Sort RELEVANCE = new Sort();
@@ -116,7 +116,7 @@ implements Serializable {
 
   /**
    * Sorts by computed relevance. This is the same sort criteria as calling
-   * {@link Searcher#search(Query,int) Searcher#search()}without a sort criteria,
+   * {@link IndexSearcher#search(Query,int) IndexSearcher#search()}without a sort criteria,
    * only with slightly more overhead.
    */
   public Sort() {

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/SpanQueryFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/SpanQueryFilter.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/SpanQueryFilter.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/SpanQueryFilter.java Thu Jan 13 19:53:21 2011
@@ -17,6 +17,7 @@ package org.apache.lucene.search;
 
 
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
 import org.apache.lucene.search.spans.SpanQuery;
 import org.apache.lucene.search.spans.Spans;
 import org.apache.lucene.util.OpenBitSet;
@@ -52,8 +53,8 @@ public class SpanQueryFilter extends Spa
   }
 
   @Override
-  public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
-    SpanFilterResult result = bitSpans(reader);
+  public DocIdSet getDocIdSet(AtomicReaderContext context) throws IOException {
+    SpanFilterResult result = bitSpans(context.reader);
     return result.getDocIdSet();
   }
 

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TermCollectingRewrite.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TermCollectingRewrite.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TermCollectingRewrite.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TermCollectingRewrite.java Thu Jan 13 19:53:21 2011
@@ -18,8 +18,6 @@ package org.apache.lucene.search;
  */
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Comparator;
 
 import org.apache.lucene.index.Fields;
@@ -27,25 +25,33 @@ import org.apache.lucene.index.IndexRead
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader.ReaderContext;
 import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.PerReaderTermState;
 import org.apache.lucene.util.ReaderUtil;
 
 abstract class TermCollectingRewrite<Q extends Query> extends MultiTermQuery.RewriteMethod {
   
+  
   /** Return a suitable top-level Query for holding all expanded terms. */
   protected abstract Q getTopLevelQuery() throws IOException;
   
   /** Add a MultiTermQuery term to the top-level query */
-  protected abstract void addClause(Q topLevel, Term term, int docCount, float boost) throws IOException;
+  protected final void addClause(Q topLevel, Term term, int docCount, float boost) throws IOException {
+    addClause(topLevel, term, docCount, boost, null);
+  }
+  
+  protected abstract void addClause(Q topLevel, Term term, int docCount, float boost, PerReaderTermState states) throws IOException;
+
   
   protected final void collectTerms(IndexReader reader, MultiTermQuery query, TermCollector collector) throws IOException {
-    final List<IndexReader> subReaders = new ArrayList<IndexReader>();
-    ReaderUtil.gatherSubReaders(subReaders, reader);
+    ReaderContext topReaderContext = reader.getTopReaderContext();
     Comparator<BytesRef> lastTermComp = null;
-    
-    for (IndexReader r : subReaders) {
-      final Fields fields = r.fields();
+    final AtomicReaderContext[] leaves = ReaderUtil.leaves(topReaderContext);
+    for (AtomicReaderContext context : leaves) {
+      final Fields fields = context.reader.fields();
       if (fields == null) {
         // reader has no fields
         continue;
@@ -68,11 +74,10 @@ abstract class TermCollectingRewrite<Q e
       if (lastTermComp != null && newTermComp != null && newTermComp != lastTermComp)
         throw new RuntimeException("term comparator should not change between segments: "+lastTermComp+" != "+newTermComp);
       lastTermComp = newTermComp;
-      
+      collector.setReaderContext(topReaderContext, context);
       collector.setNextEnum(termsEnum);
       BytesRef bytes;
       while ((bytes = termsEnum.next()) != null) {
-        termsEnum.cacheCurrentTerm();
         if (!collector.collect(bytes))
           return; // interrupt whole term collection, so also don't iterate other subReaders
       }
@@ -80,6 +85,14 @@ abstract class TermCollectingRewrite<Q e
   }
   
   protected static abstract class TermCollector {
+    
+    protected AtomicReaderContext readerContext;
+    protected ReaderContext topReaderContext;
+
+    public void setReaderContext(ReaderContext topReaderContext, AtomicReaderContext readerContext) {
+      this.readerContext = readerContext;
+      this.topReaderContext = topReaderContext;
+    }
     /** attributes used for communication with the enum */
     public final AttributeSource attributes = new AttributeSource();
   

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TermQuery.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TermQuery.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TermQuery.java Thu Jan 13 19:53:21 2011
@@ -21,9 +21,17 @@ import java.io.IOException;
 import java.util.Set;
 
 import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.Term;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.TermState;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader.ReaderContext;
+import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Explanation.IDFExplanation;
+import org.apache.lucene.search.Weight.ScorerContext;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.PerReaderTermState;
+import org.apache.lucene.util.ReaderUtil;
 import org.apache.lucene.util.ToStringUtils;
 
 /** A Query that matches documents containing a term.
@@ -31,18 +39,22 @@ import org.apache.lucene.util.ToStringUt
   */
 public class TermQuery extends Query {
   private final Term term;
-  private final int docFreq;
+  private int docFreq;
+  private transient PerReaderTermState perReaderTermState;
 
   private class TermWeight extends Weight {
     private final Similarity similarity;
     private float value;
-    private float idf;
+    private final float idf;
     private float queryNorm;
     private float queryWeight;
-    private IDFExplanation idfExp;
+    private final IDFExplanation idfExp;
+    private transient PerReaderTermState termStates;
 
-    public TermWeight(Searcher searcher)
+    public TermWeight(IndexSearcher searcher, PerReaderTermState termStates, int docFreq)
       throws IOException {
+      assert termStates != null : "PerReaderTermState must not be null";
+      this.termStates = termStates;
       this.similarity = getSimilarity(searcher);
       if (docFreq != -1) {
         idfExp = similarity.idfExplain(term, searcher, docFreq);
@@ -75,21 +87,31 @@ public class TermQuery extends Query {
     }
 
     @Override
-    public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException {
-      DocsEnum docs = reader.termDocsEnum(reader.getDeletedDocs(),
-                                          term.field(),
-                                          term.bytes());
-
-      if (docs == null) {
+    public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
+      final String field = term.field();
+      final IndexReader reader = context.reader;
+      assert termStates.topReaderContext == ReaderUtil.getTopLevelContext(context) : "The top-reader used to create Weight is not the same as the current reader's top-reader";
+      final TermState state = termStates
+          .get(context.ord);
+      if (state == null) { // term is not present in that reader
+        assert termNotInReader(reader, field, term.bytes()) : "no termstate found but term exists in reader";
         return null;
       }
-
-      return new TermScorer(this, docs, similarity, reader.norms(term.field()));
+      final DocsEnum docs = reader.termDocsEnum(reader.getDeletedDocs(), field, term.bytes(), state);
+      assert docs != null;
+      return new TermScorer(this, docs, similarity, context.reader.norms(field));
+    }
+    
+    private boolean termNotInReader(IndexReader reader, String field, BytesRef bytes) throws IOException {
+      // only called from assert
+      final Terms terms = reader.terms(field);
+      return terms == null || terms.docFreq(bytes) == 0;
     }
-
+    
     @Override
-    public Explanation explain(IndexReader reader, int doc)
+    public Explanation explain(AtomicReaderContext context, int doc)
       throws IOException {
+      final IndexReader reader = context.reader;
 
       ComplexExplanation result = new ComplexExplanation();
       result.setDescription("weight("+getQuery()+" in "+doc+"), product of:");
@@ -138,7 +160,7 @@ public class TermQuery extends Query {
       fieldExpl.addDetail(expl);
 
       Explanation fieldNormExpl = new Explanation();
-      byte[] fieldNorms = reader.norms(field);
+      final byte[] fieldNorms = reader.norms(field);
       float fieldNorm =
         fieldNorms!=null ? similarity.decodeNormValue(fieldNorms[doc]) : 1.0f;
       fieldNormExpl.setValue(fieldNorm);
@@ -174,14 +196,39 @@ public class TermQuery extends Query {
   public TermQuery(Term t, int docFreq) {
     term = t;
     this.docFreq = docFreq;
+    perReaderTermState = null;
+  }
+  
+  /** Expert: constructs a TermQuery that will use the
+   *  provided docFreq instead of looking up the docFreq
+   *  against the searcher. */
+  public TermQuery(Term t, PerReaderTermState states) {
+    assert states != null;
+    term = t;
+    docFreq = states.docFreq();
+    perReaderTermState = states;
   }
 
   /** Returns the term of this query. */
   public Term getTerm() { return term; }
 
   @Override
-  public Weight createWeight(Searcher searcher) throws IOException {
-    return new TermWeight(searcher);
+  public Weight createWeight(IndexSearcher searcher) throws IOException {
+    final ReaderContext context = searcher.getTopReaderContext();
+    final int weightDocFreq;
+    final PerReaderTermState termState;
+    if (perReaderTermState == null || perReaderTermState.topReaderContext != context) {
+      // make TermQuery single-pass if we don't have a PRTS or if the context differs!
+      termState = PerReaderTermState.build(context, term, true); // cache term lookups!
+      // we must not ignore the given docFreq - if set use the given value
+      weightDocFreq = docFreq == -1 ? termState.docFreq() : docFreq;
+    } else {
+     // PRTS was pre-build for this IS
+     termState = this.perReaderTermState;
+     weightDocFreq = docFreq;
+    }
+    
+    return new TermWeight(searcher, termState, weightDocFreq);
   }
 
   @Override

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TermScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TermScorer.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TermScorer.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TermScorer.java Thu Jan 13 19:53:21 2011
@@ -77,7 +77,7 @@ final class TermScorer extends Scorer {
 
   // firstDocID is ignored since nextDoc() sets 'doc'
   @Override
-  protected boolean score(Collector c, int end, int firstDocID) throws IOException {
+  public boolean score(Collector c, int end, int firstDocID) throws IOException {
     c.setScorer(this);
     while (doc < end) {                           // for docs in window
       c.collect(doc);                      // collect score

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TimeLimitingCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TimeLimitingCollector.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TimeLimitingCollector.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TimeLimitingCollector.java Thu Jan 13 19:53:21 2011
@@ -19,7 +19,7 @@ package org.apache.lucene.search;
 
 import java.io.IOException;
 
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
 import org.apache.lucene.util.ThreadInterruptedException;
 
 /**
@@ -213,9 +213,9 @@ public class TimeLimitingCollector exten
   }
   
   @Override
-  public void setNextReader(IndexReader reader, int base) throws IOException {
-    collector.setNextReader(reader, base);
-    this.docBase = base;
+  public void setNextReader(AtomicReaderContext context) throws IOException {
+    collector.setNextReader(context);
+    this.docBase = context.docBase;
   }
   
   @Override

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TopDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TopDocs.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TopDocs.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TopDocs.java Thu Jan 13 19:53:21 2011
@@ -18,8 +18,8 @@ package org.apache.lucene.search;
  */
 
 /** Represents hits returned by {@link
- * Searcher#search(Query,Filter,int)} and {@link
- * Searcher#search(Query,int)}. */
+ * IndexSearcher#search(Query,Filter,int)} and {@link
+ * IndexSearcher#search(Query,int)}. */
 public class TopDocs implements java.io.Serializable {
   /** The total number of hits for the query.
   */

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TopFieldCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TopFieldCollector.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TopFieldCollector.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TopFieldCollector.java Thu Jan 13 19:53:21 2011
@@ -19,7 +19,7 @@ package org.apache.lucene.search;
 
 import java.io.IOException;
 
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
 import org.apache.lucene.search.FieldValueHitQueue.Entry;
 import org.apache.lucene.util.PriorityQueue;
 
@@ -92,9 +92,9 @@ public abstract class TopFieldCollector 
     }
     
     @Override
-    public void setNextReader(IndexReader reader, int docBase) throws IOException {
-      this.docBase = docBase;
-      queue.setComparator(0, comparator.setNextReader(reader, docBase));
+    public void setNextReader(AtomicReaderContext context) throws IOException {
+      this.docBase = context.docBase;
+      queue.setComparator(0, comparator.setNextReader(context));
       comparator = queue.firstComparator;
     }
     
@@ -447,10 +447,10 @@ public abstract class TopFieldCollector 
     }
 
     @Override
-    public void setNextReader(IndexReader reader, int docBase) throws IOException {
-      this.docBase = docBase;
+    public void setNextReader(AtomicReaderContext context) throws IOException {
+      docBase = context.docBase;
       for (int i = 0; i < comparators.length; i++) {
-        queue.setComparator(i, comparators[i].setNextReader(reader, docBase));
+        queue.setComparator(i, comparators[i].setNextReader(context));
       }
     }
 

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TopFieldDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TopFieldDocs.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TopFieldDocs.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TopFieldDocs.java Thu Jan 13 19:53:21 2011
@@ -19,7 +19,7 @@ package org.apache.lucene.search;
 
 
 /** Represents hits returned by {@link
- * Searcher#search(Query,Filter,int,Sort)}.
+ * IndexSearcher#search(Query,Filter,int,Sort)}.
  */
 public class TopFieldDocs
 extends TopDocs {

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TopScoreDocCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TopScoreDocCollector.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TopScoreDocCollector.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TopScoreDocCollector.java Thu Jan 13 19:53:21 2011
@@ -19,7 +19,7 @@ package org.apache.lucene.search;
 
 import java.io.IOException;
 
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
 
 /**
  * A {@link Collector} implementation that collects the top-scoring hits,
@@ -155,8 +155,8 @@ public abstract class TopScoreDocCollect
   }
   
   @Override
-  public void setNextReader(IndexReader reader, int base) {
-    docBase = base;
+  public void setNextReader(AtomicReaderContext context) {
+    docBase = context.docBase;
   }
   
   @Override

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java Thu Jan 13 19:53:21 2011
@@ -25,9 +25,11 @@ import java.util.Comparator;
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermState;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.PerReaderTermState;
 
 /**
  * Base rewrite method for collecting only the top terms
@@ -78,12 +80,12 @@ public abstract class TopTermsRewrite<Q 
         this.termComp = termsEnum.getComparator();
         // lazy init the initial ScoreTerm because comparator is not known on ctor:
         if (st == null)
-          st = new ScoreTerm(this.termComp);
+          st = new ScoreTerm(this.termComp, new PerReaderTermState(topReaderContext));
         boostAtt = termsEnum.attributes().addAttribute(BoostAttribute.class);
       }
     
       @Override
-      public boolean collect(BytesRef bytes) {
+      public boolean collect(BytesRef bytes) throws IOException {
         final float boost = boostAtt.getBoost();
         // ignore uncompetetive hits
         if (stQueue.size() == maxSize) {
@@ -94,23 +96,27 @@ public abstract class TopTermsRewrite<Q 
             return true;
         }
         ScoreTerm t = visitedTerms.get(bytes);
+        final TermState state = termsEnum.termState();
+        assert state != null;
         if (t != null) {
           // if the term is already in the PQ, only update docFreq of term in PQ
-          t.docFreq += termsEnum.docFreq();
           assert t.boost == boost : "boost should be equal in all segment TermsEnums";
+          t.termState.register(state, readerContext.ord, termsEnum.docFreq());
         } else {
           // add new entry in PQ, we must clone the term, else it may get overwritten!
           st.bytes.copy(bytes);
           st.boost = boost;
-          st.docFreq = termsEnum.docFreq();
           visitedTerms.put(st.bytes, st);
+          assert st.termState.docFreq() == 0;
+          st.termState.register(state, readerContext.ord, termsEnum.docFreq());
           stQueue.offer(st);
           // possibly drop entries from queue
           if (stQueue.size() > maxSize) {
             st = stQueue.poll();
             visitedTerms.remove(st.bytes);
+            st.termState.clear(); // reset the termstate! 
           } else {
-            st = new ScoreTerm(termComp);
+            st = new ScoreTerm(termComp, new PerReaderTermState(topReaderContext));
           }
           assert stQueue.size() <= maxSize : "the PQ size must be limited to maxSize";
           // set maxBoostAtt with values to help FuzzyTermsEnum to optimize
@@ -120,6 +126,7 @@ public abstract class TopTermsRewrite<Q 
             maxBoostAtt.setCompetitiveTerm(t.bytes);
           }
         }
+       
         return true;
       }
     });
@@ -130,8 +137,8 @@ public abstract class TopTermsRewrite<Q 
     ArrayUtil.quickSort(scoreTerms, scoreTermSortByTermComp);
     for (final ScoreTerm st : scoreTerms) {
       final Term term = placeholderTerm.createTerm(st.bytes);
-      assert reader.docFreq(term) == st.docFreq;
-      addClause(q, term, st.docFreq, query.getBoost() * st.boost); // add to query
+      assert reader.docFreq(term) == st.termState.docFreq() : "reader DF is " + reader.docFreq(term) + " vs " + st.termState.docFreq();
+      addClause(q, term, st.termState.docFreq(), query.getBoost() * st.boost, st.termState); // add to query
     }
     query.incTotalNumberOfTerms(scoreTerms.length);
     return q;
@@ -147,7 +154,7 @@ public abstract class TopTermsRewrite<Q 
     if (this == obj) return true;
     if (obj == null) return false;
     if (getClass() != obj.getClass()) return false;
-    final TopTermsRewrite other = (TopTermsRewrite) obj;
+    final TopTermsRewrite<?> other = (TopTermsRewrite<?>) obj;
     if (size != other.size) return false;
     return true;
   }
@@ -163,13 +170,12 @@ public abstract class TopTermsRewrite<Q 
 
   static final class ScoreTerm implements Comparable<ScoreTerm> {
     public final Comparator<BytesRef> termComp;
-
     public final BytesRef bytes = new BytesRef();
     public float boost;
-    public int docFreq;
-    
-    public ScoreTerm(Comparator<BytesRef> termComp) {
+    public final PerReaderTermState termState;
+    public ScoreTerm(Comparator<BytesRef> termComp, PerReaderTermState termState) {
       this.termComp = termComp;
+      this.termState = termState;
     }
     
     public int compareTo(ScoreTerm other) {

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TotalHitCountCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TotalHitCountCollector.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TotalHitCountCollector.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/TotalHitCountCollector.java Thu Jan 13 19:53:21 2011
@@ -17,7 +17,7 @@ package org.apache.lucene.search;
  * limitations under the License.
  */
 
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
 
 /**
  * Just counts the total number of hits.
@@ -38,7 +38,7 @@ public class TotalHitCountCollector exte
     totalHits++;
   }
 
-  public void setNextReader(IndexReader reader, int docBase) {
+  public void setNextReader(AtomicReaderContext context) {
   }
 
   public boolean acceptsDocsOutOfOrder() {

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/Weight.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/Weight.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/Weight.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/Weight.java Thu Jan 13 19:53:21 2011
@@ -21,27 +21,35 @@ import java.io.IOException;
 import java.io.Serializable;
 
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader.ReaderContext;
 
 /**
  * Expert: Calculate query weights and build query scorers.
  * <p>
- * The purpose of {@link Weight} is to ensure searching does not
- * modify a {@link Query}, so that a {@link Query} instance can be reused. <br>
- * {@link Searcher} dependent state of the query should reside in the
+ * The purpose of {@link Weight} is to ensure searching does not modify a
+ * {@link Query}, so that a {@link Query} instance can be reused. <br>
+ * {@link IndexSearcher} dependent state of the query should reside in the
  * {@link Weight}. <br>
  * {@link IndexReader} dependent state should reside in the {@link Scorer}.
  * <p>
+ * Since {@link Weight} creates {@link Scorer} instances for a given
+ * {@link AtomicReaderContext} ({@link #scorer(AtomicReaderContext, boolean, boolean)})
+ * callers must maintain the relationship between the searcher's top-level
+ * {@link ReaderContext} and the context used to create a {@link Scorer}. 
+ * <p>
  * A <code>Weight</code> is used in the following way:
  * <ol>
  * <li>A <code>Weight</code> is constructed by a top-level query, given a
- * <code>Searcher</code> ({@link Query#createWeight(Searcher)}).
+ * <code>IndexSearcher</code> ({@link Query#createWeight(IndexSearcher)}).
  * <li>The {@link #sumOfSquaredWeights()} method is called on the
  * <code>Weight</code> to compute the query normalization factor
  * {@link Similarity#queryNorm(float)} of the query clauses contained in the
  * query.
  * <li>The query normalization factor is passed to {@link #normalize(float)}. At
  * this point the weighting is complete.
- * <li>A <code>Scorer</code> is constructed by {@link #scorer(IndexReader,boolean,boolean)}.
+ * <li>A <code>Scorer</code> is constructed by
+ * {@link #scorer(AtomicReaderContext, ScorerContext)}.
  * </ol>
  * 
  * @since 2.9
@@ -51,12 +59,12 @@ public abstract class Weight implements 
   /**
    * An explanation of the score computation for the named document.
    * 
-   * @param reader sub-reader containing the give doc
-   * @param doc
+   * @param context the readers context to create the {@link Explanation} for.
+   * @param doc the document's id relative to the given context's reader
    * @return an Explanation for the score
-   * @throws IOException
+   * @throws IOException if an {@link IOException} occurs
    */
-  public abstract Explanation explain(IndexReader reader, int doc) throws IOException;
+  public abstract Explanation explain(AtomicReaderContext context, int doc) throws IOException;
 
   /** The query that this concerns. */
   public abstract Query getQuery();
@@ -79,25 +87,13 @@ public abstract class Weight implements 
    * <b>NOTE:</b> null can be returned if no documents will be scored by this
    * query.
    * 
-   * @param reader
-   *          the {@link IndexReader} for which to return the {@link Scorer}.
-   * @param scoreDocsInOrder
-   *          specifies whether in-order scoring of documents is required. Note
-   *          that if set to false (i.e., out-of-order scoring is required),
-   *          this method can return whatever scoring mode it supports, as every
-   *          in-order scorer is also an out-of-order one. However, an
-   *          out-of-order scorer may not support {@link Scorer#nextDoc()}
-   *          and/or {@link Scorer#advance(int)}, therefore it is recommended to
-   *          request an in-order scorer if use of these methods is required.
-   * @param topScorer
-   *          if true, {@link Scorer#score(Collector)} will be called; if false,
-   *          {@link Scorer#nextDoc()} and/or {@link Scorer#advance(int)} will
-   *          be called.
+   * @param context
+   *          the {@link AtomicReaderContext} for which to return the {@link Scorer}.
+   * @param scorercontext the {@link ScorerContext} holding the scores context variables
    * @return a {@link Scorer} which scores documents in/out-of order.
    * @throws IOException
    */
-  public abstract Scorer scorer(IndexReader reader, boolean scoreDocsInOrder,
-      boolean topScorer) throws IOException;
+  public abstract Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException;
   
   /** The sum of squared weights of contained query clauses. */
   public abstract float sumOfSquaredWeights() throws IOException;
@@ -106,7 +102,7 @@ public abstract class Weight implements 
    * Returns true iff this implementation scores docs only out of order. This
    * method is used in conjunction with {@link Collector}'s
    * {@link Collector#acceptsDocsOutOfOrder() acceptsDocsOutOfOrder} and
-   * {@link #scorer(org.apache.lucene.index.IndexReader, boolean, boolean)} to
+   * {@link #scorer(AtomicReaderContext, boolean, boolean)} to
    * create a matching {@link Scorer} instance for a given {@link Collector}, or
    * vice versa.
    * <p>
@@ -115,4 +111,82 @@ public abstract class Weight implements 
    */
   public boolean scoresDocsOutOfOrder() { return false; }
 
+  /**
+   * A struct like class encapsulating a scorer's context variables.
+   * ScorerContex is a strictly immutable struct that follows a
+   * <tt>create on modification</tt> pattern. If a context variable changes
+   * through one of the modifiers like {@link #topScorer(boolean)} a new
+   * {@link ScorerContext} instance is creates. If the modifier call doesn't
+   * change the instance the method call has no effect and the same instance is
+   * returned from the modifier.
+   * 
+   * @lucene.experimental
+   */
+  public static final class ScorerContext {
+   
+    /**
+     * Specifies whether in-order scoring of documents is required. Note that if
+     * set to false (i.e., out-of-order scoring is required), this method can
+     * return whatever scoring mode it supports, as every in-order scorer is
+     * also an out-of-order one. However, an out-of-order scorer may not support
+     * {@link Scorer#nextDoc()} and/or {@link Scorer#advance(int)}, therefore it
+     * is recommended to request an in-order scorer if use of these methods is
+     * required.
+     */
+    public final boolean scoreDocsInOrder;
+    
+    /**
+     * if <code>true</code>, {@link Scorer#score(Collector)} will be called; if
+     * false, {@link Scorer#nextDoc()} and/or {@link Scorer#advance(int)} will
+     * be called instead.
+     */
+    public final boolean topScorer;
+    
+    
+    private static final ScorerContext DEFAULT_CONTEXT = new ScorerContext(true, false);
+
+    /**
+     * Returns a default {@link ScorerContext} template initialized with:
+     * <ul>
+     * <li>{@link #scoreDocsInOrder} = <code>true</code></li>
+     * <li>{@link #topScorer} = <code>false</code></li>
+     * </ul>
+     */
+    public static ScorerContext def() {
+      return DEFAULT_CONTEXT;
+    }
+    
+    private ScorerContext(boolean scoreDocsInOrder, boolean topScorer) {
+      this.scoreDocsInOrder = scoreDocsInOrder;
+      this.topScorer = topScorer;
+    }
+
+    /**
+     * Creates and returns a copy of this context with the given value for
+     * {@link #scoreDocsInOrder} and returns a new instance of
+     * {@link ScorerContext} iff the given value differs from the
+     * {@link #scoreDocsInOrder}. Otherwise, this method has no effect and
+     * returns this instance.
+     */
+    public ScorerContext scoreDocsInOrder(boolean scoreDocsInOrder) {
+      if (this.scoreDocsInOrder == scoreDocsInOrder) {
+        return this;
+      }
+      return new ScorerContext(scoreDocsInOrder, topScorer);
+    }
+    
+    /**
+     * Creates and returns a copy of this context with the given value for
+     * {@link #topScorer} and returns a new instance of
+     * {@link ScorerContext} iff the given value differs from the
+     * {@link #topScorer}. Otherwise, this method has no effect and
+     * returns this instance.
+     */
+    public ScorerContext topScorer(boolean topScorer) {
+      if (this.topScorer == topScorer) {
+        return this;
+      }
+      return new ScorerContext(scoreDocsInOrder, topScorer);
+    }
+  }
 }

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java Thu Jan 13 19:53:21 2011
@@ -74,7 +74,7 @@ public class ByteValuesCreator extends C
   }
 
   @Override
-  public ByteValues validate(ByteValues entry, IndexReader reader) throws IOException {
+  public synchronized ByteValues validate(ByteValues entry, IndexReader reader) throws IOException {
     boolean ok = false;
     
     if( hasOption(OPTION_CACHE_VALUES) ) {

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java Thu Jan 13 19:53:21 2011
@@ -24,8 +24,11 @@ import org.apache.lucene.index.DocsAndPo
 import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.OrdTermState;
+import org.apache.lucene.index.TermState;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.codecs.PrefixCodedTermState;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.FieldCache.DocTermsIndex;
 import org.apache.lucene.util.ArrayUtil;
@@ -240,8 +243,28 @@ public class DocTermsIndexCreator extend
 
       @Override
       public SeekStatus seek(BytesRef text, boolean useCache) throws IOException {
-        // TODO - we can support with binary search
-        throw new UnsupportedOperationException();
+        int low = 1;
+        int high = numOrd-1;
+        
+        while (low <= high) {
+          int mid = (low + high) >>> 1;
+          seek(mid);
+          int cmp = term.compareTo(text);
+
+          if (cmp < 0)
+            low = mid + 1;
+          else if (cmp > 0)
+            high = mid - 1;
+          else
+            return SeekStatus.FOUND; // key found
+        }
+        
+        if (low == numOrd) {
+          return SeekStatus.END;
+        } else {
+          seek(low);
+          return SeekStatus.NOT_FOUND;
+        }
       }
 
       @Override
@@ -284,11 +307,6 @@ public class DocTermsIndexCreator extend
       }
 
       @Override
-      public void cacheCurrentTerm() throws IOException {
-        throw new UnsupportedOperationException();
-      }
-
-      @Override
       public BytesRef term() throws IOException {
         return term;
       }
@@ -315,7 +333,20 @@ public class DocTermsIndexCreator extend
 
       @Override
       public Comparator<BytesRef> getComparator() throws IOException {
-        throw new UnsupportedOperationException();
+        return BytesRef.getUTF8SortedAsUnicodeComparator();
+      }
+
+      @Override
+      public SeekStatus seek(BytesRef term, TermState state) throws IOException {
+        assert state != null && state instanceof OrdTermState;
+        return this.seek(((OrdTermState)state).ord);
+      }
+
+      @Override
+      public TermState termState() throws IOException {
+        OrdTermState state = new OrdTermState();
+        state.ord = currentOrd;
+        return state;
       }
     }
   }

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java Thu Jan 13 19:53:21 2011
@@ -74,7 +74,7 @@ public class DoubleValuesCreator extends
   }
 
   @Override
-  public DoubleValues validate(DoubleValues entry, IndexReader reader) throws IOException {
+  public synchronized DoubleValues validate(DoubleValues entry, IndexReader reader) throws IOException {
     boolean ok = false;
     
     if( hasOption(OPTION_CACHE_VALUES) ) {

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java Thu Jan 13 19:53:21 2011
@@ -75,7 +75,7 @@ public class FloatValuesCreator extends 
   }
 
   @Override
-  public FloatValues validate(FloatValues entry, IndexReader reader) throws IOException {
+  public synchronized FloatValues validate(FloatValues entry, IndexReader reader) throws IOException {
     boolean ok = false;
     
     if( hasOption(OPTION_CACHE_VALUES) ) {

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java Thu Jan 13 19:53:21 2011
@@ -75,7 +75,7 @@ public class IntValuesCreator extends Ca
   }
 
   @Override
-  public IntValues validate(IntValues entry, IndexReader reader) throws IOException {
+  public synchronized IntValues validate(IntValues entry, IndexReader reader) throws IOException {
     boolean ok = false;
     
     if( hasOption(OPTION_CACHE_VALUES) ) {

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java Thu Jan 13 19:53:21 2011
@@ -75,7 +75,7 @@ public class LongValuesCreator extends C
   }
 
   @Override
-  public LongValues validate(LongValues entry, IndexReader reader) throws IOException {
+  public synchronized LongValues validate(LongValues entry, IndexReader reader) throws IOException {
     boolean ok = false;
     
     if( hasOption(OPTION_CACHE_VALUES) ) {

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java Thu Jan 13 19:53:21 2011
@@ -75,7 +75,7 @@ public class ShortValuesCreator extends 
   }
 
   @Override
-  public ShortValues validate(ShortValues entry, IndexReader reader) throws IOException {
+  public synchronized ShortValues validate(ShortValues entry, IndexReader reader) throws IOException {
     boolean ok = false;
     
     if( hasOption(OPTION_CACHE_VALUES) ) {

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java Thu Jan 13 19:53:21 2011
@@ -22,14 +22,16 @@ import java.util.Set;
 import java.util.Arrays;
 
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.ComplexExplanation;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.Weight.ScorerContext;
 import org.apache.lucene.util.ToStringUtils;
 
 /**
@@ -187,7 +189,7 @@ public class CustomScoreQuery extends Qu
     Weight[] valSrcWeights;
     boolean qStrict;
 
-    public CustomWeight(Searcher searcher) throws IOException {
+    public CustomWeight(IndexSearcher searcher) throws IOException {
       this.similarity = getSimilarity(searcher);
       this.subQueryWeight = subQuery.weight(searcher);
       this.valSrcWeights = new Weight[valSrcQueries.length];
@@ -239,40 +241,40 @@ public class CustomScoreQuery extends Qu
     }
 
     @Override
-    public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException {
+    public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
       // Pass true for "scoresDocsInOrder", because we
       // require in-order scoring, even if caller does not,
       // since we call advance on the valSrcScorers.  Pass
       // false for "topScorer" because we will not invoke
       // score(Collector) on these scorers:
-      Scorer subQueryScorer = subQueryWeight.scorer(reader, true, false);
+      Scorer subQueryScorer = subQueryWeight.scorer(context, ScorerContext.def());
       if (subQueryScorer == null) {
         return null;
       }
       Scorer[] valSrcScorers = new Scorer[valSrcWeights.length];
       for(int i = 0; i < valSrcScorers.length; i++) {
-         valSrcScorers[i] = valSrcWeights[i].scorer(reader, true, topScorer);
+         valSrcScorers[i] = valSrcWeights[i].scorer(context, scorerContext.scoreDocsInOrder(true));
       }
-      return new CustomScorer(similarity, reader, this, subQueryScorer, valSrcScorers);
+      return new CustomScorer(similarity, context.reader, this, subQueryScorer, valSrcScorers);
     }
 
     @Override
-    public Explanation explain(IndexReader reader, int doc) throws IOException {
-      Explanation explain = doExplain(reader, doc);
+    public Explanation explain(AtomicReaderContext context, int doc) throws IOException {
+      Explanation explain = doExplain(context, doc);
       return explain == null ? new Explanation(0.0f, "no matching docs") : explain;
     }
     
-    private Explanation doExplain(IndexReader reader, int doc) throws IOException {
-      Explanation subQueryExpl = subQueryWeight.explain(reader, doc);
+    private Explanation doExplain(AtomicReaderContext info, int doc) throws IOException {
+      Explanation subQueryExpl = subQueryWeight.explain(info, doc);
       if (!subQueryExpl.isMatch()) {
         return subQueryExpl;
       }
       // match
       Explanation[] valSrcExpls = new Explanation[valSrcWeights.length];
       for(int i = 0; i < valSrcWeights.length; i++) {
-        valSrcExpls[i] = valSrcWeights[i].explain(reader, doc);
+        valSrcExpls[i] = valSrcWeights[i].explain(info, doc);
       }
-      Explanation customExp = CustomScoreQuery.this.getCustomScoreProvider(reader).customExplain(doc,subQueryExpl,valSrcExpls);
+      Explanation customExp = CustomScoreQuery.this.getCustomScoreProvider(info.reader).customExplain(doc,subQueryExpl,valSrcExpls);
       float sc = getValue() * customExp.getValue();
       Explanation res = new ComplexExplanation(
         true, sc, CustomScoreQuery.this.toString() + ", product of:");
@@ -350,7 +352,7 @@ public class CustomScoreQuery extends Qu
   }
 
   @Override
-  public Weight createWeight(Searcher searcher) throws IOException {
+  public Weight createWeight(IndexSearcher searcher) throws IOException {
     return new CustomWeight(searcher);
   }
 

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/function/FieldCacheSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/function/FieldCacheSource.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/function/FieldCacheSource.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/function/FieldCacheSource.java Thu Jan 13 19:53:21 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.search.functio
 import java.io.IOException;
 
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
 import org.apache.lucene.search.FieldCache;
 
 /**
@@ -55,8 +56,8 @@ public abstract class FieldCacheSource e
 
   /* (non-Javadoc) @see org.apache.lucene.search.function.ValueSource#getValues(org.apache.lucene.index.IndexReader) */
   @Override
-  public final DocValues getValues(IndexReader reader) throws IOException {
-    return getCachedFieldValues(FieldCache.DEFAULT, field, reader);
+  public final DocValues getValues(AtomicReaderContext context) throws IOException {
+    return getCachedFieldValues(FieldCache.DEFAULT, field, context.reader);
   }
 
   /* (non-Javadoc) @see org.apache.lucene.search.function.ValueSource#description() */

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/function/MultiValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/function/MultiValueSource.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/function/MultiValueSource.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/function/MultiValueSource.java Thu Jan 13 19:53:21 2011
@@ -20,6 +20,9 @@ package org.apache.lucene.search.functio
 import java.io.IOException;
 
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader.ReaderContext;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.util.ReaderUtil;
 
@@ -44,16 +47,16 @@ public final class MultiValueSource exte
   }
 
   @Override
-  public DocValues getValues(IndexReader reader) throws IOException {
-
-    IndexReader[] subReaders = reader.getSequentialSubReaders();
-    if (subReaders != null) {
-      // This is a composite reader
-      return new MultiDocValues(subReaders);
-    } else {
+  public DocValues getValues(AtomicReaderContext context) throws IOException {
       // Already an atomic reader -- just delegate
-      return other.getValues(reader);
+      return other.getValues(context);
+  }
+  
+  public DocValues getValues(ReaderContext context) throws IOException {
+    if (context.isAtomic) {
+      return getValues((AtomicReaderContext) context);
     }
+    return new MultiDocValues(ReaderUtil.leaves(context));
   }
 
   @Override
@@ -78,59 +81,56 @@ public final class MultiValueSource exte
   private final class MultiDocValues extends DocValues {
 
     final DocValues[] docValues;
-    final int[] docStarts;
+    final AtomicReaderContext[] leaves;
 
-    MultiDocValues(IndexReader[] subReaders) throws IOException {
-      docValues = new DocValues[subReaders.length];
-      docStarts = new int[subReaders.length];
-      int base = 0;
-      for(int i=0;i<subReaders.length;i++) {
-        docValues[i] = other.getValues(subReaders[i]);
-        docStarts[i] = base;
-        base += subReaders[i].maxDoc();
+    MultiDocValues(AtomicReaderContext[] leaves) throws IOException {
+      this.leaves = leaves;
+      docValues = new DocValues[leaves.length];
+      for(int i=0;i<leaves.length;i++) {
+        docValues[i] = other.getValues(leaves[i]);
       }
     }
     
     @Override
     public float floatVal(int doc) {
-      final int n = ReaderUtil.subIndex(doc, docStarts);
-      return docValues[n].floatVal(doc-docStarts[n]);
+      final int n = ReaderUtil.subIndex(doc, leaves);
+      return docValues[n].floatVal(doc-leaves[n].docBase);
     }
 
     @Override
     public int intVal(int doc) {
-      final int n = ReaderUtil.subIndex(doc, docStarts);
-      return docValues[n].intVal(doc-docStarts[n]);
+      final int n = ReaderUtil.subIndex(doc, leaves);
+      return docValues[n].intVal(doc-leaves[n].docBase);
     }
 
     @Override
     public long longVal(int doc) {
-      final int n = ReaderUtil.subIndex(doc, docStarts);
-      return docValues[n].longVal(doc-docStarts[n]);
+      final int n = ReaderUtil.subIndex(doc, leaves);
+      return docValues[n].longVal(doc-leaves[n].docBase);
     }
 
     @Override
     public double doubleVal(int doc) {
-      final int n = ReaderUtil.subIndex(doc, docStarts);
-      return docValues[n].doubleVal(doc-docStarts[n]);
+      final int n = ReaderUtil.subIndex(doc, leaves);
+      return docValues[n].doubleVal(doc-leaves[n].docBase);
     }
 
     @Override
     public String strVal(int doc) {
-      final int n = ReaderUtil.subIndex(doc, docStarts);
-      return docValues[n].strVal(doc-docStarts[n]);
+      final int n = ReaderUtil.subIndex(doc, leaves);
+      return docValues[n].strVal(doc-leaves[n].docBase);
     }
 
     @Override
     public String toString(int doc) {
-      final int n = ReaderUtil.subIndex(doc, docStarts);
-      return docValues[n].toString(doc-docStarts[n]);
+      final int n = ReaderUtil.subIndex(doc, leaves);
+      return docValues[n].toString(doc-leaves[n].docBase);
     }
 
     @Override
     public Explanation explain(int doc) {
-      final int n = ReaderUtil.subIndex(doc, docStarts);
-      return docValues[n].explain(doc-docStarts[n]);
+      final int n = ReaderUtil.subIndex(doc, leaves);
+      return docValues[n].explain(doc-leaves[n].docBase);
     }
   }
 }

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/function/OrdFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/function/OrdFieldSource.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/function/OrdFieldSource.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/function/OrdFieldSource.java Thu Jan 13 19:53:21 2011
@@ -17,7 +17,7 @@
 
 package org.apache.lucene.search.function;
 
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
 import org.apache.lucene.search.FieldCache;
 import org.apache.lucene.search.FieldCache.DocTermsIndex;
 
@@ -69,8 +69,8 @@ public class OrdFieldSource extends Valu
 
   /*(non-Javadoc) @see org.apache.lucene.search.function.ValueSource#getValues(org.apache.lucene.index.IndexReader) */
   @Override
-  public DocValues getValues(IndexReader reader) throws IOException {
-    final DocTermsIndex termsIndex = FieldCache.DEFAULT.getTermsIndex(reader, field);
+  public DocValues getValues(AtomicReaderContext context) throws IOException {
+    final DocTermsIndex termsIndex = FieldCache.DEFAULT.getTermsIndex(context.reader, field);
     return new DocValues() {
       /*(non-Javadoc) @see org.apache.lucene.search.function.DocValues#floatVal(int) */
       @Override

Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/function/ReverseOrdFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/function/ReverseOrdFieldSource.java?rev=1058718&r1=1058717&r2=1058718&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/function/ReverseOrdFieldSource.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/function/ReverseOrdFieldSource.java Thu Jan 13 19:53:21 2011
@@ -17,7 +17,7 @@
 
 package org.apache.lucene.search.function;
 
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
 import org.apache.lucene.search.FieldCache;
 
 import java.io.IOException;
@@ -69,8 +69,8 @@ public class ReverseOrdFieldSource exten
 
   /*(non-Javadoc) @see org.apache.lucene.search.function.ValueSource#getValues(org.apache.lucene.index.IndexReader) */
   @Override
-  public DocValues getValues(IndexReader reader) throws IOException {
-    final FieldCache.DocTermsIndex termsIndex = FieldCache.DEFAULT.getTermsIndex(reader, field);
+  public DocValues getValues(AtomicReaderContext context) throws IOException {
+    final FieldCache.DocTermsIndex termsIndex = FieldCache.DEFAULT.getTermsIndex(context.reader, field);
 
     final int end = termsIndex.numOrd();
 



Mime
View raw message