lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sh...@apache.org
Subject svn commit: r1454862 [27/49] - in /lucene/dev/branches/lucene4258: ./ dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/demo/ dev-tools/idea/lucene/facet/ dev-tools/idea/lucene/highlighter/ dev-to...
Date Sun, 10 Mar 2013 15:05:21 GMT
Modified: lucene/dev/branches/lucene4258/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java (original)
+++ lucene/dev/branches/lucene4258/lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java Sun Mar 10 15:04:57 2013
@@ -17,9 +17,11 @@ package org.apache.lucene.search.join;
  * limitations under the License.
  */
 
+import java.io.IOException;
+
 import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.DocTermOrds;
-import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.FieldCache;
 import org.apache.lucene.search.Scorer;
@@ -27,8 +29,6 @@ import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefHash;
 
-import java.io.IOException;
-
 abstract class TermsWithScoreCollector extends Collector {
 
   private final static int INITIAL_ARRAY_SIZE = 256;
@@ -92,7 +92,7 @@ abstract class TermsWithScoreCollector e
   static class SV extends TermsWithScoreCollector {
 
     final BytesRef spare = new BytesRef();
-    FieldCache.DocTerms fromDocTerms;
+    BinaryDocValues fromDocTerms;
 
     SV(String field, ScoreMode scoreMode) {
       super(field, scoreMode);
@@ -100,7 +100,8 @@ abstract class TermsWithScoreCollector e
 
     @Override
     public void collect(int doc) throws IOException {
-      int ord = collectedTerms.add(fromDocTerms.getTerm(doc, spare));
+      fromDocTerms.get(doc, spare);
+      int ord = collectedTerms.add(spare);
       if (ord < 0) {
         ord = -ord - 1;
       } else {
@@ -141,7 +142,8 @@ abstract class TermsWithScoreCollector e
 
       @Override
       public void collect(int doc) throws IOException {
-        int ord = collectedTerms.add(fromDocTerms.getTerm(doc, spare));
+        fromDocTerms.get(doc, spare);
+        int ord = collectedTerms.add(spare);
         if (ord < 0) {
           ord = -ord - 1;
         } else {
@@ -178,9 +180,8 @@ abstract class TermsWithScoreCollector e
   // impl that works with multiple values per document
   static class MV extends TermsWithScoreCollector {
 
-    DocTermOrds fromDocTermOrds;
-    TermsEnum docTermsEnum;
-    DocTermOrds.TermOrdsIterator reuse;
+    SortedSetDocValues fromDocTermOrds;
+    final BytesRef scratch = new BytesRef();
 
     MV(String field, ScoreMode scoreMode) {
       super(field, scoreMode);
@@ -188,52 +189,33 @@ abstract class TermsWithScoreCollector e
 
     @Override
     public void collect(int doc) throws IOException {
-      reuse = fromDocTermOrds.lookup(doc, reuse);
-      int[] buffer = new int[5];
-
-      int chunk;
-      do {
-        chunk = reuse.read(buffer);
-        if (chunk == 0) {
-          return;
-        }
-
-        for (int idx = 0; idx < chunk; idx++) {
-          int key = buffer[idx];
-          docTermsEnum.seekExact((long) key);
-          int ord = collectedTerms.add(docTermsEnum.term());
-          if (ord < 0) {
-            ord = -ord - 1;
-          } else {
-            if (ord >= scoreSums.length) {
-              scoreSums = ArrayUtil.grow(scoreSums);
-            }
-          }
-
-          final float current = scorer.score();
-          final float existing = scoreSums[ord];
-          if (Float.compare(existing, 0.0f) == 0) {
-            scoreSums[ord] = current;
-          } else {
-            switch (scoreMode) {
-              case Total:
-                scoreSums[ord] = existing + current;
-                break;
-              case Max:
-                if (current > existing) {
-                  scoreSums[ord] = current;
-                }
-            }
+      fromDocTermOrds.setDocument(doc);
+      long ord;
+      while ((ord = fromDocTermOrds.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+        fromDocTermOrds.lookupOrd(ord, scratch);
+        
+        int termID = collectedTerms.add(scratch);
+        if (termID < 0) {
+          termID = -termID - 1;
+        } else {
+          if (termID >= scoreSums.length) {
+            scoreSums = ArrayUtil.grow(scoreSums);
           }
         }
-      } while (chunk >= buffer.length);
+        
+        switch (scoreMode) {
+          case Total:
+            scoreSums[termID] += scorer.score();
+            break;
+          case Max:
+            scoreSums[termID] = Math.max(scoreSums[termID], scorer.score());
+        }
+      }
     }
 
     @Override
     public void setNextReader(AtomicReaderContext context) throws IOException {
       fromDocTermOrds = FieldCache.DEFAULT.getDocTermOrds(context.reader(), field);
-      docTermsEnum = fromDocTermOrds.getOrdTermsEnum(context.reader());
-      reuse = null; // LUCENE-3377 needs to be fixed first then this statement can be removed...
     }
 
     static class Avg extends MV {
@@ -246,40 +228,23 @@ abstract class TermsWithScoreCollector e
 
       @Override
       public void collect(int doc) throws IOException {
-        reuse = fromDocTermOrds.lookup(doc, reuse);
-        int[] buffer = new int[5];
-
-        int chunk;
-        do {
-          chunk = reuse.read(buffer);
-          if (chunk == 0) {
-            return;
-          }
-
-          for (int idx = 0; idx < chunk; idx++) {
-            int key = buffer[idx];
-            docTermsEnum.seekExact((long) key);
-            int ord = collectedTerms.add(docTermsEnum.term());
-            if (ord < 0) {
-              ord = -ord - 1;
-            } else {
-              if (ord >= scoreSums.length) {
-                scoreSums = ArrayUtil.grow(scoreSums);
-                scoreCounts = ArrayUtil.grow(scoreCounts);
-              }
-            }
-
-            float current = scorer.score();
-            float existing = scoreSums[ord];
-            if (Float.compare(existing, 0.0f) == 0) {
-              scoreSums[ord] = current;
-              scoreCounts[ord] = 1;
-            } else {
-              scoreSums[ord] = scoreSums[ord] + current;
-              scoreCounts[ord]++;
+        fromDocTermOrds.setDocument(doc);
+        long ord;
+        while ((ord = fromDocTermOrds.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+          fromDocTermOrds.lookupOrd(ord, scratch);
+          
+          int termID = collectedTerms.add(scratch);
+          if (termID < 0) {
+            termID = -termID - 1;
+          } else {
+            if (termID >= scoreSums.length) {
+              scoreSums = ArrayUtil.grow(scoreSums);
             }
           }
-        } while (chunk >= buffer.length);
+          
+          scoreSums[termID] += scorer.score();
+          scoreCounts[termID]++;
+        }
       }
 
       @Override

Modified: lucene/dev/branches/lucene4258/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java (original)
+++ lucene/dev/branches/lucene4258/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java Sun Mar 10 15:04:57 2013
@@ -17,6 +17,9 @@ package org.apache.lucene.search.join;
  * limitations under the License.
  */
 
+import java.io.IOException;
+import java.util.*;
+
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.document.Document;
@@ -24,6 +27,7 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.DocTermOrds;
 import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.IndexReader;
@@ -31,6 +35,7 @@ import org.apache.lucene.index.MultiFiel
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.ReaderUtil;
 import org.apache.lucene.index.SlowCompositeReaderWrapper;
+import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
@@ -49,14 +54,11 @@ import org.apache.lucene.search.TopScore
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.LuceneTestCase.Slow;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
-import org.apache.lucene.util.LuceneTestCase.Slow;
 import org.junit.Test;
 
-import java.io.IOException;
-import java.util.*;
-
 public class TestJoinUtil extends LuceneTestCase {
 
   public void testSimple() throws Exception {
@@ -465,48 +467,26 @@ public class TestJoinUtil extends Lucene
         fromSearcher.search(new TermQuery(new Term("value", uniqueRandomValue)), new Collector() {
 
           private Scorer scorer;
-          private DocTermOrds docTermOrds;
-          private TermsEnum docTermsEnum;
-          private DocTermOrds.TermOrdsIterator reuse;
+          private SortedSetDocValues docTermOrds;
+          final BytesRef joinValue = new BytesRef();
 
           @Override
           public void collect(int doc) throws IOException {
-            if (docTermOrds.isEmpty()) {
-              return;
-            }
-
-            reuse = docTermOrds.lookup(doc, reuse);
-            int[] buffer = new int[5];
-
-            int chunk;
-            do {
-              chunk = reuse.read(buffer);
-              if (chunk == 0) {
-                return;
-              }
-
-              for (int idx = 0; idx < chunk; idx++) {
-                int key = buffer[idx];
-                docTermsEnum.seekExact((long) key);
-                BytesRef joinValue = docTermsEnum.term();
-                if (joinValue == null) {
-                  continue;
-                }
-
-                JoinScore joinScore = joinValueToJoinScores.get(joinValue);
-                if (joinScore == null) {
-                  joinValueToJoinScores.put(BytesRef.deepCopyOf(joinValue), joinScore = new JoinScore());
-                }
-                joinScore.addScore(scorer.score());
+            docTermOrds.setDocument(doc);
+            long ord;
+            while ((ord = docTermOrds.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+              docTermOrds.lookupOrd(ord, joinValue);
+              JoinScore joinScore = joinValueToJoinScores.get(joinValue);
+              if (joinScore == null) {
+                joinValueToJoinScores.put(BytesRef.deepCopyOf(joinValue), joinScore = new JoinScore());
               }
-            } while (chunk >= buffer.length);
+              joinScore.addScore(scorer.score());
+            }
           }
 
           @Override
           public void setNextReader(AtomicReaderContext context) throws IOException {
             docTermOrds = FieldCache.DEFAULT.getDocTermOrds(context.reader(), fromField);
-            docTermsEnum = docTermOrds.getOrdTermsEnum(context.reader());
-            reuse = null;
           }
 
           @Override
@@ -523,13 +503,14 @@ public class TestJoinUtil extends Lucene
         fromSearcher.search(new TermQuery(new Term("value", uniqueRandomValue)), new Collector() {
 
           private Scorer scorer;
-          private FieldCache.DocTerms terms;
+          private BinaryDocValues terms;
           private final BytesRef spare = new BytesRef();
 
           @Override
           public void collect(int doc) throws IOException {
-            BytesRef joinValue = terms.getTerm(doc, spare);
-            if (joinValue == null) {
+            terms.get(doc, spare);
+            BytesRef joinValue = spare;
+            if (joinValue.bytes == BinaryDocValues.MISSING) {
               return;
             }
 
@@ -586,50 +567,33 @@ public class TestJoinUtil extends Lucene
         } else {
           toSearcher.search(new MatchAllDocsQuery(), new Collector() {
 
-            private DocTermOrds docTermOrds;
-            private TermsEnum docTermsEnum;
-            private DocTermOrds.TermOrdsIterator reuse;
+            private SortedSetDocValues docTermOrds;
+            private final BytesRef scratch = new BytesRef();
             private int docBase;
 
             @Override
             public void collect(int doc) throws IOException {
-              if (docTermOrds.isEmpty()) {
-                return;
-              }
-
-              reuse = docTermOrds.lookup(doc, reuse);
-              int[] buffer = new int[5];
-
-              int chunk;
-              do {
-                chunk = reuse.read(buffer);
-                if (chunk == 0) {
-                  return;
+              docTermOrds.setDocument(doc);
+              long ord;
+              while ((ord = docTermOrds.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+                docTermOrds.lookupOrd(ord, scratch);
+                JoinScore joinScore = joinValueToJoinScores.get(scratch);
+                if (joinScore == null) {
+                  continue;
                 }
-
-                for (int idx = 0; idx < chunk; idx++) {
-                  int key = buffer[idx];
-                  docTermsEnum.seekExact((long) key);
-                  JoinScore joinScore = joinValueToJoinScores.get(docTermsEnum.term());
-                  if (joinScore == null) {
-                    continue;
-                  }
-                  Integer basedDoc = docBase + doc;
-                  // First encountered join value determines the score.
-                  // Something to keep in mind for many-to-many relations.
-                  if (!docToJoinScore.containsKey(basedDoc)) {
-                    docToJoinScore.put(basedDoc, joinScore);
-                  }
+                Integer basedDoc = docBase + doc;
+                // First encountered join value determines the score.
+                // Something to keep in mind for many-to-many relations.
+                if (!docToJoinScore.containsKey(basedDoc)) {
+                  docToJoinScore.put(basedDoc, joinScore);
                 }
-              } while (chunk >= buffer.length);
+              }
             }
 
             @Override
             public void setNextReader(AtomicReaderContext context) throws IOException {
               docBase = context.docBase;
               docTermOrds = FieldCache.DEFAULT.getDocTermOrds(context.reader(), toField);
-              docTermsEnum = docTermOrds.getOrdTermsEnum(context.reader());
-              reuse = null;
             }
 
             @Override
@@ -641,13 +605,14 @@ public class TestJoinUtil extends Lucene
       } else {
         toSearcher.search(new MatchAllDocsQuery(), new Collector() {
 
-          private FieldCache.DocTerms terms;
+          private BinaryDocValues terms;
           private int docBase;
           private final BytesRef spare = new BytesRef();
 
           @Override
           public void collect(int doc) {
-            JoinScore joinScore = joinValueToJoinScores.get(terms.getTerm(doc, spare));
+            terms.get(doc, spare);
+            JoinScore joinScore = joinValueToJoinScores.get(spare);
             if (joinScore == null) {
               return;
             }

Modified: lucene/dev/branches/lucene4258/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/lucene4258/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Sun Mar 10 15:04:57 2013
@@ -35,21 +35,22 @@ import org.apache.lucene.analysis.tokena
 import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
 import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.Norm;
-import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.DocsAndPositionsEnum;
 import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.FieldInvertState;
 import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.OrdTermState;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.StoredFieldVisitor;
 import org.apache.lucene.index.TermState;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
-import org.apache.lucene.index.memory.MemoryIndexNormDocValues.SingleValueSource;
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
@@ -60,13 +61,13 @@ import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.ByteBlockPool;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefHash.DirectBytesStartArray;
 import org.apache.lucene.util.BytesRefHash;
+import org.apache.lucene.util.Constants; // for javadocs
 import org.apache.lucene.util.Counter;
-import org.apache.lucene.util.IntBlockPool;
-import org.apache.lucene.util.BytesRefHash.DirectBytesStartArray;
 import org.apache.lucene.util.IntBlockPool.SliceReader;
 import org.apache.lucene.util.IntBlockPool.SliceWriter;
-import org.apache.lucene.util.Constants; // for javadocs
+import org.apache.lucene.util.IntBlockPool;
 import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util.RecyclingByteBlockAllocator;
 import org.apache.lucene.util.RecyclingIntBlockAllocator;
@@ -253,8 +254,7 @@ public class MemoryIndex {
   }
   
   /**
-   * Expert: This constructor accepts a byte and int block allocator that is used internally to allocate 
-   * int & byte blocks for term and posting storage.  
+   * Expert: This constructor accepts an upper limit for the number of bytes that should be reused if this instance is {@link #reset()}.
    * @param storeOffsets <code>true</code> if offsets should be stored
    * @param maxReusedBytes the number of bytes that should remain in the internal memory pools after {@link #reset()} is called
    */
@@ -466,7 +466,9 @@ public class MemoryIndex {
       throw new RuntimeException(e);
     } finally {
       try {
-        if (stream != null) stream.close();
+        if (stream != null) {
+          stream.close();
+        }
       } catch (IOException e2) {
         throw new RuntimeException(e2);
       }
@@ -737,6 +739,26 @@ public class MemoryIndex {
       return new FieldInfos(fieldInfos.values().toArray(new FieldInfo[fieldInfos.size()]));
     }
 
+    @Override
+    public NumericDocValues getNumericDocValues(String field) {
+      return null;
+    }
+
+    @Override
+    public BinaryDocValues getBinaryDocValues(String field) {
+      return null;
+    }
+
+    @Override
+    public SortedDocValues getSortedDocValues(String field) {
+      return null;
+    }
+    
+    @Override
+    public SortedSetDocValues getSortedSetDocValues(String field) {
+      return null;
+    }
+
     private class MemoryFields extends Fields {
       @Override
       public Iterator<String> iterator() {
@@ -1126,22 +1148,18 @@ public class MemoryIndex {
     protected void doClose() {
       if (DEBUG) System.err.println("MemoryIndexReader.doClose");
     }
-
-    @Override
-    public DocValues docValues(String field) {
-      return null;
-    }
     
     /** performance hack: cache norms to avoid repeated expensive calculations */
-    private DocValues cachedNormValues;
+    private NumericDocValues cachedNormValues;
     private String cachedFieldName;
     private Similarity cachedSimilarity;
     
     @Override
-    public DocValues normValues(String field) {
-      if (fieldInfos.get(field).omitsNorms())
+    public NumericDocValues getNormValues(String field) {
+      FieldInfo fieldInfo = fieldInfos.get(field);
+      if (fieldInfo == null || fieldInfo.omitsNorms())
         return null;
-      DocValues norms = cachedNormValues;
+      NumericDocValues norms = cachedNormValues;
       Similarity sim = getSimilarity();
       if (!field.equals(cachedFieldName) || sim != cachedSimilarity) { // not cached?
         Info info = getInfo(field);
@@ -1149,15 +1167,13 @@ public class MemoryIndex {
         int numOverlapTokens = info != null ? info.numOverlapTokens : 0;
         float boost = info != null ? info.getBoost() : 1.0f; 
         FieldInvertState invertState = new FieldInvertState(field, 0, numTokens, numOverlapTokens, 0, boost);
-        Norm norm = new Norm();
-        sim.computeNorm(invertState, norm);
-        SingleValueSource singleByteSource = new SingleValueSource(norm);
-        norms = new MemoryIndexNormDocValues(singleByteSource);
+        long value = sim.computeNorm(invertState);
+        norms = new MemoryIndexNormDocValues(value);
         // cache it for future reuse
         cachedNormValues = norms;
         cachedFieldName = field;
         cachedSimilarity = sim;
-        if (DEBUG) System.err.println("MemoryIndexReader.norms: " + field + ":" + norm + ":" + numTokens);
+        if (DEBUG) System.err.println("MemoryIndexReader.norms: " + field + ":" + value + ":" + numTokens);
       }
       return norms;
     }

Modified: lucene/dev/branches/lucene4258/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndexNormDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndexNormDocValues.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndexNormDocValues.java (original)
+++ lucene/dev/branches/lucene4258/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndexNormDocValues.java Sun Mar 10 15:04:57 2013
@@ -15,129 +15,25 @@ package org.apache.lucene.index.memory;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import java.io.IOException;
 
-import org.apache.lucene.index.Norm;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.index.NumericDocValues;
 
 /**
  * 
  * @lucene.internal
  */
-class MemoryIndexNormDocValues extends DocValues {
-
-  private final Source source;
-
-  MemoryIndexNormDocValues(Source source) {
-    this.source = source;
-  }
-  @Override
-  protected Source loadSource() throws IOException {
-    return source;
-  }
-
-  @Override
-  protected Source loadDirectSource() throws IOException {
-    return source;
+class MemoryIndexNormDocValues extends NumericDocValues {
+  private final long value;
+  public MemoryIndexNormDocValues(long value) {
+    this.value = value;
   }
 
   @Override
-  public Type getType() {
-    return source.getType();
-  }
-  
-  @Override
-  public int getValueSize() {
-    return 1;
-  }
-
-  public static class SingleValueSource extends Source {
-
-    private final Number numericValue;
-    private final BytesRef binaryValue;
-
-    protected SingleValueSource(Norm norm) {
-      super(norm.type());
-      this.numericValue = norm.field().numericValue();
-      this.binaryValue = norm.field().binaryValue();
-    }
-
-    @Override
-    public long getInt(int docID) {
-      switch (type) {
-      case FIXED_INTS_16:
-      case FIXED_INTS_32:
-      case FIXED_INTS_64:
-      case FIXED_INTS_8:
-      case VAR_INTS:
-        assert numericValue != null;
-        return numericValue.longValue();
-      }
-      return super.getInt(docID);
-    }
-
-    @Override
-    public double getFloat(int docID) {
-      switch (type) {
-      case FLOAT_32:
-      case FLOAT_64:
-        assert numericValue != null;
-        return numericValue.floatValue();
-      }
-      return super.getFloat(docID);
-    }
-
-    @Override
-    public BytesRef getBytes(int docID, BytesRef ref) {
-      switch (type) {
-      case BYTES_FIXED_DEREF:
-      case BYTES_FIXED_SORTED:
-      case BYTES_FIXED_STRAIGHT:
-      case BYTES_VAR_DEREF:
-      case BYTES_VAR_SORTED:
-      case BYTES_VAR_STRAIGHT:
-        assert binaryValue != null;
-        ref.copyBytes(binaryValue);
-        return ref;
-      }
-      return super.getBytes(docID, ref);
-    }
-
-    @Override
-    public boolean hasArray() {
-      return true;
-    }
-
-    @Override
-    public Object getArray() {
-      switch (type) {
-      case BYTES_FIXED_DEREF:
-      case BYTES_FIXED_SORTED:
-      case BYTES_FIXED_STRAIGHT:
-      case BYTES_VAR_DEREF:
-      case BYTES_VAR_SORTED:
-      case BYTES_VAR_STRAIGHT:
-        return binaryValue.bytes;
-      case FIXED_INTS_16:
-        return new short[] { numericValue.shortValue() };
-      case FIXED_INTS_32:
-        return new int[] { numericValue.intValue() };
-      case FIXED_INTS_64:
-        return new long[] { numericValue.longValue() };
-      case FIXED_INTS_8:
-        return new byte[] { numericValue.byteValue() };
-      case VAR_INTS:
-        return new long[] { numericValue.longValue() };
-      case FLOAT_32:
-        return new float[] { numericValue.floatValue() };
-      case FLOAT_64:
-        return new double[] { numericValue.doubleValue() };
-      default:
-        throw new IllegalArgumentException("unknown type " + type);
-      }
-
-    }
+  public long get(int docID) {
+    if (docID != 0)
+      throw new IndexOutOfBoundsException();
+    else
+      return value;
   }
 
 }

Modified: lucene/dev/branches/lucene4258/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (original)
+++ lucene/dev/branches/lucene4258/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java Sun Mar 10 15:04:57 2013
@@ -36,8 +36,6 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.CompositeReader;
 import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.DocValues.Source;
 import org.apache.lucene.index.DocsAndPositionsEnum;
 import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.Fields;
@@ -46,6 +44,7 @@ import org.apache.lucene.index.IndexWrit
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.SlowCompositeReaderWrapper;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.Terms;
@@ -178,12 +177,12 @@ public class MemoryIndexTest extends Bas
       if (iwTerms == null) {
         assertNull(memTerms);
       } else {
-        DocValues normValues = competitor.normValues(field);
-        DocValues memNormValues = memIndexReader.normValues(field);
+        NumericDocValues normValues = competitor.getNormValues(field);
+        NumericDocValues memNormValues = memIndexReader.getNormValues(field);
         if (normValues != null) {
           // mem idx always computes norms on the fly
           assertNotNull(memNormValues);
-          assertEquals(normValues.getDirectSource().getInt(0), memNormValues.getDirectSource().getInt(0), 0.01);
+          assertEquals(normValues.get(0), memNormValues.get(0));
         }
           
         assertNotNull(memTerms);
@@ -381,6 +380,17 @@ public class MemoryIndexTest extends Bas
     assertTrue("posGap" + mockAnalyzer.getPositionIncrementGap("field") , mindex.search(query) > 0.0001);
   }
   
+  public void testNonExistingsField() throws IOException {
+    MemoryIndex mindex = new MemoryIndex(random().nextBoolean(),  random().nextInt(50) * 1024 * 1024);
+    MockAnalyzer mockAnalyzer = new MockAnalyzer(random());
+    mindex.addField("field", "the quick brown fox", mockAnalyzer);
+    AtomicReader reader = (AtomicReader) mindex.createSearcher().getIndexReader();
+    assertNull(reader.getNumericDocValues("not-in-index"));
+    assertNull(reader.getNormValues("not-in-index"));
+    assertNull(reader.termDocsEnum(new Term("not-in-index", "foo")));
+    assertNull(reader.termPositionsEnum(new Term("not-in-index", "foo")));
+    assertNull(reader.terms("not-in-index"));
+  }
   
   public void testDuellMemIndex() throws IOException {
     LineFileDocs lineFileDocs = new LineFileDocs(random());

Modified: lucene/dev/branches/lucene4258/lucene/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java (original)
+++ lucene/dev/branches/lucene4258/lucene/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java Sun Mar 10 15:04:57 2013
@@ -40,11 +40,13 @@ import java.util.Arrays;
 import java.util.Comparator;
 
 /**
- * 
  * <code>HighFreqTerms</code> class extracts the top n most frequent terms
- * (by document frequency ) from an existing Lucene index and reports their document frequencey.
- * If the -t flag is  and reports both their document frequency and their total tf (total number of occurences) 
- * in order of highest total tf
+ * (by document frequency) from an existing Lucene index and reports their
+ * document frequency.
+ * <p>
+ * If the -t flag is given, both document frequency and total tf (total
+ * number of occurrences) are reported, ordered by descending total tf.
+ *
  */
 public class HighFreqTerms {
   

Modified: lucene/dev/branches/lucene4258/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java (original)
+++ lucene/dev/branches/lucene4258/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java Sun Mar 10 15:04:57 2013
@@ -19,24 +19,19 @@ package org.apache.lucene.misc;
 
 import org.apache.lucene.search.similarities.DefaultSimilarity;
 import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.Norm;
 
 /**
+ * <p>
  * A similarity with a lengthNorm that provides for a "plateau" of
  * equally good lengths, and tf helper functions.
- *
+ * </p>
  * <p>
- * For lengthNorm, A global min/max can be specified to define the
+ * For lengthNorm, A min/max can be specified to define the
  * plateau of lengths that should all have a norm of 1.0.
  * Below the min, and above the max the lengthNorm drops off in a
  * sqrt function.
  * </p>
  * <p>
- * A per field min/max can be specified if different fields have
- * different sweet spots.
- * </p>
- *
- * <p>
  * For tf, baselineTf and hyperbolicTf functions are provided, which
  * subclasses can choose between.
  * </p>
@@ -106,7 +101,8 @@ public class SweetSpotSimilarity extends
    * computeLengthNorm(numTokens) </code> where
    * numTokens does not count overlap tokens if
    * discountOverlaps is true by default or true for this
-   * specific field. */
+   * specific field. 
+   */
   @Override
   public float lengthNorm(FieldInvertState state) {
     final int numTokens;

Modified: lucene/dev/branches/lucene4258/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java Sun Mar 10 15:04:57 2013
@@ -73,7 +73,7 @@ public class NativeUnixDirectory extends
   private final static long ALIGN = 512;
   private final static long ALIGN_NOT_MASK = ~(ALIGN-1);
   
-  /** Default buffer size before writing to disk (256 MB);
+  /** Default buffer size before writing to disk (256 KB);
    *  larger means less IO load but more RAM and direct
    *  buffer storage space consumed during merging. */
 

Modified: lucene/dev/branches/lucene4258/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java (original)
+++ lucene/dev/branches/lucene4258/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java Sun Mar 10 15:04:57 2013
@@ -23,7 +23,6 @@ import org.apache.lucene.search.similari
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.search.similarities.TFIDFSimilarity;
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.index.Norm;
 import org.apache.lucene.index.FieldInvertState;
 
 
@@ -37,9 +36,7 @@ public class SweetSpotSimilarityTest ext
   }
   
   public static byte computeAndGetNorm(Similarity s, FieldInvertState state) {
-    Norm norm = new Norm();
-    s.computeNorm(state, norm);
-    return norm.field().numericValue().byteValue();
+    return (byte) s.computeNorm(state);
   }
 
   public void testSweetSpotComputeNorm() {
@@ -56,10 +53,6 @@ public class SweetSpotSimilarityTest ext
     invertState.setBoost(1.0f);
     for (int i = 1; i < 1000; i++) {
       invertState.setLength(i);
-      Norm lNorm = new Norm();
-      Norm rNorm = new Norm();
-      d.computeNorm(invertState, lNorm);
-      s.computeNorm(invertState, rNorm);
       assertEquals("base case: i="+i,
                    computeAndGetNorm(d, invertState),
                    computeAndGetNorm(s, invertState),
@@ -90,7 +83,7 @@ public class SweetSpotSimilarityTest ext
     }
 
 
-    // seperate sweet spot for certain fields
+    // separate sweet spot for certain fields
 
     final SweetSpotSimilarity ssBar = new SweetSpotSimilarity();
     ssBar.setLengthNormFactors(8,13, 0.5f, false);

Modified: lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/CommonTermsQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/CommonTermsQuery.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/CommonTermsQuery.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/CommonTermsQuery.java Sun Mar 10 15:04:57 2013
@@ -74,7 +74,7 @@ public class CommonTermsQuery extends Qu
   protected final Occur highFreqOccur;
   protected float lowFreqBoost = 1.0f;
   protected float highFreqBoost = 1.0f;
-  protected int minNrShouldMatch = 0;
+  protected float minNrShouldMatch = 0;
   
   /**
    * Creates a new {@link CommonTermsQuery}
@@ -84,7 +84,7 @@ public class CommonTermsQuery extends Qu
    * @param lowFreqOccur
    *          {@link Occur} used for low frequency terms
    * @param maxTermFrequency
-   *          a value in [0..1] (or absolute number >=1) representing the
+   *          a value in [0..1) (or absolute number >=1) representing the
    *          maximum threshold of a terms document frequency to be considered a
    *          low frequency term.
    * @throws IllegalArgumentException
@@ -104,7 +104,7 @@ public class CommonTermsQuery extends Qu
    * @param lowFreqOccur
    *          {@link Occur} used for low frequency terms
    * @param maxTermFrequency
-   *          a value in [0..1] (or absolute number >=1) representing the
+   *          a value in [0..1) (or absolute number >=1) representing the
    *          maximum threshold of a terms document frequency to be considered a
    *          low frequency term.
    * @param disableCoord
@@ -160,15 +160,19 @@ public class CommonTermsQuery extends Qu
     return buildQuery(maxDoc, contextArray, queryTerms);
   }
   
+  protected int calcLowFreqMinimumNumberShouldMatch(int numOptional) {
+      if (minNrShouldMatch >= 1.0f || minNrShouldMatch == 0.0f) {
+          return (int) minNrShouldMatch;
+      }
+      return (int) (Math.round(minNrShouldMatch * numOptional));
+  }
+  
   protected Query buildQuery(final int maxDoc,
       final TermContext[] contextArray, final Term[] queryTerms) {
     BooleanQuery lowFreq = new BooleanQuery(disableCoord);
     BooleanQuery highFreq = new BooleanQuery(disableCoord);
     highFreq.setBoost(highFreqBoost);
     lowFreq.setBoost(lowFreqBoost);
-    if (lowFreqOccur == Occur.SHOULD) {
-      lowFreq.setMinimumNumberShouldMatch(minNrShouldMatch);
-    }
     BooleanQuery query = new BooleanQuery(true);
     for (int i = 0; i < queryTerms.length; i++) {
       TermContext termContext = contextArray[i];
@@ -186,6 +190,11 @@ public class CommonTermsQuery extends Qu
       }
       
     }
+    final int numLowFreqClauses = lowFreq.clauses().size(); 
+    if (lowFreqOccur == Occur.SHOULD && numLowFreqClauses > 0) {
+      int minMustMatch = calcLowFreqMinimumNumberShouldMatch(numLowFreqClauses);
+      lowFreq.setMinimumNumberShouldMatch(minMustMatch);
+    }
     if (lowFreq.clauses().isEmpty()) {
       /*
        * if lowFreq is empty we rewrite the high freq terms in a conjunction to
@@ -265,7 +274,9 @@ public class CommonTermsQuery extends Qu
   /**
    * Specifies a minimum number of the optional BooleanClauses which must be
    * satisfied in order to produce a match on the low frequency terms query
-   * part.
+   * part. This method accepts a float value in the range [0..1) as a fraction
+   * of the actual query terms in the low frequent clause or a number
+   * <tt>&gt;=1</tt> as an absolut number of clauses that need to match.
    * 
    * <p>
    * By default no optional clauses are necessary for a match (unless there are
@@ -276,7 +287,7 @@ public class CommonTermsQuery extends Qu
    * @param min
    *          the number of optional clauses that must match
    */
-  public void setMinimumNumberShouldMatch(int min) {
+  public void setMinimumNumberShouldMatch(float min) {
     this.minNrShouldMatch = min;
   }
   
@@ -284,7 +295,7 @@ public class CommonTermsQuery extends Qu
    * Gets the minimum number of the optional BooleanClauses which must be
    * satisfied.
    */
-  public int getMinimumNumberShouldMatch() {
+  public float getMinimumNumberShouldMatch() {
     return minNrShouldMatch;
   }
   
@@ -332,7 +343,7 @@ public class CommonTermsQuery extends Qu
     result = prime * result
         + ((lowFreqOccur == null) ? 0 : lowFreqOccur.hashCode());
     result = prime * result + Float.floatToIntBits(maxTermFrequency);
-    result = prime * result + minNrShouldMatch;
+    result = prime * result + Float.floatToIntBits(minNrShouldMatch);
     result = prime * result + ((terms == null) ? 0 : terms.hashCode());
     return result;
   }

Modified: lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreProvider.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreProvider.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreProvider.java Sun Mar 10 15:04:57 2013
@@ -21,6 +21,7 @@ import java.io.IOException;
 
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.IndexReader; // for javadocs
+import org.apache.lucene.queries.function.FunctionQuery;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.FieldCache; // for javadocs
 
@@ -55,7 +56,7 @@ public class CustomScoreProvider {
    * <p>
    * If your custom scoring is different than the default herein you 
    * should override at least one of the two customScore() methods.
-   * If the number of ValueSourceQueries is always &lt; 2 it is 
+   * If the number of {@link FunctionQuery function queries} is always &lt; 2 it is 
    * sufficient to override the other 
    * {@link #customScore(int, float, float) customScore()} 
    * method, which is simpler. 
@@ -67,7 +68,7 @@ public class CustomScoreProvider {
    * 
    * @param doc id of scored doc. 
    * @param subQueryScore score of that doc by the subQuery.
-   * @param valSrcScores scores of that doc by the ValueSourceQuery.
+   * @param valSrcScores scores of that doc by the {@link FunctionQuery}.
    * @return custom score.
    */
   public float customScore(int doc, float subQueryScore, float valSrcScores[]) throws IOException {
@@ -85,13 +86,13 @@ public class CustomScoreProvider {
   }
 
   /**
-   * Compute a custom score by the subQuery score and the ValueSourceQuery score.
+   * Compute a custom score by the subQuery score and the {@link FunctionQuery} score.
    * <p> 
    * Subclasses can override this method to modify the custom score.
    * <p>
    * If your custom scoring is different than the default herein you 
    * should override at least one of the two customScore() methods.
-   * If the number of ValueSourceQueries is always &lt; 2 it is 
+   * If the number of {@link FunctionQuery function queries} is always &lt; 2 it is 
    * sufficient to override this customScore() method, which is simpler. 
    * <p>
    * The default computation herein is a multiplication of the two scores:
@@ -101,7 +102,7 @@ public class CustomScoreProvider {
    *
    * @param doc id of scored doc. 
    * @param subQueryScore score of that doc by the subQuery.
-   * @param valSrcScore score of that doc by the ValueSourceQuery.
+   * @param valSrcScore score of that doc by the {@link FunctionQuery}.
    * @return custom score.
    */
   public float customScore(int doc, float subQueryScore, float valSrcScore) throws IOException {

Modified: lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java Sun Mar 10 15:04:57 2013
@@ -26,6 +26,8 @@ import java.util.Arrays;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.queries.function.FunctionQuery;
+import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.search.ComplexExplanation;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.Query;
@@ -39,7 +41,7 @@ import org.apache.lucene.util.ToStringUt
  * Query that sets document score as a programmatic function of several (sub) scores:
  * <ol>
  *    <li>the score of its subQuery (any query)</li>
- *    <li>(optional) the score of its ValueSourceQuery (or queries).</li>
+ *    <li>(optional) the score of its {@link FunctionQuery} (or queries).</li>
  * </ol>
  * Subclasses can modify the computation by overriding {@link #getCustomScoreProvider}.
  * 
@@ -355,12 +357,12 @@ public class CustomScoreQuery extends Qu
 
   /**
    * Checks if this is strict custom scoring.
-   * In strict custom scoring, the ValueSource part does not participate in weight normalization.
+   * In strict custom scoring, the {@link ValueSource} part does not participate in weight normalization.
    * This may be useful when one wants full control over how scores are modified, and does 
-   * not care about normalizing by the ValueSource part.
+   * not care about normalizing by the {@link ValueSource} part.
    * One particular case where this is useful if for testing this query.   
    * <P>
-   * Note: only has effect when the ValueSource part is not null.
+   * Note: only has effect when the {@link ValueSource} part is not null.
    */
   public boolean isStrict() {
     return strict;

Modified: lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java Sun Mar 10 15:04:57 2013
@@ -17,26 +17,29 @@
 
 package org.apache.lucene.queries.function.docvalues;
 
+import java.io.IOException;
+
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.queries.function.ValueSourceScorer;
 import org.apache.lucene.search.FieldCache;
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CharsRef;
 import org.apache.lucene.util.UnicodeUtil;
 import org.apache.lucene.util.mutable.MutableValue;
 import org.apache.lucene.util.mutable.MutableValueStr;
 
-import java.io.IOException;
-
 /**
- * Internal class, subject to change.
  * Serves as base class for FunctionValues based on DocTermsIndex.
+ * @lucene.internal
  */
 public abstract class DocTermsIndexDocValues extends FunctionValues {
-  protected final FieldCache.DocTermsIndex termsIndex;
+  protected final SortedDocValues termsIndex;
+  protected final Bits valid;
   protected final ValueSource vs;
   protected final MutableValueStr val = new MutableValueStr();
   protected final BytesRef spare = new BytesRef();
@@ -45,40 +48,42 @@ public abstract class DocTermsIndexDocVa
   public DocTermsIndexDocValues(ValueSource vs, AtomicReaderContext context, String field) throws IOException {
     try {
       termsIndex = FieldCache.DEFAULT.getTermsIndex(context.reader(), field);
+      valid = FieldCache.DEFAULT.getDocsWithField(context.reader(), field);
     } catch (RuntimeException e) {
       throw new DocTermsIndexException(field, e);
     }
     this.vs = vs;
   }
 
-  public FieldCache.DocTermsIndex getDocTermsIndex() {
-    return termsIndex;
-  }
-
   protected abstract String toTerm(String readableValue);
 
   @Override
   public boolean exists(int doc) {
-    return termsIndex.getOrd(doc) != 0;
+    return valid.get(doc);
   }
 
+  @Override
+  public int ordVal(int doc) {
+    return termsIndex.getOrd(doc);
+  }
+
+  @Override
+  public int numOrd() {
+    return termsIndex.getValueCount();
+  }
 
   @Override
   public boolean bytesVal(int doc, BytesRef target) {
-    int ord=termsIndex.getOrd(doc);
-    if (ord==0) {
-      target.length = 0;
-      return false;
-    }
-    termsIndex.lookup(ord, target);
-    return true;
+    termsIndex.get(doc, target);
+    return target.length > 0;
   }
 
   @Override
   public String strVal(int doc) {
-    int ord=termsIndex.getOrd(doc);
-    if (ord==0) return null;
-    termsIndex.lookup(ord, spare);
+    termsIndex.get(doc, spare);
+    if (spare.length == 0) {
+      return null;
+    }
     UnicodeUtil.UTF8toUTF16(spare, spareChars);
     return spareChars.toString();
   }
@@ -97,11 +102,9 @@ public abstract class DocTermsIndexDocVa
     lowerVal = lowerVal == null ? null : toTerm(lowerVal);
     upperVal = upperVal == null ? null : toTerm(upperVal);
 
-    final BytesRef spare = new BytesRef();
-
     int lower = Integer.MIN_VALUE;
     if (lowerVal != null) {
-      lower = termsIndex.binarySearchLookup(new BytesRef(lowerVal), spare);
+      lower = termsIndex.lookupTerm(new BytesRef(lowerVal));
       if (lower < 0) {
         lower = -lower-1;
       } else if (!includeLower) {
@@ -111,7 +114,7 @@ public abstract class DocTermsIndexDocVa
 
     int upper = Integer.MAX_VALUE;
     if (upperVal != null) {
-      upper = termsIndex.binarySearchLookup(new BytesRef(upperVal), spare);
+      upper = termsIndex.lookupTerm(new BytesRef(upperVal));
       if (upper < 0) {
         upper = -upper-2;
       } else if (!includeUpper) {
@@ -148,9 +151,8 @@ public abstract class DocTermsIndexDocVa
 
       @Override
       public void fillValue(int doc) {
-        int ord = termsIndex.getOrd(doc);
-        mval.exists = ord != 0;
-        mval.value = termsIndex.lookup(ord, mval.value);
+        termsIndex.get(doc, mval.value);
+        mval.exists = mval.value.bytes != SortedDocValues.MISSING;
       }
     };
   }

Modified: lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ByteFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ByteFieldSource.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ByteFieldSource.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ByteFieldSource.java Sun Mar 10 15:04:57 2013
@@ -51,42 +51,42 @@ public class ByteFieldSource extends Fie
 
   @Override
   public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
-    final byte[] arr = cache.getBytes(readerContext.reader(), field, parser, false);
+    final FieldCache.Bytes arr = cache.getBytes(readerContext.reader(), field, parser, false);
     
     return new FunctionValues() {
       @Override
       public byte byteVal(int doc) {
-        return arr[doc];
+        return arr.get(doc);
       }
 
       @Override
       public short shortVal(int doc) {
-        return (short) arr[doc];
+        return (short) arr.get(doc);
       }
 
       @Override
       public float floatVal(int doc) {
-        return (float) arr[doc];
+        return (float) arr.get(doc);
       }
 
       @Override
       public int intVal(int doc) {
-        return (int) arr[doc];
+        return (int) arr.get(doc);
       }
 
       @Override
       public long longVal(int doc) {
-        return (long) arr[doc];
+        return (long) arr.get(doc);
       }
 
       @Override
       public double doubleVal(int doc) {
-        return (double) arr[doc];
+        return (double) arr.get(doc);
       }
 
       @Override
       public String strVal(int doc) {
-        return Byte.toString(arr[doc]);
+        return Byte.toString(arr.get(doc));
       }
 
       @Override
@@ -96,7 +96,7 @@ public class ByteFieldSource extends Fie
 
       @Override
       public Object objectVal(int doc) {
-        return arr[doc];  // TODO: valid?
+        return arr.get(doc);  // TODO: valid?
       }
 
     };

Modified: lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java Sun Mar 10 15:04:57 2013
@@ -17,12 +17,18 @@ package org.apache.lucene.queries.functi
  * limitations under the License.
  */
 
+import java.io.IOException;
+import java.util.Map;
+
 import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
 import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.docvalues.DocTermsIndexDocValues;
-
-import java.io.IOException;
-import java.util.Map;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
 
 /**
  * An implementation for retrieving {@link FunctionValues} instances for string based fields.
@@ -35,23 +41,59 @@ public class BytesRefFieldSource extends
 
   @Override
   public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
-    return new DocTermsIndexDocValues(this, readerContext, field) {
-
-      @Override
-      protected String toTerm(String readableValue) {
-        return readableValue;
-      }
-
-      @Override
-      public Object objectVal(int doc) {
-        return strVal(doc);
-      }
-
-      @Override
-      public String toString(int doc) {
-        return description() + '=' + strVal(doc);
-      }
-
-    };
+    final FieldInfo fieldInfo = readerContext.reader().getFieldInfos().fieldInfo(field);
+    // To be sorted or not to be sorted, that is the question
+    // TODO: do it cleaner?
+    if (fieldInfo != null && fieldInfo.getDocValuesType() == DocValuesType.BINARY) {
+      final BinaryDocValues binaryValues = FieldCache.DEFAULT.getTerms(readerContext.reader(), field);
+      return new FunctionValues() {
+
+        @Override
+        public boolean exists(int doc) {
+          return true; // doc values are dense
+        }
+
+        @Override
+        public boolean bytesVal(int doc, BytesRef target) {
+          binaryValues.get(doc, target);
+          return target.length > 0;
+        }
+
+        public String strVal(int doc) {
+          final BytesRef bytes = new BytesRef();
+          return bytesVal(doc, bytes)
+              ? bytes.utf8ToString()
+              : null;
+        }
+
+        @Override
+        public Object objectVal(int doc) {
+          return strVal(doc);
+        }
+
+        @Override
+        public String toString(int doc) {
+          return description() + '=' + strVal(doc);
+        }
+      };
+    } else {
+      return new DocTermsIndexDocValues(this, readerContext, field) {
+
+        @Override
+        protected String toTerm(String readableValue) {
+          return readableValue;
+        }
+
+        @Override
+        public Object objectVal(int doc) {
+          return strVal(doc);
+        }
+
+        @Override
+        public String toString(int doc) {
+          return description() + '=' + strVal(doc);
+        }
+      };
+    }
   }
 }

Modified: lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java Sun Mar 10 15:04:57 2013
@@ -58,12 +58,12 @@ public class DoubleFieldSource extends F
 
   @Override
   public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
-    final double[] arr = cache.getDoubles(readerContext.reader(), field, parser, true);
+    final FieldCache.Doubles arr = cache.getDoubles(readerContext.reader(), field, parser, true);
     final Bits valid = cache.getDocsWithField(readerContext.reader(), field);
     return new DoubleDocValues(this) {
       @Override
       public double doubleVal(int doc) {
-        return arr[doc];
+        return arr.get(doc);
       }
 
       @Override
@@ -132,7 +132,6 @@ public class DoubleFieldSource extends F
       @Override
       public ValueFiller getValueFiller() {
         return new ValueFiller() {
-          private final double[] doubleArr = arr;
           private final MutableValueDouble mval = new MutableValueDouble();
 
           @Override
@@ -142,7 +141,7 @@ public class DoubleFieldSource extends F
 
           @Override
           public void fillValue(int doc) {
-            mval.value = doubleArr[doc];
+            mval.value = arr.get(doc);
             mval.exists = valid.get(doc);
           }
         };

Modified: lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java Sun Mar 10 15:04:57 2013
@@ -56,18 +56,18 @@ public class FloatFieldSource extends Fi
 
   @Override
   public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
-    final float[] arr = cache.getFloats(readerContext.reader(), field, parser, true);
+    final FieldCache.Floats arr = cache.getFloats(readerContext.reader(), field, parser, true);
     final Bits valid = cache.getDocsWithField(readerContext.reader(), field);
 
     return new FloatDocValues(this) {
       @Override
       public float floatVal(int doc) {
-        return arr[doc];
+        return arr.get(doc);
       }
 
       @Override
       public Object objectVal(int doc) {
-        return valid.get(doc) ? arr[doc] : null;
+        return valid.get(doc) ? arr.get(doc) : null;
       }
 
       @Override
@@ -78,7 +78,6 @@ public class FloatFieldSource extends Fi
       @Override
       public ValueFiller getValueFiller() {
         return new ValueFiller() {
-          private final float[] floatArr = arr;
           private final MutableValueFloat mval = new MutableValueFloat();
 
           @Override
@@ -88,7 +87,7 @@ public class FloatFieldSource extends Fi
 
           @Override
           public void fillValue(int doc) {
-            mval.value = floatArr[doc];
+            mval.value = arr.get(doc);
             mval.exists = valid.get(doc);
           }
         };

Modified: lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java Sun Mar 10 15:04:57 2013
@@ -57,7 +57,7 @@ public class IntFieldSource extends Fiel
 
   @Override
   public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
-    final int[] arr = cache.getInts(readerContext.reader(), field, parser, true);
+    final FieldCache.Ints arr = cache.getInts(readerContext.reader(), field, parser, true);
     final Bits valid = cache.getDocsWithField(readerContext.reader(), field);
     
     return new IntDocValues(this) {
@@ -65,32 +65,32 @@ public class IntFieldSource extends Fiel
       
       @Override
       public float floatVal(int doc) {
-        return (float)arr[doc];
+        return (float)arr.get(doc);
       }
 
       @Override
       public int intVal(int doc) {
-        return arr[doc];
+        return arr.get(doc);
       }
 
       @Override
       public long longVal(int doc) {
-        return (long)arr[doc];
+        return (long)arr.get(doc);
       }
 
       @Override
       public double doubleVal(int doc) {
-        return (double)arr[doc];
+        return (double)arr.get(doc);
       }
 
       @Override
       public String strVal(int doc) {
-        return Float.toString(arr[doc]);
+        return Integer.toString(arr.get(doc));
       }
 
       @Override
       public Object objectVal(int doc) {
-        return valid.get(doc) ? arr[doc] : null;
+        return valid.get(doc) ? arr.get(doc) : null;
       }
 
       @Override
@@ -129,7 +129,7 @@ public class IntFieldSource extends Fiel
         return new ValueSourceScorer(reader, this) {
           @Override
           public boolean matchesValue(int doc) {
-            int val = arr[doc];
+            int val = arr.get(doc);
             // only check for deleted if it's the default value
             // if (val==0 && reader.isDeleted(doc)) return false;
             return val >= ll && val <= uu;
@@ -140,7 +140,6 @@ public class IntFieldSource extends Fiel
       @Override
       public ValueFiller getValueFiller() {
         return new ValueFiller() {
-          private final int[] intArr = arr;
           private final MutableValueInt mval = new MutableValueInt();
 
           @Override
@@ -150,7 +149,7 @@ public class IntFieldSource extends Fiel
 
           @Override
           public void fillValue(int doc) {
-            mval.value = intArr[doc];
+            mval.value = arr.get(doc);
             mval.exists = valid.get(doc);
           }
         };

Modified: lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java Sun Mar 10 15:04:57 2013
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.util.Map;
 
 import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.index.ReaderUtil;
@@ -28,7 +29,6 @@ import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.docvalues.IntDocValues;
-import org.apache.lucene.search.FieldCache.DocTerms;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.packed.PackedInts;
 
@@ -56,7 +56,7 @@ public class JoinDocFreqValueSource exte
   @Override
   public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException
   {
-    final DocTerms terms = cache.getTerms(readerContext.reader(), field, PackedInts.FAST);
+    final BinaryDocValues terms = cache.getTerms(readerContext.reader(), field, PackedInts.FAST);
     final IndexReader top = ReaderUtil.getTopLevelContext(readerContext).reader();
     Terms t = MultiFields.getTerms(top, qfield);
     final TermsEnum termsEnum = t == null ? TermsEnum.EMPTY : t.iterator(null);
@@ -68,7 +68,7 @@ public class JoinDocFreqValueSource exte
       public int intVal(int doc) 
       {
         try {
-          terms.getTerm(doc, ref);
+          terms.get(doc, ref);
           if (termsEnum.seekExact(ref, true)) {
             return termsEnum.docFreq();
           } else {

Modified: lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java Sun Mar 10 15:04:57 2013
@@ -64,15 +64,19 @@ public class LongFieldSource extends Fie
     return val;
   }
 
+  public String longToString(long val) {
+    return longToObject(val).toString();
+  }
+
   @Override
   public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
-    final long[] arr = cache.getLongs(readerContext.reader(), field, parser, true);
+    final FieldCache.Longs arr = cache.getLongs(readerContext.reader(), field, parser, true);
     final Bits valid = cache.getDocsWithField(readerContext.reader(), field);
     
     return new LongDocValues(this) {
       @Override
       public long longVal(int doc) {
-        return arr[doc];
+        return arr.get(doc);
       }
 
       @Override
@@ -82,7 +86,12 @@ public class LongFieldSource extends Fie
 
       @Override
       public Object objectVal(int doc) {
-        return valid.get(doc) ? longToObject(arr[doc]) : null;
+        return valid.get(doc) ? longToObject(arr.get(doc)) : null;
+      }
+
+      @Override
+      public String strVal(int doc) {
+        return valid.get(doc) ? longToString(arr.get(doc)) : null;
       }
 
       @Override
@@ -111,7 +120,7 @@ public class LongFieldSource extends Fie
         return new ValueSourceScorer(reader, this) {
           @Override
           public boolean matchesValue(int doc) {
-            long val = arr[doc];
+            long val = arr.get(doc);
             // only check for deleted if it's the default value
             // if (val==0 && reader.isDeleted(doc)) return false;
             return val >= ll && val <= uu;
@@ -122,7 +131,6 @@ public class LongFieldSource extends Fie
       @Override
       public ValueFiller getValueFiller() {
         return new ValueFiller() {
-          private final long[] longArr = arr;
           private final MutableValueLong mval = newMutableValueLong();
 
           @Override
@@ -132,7 +140,7 @@ public class LongFieldSource extends Fie
 
           @Override
           public void fillValue(int doc) {
-            mval.value = longArr[doc];
+            mval.value = arr.get(doc);
             mval.exists = valid.get(doc);
           }
         };

Modified: lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java Sun Mar 10 15:04:57 2013
@@ -18,7 +18,7 @@
 package org.apache.lucene.queries.function.valuesource;
 
 import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.queries.function.docvalues.FloatDocValues;
@@ -62,18 +62,16 @@ public class NormValueSource extends Val
     if (similarity == null) {
       throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as DefaultSimilarity)");
     }
-    DocValues dv = readerContext.reader().normValues(field);
+    final NumericDocValues norms = readerContext.reader().getNormValues(field);
 
-    if (dv == null) {
+    if (norms == null) {
       return new ConstDoubleDocValues(0.0, this);
     }
     
-    final byte[] norms = (byte[]) dv.getSource().getArray();
-
     return new FloatDocValues(this) {
       @Override
       public float floatVal(int doc) {
-        return similarity.decodeNormValue(norms[doc]);
+        return similarity.decodeNormValue((byte)norms.get(doc));
       }
     };
   }

Modified: lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/OrdFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/OrdFieldSource.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/OrdFieldSource.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/OrdFieldSource.java Sun Mar 10 15:04:57 2013
@@ -17,12 +17,16 @@
 
 package org.apache.lucene.queries.function.valuesource;
 
+import java.io.IOException;
+import java.util.Map;
+
 import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.CompositeReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.ReaderUtil;
 import org.apache.lucene.index.SlowCompositeReaderWrapper;
+import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.queries.function.docvalues.IntDocValues;
@@ -30,9 +34,6 @@ import org.apache.lucene.search.FieldCac
 import org.apache.lucene.util.mutable.MutableValue;
 import org.apache.lucene.util.mutable.MutableValueInt;
 
-import java.io.IOException;
-import java.util.Map;
-
 /**
  * Obtains the ordinal of the field value from the default Lucene {@link org.apache.lucene.search.FieldCache} using getStringIndex().
  * <br>
@@ -72,7 +73,7 @@ public class OrdFieldSource extends Valu
     final AtomicReader r = topReader instanceof CompositeReader 
         ? new SlowCompositeReaderWrapper((CompositeReader)topReader) 
         : (AtomicReader) topReader;
-    final FieldCache.DocTermsIndex sindex = FieldCache.DEFAULT.getTermsIndex(r, field);
+    final SortedDocValues sindex = FieldCache.DEFAULT.getTermsIndex(r, field);
     return new IntDocValues(this) {
       protected String toTerm(String readableValue) {
         return readableValue;
@@ -87,7 +88,7 @@ public class OrdFieldSource extends Valu
       }
       @Override
       public int numOrd() {
-        return sindex.numOrd();
+        return sindex.getValueCount();
       }
 
       @Override

Modified: lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReverseOrdFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReverseOrdFieldSource.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReverseOrdFieldSource.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReverseOrdFieldSource.java Sun Mar 10 15:04:57 2013
@@ -17,20 +17,21 @@
 
 package org.apache.lucene.queries.function.valuesource;
 
+import java.io.IOException;
+import java.util.Map;
+
 import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.CompositeReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.ReaderUtil;
 import org.apache.lucene.index.SlowCompositeReaderWrapper;
+import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.queries.function.docvalues.IntDocValues;
 import org.apache.lucene.search.FieldCache;
 
-import java.io.IOException;
-import java.util.Map;
-
 /**
  * Obtains the ordinal of the field value from the default Lucene {@link org.apache.lucene.search.FieldCache} using getTermsIndex()
  * and reverses the order.
@@ -73,13 +74,13 @@ public class ReverseOrdFieldSource exten
         : (AtomicReader) topReader;
     final int off = readerContext.docBase;
 
-    final FieldCache.DocTermsIndex sindex = FieldCache.DEFAULT.getTermsIndex(r, field);
-    final int end = sindex.numOrd();
+    final SortedDocValues sindex = FieldCache.DEFAULT.getTermsIndex(r, field);
+    final int end = sindex.getValueCount();
 
     return new IntDocValues(this) {
      @Override
       public int intVal(int doc) {
-        return (end - sindex.getOrd(doc+off));
+        return (end - sindex.getOrd(doc+off) - 1);
       }
     };
   }

Modified: lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ShortFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ShortFieldSource.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ShortFieldSource.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ShortFieldSource.java Sun Mar 10 15:04:57 2013
@@ -49,42 +49,42 @@ public class ShortFieldSource extends Fi
 
   @Override
   public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
-    final short[] arr = cache.getShorts(readerContext.reader(), field, parser, false);
+    final FieldCache.Shorts arr = cache.getShorts(readerContext.reader(), field, parser, false);
     
     return new FunctionValues() {
       @Override
       public byte byteVal(int doc) {
-        return (byte) arr[doc];
+        return (byte) arr.get(doc);
       }
 
       @Override
       public short shortVal(int doc) {
-        return arr[doc];
+        return arr.get(doc);
       }
 
       @Override
       public float floatVal(int doc) {
-        return (float) arr[doc];
+        return (float) arr.get(doc);
       }
 
       @Override
       public int intVal(int doc) {
-        return (int) arr[doc];
+        return (int) arr.get(doc);
       }
 
       @Override
       public long longVal(int doc) {
-        return (long) arr[doc];
+        return (long) arr.get(doc);
       }
 
       @Override
       public double doubleVal(int doc) {
-        return (double) arr[doc];
+        return (double) arr.get(doc);
       }
 
       @Override
       public String strVal(int doc) {
-        return Short.toString(arr[doc]);
+        return Short.toString(arr.get(doc));
       }
 
       @Override

Modified: lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java Sun Mar 10 15:04:57 2013
@@ -175,6 +175,90 @@ public class CommonTermsQueryTest extend
     }
   }
   
+  public void testMinShouldMatch() throws IOException {
+    Directory dir = newDirectory();
+    RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+    String[] docs = new String[] {"this is the end of the world right",
+        "is this it or maybe not",
+        "this is the end of the universe as we know it",
+        "there is the famous restaurant at the end of the universe",};
+    for (int i = 0; i < docs.length; i++) {
+      Document doc = new Document();
+      doc.add(newStringField("id", "" + i, Field.Store.YES));
+      doc.add(newTextField("field", docs[i], Field.Store.NO));
+      w.addDocument(doc);
+    }
+    
+    IndexReader r = w.getReader();
+    IndexSearcher s = newSearcher(r);
+    {
+      CommonTermsQuery query = new CommonTermsQuery(Occur.SHOULD, Occur.SHOULD,
+          random().nextBoolean() ? 2.0f : 0.5f);
+      query.add(new Term("field", "is"));
+      query.add(new Term("field", "this"));
+      query.add(new Term("field", "end"));
+      query.add(new Term("field", "world"));
+      query.add(new Term("field", "universe"));
+      query.add(new Term("field", "right"));
+      query.setMinimumNumberShouldMatch(0.5f);
+      TopDocs search = s.search(query, 10);
+      assertEquals(search.totalHits, 1);
+      assertEquals("0", r.document(search.scoreDocs[0].doc).get("id"));
+    }
+    {
+      CommonTermsQuery query = new CommonTermsQuery(Occur.SHOULD, Occur.SHOULD,
+          random().nextBoolean() ? 2.0f : 0.5f);
+      query.add(new Term("field", "is"));
+      query.add(new Term("field", "this"));
+      query.add(new Term("field", "end"));
+      query.add(new Term("field", "world"));
+      query.add(new Term("field", "universe"));
+      query.add(new Term("field", "right"));
+      query.setMinimumNumberShouldMatch(2.0f);
+      TopDocs search = s.search(query, 10);
+      assertEquals(search.totalHits, 1);
+      assertEquals("0", r.document(search.scoreDocs[0].doc).get("id"));
+    }
+    
+    {
+      CommonTermsQuery query = new CommonTermsQuery(Occur.SHOULD, Occur.SHOULD,
+          random().nextBoolean() ? 2.0f : 0.5f);
+      query.add(new Term("field", "is"));
+      query.add(new Term("field", "this"));
+      query.add(new Term("field", "end"));
+      query.add(new Term("field", "world"));
+      query.add(new Term("field", "universe"));
+      query.add(new Term("field", "right"));
+      query.setMinimumNumberShouldMatch(0.49f);
+      TopDocs search = s.search(query, 10);
+      assertEquals(search.totalHits, 3);
+      assertEquals("0", r.document(search.scoreDocs[0].doc).get("id"));
+      assertEquals("2", r.document(search.scoreDocs[1].doc).get("id"));
+      assertEquals("3", r.document(search.scoreDocs[2].doc).get("id"));
+    }
+    
+    {
+      CommonTermsQuery query = new CommonTermsQuery(Occur.SHOULD, Occur.SHOULD,
+          random().nextBoolean() ? 2.0f : 0.5f);
+      query.add(new Term("field", "is"));
+      query.add(new Term("field", "this"));
+      query.add(new Term("field", "end"));
+      query.add(new Term("field", "world"));
+      query.add(new Term("field", "universe"));
+      query.add(new Term("field", "right"));
+      query.setMinimumNumberShouldMatch(1.0f);
+      TopDocs search = s.search(query, 10);
+      assertEquals(search.totalHits, 3);
+      assertEquals("0", r.document(search.scoreDocs[0].doc).get("id"));
+      assertEquals("2", r.document(search.scoreDocs[1].doc).get("id"));
+      assertEquals("3", r.document(search.scoreDocs[2].doc).get("id"));
+    }
+   
+    r.close();
+    w.close();
+    dir.close();
+  }
+  
   public void testIllegalOccur() {
     Random random = random();
     

Modified: lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java Sun Mar 10 15:04:57 2013
@@ -183,12 +183,12 @@ public class TestCustomScoreQuery extend
 
     @Override
     protected CustomScoreProvider getCustomScoreProvider(AtomicReaderContext context) throws IOException {
-      final int[] values = FieldCache.DEFAULT.getInts(context.reader(), INT_FIELD, false);
+      final FieldCache.Ints values = FieldCache.DEFAULT.getInts(context.reader(), INT_FIELD, false);
       return new CustomScoreProvider(context) {
         @Override
         public float customScore(int doc, float subScore, float valSrcScore) {
           assertTrue(doc <= context.reader().maxDoc());
-          return values[doc];
+          return values.get(doc);
         }
       };
     }



Mime
View raw message