lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1146632 [1/2] - in /lucene/dev/branches/fieldtype: lucene/ lucene/contrib/ lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/ lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/ lucene/c...
Date Thu, 14 Jul 2011 10:22:03 GMT
Author: mikemccand
Date: Thu Jul 14 10:21:56 2011
New Revision: 1146632

URL: http://svn.apache.org/viewvc?rev=1146632&view=rev
Log:
LUCENE-3309: switch core to very minimal but expert API for retrieving stored fields, but keep sugar for simple access to load a reconstituted Document

Added:
    lucene/dev/branches/fieldtype/lucene/contrib/misc/src/java/org/apache/lucene/document/
    lucene/dev/branches/fieldtype/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelector.java
      - copied, changed from r1143927, lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/FieldSelector.java
    lucene/dev/branches/fieldtype/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelectorResult.java
      - copied unchanged from r1143927, lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/FieldSelectorResult.java
    lucene/dev/branches/fieldtype/lucene/contrib/misc/src/java/org/apache/lucene/document/LoadFirstFieldSelector.java
      - copied unchanged from r1143927, lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/LoadFirstFieldSelector.java
    lucene/dev/branches/fieldtype/lucene/contrib/misc/src/java/org/apache/lucene/document/MapFieldSelector.java
      - copied unchanged from r1143927, lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/MapFieldSelector.java
    lucene/dev/branches/fieldtype/lucene/contrib/misc/src/java/org/apache/lucene/document/SetBasedFieldSelector.java
      - copied unchanged from r1143927, lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/SetBasedFieldSelector.java
    lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldsReader.java   (with props)
    lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexReader.java   (with props)
    lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/index/TestLazyBug.java
      - copied, changed from r1145297, lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestLazyBug.java
    lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/index/TestParallelReader.java   (with props)
    lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/search/
    lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/search/TestThreadSafe.java
      - copied, changed from r1145297, lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/search/TestThreadSafe.java
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocumentStoredFieldVisitor.java   (with props)
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/StoredFieldVisitor.java   (with props)
Removed:
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/FieldSelector.java
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/FieldSelectorResult.java
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/LoadFirstFieldSelector.java
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/MapFieldSelector.java
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/SetBasedFieldSelector.java
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/FieldSelector.java
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/FieldSelectorResult.java
    lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestLazyBug.java
    lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/search/TestThreadSafe.java
Modified:
    lucene/dev/branches/fieldtype/lucene/CHANGES.txt
    lucene/dev/branches/fieldtype/lucene/contrib/CHANGES.txt
    lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java
    lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
    lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java
    lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java
    lucene/dev/branches/fieldtype/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/package.html
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/Document.java
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/NumericField.java
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldInfo.java
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldsReader.java
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldsWriter.java
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/IndexReader.java
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/MultiReader.java
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/ParallelReader.java
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/SegmentReader.java
    lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/search/IndexSearcher.java
    lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java
    lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
    lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestParallelReader.java
    lucene/dev/branches/fieldtype/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchTravRetLoadFieldSelectorTask.java
    lucene/dev/branches/fieldtype/modules/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/DocNameExtractor.java
    lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/handler/component/TermVectorComponent.java
    lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java

Modified: lucene/dev/branches/fieldtype/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/CHANGES.txt?rev=1146632&r1=1146631&r2=1146632&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/fieldtype/lucene/CHANGES.txt Thu Jul 14 10:21:56 2011
@@ -238,6 +238,11 @@ Changes in Runtime Behavior
 
 * LUCENE-3146: IndexReader.setNorm throws IllegalStateException if the field
   does not store norms. (Shai Erera, Mike McCandless)
+
+* LUCENE-3309: Stored fields no longer record whether they were
+  tokenized or not.  In general you should not rely on stored fields
+  to record any "metadata" from indexing (tokenized, omitNorms,
+  IndexOptions, boost, etc.)  (Mike McCandless)
   
 API Changes
 

Modified: lucene/dev/branches/fieldtype/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/CHANGES.txt?rev=1146632&r1=1146631&r2=1146632&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/branches/fieldtype/lucene/contrib/CHANGES.txt Thu Jul 14 10:21:56 2011
@@ -5,6 +5,14 @@ http://s.apache.org/luceneversions
 
 ======================= Trunk (not yet released) =======================
   
+Changes in Runtime Behavior
+
+ * LUCENE-3309: Fast vector highlighter now inserts the
+   MultiValuedSeparator for NOT_ANALYZED fields (in addition to
+   ANALYZED fields).  To ensure your offsets are correct you should
+   provide an analyzer that returns 1 from the offsetGap method.
+   (Mike McCandless)
+
 Build
 
  * LUCENE-2845: Moved contrib/benchmark to modules.

Modified: lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java?rev=1146632&r1=1146631&r2=1146632&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java (original)
+++ lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java Thu Jul 14 10:21:56 2011
@@ -21,15 +21,16 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.document.MapFieldSelector;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.StoredFieldVisitor;
 import org.apache.lucene.search.highlight.DefaultEncoder;
 import org.apache.lucene.search.highlight.Encoder;
-import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo;
 import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo.SubInfo;
+import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo;
 import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo.Toffs;
+import org.apache.lucene.store.IndexInput;
 
 public abstract class BaseFragmentsBuilder implements FragmentsBuilder {
 
@@ -107,10 +108,24 @@ public abstract class BaseFragmentsBuild
     return fragments.toArray( new String[fragments.size()] );
   }
   
-  protected Field[] getFields( IndexReader reader, int docId, String fieldName) throws IOException {
+  protected Field[] getFields( IndexReader reader, int docId, final String fieldName) throws IOException {
     // according to javadoc, doc.getFields(fieldName) cannot be used with lazy loaded field???
-    Document doc = reader.document( docId, new MapFieldSelector(fieldName) );
-    return doc.getFields( fieldName ); // according to Document class javadoc, this never returns null
+    final List<Field> fields = new ArrayList<Field>();
+    reader.document(docId, new StoredFieldVisitor() {
+        @Override
+        public boolean stringField(FieldInfo fieldInfo, IndexInput in, int numUTF8Bytes) throws IOException {
+          if (fieldInfo.name.equals(fieldName)) {
+            final byte[] b = new byte[numUTF8Bytes];
+            in.readBytes(b, 0, b.length);
+            Field.TermVector termVector = Field.TermVector.toTermVector(fieldInfo.storeTermVector, fieldInfo.storeOffsetWithTermVector, fieldInfo.storePositionWithTermVector);
+            fields.add(new Field(fieldInfo.name, false, new String(b, "UTF-8"), Field.Store.YES, Field.Index.ANALYZED, termVector));
+          } else {
+            in.seek(in.getFilePointer() + numUTF8Bytes);
+          }
+          return false;
+        }
+      });
+    return fields.toArray(new Field[fields.size()]);
   }
 
   protected String makeFragment( StringBuilder buffer, int[] index, Field[] values, WeightedFragInfo fragInfo,
@@ -142,8 +157,7 @@ public abstract class BaseFragmentsBuild
       int startOffset, int endOffset ){
     while( buffer.length() < endOffset && index[0] < values.length ){
       buffer.append( values[index[0]].stringValue() );
-      if( values[index[0]].isTokenized() )
-        buffer.append( multiValuedSeparator );
+      buffer.append( multiValuedSeparator );
       index[0]++;
     }
     int eo = buffer.length() < endOffset ? buffer.length() : endOffset;

Modified: lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java?rev=1146632&r1=1146631&r2=1146632&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java (original)
+++ lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java Thu Jul 14 10:21:56 2011
@@ -34,9 +34,10 @@ import org.apache.lucene.document2.Field
 import org.apache.lucene.document2.TextField;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.queryParser.QueryParser;
 import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.PhraseQuery;
@@ -88,7 +89,26 @@ public abstract class AbstractTestCase e
     super.setUp();
     analyzerW = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
     analyzerB = new BigramAnalyzer();
-    analyzerK = new MockAnalyzer(random, MockTokenizer.KEYWORD, false);
+    final Analyzer k = new MockAnalyzer(random, MockTokenizer.KEYWORD, false);
+    analyzerK = new Analyzer() {
+      @Override
+      public TokenStream tokenStream(String fieldName, Reader reader) {
+        return k.tokenStream(fieldName, reader);
+      }
+
+      @Override
+      public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
+        return k.reusableTokenStream(fieldName, reader);
+      }
+
+      @Override
+      public int getOffsetGap(IndexableField field) {
+        // Because we add single-char separator for all
+        // (even not-tokenized) fields:
+        return 1;
+      }
+    };
+
     paW = new QueryParser(TEST_VERSION_CURRENT,  F, analyzerW );
     paB = new QueryParser(TEST_VERSION_CURRENT,  F, analyzerB );
     dir = newDirectory();

Modified: lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java?rev=1146632&r1=1146631&r2=1146632&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java (original)
+++ lucene/dev/branches/fieldtype/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java Thu Jul 14 10:21:56 2011
@@ -152,9 +152,8 @@ public class SimpleFragmentsBuilderTest 
     SimpleFragListBuilder sflb = new SimpleFragListBuilder();
     FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
     SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
-    // '/' separator doesn't effect the snippet because of NOT_ANALYZED field
     sfb.setMultiValuedSeparator( '/' );
-    assertEquals( "abc<b>defg</b>hijkl", sfb.createFragment( reader, 0, F, ffl ) );
+    assertEquals( "abc/<b>defg</b>/hijkl/", sfb.createFragment( reader, 0, F, ffl ) );
   }
   
   public void testMVSeparator() throws Exception {

Modified: lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java?rev=1146632&r1=1146631&r2=1146632&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java (original)
+++ lucene/dev/branches/fieldtype/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java Thu Jul 14 10:21:56 2011
@@ -30,7 +30,6 @@ import java.util.Set;
 import java.util.Comparator;
 
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.FieldSelector;
 import org.apache.lucene.index.*;
 import org.apache.lucene.index.IndexReader.ReaderContext;
 import org.apache.lucene.store.Directory;
@@ -252,42 +251,6 @@ public class InstantiatedIndexReader ext
   }
 
   /**
-   * Return the {@link org.apache.lucene.document.Document} at the <code>n</code><sup>th</sup>
-   * position.
-     <p>
-   * <b>Warning!</b>
-   * The resulting document is the actual stored document instance
-   * and not a deserialized clone as retuned by an IndexReader
-   * over a {@link org.apache.lucene.store.Directory}.
-   * I.e., if you need to touch the document, clone it first!
-   * <p>
-   * This can also be seen as a feature for live changes of stored values,
-   * but be careful! Adding a field with an name unknown to the index
-   * or to a field with previously no stored values will make
-   * {@link org.apache.lucene.store.instantiated.InstantiatedIndexReader#getFieldNames(org.apache.lucene.index.IndexReader.FieldOption)}
-   * out of sync, causing problems for instance when merging the
-   * instantiated index to another index.
-     <p>
-   * This implementation ignores the field selector! All stored fields are always returned!
-   * <p>
-   *
-   * @param n document number
-   * @param fieldSelector ignored
-   * @return The stored fields of the {@link org.apache.lucene.document.Document} at the nth position
-   * @throws CorruptIndexException if the index is corrupt
-   * @throws IOException if there is a low-level IO error
-   * 
-   * @see org.apache.lucene.document.Fieldable
-   * @see org.apache.lucene.document.FieldSelector
-   * @see org.apache.lucene.document.SetBasedFieldSelector
-   * @see org.apache.lucene.document.LoadFirstFieldSelector
-   */
-  @Override
-  public Document document(int n, FieldSelector fieldSelector) throws CorruptIndexException, IOException {
-    return document(n);
-  }
-
-  /**
    * Returns the stored fields of the <code>n</code><sup>th</sup>
    * <code>Document</code> in this index.
    * <p>
@@ -313,6 +276,11 @@ public class InstantiatedIndexReader ext
     return getIndex().getDocumentsByNumber()[n].getDocument();
   }
 
+  @Override
+  public void document(int docID, StoredFieldVisitor visitor) throws IOException {
+    throw new UnsupportedOperationException();
+  }
+
   /**
    * never ever touch these values. it is the true values, unless norms have
    * been touched.

Modified: lucene/dev/branches/fieldtype/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1146632&r1=1146631&r2=1146632&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/fieldtype/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Thu Jul 14 10:21:56 2011
@@ -35,23 +35,23 @@ import org.apache.lucene.analysis.tokena
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.FieldInvertState;
 import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.FieldsEnum;
 import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader.ReaderContext;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.OrdTermState;
-import org.apache.lucene.index.TermState;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.FieldsEnum;
-import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.StoredFieldVisitor;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermFreqVector;
 import org.apache.lucene.index.TermPositionVector;
+import org.apache.lucene.index.TermState;
 import org.apache.lucene.index.TermVectorMapper;
-import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.IndexReader.ReaderContext;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
@@ -60,8 +60,8 @@ import org.apache.lucene.search.Similari
 import org.apache.lucene.search.SimilarityProvider;
 import org.apache.lucene.store.RAMDirectory; // for javadocs
 import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Constants; // for javadocs
 
 /**
@@ -1225,16 +1225,9 @@ public class MemoryIndex {
     }
   
     @Override
-    public Document document(int n) {
-      if (DEBUG) System.err.println("MemoryIndexReader.document");
-      return new Document(); // there are no stored fields
-    }
-
-    //When we convert to JDK 1.5 make this Set<String>
-    @Override
-    public Document document(int n, FieldSelector fieldSelector) throws IOException {
+    public void document(int docID, StoredFieldVisitor visitor) {
       if (DEBUG) System.err.println("MemoryIndexReader.document");
-      return new Document(); // there are no stored fields
+      // no-op: there are no stored fields
     }
 
     @Override

Copied: lucene/dev/branches/fieldtype/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelector.java (from r1143927, lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/FieldSelector.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelector.java?p2=lucene/dev/branches/fieldtype/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelector.java&p1=lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/FieldSelector.java&r1=1143927&r2=1146632&rev=1146632&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/FieldSelector.java (original)
+++ lucene/dev/branches/fieldtype/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelector.java Thu Jul 14 10:21:56 2011
@@ -18,7 +18,7 @@ package org.apache.lucene.document;
 
 /**
  * Similar to a {@link java.io.FileFilter}, the FieldSelector allows one to make decisions about
- * what Fields get loaded on a {@link Document} by {@link org.apache.lucene.index.IndexReader#document(int,org.apache.lucene.document.FieldSelector)}
+ * what Fields get loaded on a {@link Document} by {@link FieldSelectorVisitor}
  *
  **/
 public interface FieldSelector {

Added: lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=1146632&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldsReader.java (added)
+++ lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldsReader.java Thu Jul 14 10:21:56 2011
@@ -0,0 +1,350 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.document.FieldSelectorResult;
+import org.apache.lucene.document.FieldSelectorVisitor;
+import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document.LoadFirstFieldSelector;
+import org.apache.lucene.document.SetBasedFieldSelector;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+
+public class TestFieldsReader extends LuceneTestCase {
+  private static Directory dir;
+  private static org.apache.lucene.document2.Document testDoc = new org.apache.lucene.document2.Document();
+  private static FieldInfos fieldInfos = null;
+
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    fieldInfos = new FieldInfos();
+    DocHelper.setupDoc(testDoc);
+    _TestUtil.add(testDoc, fieldInfos);
+    dir = newDirectory();
+    IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy());
+    ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(false);
+    IndexWriter writer = new IndexWriter(dir, conf);
+    writer.addDocument(testDoc);
+    writer.close();
+  }
+
+  @AfterClass
+  public static void afterClass() throws Exception {
+    dir.close();
+    dir = null;
+    fieldInfos = null;
+    testDoc = null;
+  }
+
+  private Document getDocument(IndexReader ir, int docID, FieldSelector selector) throws IOException {
+    final FieldSelectorVisitor visitor = new FieldSelectorVisitor(selector);
+    ir.document(docID, visitor);
+    return visitor.getDocument();
+  }
+
+  public void testLazyFields() throws Exception {
+    assertTrue(dir != null);
+    assertTrue(fieldInfos != null);
+    IndexReader reader = IndexReader.open(dir);
+    Set<String> loadFieldNames = new HashSet<String>();
+    loadFieldNames.add(DocHelper.TEXT_FIELD_1_KEY);
+    loadFieldNames.add(DocHelper.TEXT_FIELD_UTF1_KEY);
+    Set<String> lazyFieldNames = new HashSet<String>();
+    //new String[]{DocHelper.LARGE_LAZY_FIELD_KEY, DocHelper.LAZY_FIELD_KEY, DocHelper.LAZY_FIELD_BINARY_KEY};
+    lazyFieldNames.add(DocHelper.LARGE_LAZY_FIELD_KEY);
+    lazyFieldNames.add(DocHelper.LAZY_FIELD_KEY);
+    lazyFieldNames.add(DocHelper.LAZY_FIELD_BINARY_KEY);
+    lazyFieldNames.add(DocHelper.TEXT_FIELD_UTF2_KEY);
+    SetBasedFieldSelector fieldSelector = new SetBasedFieldSelector(loadFieldNames, lazyFieldNames);
+    Document doc = getDocument(reader, 0, fieldSelector);
+    assertTrue("doc is null and it shouldn't be", doc != null);
+    Fieldable field = doc.getFieldable(DocHelper.LAZY_FIELD_KEY);
+    assertTrue("field is null and it shouldn't be", field != null);
+    assertTrue("field is not lazy and it should be", field.isLazy());
+    String value = field.stringValue();
+    assertTrue("value is null and it shouldn't be", value != null);
+    assertTrue(value + " is not equal to " + DocHelper.LAZY_FIELD_TEXT, value.equals(DocHelper.LAZY_FIELD_TEXT) == true);
+    assertTrue("calling stringValue() twice should give same reference", field.stringValue() == field.stringValue());
+
+    field = doc.getFieldable(DocHelper.TEXT_FIELD_1_KEY);
+    assertTrue("field is null and it shouldn't be", field != null);
+    assertTrue("Field is lazy and it should not be", field.isLazy() == false);
+    field = doc.getFieldable(DocHelper.TEXT_FIELD_UTF1_KEY);
+    assertTrue("field is null and it shouldn't be", field != null);
+    assertTrue("Field is lazy and it should not be", field.isLazy() == false);
+    assertTrue(field.stringValue() + " is not equal to " + DocHelper.FIELD_UTF1_TEXT, field.stringValue().equals(DocHelper.FIELD_UTF1_TEXT) == true);
+
+    field = doc.getFieldable(DocHelper.TEXT_FIELD_UTF2_KEY);
+    assertTrue("field is null and it shouldn't be", field != null);
+    assertTrue("Field is lazy and it should not be", field.isLazy() == true);
+    assertTrue(field.stringValue() + " is not equal to " + DocHelper.FIELD_UTF2_TEXT, field.stringValue().equals(DocHelper.FIELD_UTF2_TEXT) == true);
+
+    field = doc.getFieldable(DocHelper.LAZY_FIELD_BINARY_KEY);
+    assertTrue("field is null and it shouldn't be", field != null);
+    assertTrue("stringValue isn't null for lazy binary field", field.stringValue() == null);
+
+    byte [] bytes = field.binaryValue(null).bytes;
+    assertTrue("bytes is null and it shouldn't be", bytes != null);
+    assertTrue("", DocHelper.LAZY_FIELD_BINARY_BYTES.length == bytes.length);
+    assertTrue("calling binaryValue() twice should give same reference", field.binaryValue(null).bytes == field.binaryValue(null).bytes);
+    for (int i = 0; i < bytes.length; i++) {
+      assertTrue("byte[" + i + "] is mismatched", bytes[i] == DocHelper.LAZY_FIELD_BINARY_BYTES[i]);
+
+    }
+    reader.close();
+  }
+
+  public void testLatentFields() throws Exception {
+    assertTrue(dir != null);
+    assertTrue(fieldInfos != null);
+    IndexReader reader = IndexReader.open(dir);
+    Set<String> loadFieldNames = new HashSet<String>();
+    loadFieldNames.add(DocHelper.TEXT_FIELD_1_KEY);
+    loadFieldNames.add(DocHelper.TEXT_FIELD_UTF1_KEY);
+    Set<String> lazyFieldNames = new HashSet<String>();
+    //new String[]{DocHelper.LARGE_LAZY_FIELD_KEY, DocHelper.LAZY_FIELD_KEY, DocHelper.LAZY_FIELD_BINARY_KEY};
+    lazyFieldNames.add(DocHelper.LARGE_LAZY_FIELD_KEY);
+    lazyFieldNames.add(DocHelper.LAZY_FIELD_KEY);
+    lazyFieldNames.add(DocHelper.LAZY_FIELD_BINARY_KEY);
+    lazyFieldNames.add(DocHelper.TEXT_FIELD_UTF2_KEY);
+
+    // Use LATENT instead of LAZY
+    SetBasedFieldSelector fieldSelector = new SetBasedFieldSelector(loadFieldNames, lazyFieldNames) {
+        @Override
+        public FieldSelectorResult accept(String fieldName) {
+          final FieldSelectorResult result = super.accept(fieldName);
+          if (result == FieldSelectorResult.LAZY_LOAD) {
+            return FieldSelectorResult.LATENT;
+          } else {
+            return result;
+          }
+        }
+      };
+
+    Document doc = getDocument(reader, 0, fieldSelector);
+    assertTrue("doc is null and it shouldn't be", doc != null);
+    Fieldable field = doc.getFieldable(DocHelper.LAZY_FIELD_KEY);
+    assertTrue("field is null and it shouldn't be", field != null);
+    assertTrue("field is not lazy and it should be", field.isLazy());
+    String value = field.stringValue();
+    assertTrue("value is null and it shouldn't be", value != null);
+    assertTrue(value + " is not equal to " + DocHelper.LAZY_FIELD_TEXT, value.equals(DocHelper.LAZY_FIELD_TEXT) == true);
+    assertTrue("calling stringValue() twice should give different references", field.stringValue() != field.stringValue());
+
+    field = doc.getFieldable(DocHelper.TEXT_FIELD_1_KEY);
+    assertTrue("field is null and it shouldn't be", field != null);
+    assertTrue("Field is lazy and it should not be", field.isLazy() == false);
+    assertTrue("calling stringValue() twice should give same reference", field.stringValue() == field.stringValue());
+
+    field = doc.getFieldable(DocHelper.TEXT_FIELD_UTF1_KEY);
+    assertTrue("field is null and it shouldn't be", field != null);
+    assertTrue("Field is lazy and it should not be", field.isLazy() == false);
+    assertTrue(field.stringValue() + " is not equal to " + DocHelper.FIELD_UTF1_TEXT, field.stringValue().equals(DocHelper.FIELD_UTF1_TEXT) == true);
+    assertTrue("calling stringValue() twice should give same reference", field.stringValue() == field.stringValue());
+
+    field = doc.getFieldable(DocHelper.TEXT_FIELD_UTF2_KEY);
+    assertTrue("field is null and it shouldn't be", field != null);
+    assertTrue("Field is lazy and it should not be", field.isLazy() == true);
+    assertTrue(field.stringValue() + " is not equal to " + DocHelper.FIELD_UTF2_TEXT, field.stringValue().equals(DocHelper.FIELD_UTF2_TEXT) == true);
+    assertTrue("calling stringValue() twice should give different references", field.stringValue() != field.stringValue());
+
+    field = doc.getFieldable(DocHelper.LAZY_FIELD_BINARY_KEY);
+    assertTrue("field is null and it shouldn't be", field != null);
+    assertTrue("stringValue isn't null for lazy binary field", field.stringValue() == null);
+    assertTrue("calling binaryValue() twice should give different references", field.binaryValue(null).bytes != field.binaryValue(null).bytes);
+
+    byte [] bytes = field.binaryValue(null).bytes;
+    assertTrue("bytes is null and it shouldn't be", bytes != null);
+    assertTrue("", DocHelper.LAZY_FIELD_BINARY_BYTES.length == bytes.length);
+    for (int i = 0; i < bytes.length; i++) {
+      assertTrue("byte[" + i + "] is mismatched", bytes[i] == DocHelper.LAZY_FIELD_BINARY_BYTES[i]);
+
+    }
+    reader.close();
+  }
+
+
+
+
+  public void testLazyFieldsAfterClose() throws Exception {
+    assertTrue(dir != null);
+    assertTrue(fieldInfos != null);
+    IndexReader reader = IndexReader.open(dir);
+    Set<String> loadFieldNames = new HashSet<String>();
+    loadFieldNames.add(DocHelper.TEXT_FIELD_1_KEY);
+    loadFieldNames.add(DocHelper.TEXT_FIELD_UTF1_KEY);
+    Set<String> lazyFieldNames = new HashSet<String>();
+    lazyFieldNames.add(DocHelper.LARGE_LAZY_FIELD_KEY);
+    lazyFieldNames.add(DocHelper.LAZY_FIELD_KEY);
+    lazyFieldNames.add(DocHelper.LAZY_FIELD_BINARY_KEY);
+    lazyFieldNames.add(DocHelper.TEXT_FIELD_UTF2_KEY);
+    SetBasedFieldSelector fieldSelector = new SetBasedFieldSelector(loadFieldNames, lazyFieldNames);
+    Document doc = getDocument(reader, 0, fieldSelector);
+    assertTrue("doc is null and it shouldn't be", doc != null);
+    Fieldable field = doc.getFieldable(DocHelper.LAZY_FIELD_KEY);
+    assertTrue("field is null and it shouldn't be", field != null);
+    assertTrue("field is not lazy and it should be", field.isLazy());
+    reader.close();
+    try {
+      field.stringValue();
+      fail("did not hit AlreadyClosedException as expected");
+    } catch (AlreadyClosedException e) {
+      // expected
+    }
+  }
+
+  public void testLoadFirst() throws Exception {
+    assertTrue(dir != null);
+    assertTrue(fieldInfos != null);
+    IndexReader reader = IndexReader.open(dir);
+    LoadFirstFieldSelector fieldSelector = new LoadFirstFieldSelector();
+    Document doc = getDocument(reader, 0, fieldSelector);
+    assertTrue("doc is null and it shouldn't be", doc != null);
+    int count = 0;
+    List<Fieldable> l = doc.getFields();
+    for (final Fieldable fieldable : l ) {
+      Field field = (Field) fieldable;
+
+      assertTrue("field is null and it shouldn't be", field != null);
+      String sv = field.stringValue();
+      assertTrue("sv is null and it shouldn't be", sv != null);
+      count++;
+    }
+    assertTrue(count + " does not equal: " + 1, count == 1);
+    reader.close();
+  }
+
+  /**
+   * Not really a test per se, but we should have some way of assessing whether this is worthwhile.
+   * <p/>
+   * Must test using a File based directory
+   *
+   * @throws Exception
+   */
+  public void testLazyPerformance() throws Exception {
+    String userName = System.getProperty("user.name");
+    File file = _TestUtil.getTempDir("lazyDir" + userName);
+    Directory tmpDir = newFSDirectory(file);
+    assertTrue(tmpDir != null);
+
+    IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setMergePolicy(newLogMergePolicy());
+    ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(false);
+    IndexWriter writer = new IndexWriter(tmpDir, conf);
+    writer.addDocument(testDoc);
+    writer.close();
+
+    assertTrue(fieldInfos != null);
+    long lazyTime = 0;
+    long regularTime = 0;
+    int length = 10;
+    Set<String> lazyFieldNames = new HashSet<String>();
+    lazyFieldNames.add(DocHelper.LARGE_LAZY_FIELD_KEY);
+    SetBasedFieldSelector fieldSelector = new SetBasedFieldSelector(Collections. <String> emptySet(), lazyFieldNames);
+
+    for (int i = 0; i < length; i++) {
+      IndexReader reader = IndexReader.open(tmpDir);
+
+      Document doc;
+      doc = reader.document(0);//Load all of them
+      assertTrue("doc is null and it shouldn't be", doc != null);
+      Fieldable field = doc.getFieldable(DocHelper.LARGE_LAZY_FIELD_KEY);
+      assertTrue("field is null and it shouldn't be", field != null);
+      assertTrue("field is lazy", field.isLazy() == false);
+      String value;
+      long start;
+      long finish;
+      start = System.currentTimeMillis();
+      //On my machine this was always 0ms.
+      value = field.stringValue();
+      finish = System.currentTimeMillis();
+      assertTrue("value is null and it shouldn't be", value != null);
+      regularTime += (finish - start);
+      reader.close();
+      reader = null;
+      doc = null;
+      //Hmmm, are we still in cache???
+      System.gc();
+      reader = IndexReader.open(tmpDir);
+      doc = getDocument(reader, 0, fieldSelector);
+      field = doc.getFieldable(DocHelper.LARGE_LAZY_FIELD_KEY);
+      assertTrue("field is not lazy", field.isLazy() == true);
+      start = System.currentTimeMillis();
+      //On my machine this took around 50 - 70ms
+      value = field.stringValue();
+      finish = System.currentTimeMillis();
+      assertTrue("value is null and it shouldn't be", value != null);
+      lazyTime += (finish - start);
+      reader.close();
+
+    }
+    tmpDir.close();
+    if (VERBOSE) {
+      System.out.println("Average Non-lazy time (should be very close to zero): " + regularTime / length + " ms for " + length + " reads");
+      System.out.println("Average Lazy Time (should be greater than zero): " + lazyTime / length + " ms for " + length + " reads");
+    }
+  }
+  
+  public void testLoadSize() throws IOException {
+    IndexReader reader = IndexReader.open(dir);
+    Document doc;
+    
+    doc = getDocument(reader, 0, new FieldSelector(){
+      public FieldSelectorResult accept(String fieldName) {
+        if (fieldName.equals(DocHelper.TEXT_FIELD_1_KEY) ||
+            fieldName.equals(DocHelper.LAZY_FIELD_BINARY_KEY))
+          return FieldSelectorResult.SIZE;
+        else if (fieldName.equals(DocHelper.TEXT_FIELD_3_KEY))
+          return FieldSelectorResult.LOAD;
+        else
+          return FieldSelectorResult.NO_LOAD;
+      }
+    });
+    Fieldable f1 = doc.getFieldable(DocHelper.TEXT_FIELD_1_KEY);
+    Fieldable f3 = doc.getFieldable(DocHelper.TEXT_FIELD_3_KEY);
+    Fieldable fb = doc.getFieldable(DocHelper.LAZY_FIELD_BINARY_KEY);
+    assertTrue(f1.isBinary());
+    assertTrue(!f3.isBinary());
+    assertTrue(fb.isBinary());
+    assertSizeEquals(2*DocHelper.FIELD_1_TEXT.length(), f1.binaryValue(null).bytes);
+    assertEquals(DocHelper.FIELD_3_TEXT, f3.stringValue());
+    assertSizeEquals(DocHelper.LAZY_FIELD_BINARY_BYTES.length, fb.binaryValue(null).bytes);
+    
+    reader.close();
+  }
+  
+  private void assertSizeEquals(int size, byte[] sizebytes) {
+    assertEquals((byte) (size>>>24), sizebytes[0]);
+    assertEquals((byte) (size>>>16), sizebytes[1]);
+    assertEquals((byte) (size>>> 8), sizebytes[2]);
+    assertEquals((byte)  size      , sizebytes[3]);
+  }
+}
\ No newline at end of file

Added: lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1146632&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexReader.java (added)
+++ lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexReader.java Thu Jul 14 10:21:56 2011
@@ -0,0 +1,188 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.document.FieldSelectorVisitor;
+import org.apache.lucene.document.SetBasedFieldSelector;
+import org.apache.lucene.document2.BinaryField;
+import org.apache.lucene.document2.Document;
+import org.apache.lucene.document2.FieldType;
+import org.apache.lucene.document2.TextField;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestIndexReader extends LuceneTestCase {
+  private org.apache.lucene.document.Document getDocument(IndexReader ir, int docID, FieldSelector selector)  throws IOException {
+    final FieldSelectorVisitor visitor = new FieldSelectorVisitor(selector);
+    ir.document(docID, visitor);
+    return visitor.getDocument();
+  }
+
+  static void addDoc(IndexWriter writer, String value) throws IOException {
+    Document doc = new Document();
+    doc.add(newField("content", value, TextField.TYPE_UNSTORED));
+    writer.addDocument(doc);
+  }
+
+  static void addDocumentWithFields(IndexWriter writer) throws IOException {
+    Document doc = new Document();
+        
+    FieldType customType = new FieldType(TextField.TYPE_UNSTORED);
+    customType.setStored(true);
+    customType.setTokenized(false);
+
+    FieldType customType2 = new FieldType(TextField.TYPE_UNSTORED);
+    customType2.setStored(true);
+
+    FieldType customType3 = new FieldType();
+    customType3.setStored(true);
+    doc.add(newField("keyword", "test1", customType));
+    doc.add(newField("text", "test1", customType2));
+    doc.add(newField("unindexed", "test1", customType3));
+    doc.add(new TextField("unstored","test1"));
+    writer.addDocument(doc);
+  }
+
+
+  static void addDocumentWithDifferentFields(IndexWriter writer) throws IOException {
+    Document doc = new Document();
+      
+    FieldType customType = new FieldType(TextField.TYPE_UNSTORED);
+    customType.setStored(true);
+    customType.setTokenized(false);
+
+    FieldType customType2 = new FieldType(TextField.TYPE_UNSTORED);
+    customType2.setStored(true);
+
+    FieldType customType3 = new FieldType();
+    customType3.setStored(true);
+    doc.add(newField("keyword2", "test1", customType));
+    doc.add(newField("text2", "test1", customType2));
+    doc.add(newField("unindexed2", "test1", customType3));
+    doc.add(new TextField("unstored2","test1"));
+    writer.addDocument(doc);
+  }
+
+  static void addDocumentWithTermVectorFields(IndexWriter writer) throws IOException {
+    Document doc = new Document();
+    FieldType customType4 = new FieldType(TextField.TYPE_UNSTORED);
+    customType4.setStored(true);
+    FieldType customType5 = new FieldType(TextField.TYPE_UNSTORED);
+    customType5.setStored(true);
+    customType5.setStoreTermVectors(true);
+    FieldType customType6 = new FieldType(TextField.TYPE_UNSTORED);
+    customType6.setStored(true);
+    customType6.setStoreTermVectors(true);
+    customType6.setStoreTermVectorOffsets(true);
+    FieldType customType7 = new FieldType(TextField.TYPE_UNSTORED);
+    customType7.setStored(true);
+    customType7.setStoreTermVectors(true);
+    customType7.setStoreTermVectorPositions(true);
+    FieldType customType8 = new FieldType(TextField.TYPE_UNSTORED);
+    customType8.setStored(true);
+    customType8.setStoreTermVectors(true);
+    customType8.setStoreTermVectorOffsets(true);
+    customType8.setStoreTermVectorPositions(true);
+    doc.add(newField("tvnot","tvnot",customType4));
+    doc.add(newField("termvector","termvector",customType5));
+    doc.add(newField("tvoffset","tvoffset", customType6));
+    doc.add(newField("tvposition","tvposition", customType7));
+    doc.add(newField("tvpositionoffset","tvpositionoffset", customType8));
+        
+    writer.addDocument(doc);
+  }
+
+  public void testBinaryFields() throws IOException {
+    Directory dir = newDirectory();
+    byte[] bin = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+        
+    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
+        
+    for (int i = 0; i < 10; i++) {
+      addDoc(writer, "document number " + (i + 1));
+      addDocumentWithFields(writer);
+      addDocumentWithDifferentFields(writer);
+      addDocumentWithTermVectorFields(writer);
+    }
+    writer.close();
+    writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy()));
+    Document doc = new Document();
+    doc.add(new BinaryField("bin1", bin));
+    doc.add(new TextField("junk", "junk text"));
+    writer.addDocument(doc);
+    writer.close();
+    IndexReader reader = IndexReader.open(dir, false);
+    org.apache.lucene.document.Document doc2 = reader.document(reader.maxDoc() - 1);
+    org.apache.lucene.document.Field[] fields = doc2.getFields("bin1");
+    assertNotNull(fields);
+    assertEquals(1, fields.length);
+    org.apache.lucene.document.Field b1 = fields[0];
+    assertTrue(b1.isBinary());
+    BytesRef bytesRef = b1.binaryValue(null);
+    assertEquals(bin.length, bytesRef.length);
+    for (int i = 0; i < bin.length; i++) {
+      assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]);
+    }
+    Set<String> lazyFields = new HashSet<String>();
+    lazyFields.add("bin1");
+    FieldSelector sel = new SetBasedFieldSelector(new HashSet<String>(), lazyFields);
+    doc2 = getDocument(reader, reader.maxDoc() - 1, sel);
+    Fieldable[] fieldables = doc2.getFieldables("bin1");
+    assertNotNull(fieldables);
+    assertEquals(1, fieldables.length);
+    Fieldable fb1 = fieldables[0];
+    assertTrue(fb1.isBinary());
+    bytesRef = fb1.binaryValue(null);
+    assertEquals(bin.length, bytesRef.bytes.length);
+    assertEquals(bin.length, bytesRef.length);
+    for (int i = 0; i < bin.length; i++) {
+      assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]);
+    }
+    reader.close();
+    // force optimize
+
+
+    writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy()));
+    writer.optimize();
+    writer.close();
+    reader = IndexReader.open(dir, false);
+    doc2 = reader.document(reader.maxDoc() - 1);
+    fields = doc2.getFields("bin1");
+    assertNotNull(fields);
+    assertEquals(1, fields.length);
+    b1 = fields[0];
+    assertTrue(b1.isBinary());
+    bytesRef = b1.binaryValue(null);
+    assertEquals(bin.length, bytesRef.length);
+    for (int i = 0; i < bin.length; i++) {
+      assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]);
+    }
+    reader.close();
+    dir.close();
+  }
+}

Copied: lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/index/TestLazyBug.java (from r1145297, lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestLazyBug.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/index/TestLazyBug.java?p2=lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/index/TestLazyBug.java&p1=lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestLazyBug.java&r1=1145297&r2=1146632&rev=1146632&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestLazyBug.java (original)
+++ lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/index/TestLazyBug.java Thu Jul 14 10:21:56 2011
@@ -22,9 +22,10 @@ import java.util.List;
 import java.util.Set;
 
 import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.document.FieldSelector;
 import org.apache.lucene.document.FieldSelectorResult;
-import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document.FieldSelectorVisitor;
 import org.apache.lucene.document2.Document;
 import org.apache.lucene.document2.TextField;
 import org.apache.lucene.store.Directory;
@@ -105,7 +106,9 @@ public class TestLazyBug extends LuceneT
   public void doTest(int[] docs) throws Exception {
     IndexReader reader = IndexReader.open(directory, true);
     for (int i = 0; i < docs.length; i++) {
-      org.apache.lucene.document.Document d = reader.document(docs[i], SELECTOR);
+      final FieldSelectorVisitor visitor = new FieldSelectorVisitor(SELECTOR);
+      reader.document(docs[i], visitor);
+      org.apache.lucene.document.Document d = visitor.getDocument();
       d.get(MAGIC_FIELD);
 
       List<Fieldable> fields = d.getFields();

Added: lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/index/TestParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/index/TestParallelReader.java?rev=1146632&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/index/TestParallelReader.java (added)
+++ lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/index/TestParallelReader.java Thu Jul 14 10:21:56 2011
@@ -0,0 +1,156 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Random;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document2.Document;
+import org.apache.lucene.document2.FieldType;
+import org.apache.lucene.document2.TextField;
+import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.document.FieldSelectorVisitor;
+import org.apache.lucene.document.MapFieldSelector;
+import org.apache.lucene.search.*;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestParallelReader extends LuceneTestCase {
+
+  private IndexSearcher parallel;
+  private IndexSearcher single;
+  private Directory dir, dir1, dir2;
+  
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    single = single(random);
+    parallel = parallel(random);
+  }
+  
+  @Override
+  public void tearDown() throws Exception {
+    single.getIndexReader().close();
+    single.close();
+    parallel.getIndexReader().close();
+    parallel.close();
+    dir.close();
+    dir1.close();
+    dir2.close();
+    super.tearDown();
+  }
+
+  // Fields 1-4 indexed together:
+  private IndexSearcher single(Random random) throws IOException {
+    dir = newDirectory();
+    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+    Document d1 = new Document();
+    FieldType customType = new FieldType(TextField.TYPE_UNSTORED);
+    customType.setStored(true);
+    d1.add(newField("f1", "v1", customType));
+    d1.add(newField("f2", "v1", customType));
+    d1.add(newField("f3", "v1", customType));
+    d1.add(newField("f4", "v1", customType));
+    w.addDocument(d1);
+    Document d2 = new Document();
+    d2.add(newField("f1", "v2", customType));
+    d2.add(newField("f2", "v2", customType));
+    d2.add(newField("f3", "v2", customType));
+    d2.add(newField("f4", "v2", customType));
+    w.addDocument(d2);
+    w.close();
+
+    return new IndexSearcher(dir, false);
+  }
+
+  // Fields 1 & 2 in one index, 3 & 4 in other, with ParallelReader:
+  private IndexSearcher parallel(Random random) throws IOException {
+    dir1 = getDir1(random);
+    dir2 = getDir2(random);
+    ParallelReader pr = new ParallelReader();
+    pr.add(IndexReader.open(dir1, false));
+    pr.add(IndexReader.open(dir2, false));
+    return newSearcher(pr);
+  }
+
+  private org.apache.lucene.document.Document getDocument(IndexReader ir, int docID, FieldSelector selector) throws IOException {
+    final FieldSelectorVisitor visitor = new FieldSelectorVisitor(selector);
+    ir.document(docID, visitor);
+    return visitor.getDocument();
+  }
+
+  public void testDocument() throws IOException {
+    Directory dir1 = getDir1(random);
+    Directory dir2 = getDir2(random);
+    ParallelReader pr = new ParallelReader();
+    pr.add(IndexReader.open(dir1, false));
+    pr.add(IndexReader.open(dir2, false));
+
+    org.apache.lucene.document.Document doc11 = getDocument(pr, 0, new MapFieldSelector("f1"));
+    org.apache.lucene.document.Document doc24 = getDocument(pr, 1, new MapFieldSelector(Arrays.asList("f4")));
+    org.apache.lucene.document.Document doc223 = getDocument(pr, 1, new MapFieldSelector("f2", "f3"));
+    
+    assertEquals(1, doc11.getFields().size());
+    assertEquals(1, doc24.getFields().size());
+    assertEquals(2, doc223.getFields().size());
+    
+    assertEquals("v1", doc11.get("f1"));
+    assertEquals("v2", doc24.get("f4"));
+    assertEquals("v2", doc223.get("f2"));
+    assertEquals("v2", doc223.get("f3"));
+    pr.close();
+    dir1.close();
+    dir2.close();
+  }
+
+  private Directory getDir1(Random random) throws IOException {
+    Directory dir1 = newDirectory();
+    IndexWriter w1 = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+    Document d1 = new Document();
+    FieldType customType = new FieldType(TextField.TYPE_UNSTORED);
+    customType.setStored(true);
+    d1.add(newField("f1", "v1", customType));
+    d1.add(newField("f2", "v1", customType));
+    w1.addDocument(d1);
+    Document d2 = new Document();
+    d2.add(newField("f1", "v2", customType));
+    d2.add(newField("f2", "v2", customType));
+    w1.addDocument(d2);
+    w1.close();
+    return dir1;
+  }
+
+  private Directory getDir2(Random random) throws IOException {
+    Directory dir2 = newDirectory();
+    FieldType customType = new FieldType(TextField.TYPE_UNSTORED);
+    customType.setStored(true);
+    IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+    Document d3 = new Document();
+    d3.add(newField("f3", "v1", customType));
+    d3.add(newField("f4", "v1", customType));
+    w2.addDocument(d3);
+    Document d4 = new Document();
+    d4.add(newField("f3", "v2", customType));
+    d4.add(newField("f4", "v2", customType));
+    w2.addDocument(d4);
+    w2.close();
+    return dir2;
+  }
+}

Copied: lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/search/TestThreadSafe.java (from r1145297, lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/search/TestThreadSafe.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/search/TestThreadSafe.java?p2=lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/search/TestThreadSafe.java&p1=lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/search/TestThreadSafe.java&r1=1145297&r2=1146632&rev=1146632&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/search/TestThreadSafe.java (original)
+++ lucene/dev/branches/fieldtype/lucene/contrib/misc/src/test/org/apache/lucene/search/TestThreadSafe.java Thu Jul 14 10:21:56 2011
@@ -24,6 +24,8 @@ import org.apache.lucene.index.IndexWrit
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.document.FieldSelectorVisitor;
 import org.apache.lucene.document2.*;
 
 import java.util.Random;
@@ -49,16 +51,16 @@ public class TestThreadSafe extends Luce
     }
 
     @Override
-    public void run() {
+      public void run() {
       try {
         for (int i=0; i<iter; i++) {
           /*** future
            // pick a random index reader... a shared one, or create your own
            IndexReader ir;
-           ***/
+          ***/
 
           switch(rand.nextInt(1)) {
-            case 0: loadDoc(ir1); break;
+          case 0: loadDoc(ir1); break;
           }
 
         }
@@ -69,20 +71,26 @@ public class TestThreadSafe extends Luce
     }
 
 
+    private org.apache.lucene.document.Document getDocument(IndexReader ir, int docID, FieldSelector selector) throws IOException {
+      final FieldSelectorVisitor visitor = new FieldSelectorVisitor(selector);
+      ir.document(docID, visitor);
+      return visitor.getDocument();
+    }
+
     void loadDoc(IndexReader ir) throws IOException {
       // beware of deleted docs in the future
-      org.apache.lucene.document.Document doc = ir.document(rand.nextInt(ir.maxDoc()),
-                new org.apache.lucene.document.FieldSelector() {
-                  public org.apache.lucene.document.FieldSelectorResult accept(String fieldName) {
-                    switch(rand.nextInt(2)) {
-                      case 0: return org.apache.lucene.document.FieldSelectorResult.LAZY_LOAD;
-                      case 1: return org.apache.lucene.document.FieldSelectorResult.LOAD;
-                      // TODO: add other options
-                      default: return org.apache.lucene.document.FieldSelectorResult.LOAD;
-                    }
-                  }
-                }
-              );
+      org.apache.lucene.document.Document doc = getDocument(ir, rand.nextInt(ir.maxDoc()),
+                                                            new org.apache.lucene.document.FieldSelector() {
+                                                              public org.apache.lucene.document.FieldSelectorResult accept(String fieldName) {
+                                                                switch(rand.nextInt(2)) {
+                                                                case 0: return org.apache.lucene.document.FieldSelectorResult.LAZY_LOAD;
+                                                                case 1: return org.apache.lucene.document.FieldSelectorResult.LOAD;
+                                                                  // TODO: add other options
+                                                                default: return org.apache.lucene.document.FieldSelectorResult.LOAD;
+                                                                }
+                                                              }
+                                                            }
+                                                            );
 
       List<Fieldable> fields = doc.getFields();
       for (final Fieldable f : fields ) {
@@ -105,7 +113,7 @@ public class TestThreadSafe extends Luce
 
   void buildDir(Directory dir, int nDocs, int maxFields, int maxFieldLen) throws IOException {
     IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(
-        TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(10));
+                                                                TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(10));
     for (int j=0; j<nDocs; j++) {
       Document d = new Document();
       int nFields = random.nextInt(maxFields);

Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/package.html?rev=1146632&r1=1146631&r2=1146632&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/package.html (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/package.html Thu Jul 14 10:21:56 2011
@@ -44,13 +44,13 @@ package also provides utilities for work
 (remember, Lucene only searches text). {@link org.apache.lucene.document.NumericField} is a special helper class
 to simplify indexing of numeric values (and also dates) for fast range range queries with {@link org.apache.lucene.search.NumericRangeQuery}
 (using a special sortable string representation of numeric values).</p>
-<p>The {@link org.apache.lucene.document.FieldSelector} class provides a mechanism to tell Lucene how to load Documents from
-storage.  If no FieldSelector is used, all Fieldables on a Document will be loaded.  As an example of the FieldSelector usage, consider
+<p>The {@link org.apache.lucene.index.StoredFieldVisitor} class provides a mechanism to customize how the stored fields values are processed.
+If no StoredFieldVisitor is used, all Fields are loaded into a returned Document.  As an example of the StoredFieldVisitor usage, consider
   the common use case of
 displaying search results on a web page and then having users click through to see the full document.  In this scenario, it is often
-  the case that there are many small fields and one or two large fields (containing the contents of the original file). Before the FieldSelector,
-the full Document had to be loaded, including the large fields, in order to display the results.  Now, using the FieldSelector, one
-can {@link org.apache.lucene.document.FieldSelectorResult#LAZY_LOAD} the large fields, thus only loading the large fields
-when a user clicks on the actual link to view the original content.</p>
+  the case that there are many small fields and one or two large fields (containing the contents of the original file). Before StoredFieldVisitor,
+the full Document had to be loaded, including the large fields, in order to display the results.  Now, using the StoredFieldVisitor, one
+can choose which fields should be loaded and how their values should be recorded.  If you simply want to load only certain fields, consider using
+{@link org.apache.lucene.index.DocumentStoredFieldVisitor}.
 </body>
 </html>

Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/Document.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/Document.java?rev=1146632&r1=1146631&r2=1146632&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/Document.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/Document.java Thu Jul 14 10:21:56 2011
@@ -32,7 +32,7 @@ import org.apache.lucene.search.ScoreDoc
  * should typically contain one or more stored fields which uniquely identify
  * it.
  *
- * <p>Note that fields which are <i>not</i> {@link Fieldable#isStored() stored} are
+ * <p>Note that fields which are <i>not</i> {@link Fieldable#stored() stored} are
  * <i>not</i> available in documents retrieved from the index, e.g. with {@link
  * ScoreDoc#doc} or {@link IndexReader#document(int)}.
  */

Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/NumericField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/NumericField.java?rev=1146632&r1=1146631&r2=1146632&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/NumericField.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document2/NumericField.java Thu Jul 14 10:21:56 2011
@@ -106,7 +106,7 @@ import org.apache.lucene.search.FieldCac
  * consumes more disk space in the index but may result in faster range search
  * performance. The default value, 4, was selected for a reasonable tradeoff of
  * disk space consumption versus performance. You can use the expert constructor
- * {@link #NumericField(String,int,Field.Store,boolean)} if you'd like to change
+ * {@link #NumericField(String,int,FieldType)} if you'd like to change
  * the value. Note that you must also specify a congruent value when creating
  * {@link NumericRangeQuery} or {@link NumericRangeFilter}. For low cardinality
  * fields larger precision steps are good. If the cardinality is &lt; 100, it is

Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1146632&r1=1146631&r2=1146632&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DirectoryReader.java Thu Jul 14 10:21:56 2011
@@ -29,8 +29,6 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.FieldSelector;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.LockObtainFailedException;
@@ -557,12 +555,11 @@ class DirectoryReader extends IndexReade
     return maxDoc;
   }
 
-  // inherit javadoc
   @Override
-  public Document document(int n, FieldSelector fieldSelector) throws CorruptIndexException, IOException {
+  public void document(int docID, StoredFieldVisitor visitor) throws CorruptIndexException, IOException {
     ensureOpen();
-    int i = readerIndex(n);                          // find segment num
-    return subReaders[i].document(n - starts[i], fieldSelector);    // dispatch to segment reader
+    int i = readerIndex(docID);                          // find segment num
+    subReaders[i].document(docID - starts[i], visitor);    // dispatch to segment reader
   }
 
   @Override

Added: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocumentStoredFieldVisitor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocumentStoredFieldVisitor.java?rev=1146632&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocumentStoredFieldVisitor.java (added)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocumentStoredFieldVisitor.java Thu Jul 14 10:21:56 2011
@@ -0,0 +1,119 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericField;
+import org.apache.lucene.store.IndexInput;
+
+/** A {@link StoredFieldVisitor} that creates a {@link
+ *  Document} containing all stored fields, or only specific
+ *  requested fields provided to {@link #DocumentStoredFieldVisitor(Set)}
+ *  This is used by {@link IndexReader#document(int)} to load a
+ *  document.
+ *
+ * @lucene.experimental */
+public class DocumentStoredFieldVisitor extends StoredFieldVisitor {
+  private final Document doc = new Document();
+  private final Set<String> fieldsToAdd;
+
+  /** Load only fields named in the provided <code>Set&lt;String&gt;</code>. */
+  public DocumentStoredFieldVisitor(Set<String> fieldsToAdd) {
+    this.fieldsToAdd = fieldsToAdd;
+  }
+
+  /** Load all stored fields. */
+  public DocumentStoredFieldVisitor() {
+    this.fieldsToAdd = null;
+  }
+
+  @Override
+  public boolean binaryField(FieldInfo fieldInfo, IndexInput in, int numBytes) throws IOException {
+    if (accept(fieldInfo)) {
+      final byte[] b = new byte[numBytes];
+      in.readBytes(b, 0, b.length);
+      doc.add(new Field(fieldInfo.name, b));
+    } else {
+      in.seek(in.getFilePointer() + numBytes);
+    }
+    return false;
+  }
+
+  @Override
+  public boolean stringField(FieldInfo fieldInfo, IndexInput in, int numUTF8Bytes) throws IOException {
+    if (accept(fieldInfo)) {
+      final byte[] b = new byte[numUTF8Bytes];
+      in.readBytes(b, 0, b.length);
+      doc.add(new Field(fieldInfo.name,
+                        false,
+                        new String(b, "UTF-8"),
+                        Field.Store.YES,
+                        Field.Index.ANALYZED,  // made up!
+                        Field.TermVector.toTermVector(fieldInfo.storeTermVector,
+                                                      fieldInfo.storeOffsetWithTermVector,
+                                                      fieldInfo.storePositionWithTermVector)));
+    } else {
+      in.seek(in.getFilePointer() + numUTF8Bytes);
+    }
+    return false;
+  }
+
+  @Override
+  public boolean intField(FieldInfo fieldInfo, int value) {
+    if (accept(fieldInfo)) {
+      doc.add(new NumericField(fieldInfo.name, Field.Store.YES, fieldInfo.isIndexed).setIntValue(value));
+    }
+    return false;
+  }
+
+  @Override
+  public boolean longField(FieldInfo fieldInfo, long value) {
+    if (accept(fieldInfo)) {
+      doc.add(new NumericField(fieldInfo.name, Field.Store.YES, fieldInfo.isIndexed).setLongValue(value));
+    }
+    return false;
+  }
+
+  @Override
+  public boolean floatField(FieldInfo fieldInfo, float value) {
+    if (accept(fieldInfo)) {
+      doc.add(new NumericField(fieldInfo.name, Field.Store.YES, fieldInfo.isIndexed).setFloatValue(value));
+    }
+    return false;
+  }
+
+  @Override
+  public boolean doubleField(FieldInfo fieldInfo, double value) {
+    if (accept(fieldInfo)) {
+      doc.add(new NumericField(fieldInfo.name, Field.Store.YES, fieldInfo.isIndexed).setDoubleValue(value));
+    }
+    return false;
+  }
+
+  private boolean accept(FieldInfo fieldInfo) {
+    return fieldsToAdd == null || fieldsToAdd.contains(fieldInfo.name);
+  }
+
+  public Document getDocument() {
+    return doc;
+  }
+}

Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldInfo.java?rev=1146632&r1=1146631&r2=1146632&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldInfo.java Thu Jul 14 10:21:56 2011
@@ -26,9 +26,9 @@ public final class FieldInfo {
   public boolean isIndexed;
 
   // true if term vector for this field should be stored
-  boolean storeTermVector;
-  boolean storeOffsetWithTermVector;
-  boolean storePositionWithTermVector;
+  public boolean storeTermVector;
+  public boolean storeOffsetWithTermVector;
+  public boolean storePositionWithTermVector;
 
   public boolean omitNorms; // omit norms associated with indexed fields  
   public boolean omitTermFreqAndPositions;



Mime
View raw message