lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1162347 [2/15] - in /lucene/dev/trunk: lucene/ lucene/contrib/ lucene/contrib/demo/src/java/org/apache/lucene/demo/ lucene/contrib/demo/src/java/org/apache/lucene/demo/xmlparser/ lucene/contrib/highlighter/src/java/org/apache/lucene/search...
Date Sat, 27 Aug 2011 13:27:12 GMT
Modified: lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (original)
+++ lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java Sat Aug 27 13:27:01 2011
@@ -29,6 +29,8 @@ import org.apache.lucene.analysis.MockAn
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.MultiNorms;
@@ -204,19 +206,44 @@ public class TestIndicesEquals extends L
 
 
   private void assembleDocument(Document document, int i) {
-    document.add(new Field("a", i + " Do you really want to go and live in that house all winter?", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+    FieldType customType = new FieldType(TextField.TYPE_STORED);
+    customType.setStoreTermVectors(true);
+    customType.setStoreTermVectorOffsets(true);
+    customType.setStoreTermVectorPositions(true);
+    //document.add(new Field("a", i + " Do you really want to go and live in that house all winter?", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+    document.add(new Field("a", customType, i + " Do you really want to go and live in that house all winter?"));
     if (i > 0) {
-      document.add(new Field("b0", i + " All work and no play makes Jack a dull boy", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
-      document.add(new Field("b1", i + " All work and no play makes Jack a dull boy", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS, Field.TermVector.NO));
-      document.add(new Field("b2", i + " All work and no play makes Jack a dull boy", Field.Store.NO, Field.Index.NOT_ANALYZED, Field.TermVector.NO));
-      document.add(new Field("b3", i + " All work and no play makes Jack a dull boy", Field.Store.YES, Field.Index.NO, Field.TermVector.NO));
+      //document.add(new Field("b0", i + " All work and no play makes Jack a dull boy", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+      document.add(new Field("b0", customType, i + " All work and no play makes Jack a dull boy"));
+
+      //document.add(new Field("b1", i + " All work and no play makes Jack a dull boy", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS, Field.TermVector.NO));
+      FieldType customType2 = new FieldType(TextField.TYPE_STORED);
+      customType2.setTokenized(false);
+      customType2.setOmitNorms(true);
+      document.add(new Field("b1", customType2, i + " All work and no play makes Jack a dull boy"));
+      
+      //document.add(new Field("b2", i + " All work and no play makes Jack a dull boy", Field.Store.NO, Field.Index.NOT_ANALYZED, Field.TermVector.NO));
+      FieldType customType3 = new FieldType(TextField.TYPE_UNSTORED);
+      customType3.setTokenized(false);
+      document.add(new Field("b1", customType3, i + " All work and no play makes Jack a dull boy"));
+      
+      //document.add(new Field("b3", i + " All work and no play makes Jack a dull boy", Field.Store.YES, Field.Index.NO, Field.TermVector.NO));
+      FieldType customType4 = new FieldType(TextField.TYPE_STORED);
+      customType4.setIndexed(false);
+      customType4.setTokenized(false);
+      document.add(new Field("b1", customType4, i + " All work and no play makes Jack a dull boy"));
       if (i > 1) {
-        document.add(new Field("c", i + " Redrum redrum", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+        //document.add(new Field("c", i + " Redrum redrum", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+        document.add(new Field("c", customType, i + " Redrum redrum"));
         if (i > 2) {
-          document.add(new Field("d", i + " Hello Danny, come and play with us... forever and ever. and ever.", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+          //document.add(new Field("d", i + " Hello Danny, come and play with us... forever and ever. and ever.", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+          document.add(new Field("d", customType, i + " Hello Danny, come and play with us... forever and ever. and ever."));
           if (i > 3) {
-            Field f = new Field("e", i + " Heres Johnny!", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
-            f.setOmitNorms(true);
+            //Field f = new Field("e", i + " Heres Johnny!", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
+            //f.setOmitNorms(true);
+            FieldType customType5 = new FieldType(TextField.TYPE_UNSTORED);
+            customType5.setOmitNorms(true);
+            Field f = new Field("e", customType5, i + " Heres Johnny!");
             document.add(f);
             if (i > 4) {
               final List<Token> tokens = new ArrayList<Token>(2);
@@ -247,7 +274,8 @@ public class TestIndicesEquals extends L
                 }
               };
               
-              document.add(new Field("f", ts));
+              //document.add(new Field("f", ts));
+              document.add(new TextField("f", ts));
             }
           }
         }

Modified: lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestRealTime.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestRealTime.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestRealTime.java (original)
+++ lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestRealTime.java Sat Aug 27 13:27:01 2011
@@ -19,7 +19,7 @@ import org.apache.lucene.search.IndexSea
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.IndexReader.AtomicReaderContext;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.util.LuceneTestCase;
@@ -43,7 +43,7 @@ public class TestRealTime extends Lucene
     Collector collector;
 
     doc = new Document();
-    doc.add(new Field("f", "a", Field.Store.NO, Field.Index.NOT_ANALYZED));
+    doc.add(new StringField("f", "a"));
     writer.addDocument(doc);
     writer.commit();
 
@@ -52,7 +52,7 @@ public class TestRealTime extends Lucene
     assertEquals(1, collector.hits);
 
     doc = new Document();
-    doc.add(new Field("f", "a", Field.Store.NO, Field.Index.NOT_ANALYZED));
+    doc.add(new StringField("f", "a"));
     writer.addDocument(doc);
     writer.commit();
 

Modified: lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestUnoptimizedReaderOnConstructor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestUnoptimizedReaderOnConstructor.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestUnoptimizedReaderOnConstructor.java (original)
+++ lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestUnoptimizedReaderOnConstructor.java Sat Aug 27 13:27:01 2011
@@ -25,7 +25,7 @@ import org.apache.lucene.store.Directory
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.document.TextField;
 
 /**
  * @since 2009-mar-30 13:15:49
@@ -66,7 +66,7 @@ public class TestUnoptimizedReaderOnCons
 
   private void addDocument(IndexWriter iw, String text) throws IOException {
     Document doc = new Document();
-    doc.add(new Field("field", text, Field.Store.NO, Field.Index.ANALYZED));
+    doc.add(new TextField("field", text));
     iw.addDocument(doc);
   }
 }

Modified: lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Sat Aug 27 13:27:01 2011
@@ -35,23 +35,24 @@ import org.apache.lucene.analysis.tokena
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.FieldInvertState;
 import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.FieldsEnum;
 import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader.ReaderContext;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.OrdTermState;
-import org.apache.lucene.index.TermState;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.FieldsEnum;
-import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.StoredFieldVisitor;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermFreqVector;
 import org.apache.lucene.index.TermPositionVector;
+import org.apache.lucene.index.TermState;
 import org.apache.lucene.index.TermVectorMapper;
-import org.apache.lucene.index.FieldInvertState;
 import org.apache.lucene.index.codecs.PerDocValues;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
@@ -60,8 +61,8 @@ import org.apache.lucene.search.Similari
 import org.apache.lucene.search.SimilarityProvider;
 import org.apache.lucene.store.RAMDirectory; // for javadocs
 import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Constants; // for javadocs
 
 /**
@@ -240,11 +241,8 @@ public class MemoryIndex {
   /**
    * Convenience method; Tokenizes the given field text and adds the resulting
    * terms to the index; Equivalent to adding an indexed non-keyword Lucene
-   * {@link org.apache.lucene.document.Field} that is
-   * {@link org.apache.lucene.document.Field.Index#ANALYZED tokenized},
-   * {@link org.apache.lucene.document.Field.Store#NO not stored},
-   * {@link org.apache.lucene.document.Field.TermVector#WITH_POSITIONS termVectorStored with positions} (or
-   * {@link org.apache.lucene.document.Field.TermVector#WITH_POSITIONS termVectorStored with positions and offsets}),
+   * {@link org.apache.lucene.document.Field} that is tokenized, not stored,
+   * termVectorStored with positions (or termVectorStored with positions and offsets),
    * 
    * @param fieldName
    *            a name to be associated with the text
@@ -1237,18 +1235,11 @@ public class MemoryIndex {
     }
   
     @Override
-    public Document document(int n) {
+    public void document(int docID, StoredFieldVisitor visitor) {
       if (DEBUG) System.err.println("MemoryIndexReader.document");
-      return new Document(); // there are no stored fields
+      // no-op: there are no stored fields
     }
-
-    //When we convert to JDK 1.5 make this Set<String>
-    @Override
-    public Document document(int n, FieldSelector fieldSelector) throws IOException {
-      if (DEBUG) System.err.println("MemoryIndexReader.document");
-      return new Document(); // there are no stored fields
-    }
-
+    
     @Override
     public boolean hasDeletions() {
       if (DEBUG) System.err.println("MemoryIndexReader.hasDeletions");

Modified: lucene/dev/trunk/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (original)
+++ lucene/dev/trunk/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java Sat Aug 27 13:27:01 2011
@@ -31,6 +31,7 @@ import org.apache.lucene.analysis.MockTo
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.queryparser.classic.QueryParser;
@@ -108,8 +109,8 @@ public class MemoryIndexTest extends Bas
     IndexWriter writer = new IndexWriter(ramdir,
                                          new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer).setCodecProvider(_TestUtil.alwaysCodec("Standard")));
     Document doc = new Document();
-    Field field1 = newField("foo", fooField.toString(), Field.Store.NO, Field.Index.ANALYZED);
-    Field field2 = newField("term", termField.toString(), Field.Store.NO, Field.Index.ANALYZED);
+    Field field1 = newField("foo", fooField.toString(), TextField.TYPE_UNSTORED);
+    Field field2 = newField("term", termField.toString(), TextField.TYPE_UNSTORED);
     doc.add(field1);
     doc.add(field2);
     writer.addDocument(doc);

Copied: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelector.java (from r1161532, lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/FieldSelector.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelector.java?p2=lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelector.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/FieldSelector.java&r1=1161532&r2=1162347&rev=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/FieldSelector.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelector.java Sat Aug 27 13:27:01 2011
@@ -1,5 +1,6 @@
 package org.apache.lucene.document;
 
+
 /**
  * Copyright 2004 The Apache Software Foundation
  *
@@ -18,7 +19,7 @@ package org.apache.lucene.document;
 
 /**
  * Similar to a {@link java.io.FileFilter}, the FieldSelector allows one to make decisions about
- * what Fields get loaded on a {@link Document} by {@link org.apache.lucene.index.IndexReader#document(int,org.apache.lucene.document.FieldSelector)}
+ * what Fields get loaded on a {@link Document} by {@link FieldSelectorVisitor}
  *
  **/
 public interface FieldSelector {

Copied: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelectorResult.java (from r1161532, lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/FieldSelectorResult.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelectorResult.java?p2=lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelectorResult.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/FieldSelectorResult.java&r1=1161532&r2=1162347&rev=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/FieldSelectorResult.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelectorResult.java Sat Aug 27 13:27:01 2011
@@ -20,39 +20,41 @@ package org.apache.lucene.document;
  *  Provides information about what should be done with this Field 
  *
  **/
+import org.apache.lucene.index.IndexableField; // for javadocs
+
 public enum FieldSelectorResult {
 
     /**
      * Load this {@link Field} every time the {@link Document} is loaded, reading in the data as it is encountered.
-     *  {@link Document#getField(String)} and {@link Document#getFieldable(String)} should not return null.
+     *  {@link Document#getField(String)} should not return null.
      *<p/>
-     * {@link Document#add(Fieldable)} should be called by the Reader.
+     * {@link Document#add(IndexableField)} should be called by the Reader.
      */
   LOAD,
 
     /**
      * Lazily load this {@link Field}.  This means the {@link Field} is valid, but it may not actually contain its data until
-     * invoked.  {@link Document#getField(String)} SHOULD NOT BE USED.  {@link Document#getFieldable(String)} is safe to use and should
-     * return a valid instance of a {@link Fieldable}.
+     * invoked.  {@link Document#getField(String)} is safe to use and should
+     * return a valid instance of a {@link IndexableField}.
      *<p/>
-     * {@link Document#add(Fieldable)} should be called by the Reader.
+     * {@link Document#add(IndexableField)} should be called by the Reader.
      */
   LAZY_LOAD,
 
     /**
-     * Do not load the {@link Field}.  {@link Document#getField(String)} and {@link Document#getFieldable(String)} should return null.
-     * {@link Document#add(Fieldable)} is not called.
+     * Do not load the {@link Field}.  {@link Document#getField(String)} should return null.
+     * {@link Document#add(IndexableField)} is not called.
      * <p/>
-     * {@link Document#add(Fieldable)} should not be called by the Reader.
+     * {@link Document#add(IndexableField)} should not be called by the Reader.
      */
   NO_LOAD,
 
     /**
      * Load this field as in the {@link #LOAD} case, but immediately return from {@link Field} loading for the {@link Document}.  Thus, the
-     * Document may not have its complete set of Fields.  {@link Document#getField(String)} and {@link Document#getFieldable(String)} should
+     * Document may not have its complete set of Fields.  {@link Document#getField(String)} should
      * both be valid for this {@link Field}
      * <p/>
-     * {@link Document#add(Fieldable)} should be called by the Reader.
+     * {@link Document#add(IndexableField)} should be called by the Reader.
      */
   LOAD_AND_BREAK,
 
@@ -67,10 +69,10 @@ public enum FieldSelectorResult {
 
   /**
      * Lazily load this {@link Field}, but do not cache the result.  This means the {@link Field} is valid, but it may not actually contain its data until
-     * invoked.  {@link Document#getField(String)} SHOULD NOT BE USED.  {@link Document#getFieldable(String)} is safe to use and should
-     * return a valid instance of a {@link Fieldable}.
+     * invoked.  {@link Document#getField(String)} is safe to use and should
+     * return a valid instance of a {@link IndexableField}.
      *<p/>
-     * {@link Document#add(Fieldable)} should be called by the Reader.
+     * {@link Document#add(IndexableField)} should be called by the Reader.
      */
   LATENT
 }

Added: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelectorVisitor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelectorVisitor.java?rev=1162347&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelectorVisitor.java (added)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelectorVisitor.java Sat Aug 27 13:27:01 2011
@@ -0,0 +1,319 @@
+package org.apache.lucene.document;
+
+/**
+ * Copyright 2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.document.NumericField.DataType;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldReaderException;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.BytesRef;
+
+/** Create this, passing a legacy {@link FieldSelector} to it, then
+ *  pass this class to {@link IndexReader#document(int,
+ *  StoredFieldVisitor)}, then call {@link #getDocument} to
+ *  retrieve the loaded document.
+
+ *  <p><b>NOTE</b>:  If you use Lazy fields, you should not
+ *  access the returned document after the reader has been
+ *  closed!
+ */
+
+public class FieldSelectorVisitor extends StoredFieldVisitor {
+
+  private final FieldSelector selector;
+  private final Document doc;
+
+  public FieldSelectorVisitor(FieldSelector selector) {
+    this.selector = selector;
+    doc = new Document();
+  }
+
+  public Document getDocument() {
+    return doc;
+  }
+
+  @Override
+  public boolean binaryField(FieldInfo fieldInfo, IndexInput in, int numBytes) throws IOException {
+    final FieldSelectorResult accept = selector.accept(fieldInfo.name);
+    switch (accept) {
+    case LOAD:
+    case LOAD_AND_BREAK:
+      final byte[] b = new byte[numBytes];
+      in.readBytes(b, 0, b.length);
+      doc.add(new BinaryField(fieldInfo.name, b));
+      return accept != FieldSelectorResult.LOAD;
+    case LAZY_LOAD:
+    case LATENT:
+      addFieldLazy(in, fieldInfo, true, accept == FieldSelectorResult.LAZY_LOAD, numBytes);
+      return false;
+    case SIZE:
+    case SIZE_AND_BREAK:
+      in.seek(in.getFilePointer() + numBytes);
+      addFieldSize(fieldInfo, numBytes);
+      return accept != FieldSelectorResult.SIZE;
+    default:
+      // skip
+      in.seek(in.getFilePointer() + numBytes);
+      return false;
+    }
+  }
+
+  @Override
+  public boolean stringField(FieldInfo fieldInfo, IndexInput in, int numUTF8Bytes) throws IOException {
+    final FieldSelectorResult accept = selector.accept(fieldInfo.name);
+    switch (accept) {
+    case LOAD:
+    case LOAD_AND_BREAK:
+      final byte[] b = new byte[numUTF8Bytes];
+      in.readBytes(b, 0, b.length);
+      FieldType ft = new FieldType(TextField.TYPE_STORED);
+      ft.setStoreTermVectors(fieldInfo.storeTermVector);
+      ft.setStoreTermVectorOffsets(fieldInfo.storeOffsetWithTermVector);
+      ft.setStoreTermVectorPositions(fieldInfo.storePositionWithTermVector);
+      doc.add(new Field(fieldInfo.name, ft, new String(b, "UTF-8"))); 
+      return accept != FieldSelectorResult.LOAD;
+    case LAZY_LOAD:
+    case LATENT:
+      addFieldLazy(in, fieldInfo, false, accept == FieldSelectorResult.LAZY_LOAD, numUTF8Bytes);
+      return false;
+    case SIZE:
+    case SIZE_AND_BREAK:
+      in.seek(in.getFilePointer() + numUTF8Bytes);
+      addFieldSize(fieldInfo, 2*numUTF8Bytes);
+      return accept != FieldSelectorResult.SIZE;
+    default:
+      // skip
+      in.seek(in.getFilePointer() + numUTF8Bytes);
+      return false;
+    }
+  }
+
+  @Override
+  public boolean intField(FieldInfo fieldInfo, int value) throws IOException {
+		FieldType ft = new FieldType(NumericField.TYPE_STORED);
+		ft.setIndexed(fieldInfo.isIndexed);
+		ft.setOmitNorms(fieldInfo.omitNorms);
+		ft.setIndexOptions(fieldInfo.indexOptions);
+    return addNumericField(fieldInfo, new NumericField(fieldInfo.name, ft).setIntValue(value));
+  }
+
+  @Override
+  public boolean longField(FieldInfo fieldInfo, long value) throws IOException { 
+		FieldType ft = new FieldType(NumericField.TYPE_STORED);
+		ft.setIndexed(fieldInfo.isIndexed);
+		ft.setOmitNorms(fieldInfo.omitNorms);
+		ft.setIndexOptions(fieldInfo.indexOptions);
+    return addNumericField(fieldInfo, new NumericField(fieldInfo.name, ft).setLongValue(value));
+  }
+
+  @Override
+  public boolean floatField(FieldInfo fieldInfo, float value) throws IOException {
+		FieldType ft = new FieldType(NumericField.TYPE_STORED);
+		ft.setIndexed(fieldInfo.isIndexed);
+		ft.setOmitNorms(fieldInfo.omitNorms);
+		ft.setIndexOptions(fieldInfo.indexOptions);
+    return addNumericField(fieldInfo, new NumericField(fieldInfo.name, ft).setFloatValue(value));
+  }
+
+  @Override
+  public boolean doubleField(FieldInfo fieldInfo, double value) throws IOException {
+		FieldType ft = new FieldType(NumericField.TYPE_STORED);
+		ft.setIndexed(fieldInfo.isIndexed);
+		ft.setOmitNorms(fieldInfo.omitNorms);
+		ft.setIndexOptions(fieldInfo.indexOptions);
+    return addNumericField(fieldInfo, new NumericField(fieldInfo.name, ft).setDoubleValue(value));
+  }
+
+  private boolean addNumericField(FieldInfo fieldInfo, NumericField f) {
+    doc.add(f);
+    final FieldSelectorResult accept = selector.accept(fieldInfo.name);
+    switch (accept) {
+    case LOAD:
+      return false;
+    case LOAD_AND_BREAK:
+      return true;
+    case LAZY_LOAD:
+    case LATENT:
+      return false;
+    case SIZE:
+      return false;
+    case SIZE_AND_BREAK:
+      return true;
+    default:
+      return false;
+    }
+  }
+
+  private void addFieldLazy(IndexInput in, FieldInfo fi, boolean binary, boolean cacheResult, int numBytes) throws IOException {
+    final IndexableField f;
+    final long pointer = in.getFilePointer();
+    // Need to move the pointer ahead by toRead positions
+    in.seek(pointer+numBytes);
+    FieldType ft = new FieldType();
+    ft.setStored(true);
+    ft.setOmitNorms(fi.omitNorms);
+    ft.setIndexOptions(fi.indexOptions);
+    ft.setLazy(true);
+    
+    if (binary) {
+      f = new LazyField(in, fi.name, ft, numBytes, pointer, binary, cacheResult);
+    } else {
+      ft.setStoreTermVectors(fi.storeTermVector);
+      ft.setStoreTermVectorOffsets(fi.storeOffsetWithTermVector);
+      ft.setStoreTermVectorPositions(fi.storePositionWithTermVector);
+      f = new LazyField(in, fi.name, ft, numBytes, pointer, binary, cacheResult);
+    }
+    
+    doc.add(f);
+  }
+
+  // Add the size of field as a byte[] containing the 4 bytes of the integer byte size (high order byte first; char = 2 bytes)
+  // Read just the size -- caller must skip the field content to continue reading fields
+  // Return the size in bytes or chars, depending on field type
+  private void addFieldSize(FieldInfo fi, int numBytes) throws IOException {
+    byte[] sizebytes = new byte[4];
+    sizebytes[0] = (byte) (numBytes>>>24);
+    sizebytes[1] = (byte) (numBytes>>>16);
+    sizebytes[2] = (byte) (numBytes>>> 8);
+    sizebytes[3] = (byte)  numBytes      ;
+    doc.add(new BinaryField(fi.name, sizebytes));
+  }
+
+  /**
+   * A Lazy field implementation that defers loading of fields until asked for, instead of when the Document is
+   * loaded.
+   */
+  private static class LazyField extends Field {
+    private int toRead;
+    private long pointer;
+    private final boolean cacheResult;
+    private final IndexInput in;
+    private boolean isBinary;
+
+    public LazyField(IndexInput in, String name, FieldType ft, int toRead, long pointer, boolean isBinary, boolean cacheResult) {
+      super(name, ft);
+      this.in = in;
+      this.toRead = toRead;
+      this.pointer = pointer;
+      this.isBinary = isBinary;
+      this.cacheResult = cacheResult;
+    }
+
+    @Override
+    public Number numericValue() {
+      return null;
+    }
+
+    @Override
+    public DataType numericDataType() {
+      return null;
+    }
+
+    private IndexInput localFieldsStream;
+
+    private IndexInput getFieldStream() {
+      if (localFieldsStream == null) {
+        localFieldsStream = (IndexInput) in.clone();
+      }
+      return localFieldsStream;
+    }
+
+    /** The value of the field as a Reader, or null.  If null, the String value,
+     * binary value, or TokenStream value is used.  Exactly one of stringValue(), 
+     * readerValue(), getBinaryValue(), and tokenStreamValue() must be set. */
+    @Override
+    public Reader readerValue() {
+      return null;
+    }
+
+    /** The value of the field as a TokenStream, or null.  If null, the Reader value,
+     * String value, or binary value is used. Exactly one of stringValue(), 
+     * readerValue(), getBinaryValue(), and tokenStreamValue() must be set. */
+    @Override
+    public TokenStream tokenStreamValue() {
+      return null;
+    }
+
+    /** The value of the field as a String, or null.  If null, the Reader value,
+     * binary value, or TokenStream value is used.  Exactly one of stringValue(), 
+     * readerValue(), getBinaryValue(), and tokenStreamValue() must be set. */
+    @Override
+    synchronized public String stringValue() {
+      if (isBinary) {
+        return null;
+      } else {
+        if (fieldsData == null) {
+          String result = null;
+          IndexInput localFieldsStream = getFieldStream();
+          try {
+            localFieldsStream.seek(pointer);
+            byte[] bytes = new byte[toRead];
+            localFieldsStream.readBytes(bytes, 0, toRead);
+            result = new String(bytes, "UTF-8");
+          } catch (IOException e) {
+            throw new FieldReaderException(e);
+          }
+          if (cacheResult == true){
+            fieldsData = result;
+          }
+          return result;
+        } else {
+          return (String) fieldsData;
+        }
+      }
+    }
+
+    @Override
+    synchronized public BytesRef binaryValue() {
+      if (isBinary) {
+        if (fieldsData == null) {
+          // Allocate new buffer if result is null or too small
+          final byte[] b = new byte[toRead];
+   
+          IndexInput localFieldsStream = getFieldStream();
+
+          // Throw this IOException since IndexReader.document does so anyway, so probably not that big of a change for people
+          // since they are already handling this exception when getting the document
+          try {
+            localFieldsStream.seek(pointer);
+            localFieldsStream.readBytes(b, 0, toRead);
+          } catch (IOException e) {
+            throw new FieldReaderException(e);
+          }
+
+          final BytesRef result = new BytesRef(b);
+          result.length = toRead;
+          if (cacheResult == true){
+            fieldsData = result;
+          }
+          return result;
+        } else {
+          return (BytesRef) fieldsData;
+        }
+      } else {
+        return null;
+      }
+    }
+  }
+}

Copied: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/LoadFirstFieldSelector.java (from r1161532, lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/LoadFirstFieldSelector.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/LoadFirstFieldSelector.java?p2=lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/LoadFirstFieldSelector.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/LoadFirstFieldSelector.java&r1=1161532&r2=1162347&rev=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/LoadFirstFieldSelector.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/LoadFirstFieldSelector.java Sat Aug 27 13:27:01 2011
@@ -26,4 +26,4 @@ public class LoadFirstFieldSelector impl
   public FieldSelectorResult accept(String fieldName) {
     return FieldSelectorResult.LOAD_AND_BREAK;
   }
-}
\ No newline at end of file
+}

Copied: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/SetBasedFieldSelector.java (from r1161532, lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/SetBasedFieldSelector.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/SetBasedFieldSelector.java?p2=lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/SetBasedFieldSelector.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/SetBasedFieldSelector.java&r1=1161532&r2=1162347&rev=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/SetBasedFieldSelector.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/SetBasedFieldSelector.java Sat Aug 27 13:27:01 2011
@@ -1,6 +1,5 @@
 package org.apache.lucene.document;
 
-import java.util.Set;
 /**
  * Copyright 2004 The Apache Software Foundation
  *
@@ -17,10 +16,14 @@ import java.util.Set;
  * limitations under the License.
  */
 
+import java.util.Set;
+
+
 /**
  * Declare what fields to load normally and what fields to load lazily
  *
  **/
+
 public class SetBasedFieldSelector implements FieldSelector {
   
   private Set<String> fieldsToLoad;
@@ -55,4 +58,4 @@ public class SetBasedFieldSelector imple
     }                                           
     return result;
   }
-}
\ No newline at end of file
+}

Modified: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java Sat Aug 27 13:27:01 2011
@@ -35,7 +35,7 @@ import org.apache.lucene.util.ReaderUtil
  * 
  * If Similarity class is specified, uses its computeNorm method to set norms.
  * If -n command line argument is used, removed field norms, as if 
- * {@link org.apache.lucene.document.Field.Index}.NO_NORMS was used.
+ * {@link org.apache.lucene.document.FieldType#setOmitNorms(boolean)} was used.
  *
  * <p>
  * NOTE: This will overwrite any length normalization or field/document boosts.

Added: lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestContribFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestContribFieldsReader.java?rev=1162347&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestContribFieldsReader.java (added)
+++ lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestContribFieldsReader.java Sat Aug 27 13:27:01 2011
@@ -0,0 +1,318 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.document.FieldSelectorResult;
+import org.apache.lucene.document.FieldSelectorVisitor;
+import org.apache.lucene.document.LoadFirstFieldSelector;
+import org.apache.lucene.document.SetBasedFieldSelector;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+
+public class TestContribFieldsReader extends LuceneTestCase {
+  private static Directory dir;
+  private static org.apache.lucene.document.Document testDoc = new org.apache.lucene.document.Document();
+  private static FieldInfos fieldInfos = null;
+
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    fieldInfos = new FieldInfos();
+    DocHelper.setupDoc(testDoc);
+    _TestUtil.add(testDoc, fieldInfos);
+    dir = newDirectory();
+    IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy());
+    ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(false);
+    IndexWriter writer = new IndexWriter(dir, conf);
+    writer.addDocument(testDoc);
+    writer.close();
+  }
+
+  @AfterClass
+  public static void afterClass() throws Exception {
+    dir.close();
+    dir = null;
+    fieldInfos = null;
+    testDoc = null;
+  }
+
+  private Document getDocument(IndexReader ir, int docID, FieldSelector selector) throws IOException {
+    final FieldSelectorVisitor visitor = new FieldSelectorVisitor(selector);
+    ir.document(docID, visitor);
+    return visitor.getDocument();
+  }
+
+  public void testLazyFields() throws Exception {
+    assertTrue(dir != null);
+    assertTrue(fieldInfos != null);
+    IndexReader reader = IndexReader.open(dir);
+    Set<String> loadFieldNames = new HashSet<String>();
+    loadFieldNames.add(DocHelper.TEXT_FIELD_1_KEY);
+    loadFieldNames.add(DocHelper.TEXT_FIELD_UTF1_KEY);
+    Set<String> lazyFieldNames = new HashSet<String>();
+    //new String[]{DocHelper.LARGE_LAZY_FIELD_KEY, DocHelper.LAZY_FIELD_KEY, DocHelper.LAZY_FIELD_BINARY_KEY};
+    lazyFieldNames.add(DocHelper.LARGE_LAZY_FIELD_KEY);
+    lazyFieldNames.add(DocHelper.LAZY_FIELD_KEY);
+    lazyFieldNames.add(DocHelper.LAZY_FIELD_BINARY_KEY);
+    lazyFieldNames.add(DocHelper.TEXT_FIELD_UTF2_KEY);
+    SetBasedFieldSelector fieldSelector = new SetBasedFieldSelector(loadFieldNames, lazyFieldNames);
+    Document doc = getDocument(reader, 0, fieldSelector);
+    assertTrue("doc is null and it shouldn't be", doc != null);
+    IndexableField field = doc.getField(DocHelper.LAZY_FIELD_KEY);
+    assertTrue("field is null and it shouldn't be", field != null);
+    assertTrue("field is not lazy and it should be", field.getClass().getSimpleName().equals("LazyField"));
+    String value = field.stringValue();
+    assertTrue("value is null and it shouldn't be", value != null);
+    assertTrue(value + " is not equal to " + DocHelper.LAZY_FIELD_TEXT, value.equals(DocHelper.LAZY_FIELD_TEXT) == true);
+    assertTrue("calling stringValue() twice should give same reference", field.stringValue() == field.stringValue());
+
+    field = doc.getField(DocHelper.TEXT_FIELD_1_KEY);
+    assertTrue("field is null and it shouldn't be", field != null);
+    assertFalse("Field is lazy and it should not be", field.getClass().getSimpleName().equals("LazyField"));
+    field = doc.getField(DocHelper.TEXT_FIELD_UTF1_KEY);
+    assertTrue("field is null and it shouldn't be", field != null);
+    assertFalse("Field is lazy and it should not be", field.getClass().getSimpleName().equals("LazyField"));
+    assertTrue(field.stringValue() + " is not equal to " + DocHelper.FIELD_UTF1_TEXT, field.stringValue().equals(DocHelper.FIELD_UTF1_TEXT) == true);
+
+    field = doc.getField(DocHelper.TEXT_FIELD_UTF2_KEY);
+    assertTrue("field is null and it shouldn't be", field != null);
+    assertTrue("Field is lazy and it should not be", field.getClass().getSimpleName().equals("LazyField"));
+    assertTrue(field.stringValue() + " is not equal to " + DocHelper.FIELD_UTF2_TEXT, field.stringValue().equals(DocHelper.FIELD_UTF2_TEXT) == true);
+
+    field = doc.getField(DocHelper.LAZY_FIELD_BINARY_KEY);
+    assertTrue("field is null and it shouldn't be", field != null);
+    assertTrue("stringValue isn't null for lazy binary field", field.stringValue() == null);
+
+    byte [] bytes = field.binaryValue().bytes;
+    assertTrue("bytes is null and it shouldn't be", bytes != null);
+    assertTrue("", DocHelper.LAZY_FIELD_BINARY_BYTES.length == bytes.length);
+    assertTrue("calling binaryValue() twice should give same reference", field.binaryValue().bytes == field.binaryValue().bytes);
+    for (int i = 0; i < bytes.length; i++) {
+      assertTrue("byte[" + i + "] is mismatched", bytes[i] == DocHelper.LAZY_FIELD_BINARY_BYTES[i]);
+
+    }
+    reader.close();
+  }
+
+  public void testLatentFields() throws Exception {
+    assertTrue(dir != null);
+    assertTrue(fieldInfos != null);
+    IndexReader reader = IndexReader.open(dir);
+    Set<String> loadFieldNames = new HashSet<String>();
+    loadFieldNames.add(DocHelper.TEXT_FIELD_1_KEY);
+    loadFieldNames.add(DocHelper.TEXT_FIELD_UTF1_KEY);
+    Set<String> lazyFieldNames = new HashSet<String>();
+    //new String[]{DocHelper.LARGE_LAZY_FIELD_KEY, DocHelper.LAZY_FIELD_KEY, DocHelper.LAZY_FIELD_BINARY_KEY};
+    lazyFieldNames.add(DocHelper.LARGE_LAZY_FIELD_KEY);
+    lazyFieldNames.add(DocHelper.LAZY_FIELD_KEY);
+    lazyFieldNames.add(DocHelper.LAZY_FIELD_BINARY_KEY);
+    lazyFieldNames.add(DocHelper.TEXT_FIELD_UTF2_KEY);
+
+    // Use LATENT instead of LAZY
+    SetBasedFieldSelector fieldSelector = new SetBasedFieldSelector(loadFieldNames, lazyFieldNames) {
+        @Override
+        public FieldSelectorResult accept(String fieldName) {
+          final FieldSelectorResult result = super.accept(fieldName);
+          if (result == FieldSelectorResult.LAZY_LOAD) {
+            return FieldSelectorResult.LATENT;
+          } else {
+            return result;
+          }
+        }
+      };
+
+    Document doc = getDocument(reader, 0, fieldSelector);
+    assertTrue("doc is null and it shouldn't be", doc != null);
+    IndexableField field = doc.getField(DocHelper.LAZY_FIELD_KEY);
+    assertTrue("field is null and it shouldn't be", field != null);
+    assertTrue("field is not lazy and it should be", field.getClass().getSimpleName().equals("LazyField"));
+    String value = field.stringValue();
+    assertTrue("value is null and it shouldn't be", value != null);
+    assertTrue(value + " is not equal to " + DocHelper.LAZY_FIELD_TEXT, value.equals(DocHelper.LAZY_FIELD_TEXT) == true);
+    assertTrue("calling stringValue() twice should give different references", field.stringValue() != field.stringValue());
+
+    field = doc.getField(DocHelper.TEXT_FIELD_1_KEY);
+    assertTrue("field is null and it shouldn't be", field != null);
+    assertFalse("Field is lazy and it should not be", field.getClass().getSimpleName().equals("LazyField"));
+    assertTrue("calling stringValue() twice should give same reference", field.stringValue() == field.stringValue());
+
+    field = doc.getField(DocHelper.TEXT_FIELD_UTF1_KEY);
+    assertTrue("field is null and it shouldn't be", field != null);
+    assertFalse("Field is lazy and it should not be", field.getClass().getSimpleName().equals("LazyField"));
+    assertTrue(field.stringValue() + " is not equal to " + DocHelper.FIELD_UTF1_TEXT, field.stringValue().equals(DocHelper.FIELD_UTF1_TEXT) == true);
+    assertTrue("calling stringValue() twice should give same reference", field.stringValue() == field.stringValue());
+
+    field = doc.getField(DocHelper.TEXT_FIELD_UTF2_KEY);
+    assertTrue("field is null and it shouldn't be", field != null);
+    assertTrue("Field is lazy and it should not be", field.getClass().getSimpleName().equals("LazyField"));
+    assertTrue(field.stringValue() + " is not equal to " + DocHelper.FIELD_UTF2_TEXT, field.stringValue().equals(DocHelper.FIELD_UTF2_TEXT) == true);
+    assertTrue("calling stringValue() twice should give different references", field.stringValue() != field.stringValue());
+
+    field = doc.getField(DocHelper.LAZY_FIELD_BINARY_KEY);
+    assertTrue("field is null and it shouldn't be", field != null);
+    assertTrue("stringValue isn't null for lazy binary field", field.stringValue() == null);
+    assertTrue("calling binaryValue() twice should give different references", field.binaryValue().bytes != field.binaryValue().bytes);
+
+    byte [] bytes = field.binaryValue().bytes;
+    assertTrue("bytes is null and it shouldn't be", bytes != null);
+    assertTrue("", DocHelper.LAZY_FIELD_BINARY_BYTES.length == bytes.length);
+    for (int i = 0; i < bytes.length; i++) {
+      assertTrue("byte[" + i + "] is mismatched", bytes[i] == DocHelper.LAZY_FIELD_BINARY_BYTES[i]);
+
+    }
+    reader.close();
+  }
+
+  public void testLoadFirst() throws Exception {
+    assertTrue(dir != null);
+    assertTrue(fieldInfos != null);
+    IndexReader reader = IndexReader.open(dir);
+    LoadFirstFieldSelector fieldSelector = new LoadFirstFieldSelector();
+    Document doc = getDocument(reader, 0, fieldSelector);
+    assertTrue("doc is null and it shouldn't be", doc != null);
+    int count = 0;
+    List<IndexableField> l = doc.getFields();
+    for (final IndexableField IndexableField : l ) {
+      Field field = (Field) IndexableField;
+
+      assertTrue("field is null and it shouldn't be", field != null);
+      String sv = field.stringValue();
+      assertTrue("sv is null and it shouldn't be", sv != null);
+      count++;
+    }
+    assertTrue(count + " does not equal: " + 1, count == 1);
+    reader.close();
+  }
+
+  /**
+   * Not really a test per se, but we should have some way of assessing whether this is worthwhile.
+   * <p/>
+   * Must test using a File based directory
+   *
+   * @throws Exception
+   */
+  public void testLazyPerformance() throws Exception {
+    String userName = System.getProperty("user.name");
+    File file = _TestUtil.getTempDir("lazyDir" + userName);
+    Directory tmpDir = newFSDirectory(file);
+    assertTrue(tmpDir != null);
+
+    IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setMergePolicy(newLogMergePolicy());
+    ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(false);
+    IndexWriter writer = new IndexWriter(tmpDir, conf);
+    writer.addDocument(testDoc);
+    writer.close();
+
+    assertTrue(fieldInfos != null);
+    long lazyTime = 0;
+    long regularTime = 0;
+    int length = 10;
+    Set<String> lazyFieldNames = new HashSet<String>();
+    lazyFieldNames.add(DocHelper.LARGE_LAZY_FIELD_KEY);
+    SetBasedFieldSelector fieldSelector = new SetBasedFieldSelector(Collections. <String> emptySet(), lazyFieldNames);
+
+    for (int i = 0; i < length; i++) {
+      IndexReader reader = IndexReader.open(tmpDir);
+
+      Document doc;
+      doc = reader.document(0);//Load all of them
+      assertTrue("doc is null and it shouldn't be", doc != null);
+      IndexableField field = doc.getField(DocHelper.LARGE_LAZY_FIELD_KEY);
+      assertTrue("field is null and it shouldn't be", field != null);
+      assertFalse("field is lazy", field.getClass().getSimpleName().equals("LazyField"));
+      String value;
+      long start;
+      long finish;
+      start = System.currentTimeMillis();
+      //On my machine this was always 0ms.
+      value = field.stringValue();
+      finish = System.currentTimeMillis();
+      assertTrue("value is null and it shouldn't be", value != null);
+      regularTime += (finish - start);
+      reader.close();
+      reader = null;
+      doc = null;
+      //Hmmm, are we still in cache???
+      System.gc();
+      reader = IndexReader.open(tmpDir);
+      doc = getDocument(reader, 0, fieldSelector);
+      field = doc.getField(DocHelper.LARGE_LAZY_FIELD_KEY);
+      assertTrue("field is not lazy", field.getClass().getSimpleName().equals("LazyField"));
+      start = System.currentTimeMillis();
+      //On my machine this took around 50 - 70ms
+      value = field.stringValue();
+      finish = System.currentTimeMillis();
+      assertTrue("value is null and it shouldn't be", value != null);
+      lazyTime += (finish - start);
+      reader.close();
+
+    }
+    tmpDir.close();
+    if (VERBOSE) {
+      System.out.println("Average Non-lazy time (should be very close to zero): " + regularTime / length + " ms for " + length + " reads");
+      System.out.println("Average Lazy Time (should be greater than zero): " + lazyTime / length + " ms for " + length + " reads");
+    }
+  }
+  
+  public void testLoadSize() throws IOException {
+    IndexReader reader = IndexReader.open(dir);
+    Document doc;
+    
+    doc = getDocument(reader, 0, new FieldSelector(){
+      public FieldSelectorResult accept(String fieldName) {
+        if (fieldName.equals(DocHelper.TEXT_FIELD_1_KEY) ||
+            fieldName.equals(DocHelper.LAZY_FIELD_BINARY_KEY))
+          return FieldSelectorResult.SIZE;
+        else if (fieldName.equals(DocHelper.TEXT_FIELD_3_KEY))
+          return FieldSelectorResult.LOAD;
+        else
+          return FieldSelectorResult.NO_LOAD;
+      }
+    });
+    IndexableField f1 = doc.getField(DocHelper.TEXT_FIELD_1_KEY);
+    IndexableField f3 = doc.getField(DocHelper.TEXT_FIELD_3_KEY);
+    IndexableField fb = doc.getField(DocHelper.LAZY_FIELD_BINARY_KEY);
+    assertTrue(f1.binaryValue()!=null);
+    assertTrue(f3.binaryValue()==null);
+    assertTrue(fb.binaryValue()!=null);
+    assertSizeEquals(2*DocHelper.FIELD_1_TEXT.length(), f1.binaryValue().bytes);
+    assertEquals(DocHelper.FIELD_3_TEXT, f3.stringValue());
+    assertSizeEquals(DocHelper.LAZY_FIELD_BINARY_BYTES.length, fb.binaryValue().bytes);
+    
+    reader.close();
+  }
+  
+  private void assertSizeEquals(int size, byte[] sizebytes) {
+    assertEquals((byte) (size>>>24), sizebytes[0]);
+    assertEquals((byte) (size>>>16), sizebytes[1]);
+    assertEquals((byte) (size>>> 8), sizebytes[2]);
+    assertEquals((byte)  size      , sizebytes[3]);
+  }
+}

Added: lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestContribIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestContribIndexReader.java?rev=1162347&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestContribIndexReader.java (added)
+++ lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestContribIndexReader.java Sat Aug 27 13:27:01 2011
@@ -0,0 +1,169 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.BinaryField;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.document.FieldSelectorVisitor;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.SetBasedFieldSelector;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestContribIndexReader extends LuceneTestCase {
+  private Document getDocument(IndexReader ir, int docID, FieldSelector selector)  throws IOException {
+    final FieldSelectorVisitor visitor = new FieldSelectorVisitor(selector);
+    ir.document(docID, visitor);
+    return visitor.getDocument();
+  }
+
+  static void addDoc(IndexWriter writer, String value) throws IOException {
+    Document doc = new Document();
+    doc.add(newField("content", value, TextField.TYPE_UNSTORED));
+    writer.addDocument(doc);
+  }
+
+  static void addDocumentWithFields(IndexWriter writer) throws IOException {
+    Document doc = new Document();
+        
+    FieldType customType3 = new FieldType();
+    customType3.setStored(true);
+    doc.add(newField("keyword", "test1", StringField.TYPE_STORED));
+    doc.add(newField("text", "test1", TextField.TYPE_STORED));
+    doc.add(newField("unindexed", "test1", customType3));
+    doc.add(new TextField("unstored","test1"));
+    writer.addDocument(doc);
+  }
+
+
+  static void addDocumentWithDifferentFields(IndexWriter writer) throws IOException {
+    Document doc = new Document();
+      
+    FieldType customType3 = new FieldType();
+    customType3.setStored(true);
+    doc.add(newField("keyword2", "test1", StringField.TYPE_STORED));
+    doc.add(newField("text2", "test1", TextField.TYPE_STORED));
+    doc.add(newField("unindexed2", "test1", customType3));
+    doc.add(new TextField("unstored2","test1"));
+    writer.addDocument(doc);
+  }
+
+  static void addDocumentWithTermVectorFields(IndexWriter writer) throws IOException {
+    Document doc = new Document();
+    FieldType customType5 = new FieldType(TextField.TYPE_STORED);
+    customType5.setStoreTermVectors(true);
+    FieldType customType6 = new FieldType(TextField.TYPE_STORED);
+    customType6.setStoreTermVectors(true);
+    customType6.setStoreTermVectorOffsets(true);
+    FieldType customType7 = new FieldType(TextField.TYPE_STORED);
+    customType7.setStoreTermVectors(true);
+    customType7.setStoreTermVectorPositions(true);
+    FieldType customType8 = new FieldType(TextField.TYPE_STORED);
+    customType8.setStoreTermVectors(true);
+    customType8.setStoreTermVectorOffsets(true);
+    customType8.setStoreTermVectorPositions(true);
+    doc.add(newField("tvnot","tvnot",TextField.TYPE_STORED));
+    doc.add(newField("termvector","termvector",customType5));
+    doc.add(newField("tvoffset","tvoffset", customType6));
+    doc.add(newField("tvposition","tvposition", customType7));
+    doc.add(newField("tvpositionoffset","tvpositionoffset", customType8));
+        
+    writer.addDocument(doc);
+  }
+
+  public void testBinaryFields() throws IOException {
+    Directory dir = newDirectory();
+    byte[] bin = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+        
+    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
+        
+    for (int i = 0; i < 10; i++) {
+      addDoc(writer, "document number " + (i + 1));
+      addDocumentWithFields(writer);
+      addDocumentWithDifferentFields(writer);
+      addDocumentWithTermVectorFields(writer);
+    }
+    writer.close();
+    writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy()));
+    Document doc = new Document();
+    doc.add(new BinaryField("bin1", bin));
+    doc.add(new TextField("junk", "junk text"));
+    writer.addDocument(doc);
+    writer.close();
+    IndexReader reader = IndexReader.open(dir, false);
+    Document doc2 = reader.document(reader.maxDoc() - 1);
+    IndexableField[] fields = doc2.getFields("bin1");
+    assertNotNull(fields);
+    assertEquals(1, fields.length);
+    Field b1 = (Field) fields[0];
+    assertTrue(b1.isBinary());
+    BytesRef bytesRef = b1.binaryValue();
+    assertEquals(bin.length, bytesRef.length);
+    for (int i = 0; i < bin.length; i++) {
+      assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]);
+    }
+    Set<String> lazyFields = new HashSet<String>();
+    lazyFields.add("bin1");
+    FieldSelector sel = new SetBasedFieldSelector(new HashSet<String>(), lazyFields);
+    doc2 = getDocument(reader, reader.maxDoc() - 1, sel);
+    fields = doc2.getFields("bin1");
+    assertNotNull(fields);
+    assertEquals(1, fields.length);
+    IndexableField fb1 = fields[0];
+    assertTrue(fb1.binaryValue()!=null);
+    bytesRef = fb1.binaryValue();
+    assertEquals(bin.length, bytesRef.bytes.length);
+    assertEquals(bin.length, bytesRef.length);
+    for (int i = 0; i < bin.length; i++) {
+      assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]);
+    }
+    reader.close();
+    // force optimize
+
+
+    writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy()));
+    writer.optimize();
+    writer.close();
+    reader = IndexReader.open(dir, false);
+    doc2 = reader.document(reader.maxDoc() - 1);
+    fields = doc2.getFields("bin1");
+    assertNotNull(fields);
+    assertEquals(1, fields.length);
+    b1 = (Field) fields[0];
+    assertTrue(b1.isBinary());
+    bytesRef = b1.binaryValue();
+    assertEquals(bin.length, bytesRef.length);
+    for (int i = 0; i < bin.length; i++) {
+      assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]);
+    }
+    reader.close();
+    dir.close();
+  }
+}

Added: lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestContribParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestContribParallelReader.java?rev=1162347&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestContribParallelReader.java (added)
+++ lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestContribParallelReader.java Sat Aug 27 13:27:01 2011
@@ -0,0 +1,149 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Random;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.document.FieldSelectorVisitor;
+import org.apache.lucene.document.MapFieldSelector;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.search.*;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestContribParallelReader extends LuceneTestCase {
+
+  private IndexSearcher parallel;
+  private IndexSearcher single;
+  private Directory dir, dir1, dir2;
+  
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    single = single(random);
+    parallel = parallel(random);
+  }
+  
+  @Override
+  public void tearDown() throws Exception {
+    single.getIndexReader().close();
+    single.close();
+    parallel.getIndexReader().close();
+    parallel.close();
+    dir.close();
+    dir1.close();
+    dir2.close();
+    super.tearDown();
+  }
+
+  // Fields 1-4 indexed together:
+  private IndexSearcher single(Random random) throws IOException {
+    dir = newDirectory();
+    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+    Document d1 = new Document();
+    d1.add(newField("f1", "v1", TextField.TYPE_STORED));
+    d1.add(newField("f2", "v1", TextField.TYPE_STORED));
+    d1.add(newField("f3", "v1", TextField.TYPE_STORED));
+    d1.add(newField("f4", "v1", TextField.TYPE_STORED));
+    w.addDocument(d1);
+    Document d2 = new Document();
+    d2.add(newField("f1", "v2", TextField.TYPE_STORED));
+    d2.add(newField("f2", "v2", TextField.TYPE_STORED));
+    d2.add(newField("f3", "v2", TextField.TYPE_STORED));
+    d2.add(newField("f4", "v2", TextField.TYPE_STORED));
+    w.addDocument(d2);
+    w.close();
+
+    return new IndexSearcher(dir, false);
+  }
+
+  // Fields 1 & 2 in one index, 3 & 4 in other, with ParallelReader:
+  private IndexSearcher parallel(Random random) throws IOException {
+    dir1 = getDir1(random);
+    dir2 = getDir2(random);
+    ParallelReader pr = new ParallelReader();
+    pr.add(IndexReader.open(dir1, false));
+    pr.add(IndexReader.open(dir2, false));
+    return newSearcher(pr);
+  }
+
+  private Document getDocument(IndexReader ir, int docID, FieldSelector selector) throws IOException {
+    final FieldSelectorVisitor visitor = new FieldSelectorVisitor(selector);
+    ir.document(docID, visitor);
+    return visitor.getDocument();
+  }
+
+  public void testDocument() throws IOException {
+    Directory dir1 = getDir1(random);
+    Directory dir2 = getDir2(random);
+    ParallelReader pr = new ParallelReader();
+    pr.add(IndexReader.open(dir1, false));
+    pr.add(IndexReader.open(dir2, false));
+
+    Document doc11 = getDocument(pr, 0, new MapFieldSelector("f1"));
+    Document doc24 = getDocument(pr, 1, new MapFieldSelector(Arrays.asList("f4")));
+    Document doc223 = getDocument(pr, 1, new MapFieldSelector("f2", "f3"));
+    
+    assertEquals(1, doc11.getFields().size());
+    assertEquals(1, doc24.getFields().size());
+    assertEquals(2, doc223.getFields().size());
+    
+    assertEquals("v1", doc11.get("f1"));
+    assertEquals("v2", doc24.get("f4"));
+    assertEquals("v2", doc223.get("f2"));
+    assertEquals("v2", doc223.get("f3"));
+    pr.close();
+    dir1.close();
+    dir2.close();
+  }
+
+  private Directory getDir1(Random random) throws IOException {
+    Directory dir1 = newDirectory();
+    IndexWriter w1 = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+    Document d1 = new Document();
+    d1.add(newField("f1", "v1", TextField.TYPE_STORED));
+    d1.add(newField("f2", "v1", TextField.TYPE_STORED));
+    w1.addDocument(d1);
+    Document d2 = new Document();
+    d2.add(newField("f1", "v2", TextField.TYPE_STORED));
+    d2.add(newField("f2", "v2", TextField.TYPE_STORED));
+    w1.addDocument(d2);
+    w1.close();
+    return dir1;
+  }
+
+  private Directory getDir2(Random random) throws IOException {
+    Directory dir2 = newDirectory();
+    IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+    Document d3 = new Document();
+    d3.add(newField("f3", "v1", TextField.TYPE_STORED));
+    d3.add(newField("f4", "v1", TextField.TYPE_STORED));
+    w2.addDocument(d3);
+    Document d4 = new Document();
+    d4.add(newField("f3", "v2", TextField.TYPE_STORED));
+    d4.add(newField("f4", "v2", TextField.TYPE_STORED));
+    w2.addDocument(d4);
+    w2.close();
+    return dir2;
+  }
+}

Modified: lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java Sat Aug 27 13:27:01 2011
@@ -22,7 +22,8 @@ import java.util.Arrays;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.IndexReader.AtomicReaderContext;
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.DefaultSimilarity;
@@ -65,13 +66,15 @@ public class TestFieldNormModifier exten
     
     for (int i = 0; i < NUM_DOCS; i++) {
       Document d = new Document();
-      d.add(newField("field", "word", Field.Store.YES, Field.Index.ANALYZED));
-      d.add(newField("nonorm", "word", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
-      d.add(newField("untokfield", "20061212 20071212", Field.Store.YES, Field.Index.ANALYZED));
+      
+      d.add(newField("field", "word", TextField.TYPE_STORED));
+
+      d.add(newField("nonorm", "word", StringField.TYPE_STORED));
+      d.add(newField("untokfield", "20061212 20071212", TextField.TYPE_STORED));
       
       for (int j = 1; j <= i; j++) {
-        d.add(newField("field", "crap", Field.Store.YES, Field.Index.ANALYZED));
-        d.add(newField("nonorm", "more words", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
+        d.add(newField("field", "crap", TextField.TYPE_STORED));
+        d.add(newField("nonorm", "more words", StringField.TYPE_STORED));
       }
       writer.addDocument(d);
     }

Modified: lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java Sat Aug 27 13:27:01 2011
@@ -21,6 +21,7 @@ import java.io.File;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.LuceneTestCase;
@@ -112,10 +113,10 @@ public class TestIndexSplitter extends L
     Directory fsDir = newFSDirectory(indexPath);
     IndexWriter indexWriter = new IndexWriter(fsDir, iwConfig);
     Document doc = new Document();
-    doc.add(new Field("content", "doc 1", Field.Store.YES, Field.Index.ANALYZED_NO_NORMS));
+    doc.add(new Field("content", StringField.TYPE_STORED, "doc 1"));
     indexWriter.addDocument(doc);
     doc = new Document();
-    doc.add(new Field("content", "doc 2", Field.Store.YES, Field.Index.ANALYZED_NO_NORMS));
+    doc.add(new Field("content", StringField.TYPE_STORED, "doc 2"));
     indexWriter.addDocument(doc);
     indexWriter.close();
     fsDir.close();

Copied: lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestLazyBug.java (from r1161532, lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLazyBug.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestLazyBug.java?p2=lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestLazyBug.java&p1=lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLazyBug.java&r1=1161532&r2=1162347&rev=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLazyBug.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestLazyBug.java Sat Aug 27 13:27:01 2011
@@ -22,7 +22,11 @@ import java.util.List;
 import java.util.Set;
 
 import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.*;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.document.FieldSelectorResult;
+import org.apache.lucene.document.FieldSelectorVisitor;
+import org.apache.lucene.document.TextField;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.LuceneTestCase;
 import org.junit.AfterClass;
@@ -87,8 +91,7 @@ public class TestLazyBug extends LuceneT
           doc.add(newField("f"+f,
                             data[f % data.length]
                             + '#' + data[random.nextInt(data.length)],
-                            Field.Store.NO,
-                            Field.Index.ANALYZED));
+                            TextField.TYPE_UNSTORED));
         }
         writer.addDocument(doc);
       }
@@ -102,12 +105,14 @@ public class TestLazyBug extends LuceneT
   public void doTest(int[] docs) throws Exception {
     IndexReader reader = IndexReader.open(directory, true);
     for (int i = 0; i < docs.length; i++) {
-      Document d = reader.document(docs[i], SELECTOR);
+      final FieldSelectorVisitor visitor = new FieldSelectorVisitor(SELECTOR);
+      reader.document(docs[i], visitor);
+      Document d = visitor.getDocument();
       d.get(MAGIC_FIELD);
 
-      List<Fieldable> fields = d.getFields();
-      for (Iterator<Fieldable> fi = fields.iterator(); fi.hasNext(); ) {
-        Fieldable f=null;
+      List<IndexableField> fields = d.getFields();
+      for (Iterator<IndexableField> fi = fields.iterator(); fi.hasNext(); ) {
+        IndexableField f=null;
         try {
           f =  fi.next();
           String fname = f.name();
@@ -136,5 +141,4 @@ public class TestLazyBug extends LuceneT
   public void testLazyBroken() throws Exception {
     doTest(new int[] { NUM_DOCS/2, NUM_DOCS-1 });
   }
-
 }

Modified: lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java Sat Aug 27 13:27:01 2011
@@ -18,10 +18,11 @@ package org.apache.lucene.index;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
 
 public class TestMultiPassIndexSplitter extends LuceneTestCase {
   IndexReader input;
@@ -36,8 +37,8 @@ public class TestMultiPassIndexSplitter 
     Document doc;
     for (int i = 0; i < NUM_DOCS; i++) {
       doc = new Document();
-      doc.add(newField("id", i + "", Field.Store.YES, Field.Index.NOT_ANALYZED));
-      doc.add(newField("f", i + " " + i, Field.Store.YES, Field.Index.ANALYZED));
+      doc.add(newField("id", i + "", StringField.TYPE_STORED));
+      doc.add(newField("f", i + " " + i, TextField.TYPE_STORED));
       w.addDocument(doc);
     }
     w.close();

Modified: lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java Sat Aug 27 13:27:01 2011
@@ -33,7 +33,8 @@ import java.util.concurrent.atomic.Atomi
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.PhraseQuery;
@@ -77,17 +78,12 @@ public class TestNRTManager extends Luce
   // TODO: is there a pre-existing way to do this!!!
   private Document cloneDoc(Document doc1) {
     final Document doc2 = new Document();
-    for(Fieldable f : doc1.getFields()) {
+    for(IndexableField f : doc1) {
       Field field1 = (Field) f;
       
       Field field2 = new Field(field1.name(),
-                               field1.stringValue(),
-                               field1.isStored() ? Field.Store.YES : Field.Store.NO,
-                               field1.isIndexed() ? (field1.isTokenized() ? Field.Index.ANALYZED : Field.Index.NOT_ANALYZED) : Field.Index.NO);
-      if (field1.getOmitNorms()) {
-        field2.setOmitNorms(true);
-      }
-      field2.setIndexOptions(field1.getIndexOptions());
+                              ((Field) f).getFieldType(),
+                               field1.stringValue());
       doc2.add(field2);
     }
 
@@ -240,7 +236,7 @@ public class TestNRTManager extends Luce
                 final String addedField;
                 if (random.nextBoolean()) {
                   addedField = "extra" + random.nextInt(10);
-                  doc.add(new Field(addedField, "a random field", Field.Store.NO, Field.Index.ANALYZED));
+                  doc.add(new TextField(addedField, "a random field"));
                 } else {
                   addedField = null;
                 }
@@ -262,7 +258,7 @@ public class TestNRTManager extends Luce
                       packID = packCount.getAndIncrement() + "";
                     }
 
-                    final Field packIDField = newField("packID", packID, Field.Store.YES, Field.Index.NOT_ANALYZED);
+                    final Field packIDField = newField("packID", packID, StringField.TYPE_STORED);
                     final List<String> docIDs = new ArrayList<String>();
                     final SubDocs subDocs = new SubDocs(packID, docIDs);
                     final List<Document> docsList = new ArrayList<Document>();

Modified: lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestPKIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestPKIndexSplitter.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestPKIndexSplitter.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestPKIndexSplitter.java Sat Aug 27 13:27:01 2011
@@ -23,8 +23,8 @@ import java.text.NumberFormat;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Bits;
@@ -102,15 +102,15 @@ public class TestPKIndexSplitter extends
     StringBuilder sb = new StringBuilder();
     Document doc = new Document();
     String id = format.format(n);
-    doc.add(newField("id", id, Store.YES, Index.NOT_ANALYZED));
-    doc.add(newField("indexname", indexName, Store.YES, Index.NOT_ANALYZED));
+    doc.add(newField("id", id, StringField.TYPE_STORED));
+    doc.add(newField("indexname", indexName, StringField.TYPE_STORED));
     sb.append("a");
     sb.append(n);
-    doc.add(newField("field1", sb.toString(), Store.YES, Index.ANALYZED));
+    doc.add(newField("field1", sb.toString(), TextField.TYPE_STORED));
     sb.append(" b");
     sb.append(n);
     for (int i = 1; i < numFields; i++) {
-      doc.add(newField("field" + (i + 1), sb.toString(), Store.YES, Index.ANALYZED));
+      doc.add(newField("field" + (i + 1), sb.toString(), TextField.TYPE_STORED));
     }
     return doc;
   }

Modified: lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestTermVectorAccessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestTermVectorAccessor.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestTermVectorAccessor.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestTermVectorAccessor.java Sat Aug 27 13:27:01 2011
@@ -2,7 +2,8 @@ package org.apache.lucene.index;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.TextField;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.LuceneTestCase;
 
@@ -30,33 +31,42 @@ public class TestTermVectorAccessor exte
     Document doc;
 
     doc = new Document();
-    doc.add(newField("a", "a b a c a d a e a f a g a h a", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
-    doc.add(newField("b", "a b c b d b e b f b g b h b", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
-    doc.add(newField("c", "a c b c d c e c f c g c h c", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+    FieldType customType = new FieldType(TextField.TYPE_UNSTORED);
+    customType.setStoreTermVectors(true);
+    customType.setStoreTermVectorPositions(true);
+    customType.setStoreTermVectorOffsets(true);
+    doc.add(newField("a", "a b a c a d a e a f a g a h a", customType));
+    doc.add(newField("b", "a b c b d b e b f b g b h b", customType));
+    doc.add(newField("c", "a c b c d c e c f c g c h c", customType));
     iw.addDocument(doc);
 
     doc = new Document();
-    doc.add(newField("a", "a b a c a d a e a f a g a h a", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS));
-    doc.add(newField("b", "a b c b d b e b f b g b h b", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS));
-    doc.add(newField("c", "a c b c d c e c f c g c h c", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS));
+    FieldType customType2 = new FieldType(TextField.TYPE_UNSTORED);
+    customType2.setStoreTermVectors(true);
+    customType2.setStoreTermVectorPositions(true);
+    doc.add(newField("a", "a b a c a d a e a f a g a h a", customType2));
+    doc.add(newField("b", "a b c b d b e b f b g b h b", customType2));
+    doc.add(newField("c", "a c b c d c e c f c g c h c", customType2));
     iw.addDocument(doc);
 
     doc = new Document();
-    doc.add(newField("a", "a b a c a d a e a f a g a h a", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.YES));
-    doc.add(newField("b", "a b c b d b e b f b g b h b", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.YES));
-    doc.add(newField("c", "a c b c d c e c f c g c h c", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.YES));
+    FieldType customType3 = new FieldType(TextField.TYPE_UNSTORED);
+    customType3.setStoreTermVectors(true);
+    doc.add(newField("a", "a b a c a d a e a f a g a h a", customType3));
+    doc.add(newField("b", "a b c b d b e b f b g b h b", customType3));
+    doc.add(newField("c", "a c b c d c e c f c g c h c", customType3));
     iw.addDocument(doc);
 
     doc = new Document();
-    doc.add(newField("a", "a b a c a d a e a f a g a h a", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.NO));
-    doc.add(newField("b", "a b c b d b e b f b g b h b", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.NO));
-    doc.add(newField("c", "a c b c d c e c f c g c h c", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.NO));
+    doc.add(newField("a", "a b a c a d a e a f a g a h a", TextField.TYPE_UNSTORED));
+    doc.add(newField("b", "a b c b d b e b f b g b h b", TextField.TYPE_UNSTORED));
+    doc.add(newField("c", "a c b c d c e c f c g c h c", TextField.TYPE_UNSTORED));
     iw.addDocument(doc);
 
     doc = new Document();
-    doc.add(newField("a", "a b a c a d a e a f a g a h a", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
-    doc.add(newField("b", "a b c b d b e b f b g b h b", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.NO));
-    doc.add(newField("c", "a c b c d c e c f c g c h c", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.YES));
+    doc.add(newField("a", "a b a c a d a e a f a g a h a", customType));
+    doc.add(newField("b", "a b c b d b e b f b g b h b", TextField.TYPE_UNSTORED));
+    doc.add(newField("c", "a c b c d c e c f c g c h c", customType3));
     iw.addDocument(doc);
 
     iw.close();

Modified: lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java Sat Aug 27 13:27:01 2011
@@ -22,9 +22,8 @@ import java.util.Random;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.Field.TermVector;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.IndexReader;
@@ -141,7 +140,11 @@ public class TestAppendingCodec extends 
     ((TieredMergePolicy)cfg.getMergePolicy()).setUseCompoundFile(false);
     IndexWriter writer = new IndexWriter(dir, cfg);
     Document doc = new Document();
-    doc.add(newField("f", text, Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
+    FieldType storedTextType = new FieldType(TextField.TYPE_STORED);
+    storedTextType.setStoreTermVectors(true);
+    storedTextType.setStoreTermVectorPositions(true);
+    storedTextType.setStoreTermVectorOffsets(true);
+    doc.add(newField("f", text, storedTextType));
     writer.addDocument(doc);
     writer.commit();
     writer.addDocument(doc);
@@ -149,8 +152,8 @@ public class TestAppendingCodec extends 
     writer.close();
     IndexReader reader = IndexReader.open(dir, null, true, 1, new AppendingCodecProvider());
     assertEquals(2, reader.numDocs());
-    doc = reader.document(0);
-    assertEquals(text, doc.get("f"));
+    Document doc2 = reader.document(0);
+    assertEquals(text, doc2.get("f"));
     Fields fields = MultiFields.getFields(reader);
     Terms terms = fields.terms("f");
     assertNotNull(terms);



Mime
View raw message