lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject svn commit: r1233096 [9/13] - in /lucene/dev/branches/solrcloud: ./ dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/ dev-tools/idea/modules/analysis/kuromoji/ dev-tools/idea/solr/contrib/analysis-extras/ dev-tools/maven/modules/a...
Date Wed, 18 Jan 2012 22:28:20 GMT
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java Wed Jan 18 22:28:07 2012
@@ -27,7 +27,6 @@ import org.apache.lucene.analysis.Analyz
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.index.IndexReader.FieldOption;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Bits;
@@ -35,6 +34,7 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.LineFileDocs;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.ReaderUtil;
 import org.apache.lucene.util.automaton.AutomatonTestUtil;
 import org.apache.lucene.util.automaton.CompiledAutomaton;
 import org.apache.lucene.util.automaton.RegExp;
@@ -260,17 +260,17 @@ public class TestDuelingCodecs extends L
       assertEquals(info, term, rightTermsEnum.next());
       assertTermStats(leftTermsEnum, rightTermsEnum);
       if (deep) {
-        assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions),
-            rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions));
-        assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions),
-            rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions));
+        assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions, false),
+                                   rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions, false));
+        assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions, false),
+                                   rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions, false));
 
         assertPositionsSkipping(leftTermsEnum.docFreq(), 
-            leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions),
-            rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions));
+                                leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions, false),
+                                rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions, false));
         assertPositionsSkipping(leftTermsEnum.docFreq(), 
-            leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions),
-            rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions));
+                                leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions, false),
+                                rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions, false));
 
         // with freqs:
         assertDocsEnum(leftDocs = leftTermsEnum.docs(null, leftDocs, true),
@@ -341,6 +341,8 @@ public class TestDuelingCodecs extends L
       for (int i = 0; i < freq; i++) {
         assertEquals(info, leftDocs.nextPosition(), rightDocs.nextPosition());
         assertEquals(info, leftDocs.hasPayload(), rightDocs.hasPayload());
+        assertEquals(info, leftDocs.startOffset(), rightDocs.startOffset());
+        assertEquals(info, leftDocs.endOffset(), rightDocs.endOffset());
         if (leftDocs.hasPayload()) {
           assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload());
         }
@@ -500,8 +502,6 @@ public class TestDuelingCodecs extends L
     assertEquals(info, leftField.binaryValue(), rightField.binaryValue());
     assertEquals(info, leftField.stringValue(), rightField.stringValue());
     assertEquals(info, leftField.numericValue(), rightField.numericValue());
-    assertEquals(info, leftField.numeric(), rightField.numeric());
-    assertEquals(info, leftField.numericDataType(), rightField.numericDataType());
     // TODO: should we check the FT at all?
   }
   
@@ -516,13 +516,24 @@ public class TestDuelingCodecs extends L
       assertFields(leftFields, rightFields, rarely());
     }
   }
+
+  private static Set<String> getDVFields(IndexReader reader) {
+    Set<String> fields = new HashSet<String>();
+    for(FieldInfo fi : ReaderUtil.getMergedFieldInfos(reader)) {
+      if (fi.hasDocValues()) {
+        fields.add(fi.name);
+      }
+    }
+
+    return fields;
+  }
   
   /**
    * checks that docvalues across all fields are equivalent
    */
   public void assertDocValues(IndexReader leftReader, IndexReader rightReader) throws Exception {
-    Set<String> leftValues = new HashSet<String>(leftReader.getFieldNames(FieldOption.DOC_VALUES));
-    Set<String> rightValues = new HashSet<String>(rightReader.getFieldNames(FieldOption.DOC_VALUES));
+    Set<String> leftValues = getDVFields(leftReader);
+    Set<String> rightValues = getDVFields(rightReader);
     assertEquals(info, leftValues, rightValues);
 
     for (String field : leftValues) {

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java Wed Jan 18 22:28:07 2012
@@ -133,22 +133,15 @@ public class TestFieldInfos extends Luce
     }
     try {
       readOnly.addOrUpdate("bogus", random.nextBoolean(), random.nextBoolean(),
-          random.nextBoolean(), random.nextBoolean());
+          random.nextBoolean());
       fail("instance should be read only");
     } catch (IllegalStateException e) {
       // expected
     }
     try {
       readOnly.addOrUpdate("bogus", random.nextBoolean(), random.nextBoolean(),
-          random.nextBoolean(), random.nextBoolean(), random.nextBoolean());
-      fail("instance should be read only");
-    } catch (IllegalStateException e) {
-      // expected
-    }
-    try {
-      readOnly.addOrUpdate("bogus", random.nextBoolean(), random.nextBoolean(),
-          random.nextBoolean(), random.nextBoolean(), random.nextBoolean(),
-          random.nextBoolean(), random.nextBoolean() ? IndexOptions.DOCS_ONLY : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, null);
+          random.nextBoolean(),
+          random.nextBoolean(), random.nextBoolean() ? IndexOptions.DOCS_ONLY : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, null, null);
       fail("instance should be read only");
     } catch (IllegalStateException e) {
       // expected

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java Wed Jan 18 22:28:07 2012
@@ -27,6 +27,7 @@ import org.apache.lucene.document.Docume
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.NumericField;
+import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
@@ -80,24 +81,18 @@ public class TestFieldsReader extends Lu
     assertTrue(field != null);
     assertTrue(field.fieldType().storeTermVectors());
 
-    assertTrue(field.fieldType().storeTermVectorOffsets());
-    assertTrue(field.fieldType().storeTermVectorPositions());
     assertFalse(field.fieldType().omitNorms());
     assertTrue(field.fieldType().indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
 
     field = (Field) doc.getField(DocHelper.TEXT_FIELD_3_KEY);
     assertTrue(field != null);
     assertFalse(field.fieldType().storeTermVectors());
-    assertFalse(field.fieldType().storeTermVectorOffsets());
-    assertFalse(field.fieldType().storeTermVectorPositions());
     assertTrue(field.fieldType().omitNorms());
     assertTrue(field.fieldType().indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
 
     field = (Field) doc.getField(DocHelper.NO_TF_KEY);
     assertTrue(field != null);
     assertFalse(field.fieldType().storeTermVectors());
-    assertFalse(field.fieldType().storeTermVectorOffsets());
-    assertFalse(field.fieldType().storeTermVectorPositions());
     assertFalse(field.fieldType().omitNorms());
     assertTrue(field.fieldType().indexOptions() == IndexOptions.DOCS_ONLY);
 
@@ -243,40 +238,42 @@ public class TestFieldsReader extends Lu
     final NumericField.DataType[] typeAnswers = new NumericField.DataType[numDocs];
     for(int id=0;id<numDocs;id++) {
       Document doc = new Document();
-      NumericField nf = new NumericField("nf", NumericField.TYPE_STORED);
-      doc.add(nf);
+      final NumericField nf;
       final Number answer;
       final NumericField.DataType typeAnswer;
       if (random.nextBoolean()) {
         // float/double
         if (random.nextBoolean()) {
           final float f = random.nextFloat();
-          nf.setFloatValue(f);
           answer = Float.valueOf(f);
+          nf = new NumericField("nf", answer, NumericField.getFieldType(NumericField.DataType.FLOAT, true));
           typeAnswer = NumericField.DataType.FLOAT;
         } else {
           final double d = random.nextDouble();
-          nf.setDoubleValue(d);
           answer = Double.valueOf(d);
+          nf = new NumericField("nf", answer, NumericField.getFieldType(NumericField.DataType.DOUBLE, true));
           typeAnswer = NumericField.DataType.DOUBLE;
         }
       } else {
         // int/long
         if (random.nextBoolean()) {
           final int i = random.nextInt();
-          nf.setIntValue(i);
           answer = Integer.valueOf(i);
+          nf = new NumericField("nf", answer, NumericField.getFieldType(NumericField.DataType.INT, true));
           typeAnswer = NumericField.DataType.INT;
         } else {
           final long l = random.nextLong();
-          nf.setLongValue(l);
           answer = Long.valueOf(l);
+          nf = new NumericField("nf", answer, NumericField.getFieldType(NumericField.DataType.LONG, true));
           typeAnswer = NumericField.DataType.LONG;
         }
       }
+      doc.add(nf);
       answers[id] = answer;
       typeAnswers[id] = typeAnswer;
-      doc.add(new NumericField("id", Integer.MAX_VALUE).setIntValue(id));
+      FieldType ft = new FieldType(NumericField.getFieldType(NumericField.DataType.INT, false));
+      ft.setNumericPrecisionStep(Integer.MAX_VALUE);
+      doc.add(new NumericField("id", id, ft));
       w.addDocument(doc);
     }
     final IndexReader r = w.getReader();
@@ -289,10 +286,8 @@ public class TestFieldsReader extends Lu
       for(int docID=0;docID<sub.numDocs();docID++) {
         final Document doc = sub.document(docID);
         final Field f = (Field) doc.getField("nf");
-        assertTrue("got f=" + f, f instanceof NumericField);
-        final NumericField nf = (NumericField) f;
-        assertEquals(answers[ids[docID]], nf.numericValue());
-        assertSame(typeAnswers[ids[docID]], nf.numericDataType());
+        assertTrue("got f=" + f, f instanceof StoredField);
+        assertEquals(answers[ids[docID]], f.numericValue());
       }
     }
     r.close();

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java Wed Jan 18 22:28:07 2012
@@ -18,19 +18,20 @@ package org.apache.lucene.index;
  */
 
 
-import org.apache.lucene.util.LuceneTestCase;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.HashSet;
 
-import org.apache.lucene.store.Directory;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.TextField;
-import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.Bits;
-
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.lang.reflect.Modifier;
-import java.util.HashSet;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.ReaderUtil;
 
 public class TestFilterIndexReader extends LuceneTestCase {
 
@@ -90,8 +91,8 @@ public class TestFilterIndexReader exten
       }
 
       @Override
-      public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException {
-        return new TestPositions(super.docsAndPositions(liveDocs, reuse == null ? null : ((FilterDocsAndPositionsEnum) reuse).in));
+      public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
+        return new TestPositions(super.docsAndPositions(liveDocs, reuse == null ? null : ((FilterDocsAndPositionsEnum) reuse).in, needsOffsets));
       }
     }
 
@@ -121,6 +122,11 @@ public class TestFilterIndexReader exten
     public Fields fields() throws IOException {
       return new TestFields(super.fields());
     }
+
+    @Override
+    public FieldInfos getFieldInfos() {
+      return ReaderUtil.getMergedFieldInfos(in);
+    }
   }
     
   /**
@@ -129,6 +135,7 @@ public class TestFilterIndexReader exten
    */
   public void testFilterIndexReader() throws Exception {
     Directory directory = newDirectory();
+
     IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
 
     Document d1 = new Document();
@@ -146,6 +153,10 @@ public class TestFilterIndexReader exten
     writer.close();
 
     Directory target = newDirectory();
+
+    // We mess with the postings so this can fail:
+    ((MockDirectoryWrapper) target).setCrossCheckTermVectorsOnClose(false);
+
     writer = new IndexWriter(target, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
     IndexReader reader = new TestReader(IndexReader.open(directory));
     writer.addIndexes(reader);
@@ -161,7 +172,7 @@ public class TestFilterIndexReader exten
     assertEquals(TermsEnum.SeekStatus.FOUND, terms.seekCeil(new BytesRef("one")));
     
     DocsAndPositionsEnum positions = terms.docsAndPositions(MultiFields.getLiveDocs(reader),
-                                                            null);
+                                                            null, false);
     while (positions.nextDoc() != DocsEnum.NO_MORE_DOCS) {
       assertTrue((positions.docID() % 2) == 1);
     }

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexReader.java Wed Jan 18 22:28:07 2012
@@ -30,13 +30,12 @@ import java.util.Set;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat;
-import org.apache.lucene.document.BinaryField;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.IndexReader.FieldOption;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.FieldCache;
@@ -46,6 +45,7 @@ import org.apache.lucene.store.NoSuchDir
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.ReaderUtil;
 import org.apache.lucene.util._TestUtil;
 import org.junit.Assume;
 
@@ -102,11 +102,11 @@ public class TestIndexReader extends Luc
         writer.close();
         // set up reader
         IndexReader reader = IndexReader.open(d);
-        Collection<String> fieldNames = reader.getFieldNames(IndexReader.FieldOption.ALL);
-        assertTrue(fieldNames.contains("keyword"));
-        assertTrue(fieldNames.contains("text"));
-        assertTrue(fieldNames.contains("unindexed"));
-        assertTrue(fieldNames.contains("unstored"));
+        FieldInfos fieldInfos = ReaderUtil.getMergedFieldInfos(reader);
+        assertNotNull(fieldInfos.fieldInfo("keyword"));
+        assertNotNull(fieldInfos.fieldInfo("text"));
+        assertNotNull(fieldInfos.fieldInfo("unindexed"));
+        assertNotNull(fieldInfos.fieldInfo("unstored"));
         reader.close();
         // add more documents
         writer = new IndexWriter(
@@ -160,61 +160,66 @@ public class TestIndexReader extends Luc
         }
         
         writer.close();
+
         // verify fields again
         reader = IndexReader.open(d);
-        fieldNames = reader.getFieldNames(IndexReader.FieldOption.ALL);
-        assertEquals(13, fieldNames.size());    // the following fields
-        assertTrue(fieldNames.contains("keyword"));
-        assertTrue(fieldNames.contains("text"));
-        assertTrue(fieldNames.contains("unindexed"));
-        assertTrue(fieldNames.contains("unstored"));
-        assertTrue(fieldNames.contains("keyword2"));
-        assertTrue(fieldNames.contains("text2"));
-        assertTrue(fieldNames.contains("unindexed2"));
-        assertTrue(fieldNames.contains("unstored2"));
-        assertTrue(fieldNames.contains("tvnot"));
-        assertTrue(fieldNames.contains("termvector"));
-        assertTrue(fieldNames.contains("tvposition"));
-        assertTrue(fieldNames.contains("tvoffset"));
-        assertTrue(fieldNames.contains("tvpositionoffset"));
+        fieldInfos = ReaderUtil.getMergedFieldInfos(reader);
+
+        Collection<String> allFieldNames = new HashSet<String>();
+        Collection<String> indexedFieldNames = new HashSet<String>();
+        Collection<String> notIndexedFieldNames = new HashSet<String>();
+        Collection<String> tvFieldNames = new HashSet<String>();
+
+        for(FieldInfo fieldInfo : fieldInfos) {
+          final String name = fieldInfo.name;
+          allFieldNames.add(name);
+          if (fieldInfo.isIndexed) {
+            indexedFieldNames.add(name);
+          } else {
+            notIndexedFieldNames.add(name);
+          }
+          if (fieldInfo.storeTermVector) {
+            tvFieldNames.add(name);
+          }
+        }
+
+        assertTrue(allFieldNames.contains("keyword"));
+        assertTrue(allFieldNames.contains("text"));
+        assertTrue(allFieldNames.contains("unindexed"));
+        assertTrue(allFieldNames.contains("unstored"));
+        assertTrue(allFieldNames.contains("keyword2"));
+        assertTrue(allFieldNames.contains("text2"));
+        assertTrue(allFieldNames.contains("unindexed2"));
+        assertTrue(allFieldNames.contains("unstored2"));
+        assertTrue(allFieldNames.contains("tvnot"));
+        assertTrue(allFieldNames.contains("termvector"));
+        assertTrue(allFieldNames.contains("tvposition"));
+        assertTrue(allFieldNames.contains("tvoffset"));
+        assertTrue(allFieldNames.contains("tvpositionoffset"));
         
         // verify that only indexed fields were returned
-        fieldNames = reader.getFieldNames(IndexReader.FieldOption.INDEXED);
-        assertEquals(11, fieldNames.size());    // 6 original + the 5 termvector fields 
-        assertTrue(fieldNames.contains("keyword"));
-        assertTrue(fieldNames.contains("text"));
-        assertTrue(fieldNames.contains("unstored"));
-        assertTrue(fieldNames.contains("keyword2"));
-        assertTrue(fieldNames.contains("text2"));
-        assertTrue(fieldNames.contains("unstored2"));
-        assertTrue(fieldNames.contains("tvnot"));
-        assertTrue(fieldNames.contains("termvector"));
-        assertTrue(fieldNames.contains("tvposition"));
-        assertTrue(fieldNames.contains("tvoffset"));
-        assertTrue(fieldNames.contains("tvpositionoffset"));
+        assertEquals(11, indexedFieldNames.size());    // 6 original + the 5 termvector fields 
+        assertTrue(indexedFieldNames.contains("keyword"));
+        assertTrue(indexedFieldNames.contains("text"));
+        assertTrue(indexedFieldNames.contains("unstored"));
+        assertTrue(indexedFieldNames.contains("keyword2"));
+        assertTrue(indexedFieldNames.contains("text2"));
+        assertTrue(indexedFieldNames.contains("unstored2"));
+        assertTrue(indexedFieldNames.contains("tvnot"));
+        assertTrue(indexedFieldNames.contains("termvector"));
+        assertTrue(indexedFieldNames.contains("tvposition"));
+        assertTrue(indexedFieldNames.contains("tvoffset"));
+        assertTrue(indexedFieldNames.contains("tvpositionoffset"));
         
         // verify that only unindexed fields were returned
-        fieldNames = reader.getFieldNames(IndexReader.FieldOption.UNINDEXED);
-        assertEquals(2, fieldNames.size());    // the following fields
-        assertTrue(fieldNames.contains("unindexed"));
-        assertTrue(fieldNames.contains("unindexed2"));
+        assertEquals(2, notIndexedFieldNames.size());    // the following fields
+        assertTrue(notIndexedFieldNames.contains("unindexed"));
+        assertTrue(notIndexedFieldNames.contains("unindexed2"));
                 
         // verify index term vector fields  
-        fieldNames = reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR);
-        assertEquals(1, fieldNames.size());    // 1 field has term vector only
-        assertTrue(fieldNames.contains("termvector"));
-        
-        fieldNames = reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION);
-        assertEquals(1, fieldNames.size());    // 4 fields are indexed with term vectors
-        assertTrue(fieldNames.contains("tvposition"));
-        
-        fieldNames = reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET);
-        assertEquals(1, fieldNames.size());    // 4 fields are indexed with term vectors
-        assertTrue(fieldNames.contains("tvoffset"));
-                
-        fieldNames = reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET);
-        assertEquals(1, fieldNames.size());    // 4 fields are indexed with term vectors
-        assertTrue(fieldNames.contains("tvpositionoffset"));
+        assertEquals(tvFieldNames.toString(), 4, tvFieldNames.size());    // 4 field has term vector only
+        assertTrue(tvFieldNames.contains("termvector"));
+
         reader.close();
         d.close();
     }
@@ -292,7 +297,7 @@ public class TestIndexReader extends Luc
         writer.close();
         writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy()));
         Document doc = new Document();
-        doc.add(new BinaryField("bin1", bin));
+        doc.add(new StoredField("bin1", bin));
         doc.add(new TextField("junk", "junk text"));
         writer.addDocument(doc);
         writer.close();
@@ -519,19 +524,19 @@ public class TestIndexReader extends Luc
       }
       
       // check field names
-      Collection<String> fields1 = index1.getFieldNames(FieldOption.ALL);
-      Collection<String> fields2 = index1.getFieldNames(FieldOption.ALL);
-      assertEquals("IndexReaders have different numbers of fields.", fields1.size(), fields2.size());
-      Iterator<String> it1 = fields1.iterator();
-      Iterator<String> it2 = fields1.iterator();
-      while (it1.hasNext()) {
-        assertEquals("Different field names.", it1.next(), it2.next());
+      FieldInfos fieldInfos1 = ReaderUtil.getMergedFieldInfos(index1);
+      FieldInfos fieldInfos2 = ReaderUtil.getMergedFieldInfos(index2);
+      assertEquals("IndexReaders have different numbers of fields.", fieldInfos1.size(), fieldInfos2.size());
+      final int numFields = fieldInfos1.size();
+      for(int fieldID=0;fieldID<numFields;fieldID++) {
+        final FieldInfo fieldInfo1 = fieldInfos1.fieldInfo(fieldID);
+        final FieldInfo fieldInfo2 = fieldInfos2.fieldInfo(fieldID);
+        assertEquals("Different field names.", fieldInfo1.name, fieldInfo2.name);
       }
       
       // check norms
-      it1 = fields1.iterator();
-      while (it1.hasNext()) {
-        String curField = it1.next();
+      for(FieldInfo fieldInfo : fieldInfos1) {
+        String curField = fieldInfo.name;
         DocValues norms1 = MultiDocValues.getNormDocValues(index1, curField);
         DocValues norms2 = MultiDocValues.getNormDocValues(index2, curField);
         if (norms1 != null && norms2 != null)
@@ -598,8 +603,8 @@ public class TestIndexReader extends Luc
 
         while(enum1.next() != null) {
           assertEquals("Different terms", enum1.term(), enum2.next());
-          DocsAndPositionsEnum tp1 = enum1.docsAndPositions(liveDocs, null);
-          DocsAndPositionsEnum tp2 = enum2.docsAndPositions(liveDocs, null);
+          DocsAndPositionsEnum tp1 = enum1.docsAndPositions(liveDocs, null, false);
+          DocsAndPositionsEnum tp2 = enum2.docsAndPositions(liveDocs, null, false);
 
           while(tp1.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
             assertTrue(tp2.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Wed Jan 18 22:28:07 2012
@@ -21,15 +21,10 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.Reader;
 import java.io.StringReader;
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
-import java.util.Random;
 import java.util.Set;
 
 import org.apache.lucene.analysis.*;
@@ -37,13 +32,14 @@ import org.apache.lucene.analysis.tokena
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.simpletext.SimpleTextCodec;
-import org.apache.lucene.document.BinaryField;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.FieldCache;
 import org.apache.lucene.search.IndexSearcher;
@@ -51,7 +47,6 @@ import org.apache.lucene.search.PhraseQu
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
@@ -905,7 +900,8 @@ public class TestIndexWriter extends Luc
     DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(s.getIndexReader(),
                                                                 MultiFields.getLiveDocs(s.getIndexReader()),
                                                                 "field",
-                                                                new BytesRef("a"));
+                                                                new BytesRef("a"),
+                                                                false);
 
     assertTrue(tps.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
     assertEquals(1, tps.freq());
@@ -927,7 +923,7 @@ public class TestIndexWriter extends Luc
       b[i] = (byte) (i+77);
 
     Document doc = new Document();
-    Field f = new BinaryField("binary", b, 10, 17);
+    Field f = new StoredField("binary", b, 10, 17);
     byte[] bx = f.binaryValue().bytes;
     assertTrue(bx != null);
     assertEquals(50, bx.length);
@@ -970,14 +966,14 @@ public class TestIndexWriter extends Luc
     Terms tpv = r.getTermVectors(0).terms("field");
     TermsEnum termsEnum = tpv.iterator(null);
     assertNotNull(termsEnum.next());
-    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null);
+    DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, false);
     assertNotNull(dpEnum);
     assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
     assertEquals(1, dpEnum.freq());
     assertEquals(100, dpEnum.nextPosition());
 
     assertNotNull(termsEnum.next());
-    dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+    dpEnum = termsEnum.docsAndPositions(null, dpEnum, false);
     assertNotNull(dpEnum);
     assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
     assertEquals(1, dpEnum.freq());
@@ -1183,11 +1179,11 @@ public class TestIndexWriter extends Luc
 
     Document doc = new Document();
 
-    FieldType customType = new FieldType(BinaryField.TYPE_STORED);
+    FieldType customType = new FieldType(StoredField.TYPE);
     customType.setTokenized(true);
-    customType.setIndexed(true);
     
     Field f = new Field("binary", b, 10, 17, customType);
+    customType.setIndexed(true);
     f.setTokenStream(new MockTokenizer(new StringReader("doc1field1"), MockTokenizer.WHITESPACE, false));
 
     FieldType customType2 = new FieldType(TextField.TYPE_STORED);
@@ -1640,7 +1636,7 @@ public class TestIndexWriter extends Luc
 
     // Make sure position is still incremented when
     // massive term is skipped:
-    DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(reader, null, "content", new BytesRef("another"));
+    DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(reader, null, "content", new BytesRef("another"), false);
     assertEquals(0, tps.nextDoc());
     assertEquals(1, tps.freq());
     assertEquals(3, tps.nextPosition());
@@ -1679,7 +1675,7 @@ public class TestIndexWriter extends Luc
     w.close();
     assertEquals(1, reader.docFreq(new Term("content", bigTerm)));
 
-    FieldCache.DocTermsIndex dti = FieldCache.DEFAULT.getTermsIndex(reader, "content", random.nextBoolean());
+    FieldCache.DocTermsIndex dti = FieldCache.DEFAULT.getTermsIndex(new SlowMultiReaderWrapper(reader), "content", random.nextBoolean());
     assertEquals(5, dti.numOrd());                // +1 for null ord
     assertEquals(4, dti.size());
     assertEquals(bigTermBytesRef, dti.lookup(3, new BytesRef()));
@@ -1767,4 +1763,27 @@ public class TestIndexWriter extends Luc
     w1.close();
     d.close();
   }
+
+  public void testChangeIndexOptions() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriter w = new IndexWriter(dir,
+                                    new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+
+    FieldType docsAndFreqs = new FieldType(TextField.TYPE_UNSTORED);
+    docsAndFreqs.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
+
+    FieldType docsOnly = new FieldType(TextField.TYPE_UNSTORED);
+    docsOnly.setIndexOptions(IndexOptions.DOCS_ONLY);
+
+    Document doc = new Document();
+    doc.add(new Field("field", "a b c", docsAndFreqs));
+    w.addDocument(doc);
+    w.addDocument(doc);
+
+    doc = new Document();
+    doc.add(new Field("field", "a b c", docsOnly));
+    w.addDocument(doc);
+    w.close();
+    dir.close();
+  }
 }

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java Wed Jan 18 22:28:07 2012
@@ -18,10 +18,12 @@ package org.apache.lucene.index;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Random;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.codecs.Codec;
@@ -29,19 +31,18 @@ import org.apache.lucene.document.Docume
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
-import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MockDirectoryWrapper;
-import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util._TestUtil;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.ThreadInterruptedException;
-import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.lucene.util._TestUtil;
 
 public class TestIndexWriterReader extends LuceneTestCase {
   
@@ -780,7 +781,8 @@ public class TestIndexWriterReader exten
 
     assertEquals(0, excs.size());
     r.close();
-    assertEquals(0, dir1.getOpenDeletedFiles().size());
+    final Collection<String> openDeletedFiles = dir1.getOpenDeletedFiles();
+    assertEquals("openDeleted=" + openDeletedFiles, 0, openDeletedFiles.size());
 
     writer.close();
 

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexableField.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexableField.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexableField.java Wed Jan 18 22:28:07 2012
@@ -25,14 +25,10 @@ import java.util.Iterator;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.NumericField.DataType;
-import org.apache.lucene.document.NumericField;
 import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.DocValues;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.NumericRangeQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
@@ -85,6 +81,11 @@ public class TestIndexableField extends 
       public FieldInfo.IndexOptions indexOptions() {
         return FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
       }
+
+      @Override
+      public DocValues.Type docValueType() {
+        return null;
+      }
     };
 
     public MyField(int counter) {
@@ -117,7 +118,7 @@ public class TestIndexableField extends 
     @Override
     public String stringValue() {
       final int fieldID = counter%10;
-      if (fieldID != 3 && fieldID != 7 && fieldID != 9) {
+      if (fieldID != 3 && fieldID != 7) {
         return "text " + counter;
       } else {
         return null;
@@ -133,20 +134,9 @@ public class TestIndexableField extends 
       }
     }
 
-    // Numeric field:
-    @Override
-    public boolean numeric() {
-      return counter%10 == 9;
-    }
-
-    @Override
-    public DataType numericDataType() {
-      return DataType.INT;
-    }
-
     @Override
     public Number numericValue() {
-      return counter;
+      return null;
     }
 
     @Override
@@ -154,24 +144,10 @@ public class TestIndexableField extends 
       return fieldType;
     }
 
-    // TODO: randomly enable doc values
-    @Override
-    public DocValue docValue() {
-      return null;
-    }
-
-    @Override
-    public DocValues.Type docValueType() {
-      return null;
-    }
-
     @Override
     public TokenStream tokenStream(Analyzer analyzer) throws IOException {
-      if (numeric()) {
-        return new NumericField(name()).setIntValue(counter).tokenStream(analyzer);
-      }
       return readerValue() != null ? analyzer.tokenStream(name(), readerValue()) :
-          analyzer.tokenStream(name(), new StringReader(stringValue()));
+        analyzer.tokenStream(name(), new StringReader(stringValue()));
     }
   }
 
@@ -253,7 +229,6 @@ public class TestIndexableField extends 
         final boolean stored = (counter&1) == 0 || fieldID == 3;
         final boolean binary = fieldID == 3;
         final boolean indexed = fieldID != 3;
-        final boolean numeric = fieldID == 9;
 
         final String stringValue;
         if (fieldID != 3 && fieldID != 9) {
@@ -274,11 +249,6 @@ public class TestIndexableField extends 
             for(int idx=0;idx<10;idx++) {
               assertEquals((byte) (idx+counter), b.bytes[b.offset+idx]);
             }
-          } else if (numeric) {
-            assertTrue(f instanceof NumericField);
-            final NumericField nf = (NumericField) f;
-            assertEquals(NumericField.DataType.INT, nf.numericDataType());
-            assertEquals(counter, nf.numericValue().intValue());
           } else {
             assert stringValue != null;
             assertEquals(stringValue, f.stringValue());
@@ -293,14 +263,14 @@ public class TestIndexableField extends 
             TermsEnum termsEnum = tfv.iterator(null);
             assertEquals(new BytesRef(""+counter), termsEnum.next());
             assertEquals(1, termsEnum.totalTermFreq());
-            DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null);
+            DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, false);
             assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
             assertEquals(1, dpEnum.freq());
             assertEquals(1, dpEnum.nextPosition());
 
             assertEquals(new BytesRef("text"), termsEnum.next());
             assertEquals(1, termsEnum.totalTermFreq());
-            dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+            dpEnum = termsEnum.docsAndPositions(null, dpEnum, false);
             assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
             assertEquals(1, dpEnum.freq());
             assertEquals(0, dpEnum.nextPosition());
@@ -314,26 +284,19 @@ public class TestIndexableField extends 
             assertTrue(vectors == null || vectors.terms(name) == null);
           }
 
-          if (numeric) {
-            NumericRangeQuery nrq = NumericRangeQuery.newIntRange(name, counter, counter, true, true);
-            final TopDocs hits2 = s.search(nrq, 1);
-            assertEquals(1, hits2.totalHits);
-            assertEquals(docID, hits2.scoreDocs[0].doc);
-          } else {
-            BooleanQuery bq = new BooleanQuery();
-            bq.add(new TermQuery(new Term("id", ""+id)), BooleanClause.Occur.MUST);
-            bq.add(new TermQuery(new Term(name, "text")), BooleanClause.Occur.MUST);
-            final TopDocs hits2 = s.search(bq, 1);
-            assertEquals(1, hits2.totalHits);
-            assertEquals(docID, hits2.scoreDocs[0].doc);
-
-            bq = new BooleanQuery();
-            bq.add(new TermQuery(new Term("id", ""+id)), BooleanClause.Occur.MUST);
-            bq.add(new TermQuery(new Term(name, ""+counter)), BooleanClause.Occur.MUST);
-            final TopDocs hits3 = s.search(bq, 1);
-            assertEquals(1, hits3.totalHits);
-            assertEquals(docID, hits3.scoreDocs[0].doc);
-          }
+          BooleanQuery bq = new BooleanQuery();
+          bq.add(new TermQuery(new Term("id", ""+id)), BooleanClause.Occur.MUST);
+          bq.add(new TermQuery(new Term(name, "text")), BooleanClause.Occur.MUST);
+          final TopDocs hits2 = s.search(bq, 1);
+          assertEquals(1, hits2.totalHits);
+          assertEquals(docID, hits2.scoreDocs[0].doc);
+
+          bq = new BooleanQuery();
+          bq.add(new TermQuery(new Term("id", ""+id)), BooleanClause.Occur.MUST);
+          bq.add(new TermQuery(new Term(name, ""+counter)), BooleanClause.Occur.MUST);
+          final TopDocs hits3 = s.search(bq, 1);
+          assertEquals(1, hits3.totalHits);
+          assertEquals(docID, hits3.scoreDocs[0].doc);
         }
 
         counter++;

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java Wed Jan 18 22:28:07 2012
@@ -156,7 +156,8 @@ public class TestLazyProxSkipping extend
         DocsAndPositionsEnum tp = MultiFields.getTermPositionsEnum(reader,
                                                                    MultiFields.getLiveDocs(reader),
                                                                    this.field,
-                                                                   new BytesRef("b"));
+                                                                   new BytesRef("b"),
+                                                                   false);
 
         for (int i = 0; i < 10; i++) {
             tp.nextDoc();
@@ -167,7 +168,8 @@ public class TestLazyProxSkipping extend
         tp = MultiFields.getTermPositionsEnum(reader,
                                               MultiFields.getLiveDocs(reader),
                                               this.field,
-                                              new BytesRef("a"));
+                                              new BytesRef("a"),
+                                              false);
 
         for (int i = 0; i < 10; i++) {
             tp.nextDoc();

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestLongPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestLongPostings.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestLongPostings.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestLongPostings.java Wed Jan 18 22:28:07 2012
@@ -173,7 +173,7 @@ public class TestLongPostings extends Lu
         System.out.println("\nTEST: iter=" + iter + " doS1=" + doS1);
       }
         
-      final DocsAndPositionsEnum postings = MultiFields.getTermPositionsEnum(r, null, "field", new BytesRef(term));
+      final DocsAndPositionsEnum postings = MultiFields.getTermPositionsEnum(r, null, "field", new BytesRef(term), false);
 
       int docID = -1;
       while(docID < DocsEnum.NO_MORE_DOCS) {

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java Wed Jan 18 22:28:07 2012
@@ -117,8 +117,8 @@ public class TestMaxTermFrequency extend
     }
 
     @Override
-    public byte computeNorm(FieldInvertState state) {
-      return encodeNormValue((float) state.getMaxTermFrequency());
+    public void computeNorm(FieldInvertState state, Norm norm) {
+      norm.setByte(encodeNormValue((float) state.getMaxTermFrequency()));
     }
   }
 }

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java Wed Jan 18 22:28:07 2012
@@ -86,7 +86,8 @@ public class TestMultiLevelSkipList exte
       counter = 0;
       DocsAndPositionsEnum tp = reader.termPositionsEnum(reader.getLiveDocs(),
                                                          term.field(),
-                                                         new BytesRef(term.text()));
+                                                         new BytesRef(term.text()),
+                                                         false);
 
       checkSkipTo(tp, 14, 185); // no skips
       checkSkipTo(tp, 17, 190); // one skip on level 0

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestNorms.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestNorms.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestNorms.java Wed Jan 18 22:28:07 2012
@@ -18,20 +18,20 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Random;
 
-import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.DocValues.Source;
+import org.apache.lucene.index.DocValues.Type;
 import org.apache.lucene.search.similarities.DefaultSimilarity;
 import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.search.similarities.SimilarityProvider;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.util.LineFileDocs;
 import org.apache.lucene.util.LuceneTestCase;
 
 /**
@@ -39,7 +39,8 @@ import org.apache.lucene.util.LuceneTest
  * separate norms, addDocument, addIndexes, forceMerge.
  */
 public class TestNorms extends LuceneTestCase {
-  
+  final String byteTestField = "normsTestByte";
+
   class CustomNormEncodingSimilarity extends DefaultSimilarity {
     @Override
     public byte encodeNormValue(float f) {
@@ -52,8 +53,8 @@ public class TestNorms extends LuceneTes
     }
 
     @Override
-    public byte computeNorm(FieldInvertState state) {
-      return encodeNormValue((float) state.getLength());
+    public void computeNorm(FieldInvertState state, Norm norm) {
+      norm.setByte(encodeNormValue((float) state.getLength()));
     }
   }
   
@@ -93,4 +94,160 @@ public class TestNorms extends LuceneTes
     reader.close();
     dir.close();
   }
+  
+  public void testMaxByteNorms() throws IOException {
+    Directory dir = newDirectory();
+    buildIndex(dir, true);
+    IndexReader open = new SlowMultiReaderWrapper(IndexReader.open(dir));
+    DocValues normValues = open.normValues(byteTestField);
+    assertNotNull(normValues);
+    Source source = normValues.getSource();
+    assertTrue(source.hasArray());
+    assertEquals(Type.FIXED_INTS_8, normValues.type());
+    byte[] norms = (byte[]) source.getArray();
+    for (int i = 0; i < open.maxDoc(); i++) {
+      Document document = open.document(i);
+      int expected = Integer.parseInt(document.get(byteTestField));
+      assertEquals((byte)expected, norms[i]);
+    }
+    open.close();
+    dir.close();
+  }
+  
+  /**
+   * this test randomly creates segments with or without norms but not omitting
+   * norms. The similarity used doesn't write a norm value if writeNorms = false is
+   * passed. This differs from omitNorm since norms are simply not written for this segment
+   * while merging fills in default values based on the Norm {@link Type}
+   */
+  public void testNormsNotPresent() throws IOException {
+    Directory dir = newDirectory();
+    boolean firstWriteNorm = random.nextBoolean();
+    buildIndex(dir, firstWriteNorm);
+
+    Directory otherDir = newDirectory();
+    boolean secondWriteNorm = random.nextBoolean();
+    buildIndex(otherDir, secondWriteNorm);
+
+    IndexReader reader = new SlowMultiReaderWrapper(IndexReader.open(otherDir));
+    FieldInfos fieldInfos = reader.getFieldInfos();
+    FieldInfo fieldInfo = fieldInfos.fieldInfo(byteTestField);
+    assertFalse(fieldInfo.omitNorms);
+    assertTrue(fieldInfo.isIndexed);
+    if (secondWriteNorm) {
+      assertTrue(fieldInfo.normsPresent());
+    } else {
+      assertFalse(fieldInfo.normsPresent());  
+    }
+    
+    IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,
+        new MockAnalyzer(random));
+    RandomIndexWriter writer = new RandomIndexWriter(random, dir, config);
+    writer.addIndexes(reader);
+    IndexReader mergedReader = new SlowMultiReaderWrapper(writer.getReader());
+    if (!firstWriteNorm && !secondWriteNorm) {
+      DocValues normValues = mergedReader.normValues(byteTestField);
+      assertNull(normValues);
+      FieldInfo fi = mergedReader.getFieldInfos().fieldInfo(byteTestField);
+      assertFalse(fi.omitNorms);
+      assertTrue(fi.isIndexed);
+      assertFalse(fi.normsPresent());
+    } else {
+      FieldInfo fi = mergedReader.getFieldInfos().fieldInfo(byteTestField);
+      assertFalse(fi.omitNorms);
+      assertTrue(fi.isIndexed);
+      assertTrue(fi.normsPresent());
+      
+      DocValues normValues = mergedReader.normValues(byteTestField);
+      assertNotNull(normValues);
+      Source source = normValues.getSource();
+      assertTrue(source.hasArray());
+      assertEquals(Type.FIXED_INTS_8, normValues.type());
+      byte[] norms = (byte[]) source.getArray();
+      for (int i = 0; i < mergedReader.maxDoc(); i++) {
+        Document document = mergedReader.document(i);
+        int expected = Integer.parseInt(document.get(byteTestField));
+        assertEquals((byte) expected, norms[i]);
+      }
+    }
+    mergedReader.close();
+    reader.close();
+
+    writer.close();
+    dir.close();
+    otherDir.close();
+  }
+
+  public void buildIndex(Directory dir, boolean writeNorms) throws IOException,
+      CorruptIndexException {
+    IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,
+        new MockAnalyzer(random));
+    SimilarityProvider provider = new MySimProvider(writeNorms);
+    config.setSimilarityProvider(provider);
+    RandomIndexWriter writer = new RandomIndexWriter(random, dir, config);
+    final LineFileDocs docs = new LineFileDocs(random);
+    int num = atLeast(100);
+    for (int i = 0; i < num; i++) {
+      Document doc = docs.nextDoc();
+      int boost = writeNorms ? 1 + random.nextInt(255) : 0;
+      Field f = new Field(byteTestField, "" + boost,
+          TextField.TYPE_STORED);
+      f.setBoost(boost);
+      doc.add(f);
+      writer.addDocument(doc);
+      doc.removeField(byteTestField);
+      if (rarely()) {
+        writer.commit();
+      }
+    }
+    writer.commit();
+    writer.close();
+  }
+
+
+  public class MySimProvider implements SimilarityProvider {
+    SimilarityProvider delegate = new DefaultSimilarityProvider();
+    private boolean writeNorms;
+    public MySimProvider(boolean writeNorms) {
+      this.writeNorms = writeNorms;
+    }
+    @Override
+    public float queryNorm(float sumOfSquaredWeights) {
+
+      return delegate.queryNorm(sumOfSquaredWeights);
+    }
+
+    @Override
+    public Similarity get(String field) {
+      if (byteTestField.equals(field)) {
+        return new ByteEncodingBoostSimilarity(writeNorms);
+      } else {
+        return delegate.get(field);
+      }
+    }
+
+    @Override
+    public float coord(int overlap, int maxOverlap) {
+      return delegate.coord(overlap, maxOverlap);
+    }
+  }
+
+  
+  public static class ByteEncodingBoostSimilarity extends DefaultSimilarity {
+
+    private boolean writeNorms;
+
+    public ByteEncodingBoostSimilarity(boolean writeNorms) {
+      this.writeNorms = writeNorms;
+    }
+
+    @Override
+    public void computeNorm(FieldInvertState state, Norm norm) {
+      if (writeNorms) {
+        int boost = (int) state.getBoost();
+        norm.setByte((byte) (0xFF & boost));
+      }
+    }
+  }
+ 
 }

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java Wed Jan 18 22:28:07 2012
@@ -67,7 +67,7 @@ public class TestOmitNorms extends Lucen
     writer.close();
 
     SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
-    FieldInfos fi = reader.fieldInfos();
+    FieldInfos fi = reader.getFieldInfos();
     assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f1").omitNorms);
     assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f2").omitNorms);
         
@@ -121,7 +121,7 @@ public class TestOmitNorms extends Lucen
     writer.close();
 
     SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
-    FieldInfos fi = reader.fieldInfos();
+    FieldInfos fi = reader.getFieldInfos();
     assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f1").omitNorms);
     assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f2").omitNorms);
         
@@ -169,7 +169,7 @@ public class TestOmitNorms extends Lucen
     writer.close();
 
     SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
-    FieldInfos fi = reader.fieldInfos();
+    FieldInfos fi = reader.getFieldInfos();
     assertTrue("OmitNorms field bit should not be set.", !fi.fieldInfo("f1").omitNorms);
     assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f2").omitNorms);
         

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitPositions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitPositions.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitPositions.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitPositions.java Wed Jan 18 22:28:07 2012
@@ -51,7 +51,7 @@ public class TestOmitPositions extends L
     IndexReader reader = w.getReader();
     w.close();
     
-    assertNull(MultiFields.getTermPositionsEnum(reader, null, "foo", new BytesRef("test")));
+    assertNull(MultiFields.getTermPositionsEnum(reader, null, "foo", new BytesRef("test"), false));
     
     DocsEnum de = _TestUtil.docs(random, reader, "foo", new BytesRef("test"), null, null, true);
     while (de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
@@ -154,7 +154,7 @@ public class TestOmitPositions extends L
     writer.close();
 
     SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
-    FieldInfos fi = reader.fieldInfos();
+    FieldInfos fi = reader.getFieldInfos();
     // docs + docs = docs
     assertEquals(IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").indexOptions);
     // docs + docs/freqs = docs

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitTf.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitTf.java Wed Jan 18 22:28:07 2012
@@ -44,8 +44,7 @@ public class TestOmitTf extends LuceneTe
     public float coord(int overlap, int maxOverlap) { return 1.0f; }
     public Similarity get(String field) {
       return new TFIDFSimilarity() {
-
-        @Override public byte computeNorm(FieldInvertState state) { return encodeNormValue(state.getBoost()); }
+        @Override public void computeNorm(FieldInvertState state, Norm norm) { norm.setByte(encodeNormValue(state.getBoost())); }
         @Override public float tf(float freq) { return freq; }
         @Override public float sloppyFreq(int distance) { return 2.0f; }
         @Override public float idf(int docFreq, int numDocs) { return 1.0f; }
@@ -101,7 +100,7 @@ public class TestOmitTf extends LuceneTe
     writer.close();
 
     SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
-    FieldInfos fi = reader.fieldInfos();
+    FieldInfos fi = reader.getFieldInfos();
     assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").indexOptions);
     assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions);
         
@@ -153,7 +152,7 @@ public class TestOmitTf extends LuceneTe
     writer.close();
 
     SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
-    FieldInfos fi = reader.fieldInfos();
+    FieldInfos fi = reader.getFieldInfos();
     assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").indexOptions);
     assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions);
         
@@ -196,7 +195,7 @@ public class TestOmitTf extends LuceneTe
     writer.close();
 
     SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
-    FieldInfos fi = reader.fieldInfos();
+    FieldInfos fi = reader.getFieldInfos();
     assertEquals("OmitTermFreqAndPositions field bit should not be set.", IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, fi.fieldInfo("f1").indexOptions);
     assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions);
         

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestParallelReader.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestParallelReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestParallelReader.java Wed Jan 18 22:28:07 2012
@@ -18,14 +18,12 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Random;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
 import org.apache.lucene.search.*;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.LuceneTestCase;
@@ -75,12 +73,12 @@ public class TestParallelReader extends 
     ParallelReader pr = new ParallelReader();
     pr.add(IndexReader.open(dir1));
     pr.add(IndexReader.open(dir2));
-    Collection<String> fieldNames = pr.getFieldNames(IndexReader.FieldOption.ALL);
-    assertEquals(4, fieldNames.size());
-    assertTrue(fieldNames.contains("f1"));
-    assertTrue(fieldNames.contains("f2"));
-    assertTrue(fieldNames.contains("f3"));
-    assertTrue(fieldNames.contains("f4"));
+    FieldInfos fieldInfos = pr.getFieldInfos();
+    assertEquals(4, fieldInfos.size());
+    assertNotNull(fieldInfos.fieldInfo("f1"));
+    assertNotNull(fieldInfos.fieldInfo("f2"));
+    assertNotNull(fieldInfos.fieldInfo("f3"));
+    assertNotNull(fieldInfos.fieldInfo("f4"));
     pr.close();
     dir1.close();
     dir2.close();

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java Wed Jan 18 22:28:07 2012
@@ -155,7 +155,7 @@ public class TestPayloadProcessorProvide
     IndexReader reader = IndexReader.open(dir);
     try {
       int numPayloads = 0;
-      DocsAndPositionsEnum tpe = MultiFields.getTermPositionsEnum(reader, null, field, text);
+      DocsAndPositionsEnum tpe = MultiFields.getTermPositionsEnum(reader, null, field, text, false);
       while (tpe.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
         tpe.nextPosition();
         if (tpe.hasPayload()) {

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestPayloads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestPayloads.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestPayloads.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestPayloads.java Wed Jan 18 22:28:07 2012
@@ -112,7 +112,7 @@ public class TestPayloads extends Lucene
         writer.close();
 
       SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
-        FieldInfos fi = reader.fieldInfos();
+        FieldInfos fi = reader.getFieldInfos();
         assertFalse("Payload field bit should not be set.", fi.fieldInfo("f1").storePayloads);
         assertTrue("Payload field bit should be set.", fi.fieldInfo("f2").storePayloads);
         assertFalse("Payload field bit should not be set.", fi.fieldInfo("f3").storePayloads);
@@ -139,7 +139,7 @@ public class TestPayloads extends Lucene
         writer.close();
 
       reader = getOnlySegmentReader(IndexReader.open(ram));
-        fi = reader.fieldInfos();
+        fi = reader.getFieldInfos();
         assertFalse("Payload field bit should not be set.", fi.fieldInfo("f1").storePayloads);
         assertTrue("Payload field bit should be set.", fi.fieldInfo("f2").storePayloads);
         assertTrue("Payload field bit should be set.", fi.fieldInfo("f3").storePayloads);
@@ -222,7 +222,8 @@ public class TestPayloads extends Lucene
           tps[i] = MultiFields.getTermPositionsEnum(reader,
                                                     MultiFields.getLiveDocs(reader),
                                                     terms[i].field(),
-                                                    new BytesRef(terms[i].text()));
+                                                    new BytesRef(terms[i].text()),
+                                                    false);
         }
         
         while (tps[0].nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
@@ -259,7 +260,8 @@ public class TestPayloads extends Lucene
         DocsAndPositionsEnum tp = MultiFields.getTermPositionsEnum(reader,
                                                                    MultiFields.getLiveDocs(reader),
                                                                    terms[0].field(),
-                                                                   new BytesRef(terms[0].text()));
+                                                                   new BytesRef(terms[0].text()),
+                                                                   false);
         tp.nextDoc();
         tp.nextPosition();
         // NOTE: prior rev of this test was failing to first
@@ -287,7 +289,8 @@ public class TestPayloads extends Lucene
         tp = MultiFields.getTermPositionsEnum(reader,
                                               MultiFields.getLiveDocs(reader),
                                               terms[1].field(),
-                                              new BytesRef(terms[1].text()));
+                                              new BytesRef(terms[1].text()),
+                                              false);
         tp.nextDoc();
         tp.nextPosition();
         assertEquals("Wrong payload length.", 1, tp.getPayload().length);
@@ -330,7 +333,8 @@ public class TestPayloads extends Lucene
         tp = MultiFields.getTermPositionsEnum(reader,
                                               MultiFields.getLiveDocs(reader),
                                               fieldName,
-                                              new BytesRef(singleTerm));
+                                              new BytesRef(singleTerm),
+                                              false);
         tp.nextDoc();
         tp.nextPosition();
         
@@ -532,7 +536,7 @@ public class TestPayloads extends Lucene
         DocsAndPositionsEnum tp = null;
         while (terms.next() != null) {
           String termText = terms.term().utf8ToString();
-          tp = terms.docsAndPositions(liveDocs, tp);
+          tp = terms.docsAndPositions(liveDocs, tp, false);
           while(tp.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
             int freq = tp.freq();
             for (int i = 0; i < freq; i++) {

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java Wed Jan 18 22:28:07 2012
@@ -18,7 +18,6 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
-import java.util.Collection;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.codecs.Codec;
@@ -30,6 +29,7 @@ import org.apache.lucene.util.InfoStream
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
 
+
 public class TestSegmentMerger extends LuceneTestCase {
   //The variables for the new merged segment
   private Directory mergedDir;
@@ -107,10 +107,15 @@ public class TestSegmentMerger extends L
     assertTrue(termDocs != null);
     assertTrue(termDocs.nextDoc() != DocsEnum.NO_MORE_DOCS);
 
-    Collection<String> stored = mergedReader.getFieldNames(IndexReader.FieldOption.INDEXED_WITH_TERMVECTOR);
-    assertTrue(stored != null);
+    int tvCount = 0;
+    for(FieldInfo fieldInfo : mergedReader.getFieldInfos()) {
+      if (fieldInfo.storeTermVector) {
+        tvCount++;
+      }
+    }
+    
     //System.out.println("stored size: " + stored.size());
-    assertTrue("We do not have 3 fields that were indexed with term vector",stored.size() == 3);
+    assertEquals("We do not have 3 fields that were indexed with term vector", 3, tvCount);
 
     Terms vector = mergedReader.getTermVectors(0).terms(DocHelper.TEXT_FIELD_2_KEY);
     assertNotNull(vector);

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java Wed Jan 18 22:28:07 2012
@@ -19,7 +19,7 @@ package org.apache.lucene.index;
 
 import java.io.IOException;
 import java.util.Collection;
-import java.util.Iterator;
+import java.util.HashSet;
 import java.util.List;
 
 import org.apache.lucene.document.Document;
@@ -74,33 +74,42 @@ public class TestSegmentReader extends L
   }
   
   public void testGetFieldNameVariations() {
-    Collection<String> result = reader.getFieldNames(IndexReader.FieldOption.ALL);
-    assertTrue(result != null);
-    assertTrue(result.size() == DocHelper.all.size());
-    for (Iterator<String> iter = result.iterator(); iter.hasNext();) {
-      String s =  iter.next();
-      //System.out.println("Name: " + s);
+    Collection<String> allFieldNames = new HashSet<String>();
+    Collection<String> indexedFieldNames = new HashSet<String>();
+    Collection<String> notIndexedFieldNames = new HashSet<String>();
+    Collection<String> tvFieldNames = new HashSet<String>();
+    Collection<String> noTVFieldNames = new HashSet<String>();
+
+    for(FieldInfo fieldInfo : reader.getFieldInfos()) {
+      final String name = fieldInfo.name;
+      allFieldNames.add(name);
+      if (fieldInfo.isIndexed) {
+        indexedFieldNames.add(name);
+      } else {
+        notIndexedFieldNames.add(name);
+      }
+      if (fieldInfo.storeTermVector) {
+        tvFieldNames.add(name);
+      } else if (fieldInfo.isIndexed) {
+        noTVFieldNames.add(name);
+      }
+    }
+
+    assertTrue(allFieldNames.size() == DocHelper.all.size());
+    for (String s : allFieldNames) {
       assertTrue(DocHelper.nameValues.containsKey(s) == true || s.equals(""));
     }                                                                               
-    result = reader.getFieldNames(IndexReader.FieldOption.INDEXED);
-    assertTrue(result != null);
-    assertTrue(result.size() == DocHelper.indexed.size());
-    for (Iterator<String> iter = result.iterator(); iter.hasNext();) {
-      String s = iter.next();
+
+    assertTrue(indexedFieldNames.size() == DocHelper.indexed.size());
+    for (String s : indexedFieldNames) {
       assertTrue(DocHelper.indexed.containsKey(s) == true || s.equals(""));
     }
     
-    result = reader.getFieldNames(IndexReader.FieldOption.UNINDEXED);
-    assertTrue(result != null);
-    assertTrue(result.size() == DocHelper.unindexed.size());
+    assertTrue(notIndexedFieldNames.size() == DocHelper.unindexed.size());
     //Get all indexed fields that are storing term vectors
-    result = reader.getFieldNames(IndexReader.FieldOption.INDEXED_WITH_TERMVECTOR);
-    assertTrue(result != null);
-    assertTrue(result.size() == DocHelper.termvector.size());
-    
-    result = reader.getFieldNames(IndexReader.FieldOption.INDEXED_NO_TERMVECTOR);
-    assertTrue(result != null);
-    assertTrue(result.size() == DocHelper.notermvector.size());
+    assertTrue(tvFieldNames.size() == DocHelper.termvector.size());
+
+    assertTrue(noTVFieldNames.size() == DocHelper.notermvector.size());
   } 
   
   public void testTerms() throws IOException {
@@ -139,7 +148,8 @@ public class TestSegmentReader extends L
     DocsAndPositionsEnum positions = MultiFields.getTermPositionsEnum(reader,
                                                                       MultiFields.getLiveDocs(reader),
                                                                       DocHelper.TEXT_FIELD_1_KEY,
-                                                                      new BytesRef("field"));
+                                                                      new BytesRef("field"),
+                                                                      false);
     // NOTE: prior rev of this test was failing to first
     // call next here:
     assertTrue(positions.nextDoc() != DocsEnum.NO_MORE_DOCS);

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java Wed Jan 18 22:28:07 2012
@@ -406,7 +406,7 @@ public class TestStressIndexing2 extends
             BytesRef term2;
             while((term2 = termsEnum3.next()) != null) {
               System.out.println("      " + term2.utf8ToString() + ": freq=" + termsEnum3.totalTermFreq());
-              dpEnum = termsEnum3.docsAndPositions(null, dpEnum);
+              dpEnum = termsEnum3.docsAndPositions(null, dpEnum, false);
               if (dpEnum != null) {
                 assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
                 final int freq = dpEnum.freq();
@@ -440,7 +440,7 @@ public class TestStressIndexing2 extends
             BytesRef term2;
             while((term2 = termsEnum3.next()) != null) {
               System.out.println("      " + term2.utf8ToString() + ": freq=" + termsEnum3.totalTermFreq());
-              dpEnum = termsEnum3.docsAndPositions(null, dpEnum);
+              dpEnum = termsEnum3.docsAndPositions(null, dpEnum, false);
               if (dpEnum != null) {
                 assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
                 final int freq = dpEnum.freq();
@@ -630,8 +630,8 @@ public class TestStressIndexing2 extends
         assertEquals(termsEnum1.totalTermFreq(),
                      termsEnum2.totalTermFreq());
         
-        dpEnum1 = termsEnum1.docsAndPositions(null, dpEnum1);
-        dpEnum2 = termsEnum2.docsAndPositions(null, dpEnum2);
+        dpEnum1 = termsEnum1.docsAndPositions(null, dpEnum1, false);
+        dpEnum2 = termsEnum2.docsAndPositions(null, dpEnum2, false);
         if (dpEnum1 != null) {
           assertNotNull(dpEnum2);
           int docID1 = dpEnum1.nextDoc();

Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java Wed Jan 18 22:28:07 2012
@@ -259,7 +259,7 @@ public class TestTermVectorsReader exten
       //System.out.println("Term: " + term);
       assertEquals(testTerms[i], term);
 
-      dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+      dpEnum = termsEnum.docsAndPositions(null, dpEnum, false);
       assertNotNull(dpEnum);
       int doc = dpEnum.docID();
       assertTrue(doc == -1 || doc == DocIdSetIterator.NO_MORE_DOCS);
@@ -270,18 +270,16 @@ public class TestTermVectorsReader exten
       }
       assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
 
-      dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+      dpEnum = termsEnum.docsAndPositions(null, dpEnum, true);
       doc = dpEnum.docID();
       assertTrue(doc == -1 || doc == DocIdSetIterator.NO_MORE_DOCS);
       assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
       assertNotNull(dpEnum);
-      final OffsetAttribute offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
-      assertNotNull(offsetAtt);
       assertEquals(dpEnum.freq(), positions[i].length);
       for (int j = 0; j < positions[i].length; j++) {
         assertEquals(positions[i][j], dpEnum.nextPosition());
-        assertEquals(j*10, offsetAtt.startOffset());
-        assertEquals(j*10 + testTerms[i].length(), offsetAtt.endOffset());
+        assertEquals(j*10, dpEnum.startOffset());
+        assertEquals(j*10 + testTerms[i].length(), dpEnum.endOffset());
       }
       assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
     }
@@ -315,7 +313,7 @@ public class TestTermVectorsReader exten
       String term = text.utf8ToString();
       assertEquals(testTerms[i], term);
 
-      dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+      dpEnum = termsEnum.docsAndPositions(null, dpEnum, false);
       assertNotNull(dpEnum);
       assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
       assertEquals(dpEnum.freq(), positions[i].length);
@@ -324,16 +322,14 @@ public class TestTermVectorsReader exten
       }
       assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
 
-      dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+      dpEnum = termsEnum.docsAndPositions(null, dpEnum, true);
       assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
-      final OffsetAttribute offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
-      assertNotNull(offsetAtt);
       assertNotNull(dpEnum);
       assertEquals(dpEnum.freq(), positions[i].length);
       for (int j = 0; j < positions[i].length; j++) {
         assertEquals(positions[i][j], dpEnum.nextPosition());
-        assertEquals(j*10, offsetAtt.startOffset());
-        assertEquals(j*10 + testTerms[i].length(), offsetAtt.endOffset());
+        assertEquals(j*10, dpEnum.startOffset());
+        assertEquals(j*10 + testTerms[i].length(), dpEnum.endOffset());
       }
       assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
     }



Mime
View raw message