lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1638066 [1/4] - in /lucene/dev/branches/lucene6005/lucene/core/src: java/ java/org/apache/lucene/document/ java/org/apache/lucene/index/ test/org/apache/lucene/codecs/lucene50/ test/org/apache/lucene/codecs/perfield/ test/org/apache/lucene...
Date Tue, 11 Nov 2014 11:24:56 GMT
Author: mikemccand
Date: Tue Nov 11 11:24:51 2014
New Revision: 1638066

URL: http://svn.apache.org/r1638066
Log:
LUCENE-6005: cutover more tests

Modified:
    lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document2.java
    lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java
    lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
    lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
    lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/dev/branches/lucene6005/lucene/core/src/java/overview.html
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat3.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50DocValuesFormat.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestDocument.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestMultiFields.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum2.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestUniqueAtom.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/FuzzyTermOnShortTermsTest.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestAutomatonQueryUnicode.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestBooleanCoord.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestBooleanUnevenly.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestCustomSearcherSort.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestDateFilter.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestDateSort.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestDocBoost.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestDocIdSet.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestDocTermOrdsRangeFilter.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestDocTermOrdsRewriteMethod.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestDocValuesScoring.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestEarlyTermination.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestElevationComparator.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestFieldCacheTermsFilter.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestIndexSearcher.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestLiveFieldValues.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestMultiTermQueryRewrites.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestMultiThreadTermVectors.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestMultiValuedNumericRangeQuery.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestNot.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestPhrasePrefixQuery.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestPositiveScoresOnlyCollector.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestPrefixFilter.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestPrefixInBooleanQuery.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestQueryWrapperFilter.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestRegexpRandom.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestRegexpRandom2.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSearchWithThreads.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSort.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSortRescorer.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSortedNumericSortField.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestTermVectors.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestTimeLimitingCollector.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestTopScoreDocCollector.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestWildcardRandom.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/payloads/PayloadHelper.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarity2.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/spans/TestBasics.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanFirstQuery.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java
    lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfUnreferencedFiles.java

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document2.java?rev=1638066&r1=1638065&r2=1638066&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document2.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document2.java Tue Nov 11 11:24:51 2014
@@ -141,11 +141,11 @@ public class Document2 implements Iterab
       case INT:
         return getReusedBinaryTokenStream(intToBytes(((Number) value).intValue()), reuse);
       case FLOAT:
-        return getReusedBinaryTokenStream(intToBytes(Float.floatToIntBits(((Number) value).floatValue())), reuse);
+        return getReusedBinaryTokenStream(intToBytes(sortableFloatBits(Float.floatToIntBits(((Number) value).floatValue()))), reuse);
       case LONG:
         return getReusedBinaryTokenStream(longToBytes(((Number) value).longValue()), reuse);
       case DOUBLE:
-        return getReusedBinaryTokenStream(longToBytes(Double.doubleToLongBits(((Number) value).doubleValue())), reuse);
+        return getReusedBinaryTokenStream(longToBytes(sortableDoubleBits(Double.doubleToLongBits(((Number) value).doubleValue()))), reuse);
       case DATE:
         return getReusedBinaryTokenStream(longToBytes(((Date) value).getTime()), reuse);
       case ATOM:
@@ -268,9 +268,11 @@ public class Document2 implements Iterab
       case LONG:
         return (Number) value;
       case FLOAT:
-        return Integer.valueOf(Float.floatToIntBits((Float) value));
+        // nocommit i shouldn't do sortableFloatBits?  but why does ot TestSortedNumericSortField.testFloat fail?
+        return Integer.valueOf(sortableFloatBits(Float.floatToIntBits((Float) value)));
       case DOUBLE:
-        return Long.valueOf(Double.doubleToLongBits((Double) value));
+        // nocommit i shouldn't do sortableDoubleBits?
+        return Long.valueOf(sortableDoubleBits(Double.doubleToLongBits((Double) value)));
       case DATE:
         return Long.valueOf(((Date) value).getTime());
       case BOOLEAN:
@@ -303,6 +305,7 @@ public class Document2 implements Iterab
           return null;
         }
       case ATOM:
+      case UNIQUE_ATOM:
         if (value instanceof String) {
           return (String) value;
         } else {
@@ -486,6 +489,11 @@ public class Document2 implements Iterab
   }
 
   /** Default: store this value. */
+  public void addStored(String fieldName, byte[] value) {
+    addStored(fieldName, new BytesRef(value));
+  }
+
+  /** Default: store & DV this value. */
   public void addBinary(String fieldName, BytesRef value) {
     if (changeSchema) {
       fieldTypes.recordValueType(fieldName, FieldTypes.ValueType.BINARY);
@@ -494,6 +502,11 @@ public class Document2 implements Iterab
   }
 
   /** Default: store this value. */
+  public void addBinary(String fieldName, byte[] value) {
+    addBinary(fieldName, new BytesRef(value));
+  }
+
+  /** Default: store this value. */
   public void addStored(String fieldName, String value) {
     // nocommit akward we inferred large_text here?
     if (changeSchema) {
@@ -701,6 +714,16 @@ public class Document2 implements Iterab
     return token;
   }
 
+  /** Converts IEEE 754 representation of a double to sortable order (or back to the original) */
+  public static long sortableDoubleBits(long bits) {
+    return bits ^ (bits >> 63) & 0x7fffffffffffffffL;
+  }
+  
+  /** Converts IEEE 754 representation of a float to sortable order (or back to the original) */
+  public static int sortableFloatBits(int bits) {
+    return bits ^ (bits >> 31) & 0x7fffffff;
+  }
+
   public Boolean getBoolean(String fieldName) {
     // nocommit can we assert this is a known field and that its type is correct?
     for(FieldValue fieldValue : fields) {
@@ -712,6 +735,8 @@ public class Document2 implements Iterab
     return null;
   }
 
+  // nocommit getFloat, getDouble, getLong
+
   public Date getDate(String fieldName) {
     // nocommit can we assert this is a known field and that its type is correct?
     for(FieldValue fieldValue : fields) {
@@ -779,6 +804,17 @@ public class Document2 implements Iterab
     return null;
   }
 
+  public Double getDouble(String fieldName) {
+    // nocommit can we assert this is a known field and that its type is correct?
+    for(FieldValue fieldValue : fields) {
+      if (fieldValue.fieldName.equals(fieldName)) {
+        return (Double) fieldValue.value;
+      }
+    }
+
+    return null;
+  }
+
   public Object get(String fieldName) {
     for(FieldValue fieldValue : fields) {
       if (fieldValue.fieldName.equals(fieldName)) {
@@ -807,6 +843,10 @@ public class Document2 implements Iterab
     return b.toString();
   }
 
+  public FieldTypes getFieldTypes() {
+    return fieldTypes;
+  }
+
   private static final TokenStream EMPTY_TOKEN_STREAM = new TokenStream() {
       @Override
       public final boolean incrementToken() {

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java?rev=1638066&r1=1638065&r2=1638066&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java Tue Nov 11 11:24:51 2014
@@ -56,7 +56,6 @@ import org.apache.lucene.search.SortedNu
 import org.apache.lucene.search.SortedSetSortField;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TermRangeFilter;
-import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.store.BufferedChecksumIndexInput;
@@ -102,12 +101,25 @@ import org.apache.lucene.util.Version;
 //   PerFieldAnalyzerWrapper
 //   oal.document
 
+// tie into query parser
+//   default operators?
+//   default search field
+
+// tie into highlighter
+// tie into faceting
+
 // nocommit byte, short?
 
+// nocommit iterator over all fields / types?
+
+// nocommit proxy sort field
+
 // nocommit allow adding array of atom values?  fieldnamesfield would use it?
 
 // nocommit optimize field exists filter to MatchAllBits when all docs in the seg have the field; same opto as range query when min < terms.min & max > terms.max
 
+// nocomit should exists filter use docsWithField?
+
 // nocommit use better pf when field is unique
 
 // nocommit filter caching?  parent docs filter?
@@ -191,10 +203,25 @@ import org.apache.lucene.util.Version;
 
 // nocommit can we move multi-field-ness out of IW?  so IW only gets a single instance of each field
 
+// nocommit nested/parent/child docs?
+
+// nocommit "all" field:
+
+// nocommit doc blocks?
+
+// nocomit hierarchical fields?
+
+// nocommit required?  not null?
+
 /** Records how each field is indexed, stored, etc.  This class persists
  *  its state using {@link IndexWriter#setCommitData}, using the
  *  {@link FieldTypes#FIELD_PROPERTIES_KEY} key. */
 
+// nocommit what about uniqueAtom number int/long?  maybe break out isUnique?  then, e.g. like norms, you could have unique set, but maybe
+// later turn it off
+
+// nocommit IW should detect if incoming document's fieldTypes != its own
+
 public class FieldTypes {
 
   enum ValueType {
@@ -214,6 +241,8 @@ public class FieldTypes {
     // nocommit primary_key?
   }
 
+  // nocommit should we have a "resolution" for Date field?
+
   private final boolean readOnly;
 
   public static final String FIELD_NAMES_FIELD = "$fieldnames";
@@ -237,10 +266,6 @@ public class FieldTypes {
   /** Used only in memory to record when something changed. */
   private long changeCount;
 
-  // nocommit nested docs?
-
-  // nocommit required?  not null?
-
   /** Just like current oal.document.FieldType, except for each setting it can also record "not-yet-set". */
   static class FieldType implements IndexableFieldType {
     private final String name;
@@ -257,7 +282,6 @@ public class FieldTypes {
       this.createdVersion = version;
     }
 
-    // nocommit don't use null here:
     volatile ValueType valueType = ValueType.NONE;
     volatile DocValuesType docValuesType = DocValuesType.NONE;
     private volatile boolean docValuesTypeSet;
@@ -393,7 +417,7 @@ public class FieldTypes {
         }
         if (indexOptions != IndexOptions.NONE && indexOptions.compareTo(IndexOptions.DOCS) > 0) {
           // nocommit too anal?
-          illegalState(name, "type " + valueType + " can only be indexed as DOCS_ONLY; got " + indexOptions);
+          illegalState(name, "type " + valueType + " can only be indexed as DOCS; got " + indexOptions);
         }
         if (minTokenLength != null) {
           illegalState(name, "type " + valueType + " cannot set min/max token length");
@@ -461,7 +485,20 @@ public class FieldTypes {
         illegalState(name, "DocValuesType=" + docValuesType + " cannot be multi-valued");
       }
 
-      if (sortable == Boolean.TRUE && (docValuesTypeSet && (docValuesType == DocValuesType.NONE || docValuesType == DocValuesType.BINARY))) {
+      if (storeTermVectors == Boolean.TRUE) {
+        if (indexOptionsSet && indexOptions == IndexOptions.NONE) {
+          illegalState(name, "cannot enable term vectors when indexOptions is NONE");
+        }
+      } else {
+        if (storeTermVectorOffsets == Boolean.TRUE) {
+          illegalState(name, "cannot enable term vector offsets when term vectors are not enabled");
+        }
+        if (storeTermVectorPositions == Boolean.TRUE) {
+          illegalState(name, "cannot enable term vector positions when term vectors are not enabled");
+        }
+      }
+
+      if (sortable == Boolean.TRUE && (docValuesTypeSet && docValuesType == DocValuesType.NONE)) {
         illegalState(name, "cannot sort when DocValuesType=" + docValuesType);
       }
 
@@ -1113,7 +1150,7 @@ public class FieldTypes {
   /** Key used to store the field types inside {@link IndexWriter#setCommitData}. */
   public static final String FIELD_TYPES_KEY = "FieldTypes";
   
-  private Version loadFields(Map<String,String> commitUserData, boolean isNewIndex) throws IOException {
+  private synchronized Version loadFields(Map<String,String> commitUserData, boolean isNewIndex) throws IOException {
     // nocommit must deserialize current fields from commit data
     String currentFieldTypes = commitUserData.get(FIELD_TYPES_KEY);
     if (currentFieldTypes != null) {
@@ -1135,7 +1172,7 @@ public class FieldTypes {
     }
   }
 
-  private FieldType newFieldType(String fieldName) {
+  private synchronized FieldType newFieldType(String fieldName) {
     if (fieldName.equals(FIELD_NAMES_FIELD)) {
       throw new IllegalArgumentException("field name \"" + fieldName + "\" is reserved");
     }
@@ -1345,6 +1382,8 @@ public class FieldTypes {
         return field.analyzerPositionGap.intValue();
       } else if (field.indexAnalyzer != null) {
         return field.indexAnalyzer.getPositionIncrementGap(fieldName);
+      } else if (defaultIndexAnalyzer == null) {
+        return 0;
       } else {
         return defaultIndexAnalyzer.getPositionIncrementGap(fieldName);
       }
@@ -1365,6 +1404,8 @@ public class FieldTypes {
         return field.analyzerOffsetGap.intValue();
       } else if (field.indexAnalyzer != null) {
         return field.indexAnalyzer.getOffsetGap(fieldName);
+      } else if (defaultIndexAnalyzer == null) {
+        return 1;
       } else {
         return defaultIndexAnalyzer.getOffsetGap(fieldName);
       }
@@ -1854,50 +1895,52 @@ public class FieldTypes {
   }
 
   public synchronized void setSortMissingFirst(String fieldName) {
-    // Field must exist
-    FieldType current = getFieldType(fieldName);
-
-    if (current.sortable != Boolean.TRUE) {
-      illegalState(fieldName, "cannot setSortMissingFirst: field is not enabled for sorting");
-    }
-
-    Boolean currentValue = current.sortMissingLast;
-    if (currentValue != Boolean.FALSE) {
+    FieldType current = fields.get(fieldName);
+    if (current == null) {
+      current = newFieldType(fieldName);
       current.sortMissingLast = Boolean.FALSE;
-      boolean success = false;
-      try {
-        current.validate();
-        success = true;
-      } finally {
-        if (success == false) {
-          current.sortMissingLast = currentValue;
+      fields.put(fieldName, current);
+      changed();
+    } else {
+      Boolean currentValue = current.sortMissingLast;
+      if (currentValue != Boolean.FALSE) {
+        current.sortMissingLast = Boolean.FALSE;
+        boolean success = false;
+        try {
+          current.validate();
+          success = true;
+        } finally {
+          if (success == false) {
+            current.sortMissingLast = currentValue;
+          }
         }
+        changed();
       }
-      changed();
     }
   }
 
   public synchronized void setSortMissingLast(String fieldName) {
-    // Field must exist
-    FieldType current = getFieldType(fieldName);
-
-    if (current.sortable != Boolean.TRUE) {
-      illegalState(fieldName, "cannot setSortMissingLast: field is not enabled for sorting");
-    }
-
-    Boolean currentValue = current.sortMissingLast;
-    if (currentValue != Boolean.TRUE) {
+    FieldType current = fields.get(fieldName);
+    if (current == null) {
+      current = newFieldType(fieldName);
       current.sortMissingLast = Boolean.TRUE;
-      boolean success = false;
-      try {
-        current.validate();
-        success = true;
-      } finally {
-        if (success == false) {
-          current.sortMissingLast = currentValue;
+      fields.put(fieldName, current);
+      changed();
+    } else {
+      Boolean currentValue = current.sortMissingLast;
+      if (currentValue != Boolean.TRUE) {
+        current.sortMissingLast = Boolean.TRUE;
+        boolean success = false;
+        try {
+          current.validate();
+          success = true;
+        } finally {
+          if (success == false) {
+            current.sortMissingLast = currentValue;
+          }
         }
+        changed();
       }
-      changed();
     }
   }
 
@@ -2077,8 +2120,6 @@ public class FieldTypes {
     return getFieldType(fieldName).stored == Boolean.TRUE;
   }
 
-  // nocommit iterator over all fields / types?
-
   // nocommit should we make a single method to enable the different combinations...?
   public synchronized void enableTermVectors(String fieldName) {
     FieldType current = fields.get(fieldName);
@@ -2540,6 +2581,8 @@ public class FieldTypes {
           } else {
             field.docValuesType = DocValuesType.SORTED;
           }
+        } else {
+          field.docValuesType = DocValuesType.BINARY;
         }
         field.docValuesTypeSet = true;
       }
@@ -2840,6 +2883,10 @@ public class FieldTypes {
     return newRangeFilter(fieldName, new BytesRef(minTerm), minInclusive, new BytesRef(maxTerm), maxInclusive);
   }
 
+  public Filter newRangeFilter(String fieldName, String minTerm, boolean minInclusive, String maxTerm, boolean maxInclusive) {
+    return newRangeFilter(fieldName, new BytesRef(minTerm), minInclusive, new BytesRef(maxTerm), maxInclusive);
+  }
+
   public Filter newRangeFilter(String fieldName, BytesRef minTerm, boolean minInclusive, BytesRef maxTerm, boolean maxInclusive) {
 
     // Field must exist:
@@ -3070,6 +3117,8 @@ public class FieldTypes {
         if (fieldType.multiValued == Boolean.TRUE) {
           // nocommit need to be able to set selector...
           sortField = new SortedSetSortField(fieldName, reverse);
+        } else if (fieldType.docValuesType == DocValuesType.BINARY) {
+          sortField = new SortField(fieldName, SortField.Type.STRING_VAL, reverse);
         } else {
           sortField = new SortField(fieldName, SortField.Type.STRING, reverse);
         }
@@ -3104,6 +3153,7 @@ public class FieldTypes {
     if (enableExistsFilters == false) {
       throw new IllegalStateException("field exists filter was disabled");
     }
+    // nocommit just use FieldValueFilter if field is DV'd? and then don't index such fields into FIELD_NAMES_FIELD?
 
     // nocommit TermFilter?
     // nocommit optimize this filter to MatchAllDocs when Terms.getDocCount() == maxDoc
@@ -3146,10 +3196,14 @@ public class FieldTypes {
     writeBoolean(out, enableExistsFilters);
     writeBoolean(out, indexedDocs);
 
-    out.writeVInt(fields.size());
+    int count = fields.size();
+    out.writeVInt(count);
+    int count2 = 0;
     for(FieldType fieldType : fields.values()) {
       fieldType.write(out);
+      count2++;
     }
+    assert count == count2;
 
     CodecUtil.writeFooter(out);
 
@@ -3223,7 +3277,7 @@ public class FieldTypes {
   }
 
   // nocommit on exception (mismatched schema), this should ensure no changes were actually made:
-  public void addAll(FieldTypes in) {
+  public synchronized void addAll(FieldTypes in) {
     for (FieldType fieldType : in.fields.values()) {
       FieldType curFieldType = fields.get(fieldType.name);
       if (curFieldType == null) {
@@ -3281,4 +3335,18 @@ public class FieldTypes {
       throw new CorruptIndexException("invalid byte for boolean: " + b, in);
     }
   }
+
+  public synchronized void clear() {
+    fields.clear();
+    enableExistsFilters = true;
+    indexedDocs = false;
+
+    FieldType fieldType = new FieldType(FIELD_NAMES_FIELD);
+    fields.put(FIELD_NAMES_FIELD, fieldType);
+    fieldType.multiValued = Boolean.TRUE;
+    fieldType.valueType = ValueType.ATOM;
+    fieldType.sortable = Boolean.TRUE;
+    fieldType.stored = Boolean.FALSE;
+    setDefaults(fieldType);
+  }
 }

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1638066&r1=1638065&r2=1638066&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Tue Nov 11 11:24:51 2014
@@ -741,6 +741,8 @@ public class CheckIndex implements Close
     if (onlySegments == null && result.numBadSegments == 0) {
       MultiReader topReader = new MultiReader(segmentReaders);
       try {
+        int nonUniqueCount = 0;
+        String nonUniqueMessage = null;
         for(String fieldName : fieldTypes.getFieldNames()) {
           if (fieldTypes.isUniqueAtom(fieldName)) {
             Terms terms = MultiFields.getTerms(topReader, fieldName);
@@ -752,15 +754,22 @@ public class CheckIndex implements Close
                 docsEnum = termsEnum.docs(liveDocs, docsEnum, DocsEnum.FLAG_NONE);
                 int docID = docsEnum.nextDoc();
                 if (docID != DocsEnum.NO_MORE_DOCS) {
-                  int docID2 = docsEnum.nextDoc();
-                  if (docID2 != DocsEnum.NO_MORE_DOCS) {
-                    msg(infoStream, "FAILED");
-                    // nocommit should "isUnique" be in low schema?
-                    // nocommit have -fix delete the offenders:
-                    String comment = "UNIQUE_ATOM field=\"" + fieldName + "\" is not unique: term=" + termsEnum.term() + " matches both docID=" + docID + " and docID=" + docID2 + "; unable to fix this index";
-                    msg(infoStream, comment);
-                    if (failFast) {
-                      throw new RuntimeException(comment);
+                  while (true) {
+                    int docID2 = docsEnum.nextDoc();
+                    if (docID2 != DocsEnum.NO_MORE_DOCS) {
+                      if (nonUniqueCount == 0) {
+                        // nocommit should "isUnique" be in low schema?
+                        // nocommit have -fix delete the offenders:
+                        nonUniqueMessage = "UNIQUE_ATOM field=\"" + fieldName + "\" is not unique: e.g. term=" + termsEnum.term() + " matches both docID=" + docID + " and docID=" + docID2;
+                        if (failFast) {
+                          msg(infoStream, "FAILED");
+                          msg(infoStream, nonUniqueMessage);
+                          throw new RuntimeException(nonUniqueMessage);
+                        }
+                      }
+                      nonUniqueCount++;
+                    } else {
+                      break;
                     }
                   }
                 }
@@ -768,6 +777,13 @@ public class CheckIndex implements Close
             }
           }
         }
+
+        if (nonUniqueCount != 0) {
+          nonUniqueMessage += "; total " + nonUniqueCount + " non-unique documents would be deleted";
+          msg(infoStream, "FAILED");
+          msg(infoStream, nonUniqueMessage);
+          throw new RuntimeException(nonUniqueMessage);
+        }
       } finally {
         topReader.close();
       }

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1638066&r1=1638065&r2=1638066&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Tue Nov 11 11:24:51 2014
@@ -26,6 +26,7 @@ import java.util.concurrent.atomic.Atomi
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.document.Document2;
 import org.apache.lucene.document.FieldTypes;
 import org.apache.lucene.index.DocumentsWriterDeleteQueue.DeleteSlice;
 import org.apache.lucene.search.similarities.Similarity;
@@ -226,6 +227,12 @@ class DocumentsWriterPerThread {
 
   public void updateDocument(Iterable<? extends IndexableField> doc, Analyzer analyzer, Term delTerm) throws IOException {
     testPoint("DocumentsWriterPerThread addDocument start");
+    if (doc instanceof Document2) {
+      Document2 doc2 = (Document2) doc;
+      if (doc2.getFieldTypes() != fieldTypes) {
+        throw new IllegalArgumentException("this document wasn't created by this writer (fieldTypes are different)");
+      }
+    }
     assert deleteQueue != null;
     docState.doc = doc;
     docState.analyzer = analyzer;
@@ -280,6 +287,12 @@ class DocumentsWriterPerThread {
         // document, so the counter will be "wrong" in that case, but
         // it's very hard to fix (we can't easily distinguish aborting
         // vs non-aborting exceptions):
+        if (doc instanceof Document2) {
+          Document2 doc2 = (Document2) doc;
+          if (doc2.getFieldTypes() != fieldTypes) {
+            throw new IllegalArgumentException("this document wasn't created by this writer (fieldTypes are different)");
+          }
+        }
         reserveDoc();
         docState.doc = doc;
         docState.docID = numDocsInRAM;

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1638066&r1=1638065&r2=1638066&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Tue Nov 11 11:24:51 2014
@@ -32,8 +32,8 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
-import java.util.Map.Entry;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Queue;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -2173,6 +2173,7 @@ public class IndexWriter implements Clos
              */
             // Don't bother saving any changes in our segmentInfos
             readerPool.dropAll(false);
+            fieldTypes.clear();
             // Mark that the index has changed
             ++changeCount;
             segmentInfos.changed();

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/overview.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/overview.html?rev=1638066&r1=1638065&r2=1638066&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/overview.html (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/overview.html Tue Nov 11 11:24:51 2014
@@ -38,9 +38,8 @@ to check if the results are what we expe
     //Directory directory = FSDirectory.open("/tmp/testindex");
     IndexWriterConfig config = new IndexWriterConfig(analyzer);
     IndexWriter iwriter = new IndexWriter(directory, config);
-    Document doc = new Document();
-    String text = "This is the text to be indexed.";
-    doc.add(new Field("fieldname", text, TextField.TYPE_STORED));
+    Document doc = iwriter.newDocument();
+    doc.addLargeText("fieldname", "This is the text to be indexed.");
     iwriter.addDocument(doc);
     iwriter.close();
     

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat3.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat3.java?rev=1638066&r1=1638065&r2=1638066&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat3.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat3.java Tue Nov 11 11:24:51 2014
@@ -31,20 +31,21 @@ import org.apache.lucene.analysis.Tokeni
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.FieldTypes;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.DocsAndPositionsEnum;
 import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum.SeekStatus;
 import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.TermsEnum.SeekStatus;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Bits;
@@ -86,6 +87,7 @@ public class TestBlockPostingsFormat3 ex
     // TODO we could actually add more fields implemented with different PFs
     // or, just put this test into the usual rotation?
     RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+    FieldTypes fieldTypes = iw.getFieldTypes();
     Document doc = new Document();
     FieldType docsOnlyType = new FieldType(TextField.TYPE_NOT_STORED);
     // turn this on for a cross-check

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50DocValuesFormat.java?rev=1638066&r1=1638065&r2=1638066&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50DocValuesFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50DocValuesFormat.java Tue Nov 11 11:24:51 2014
@@ -26,15 +26,12 @@ import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.asserting.AssertingCodec;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.SortedSetDocValuesField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.document.Document2;
+import org.apache.lucene.document.FieldTypes;
 import org.apache.lucene.index.BaseCompressingDocValuesFormatTestCase;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.SerialMergeScheduler;
 import org.apache.lucene.index.Term;
@@ -133,12 +130,14 @@ public class TestLucene50DocValuesFormat
       }
     });
     RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
-    
+    FieldTypes fieldTypes = writer.getFieldTypes();
+    fieldTypes.setMultiValued("dv");
+    fieldTypes.setMultiValued("indexed");
+
     // index some docs
     for (int i = 0; i < numDocs; i++) {
-      Document doc = new Document();
-      Field idField = new StringField("id", Integer.toString(i), Field.Store.NO);
-      doc.add(idField);
+      Document2 doc = writer.newDocument();
+      doc.addUniqueAtom("id", Integer.toString(i));
       final int length = TestUtil.nextInt(random(), minLength, maxLength);
       int numValues = random().nextInt(17);
       // create a random list of strings
@@ -151,14 +150,14 @@ public class TestLucene50DocValuesFormat
       ArrayList<String> unordered = new ArrayList<>(values);
       Collections.shuffle(unordered, random());
       for (String v : values) {
-        doc.add(newStringField("indexed", v, Field.Store.NO));
+        doc.addAtom("indexed", v);
       }
 
       // add in any order to the dv field
       ArrayList<String> unordered2 = new ArrayList<>(values);
       Collections.shuffle(unordered2, random());
       for (String v : unordered2) {
-        doc.add(new SortedSetDocValuesField("dv", new BytesRef(v)));
+        doc.addBinary("dv", new BytesRef(v));
       }
 
       writer.addDocument(doc);

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java?rev=1638066&r1=1638065&r2=1638066&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java Tue Nov 11 11:24:51 2014
@@ -26,14 +26,12 @@ import org.apache.lucene.analysis.MockAn
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.asserting.AssertingCodec;
-import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Document2;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.FieldTypes;
 import org.apache.lucene.index.BaseDocValuesFormatTestCase;
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
@@ -92,12 +90,14 @@ public class TestPerFieldDocValuesFormat
       }
     });
     IndexWriter iwriter = new IndexWriter(directory, iwc);
-    Document doc = new Document();
+    Document2 doc = iwriter.newDocument();
+    FieldTypes fieldTypes = iwriter.getFieldTypes();
+    fieldTypes.setDocValuesType("dv2", DocValuesType.BINARY);
     String longTerm = "longtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongterm";
     String text = "This is the text to be indexed. " + longTerm;
-    doc.add(newTextField("fieldname", text, Field.Store.YES));
-    doc.add(new NumericDocValuesField("dv1", 5));
-    doc.add(new BinaryDocValuesField("dv2", new BytesRef("hello world")));
+    doc.addLargeText("fieldname", text);
+    doc.addInt("dv1", 5);
+    doc.addBinary("dv2", new BytesRef("hello world"));
     iwriter.addDocument(doc);
     iwriter.close();
     
@@ -117,6 +117,7 @@ public class TestPerFieldDocValuesFormat
       NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv1");
       assertEquals(5, dv.get(hits.scoreDocs[i].doc));
       BinaryDocValues dv2 = ireader.leaves().get(0).reader().getBinaryDocValues("dv2");
+      assertNotNull(dv2);
       final BytesRef term = dv2.get(hits.scoreDocs[i].doc);
       assertEquals(new BytesRef("hello world"), term);
     }

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java?rev=1638066&r1=1638065&r2=1638066&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java Tue Nov 11 11:24:51 2014
@@ -16,6 +16,7 @@ package org.apache.lucene.codecs.perfiel
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 import java.io.IOException;
 
 import org.apache.lucene.analysis.MockAnalyzer;
@@ -25,10 +26,8 @@ import org.apache.lucene.codecs.assertin
 import org.apache.lucene.codecs.blockterms.LuceneVarGapFixedInterval;
 import org.apache.lucene.codecs.memory.MemoryPostingsFormat;
 import org.apache.lucene.codecs.simpletext.SimpleTextPostingsFormat;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.TextField;
+import org.apache.lucene.document.Document2;
+import org.apache.lucene.document.FieldTypes;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
@@ -67,25 +66,25 @@ public class TestPerFieldPostingsFormat2
 
   private void addDocs(IndexWriter writer, int numDocs) throws IOException {
     for (int i = 0; i < numDocs; i++) {
-      Document doc = new Document();
-      doc.add(newTextField("content", "aaa", Field.Store.NO));
+      Document2 doc = writer.newDocument();
+      doc.addLargeText("content", "aaa");
       writer.addDocument(doc);
     }
   }
 
   private void addDocs2(IndexWriter writer, int numDocs) throws IOException {
     for (int i = 0; i < numDocs; i++) {
-      Document doc = new Document();
-      doc.add(newTextField("content", "bbb", Field.Store.NO));
+      Document2 doc = writer.newDocument();
+      doc.addLargeText("content", "bbb");
       writer.addDocument(doc);
     }
   }
 
   private void addDocs3(IndexWriter writer, int numDocs) throws IOException {
     for (int i = 0; i < numDocs; i++) {
-      Document doc = new Document();
-      doc.add(newTextField("content", "ccc", Field.Store.NO));
-      doc.add(newStringField("id", "" + i, Field.Store.YES));
+      Document2 doc = writer.newDocument();
+      doc.addLargeText("content", "ccc");
+      doc.addAtom("id", "" + i);
       writer.addDocument(doc);
     }
   }
@@ -244,15 +243,14 @@ public class TestPerFieldPostingsFormat2
           new MockAnalyzer(random()));
       config.setOpenMode(OpenMode.CREATE_OR_APPEND);
       IndexWriter writer = newWriter(dir, config);
+      FieldTypes fieldTypes = writer.getFieldTypes();
       for (int j = 0; j < docsPerRound; j++) {
-        final Document doc = new Document();
+        final Document2 doc = writer.newDocument();
         for (int k = 0; k < num; k++) {
-          FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
-          customType.setTokenized(random().nextBoolean());
-          customType.setOmitNorms(random().nextBoolean());
-          Field field = newField("" + k, TestUtil
-              .randomRealisticUnicodeString(random(), 128), customType);
-          doc.add(field);
+          if (random().nextBoolean()) {
+            fieldTypes.disableNorms("" + k);
+          }
+          doc.addLargeText("" + k, TestUtil.randomRealisticUnicodeString(random(), 128));
         }
         writer.addDocument(doc);
       }
@@ -303,19 +301,18 @@ public class TestPerFieldPostingsFormat2
     IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
     iwc.setCodec(codec);
     RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
-    Document doc = new Document();
-    FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
+    FieldTypes fieldTypes = iw.getFieldTypes();
     // turn on vectors for the checkindex cross-check
-    ft.setStoreTermVectors(true);
-    ft.setStoreTermVectorOffsets(true);
-    ft.setStoreTermVectorPositions(true);
-    Field idField = new Field("id", "", ft);
-    Field dateField = new Field("date", "", ft);
-    doc.add(idField);
-    doc.add(dateField);
+    fieldTypes.enableTermVectors("id");
+    fieldTypes.enableTermVectorOffsets("id");
+    fieldTypes.enableTermVectorPositions("id");
+    fieldTypes.enableTermVectors("date");
+    fieldTypes.enableTermVectorOffsets("date");
+    fieldTypes.enableTermVectorPositions("date");
     for (int i = 0; i < 100; i++) {
-      idField.setStringValue(Integer.toString(random().nextInt(50)));
-      dateField.setStringValue(Integer.toString(random().nextInt(100)));
+      Document2 doc = iw.newDocument();
+      doc.addLargeText("id", Integer.toString(random().nextInt(50)));
+      doc.addLargeText("date", Integer.toString(random().nextInt(100)));
       iw.addDocument(doc);
     }
     iw.close();

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.java?rev=1638066&r1=1638065&r2=1638066&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.java Tue Nov 11 11:24:51 2014
@@ -36,23 +36,18 @@ public class TestBinaryDocument extends 
   public void testBinaryFieldInIndex()
     throws Exception
   {
-    FieldType ft = new FieldType();
-    ft.setStored(true);
-    StoredField binaryFldStored = new StoredField("binaryStored", binaryValStored.getBytes(StandardCharsets.UTF_8));
-    Field stringFldStored = new Field("stringStored", binaryValStored, ft);
+    Directory dir = newDirectory();
+    RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
 
-    Document doc = new Document();
-    
-    doc.add(binaryFldStored);
+    Document2 doc = writer.newDocument();
     
-    doc.add(stringFldStored);
+    doc.addStored("binaryStored", binaryValStored.getBytes(StandardCharsets.UTF_8));
+    doc.addStored("stringStored", binaryValStored);
 
     /** test for field count */
     assertEquals(2, doc.getFields().size());
     
     /** add the doc to a ram index */
-    Directory dir = newDirectory();
-    RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
     writer.addDocument(doc);
     
     /** open a reader and fetch the document */ 
@@ -76,17 +71,15 @@ public class TestBinaryDocument extends 
   }
   
   public void testCompressionTools() throws Exception {
-    StoredField binaryFldCompressed = new StoredField("binaryCompressed", CompressionTools.compress(binaryValCompressed.getBytes(StandardCharsets.UTF_8)));
-    StoredField stringFldCompressed = new StoredField("stringCompressed", CompressionTools.compressString(binaryValCompressed));
-    
-    Document doc = new Document();
+    Directory dir = newDirectory();
+    RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
+
+    Document2 doc = writer.newDocument();
     
-    doc.add(binaryFldCompressed);
-    doc.add(stringFldCompressed);
+    doc.addStored("binaryCompressed", CompressionTools.compress(binaryValCompressed.getBytes(StandardCharsets.UTF_8)));
+    doc.addStored("stringCompressed", CompressionTools.compressString(binaryValCompressed));
     
     /** add the doc to a ram index */
-    Directory dir = newDirectory();
-    RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
     writer.addDocument(doc);
     
     /** open a reader and fetch the document */ 

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestDocument.java?rev=1638066&r1=1638065&r2=1638066&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestDocument.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestDocument.java Tue Nov 11 11:24:51 2014
@@ -39,7 +39,6 @@ import org.apache.lucene.store.Directory
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
 
-
 /**
  * Tests {@link Document} class.
  */
@@ -340,68 +339,39 @@ public class TestDocument extends Lucene
     }
   }
   
-  public void testFieldSetValue() throws Exception {
-    
-    Field field = new StringField("id", "id1", Field.Store.YES);
-    Document doc = new Document();
-    doc.add(field);
-    doc.add(new StringField("keyword", "test", Field.Store.YES));
-    
+  // LUCENE-3616
+  public void testInvalidFields() throws IOException {
     Directory dir = newDirectory();
-    RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
-    writer.addDocument(doc);
-    field.setStringValue("id2");
-    writer.addDocument(doc);
-    field.setStringValue("id3");
-    writer.addDocument(doc);
-    
-    IndexReader reader = writer.getReader();
-    IndexSearcher searcher = newSearcher(reader);
-    
-    Query query = new TermQuery(new Term("keyword", "test"));
+    RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+    FieldTypes fieldTypes = iw.getFieldTypes();
+
+    fieldTypes.enableStored("foo");
+    Tokenizer tok = new MockTokenizer();
+    tok.setReader(new StringReader(""));
+    Document2 doc = iw.newDocument();
     
-    // ensure that queries return expected results without DateFilter first
-    ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
-    assertEquals(3, hits.length);
-    int result = 0;
-    for (int i = 0; i < 3; i++) {
-      Document2 doc2 = searcher.doc(hits[i].doc);
-      IndexableField f = doc2.getField("id");
-      if (f.stringValue().equals("id1")) result |= 1;
-      else if (f.stringValue().equals("id2")) result |= 2;
-      else if (f.stringValue().equals("id3")) result |= 4;
-      else fail("unexpected id field");
-    }
-    writer.close();
-    reader.close();
-    dir.close();
-    assertEquals("did not see all IDs", 7, result);
-  }
-  
-  // LUCENE-3616
-  public void testInvalidFields() {
     try {
-      Tokenizer tok = new MockTokenizer();
-      tok.setReader(new StringReader(""));
-      new Field("foo", tok, StringField.TYPE_STORED);
+      doc.addLargeText("foo", tok);
       fail("did not hit expected exc");
-    } catch (IllegalArgumentException iae) {
+    } catch (IllegalStateException ise) {
       // expected
-    } catch (IOException ioe) {
-      throw new RuntimeException(ioe);
     }
+    iw.close();
+    dir.close();
   }
   
   public void testNumericFieldAsString() throws Exception {
-    Document doc = new Document();
-    doc.add(new IntField("int", 5, Field.Store.YES));
-    assertEquals("5", doc.get("int"));
-    assertNull(doc.get("somethingElse"));
-    doc.add(new IntField("int", 4, Field.Store.YES));
-    assertArrayEquals(new String[] { "5", "4" }, doc.getValues("int"));
-    
     Directory dir = newDirectory();
     RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+    FieldTypes fieldTypes = iw.getFieldTypes();
+    fieldTypes.setMultiValued("int");
+    Document2 doc = iw.newDocument();
+    doc.addInt("int", 5);
+    assertEquals("5", doc.getString("int"));
+    assertNull(doc.get("somethingElse"));
+    doc.addInt("int", 4);
+    assertArrayEquals(new String[] { "5", "4" }, doc.getStrings("int"));
+    
     iw.addDocument(doc);
     DirectoryReader ir = iw.getReader();
     Document2 sdoc = ir.document(0);

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java?rev=1638066&r1=1638065&r2=1638066&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java Tue Nov 11 11:24:51 2014
@@ -24,21 +24,17 @@ import java.util.concurrent.atomic.Atomi
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.document.BinaryDocValuesField;
-import org.apache.lucene.document.Document2;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.Document2;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.document.SortedDocValuesField;
-import org.apache.lucene.document.SortedSetDocValuesField;
-import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.FieldTypes;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Ignore;
 
 /**
  * 
@@ -54,18 +50,18 @@ public class TestDocValuesIndexing exten
   public void testAddIndexes() throws IOException {
     Directory d1 = newDirectory();
     RandomIndexWriter w = new RandomIndexWriter(random(), d1);
-    Document doc = new Document();
-    doc.add(newStringField("id", "1", Field.Store.YES));
-    doc.add(new NumericDocValuesField("dv", 1));
+    Document2 doc = w.newDocument();
+    doc.addUniqueAtom("id", "1");
+    doc.addInt("dv", 1);
     w.addDocument(doc);
     IndexReader r1 = w.getReader();
     w.close();
 
     Directory d2 = newDirectory();
     w = new RandomIndexWriter(random(), d2);
-    doc = new Document();
-    doc.add(newStringField("id", "2", Field.Store.YES));
-    doc.add(new NumericDocValuesField("dv", 2));
+    doc = w.newDocument();
+    doc.addUniqueAtom("id", "2");
+    doc.addInt("dv", 2);
     w.addDocument(doc);
     IndexReader r2 = w.getReader();
     w.close();
@@ -92,16 +88,15 @@ public class TestDocValuesIndexing exten
   public void testMultiValuedDocValuesField() throws Exception {
     Directory d = newDirectory();
     RandomIndexWriter w = new RandomIndexWriter(random(), d);
-    Document doc = new Document();
-    Field f = new NumericDocValuesField("field", 17);
-    doc.add(f);
+    Document2 doc = w.newDocument();
+    doc.addInt("field", 17);
     
     // add the doc
     w.addDocument(doc);
     
     // Index doc values are single-valued so we should not
     // be able to add same field more than once:
-    doc.add(f);
+    doc.addInt("field", 16);
     try {
       w.addDocument(doc);
       fail("didn't hit expected exception");
@@ -119,17 +114,16 @@ public class TestDocValuesIndexing exten
   public void testDifferentTypedDocValuesField() throws Exception {
     Directory d = newDirectory();
     RandomIndexWriter w = new RandomIndexWriter(random(), d);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("field", 17));
+    Document2 doc = w.newDocument();
+    doc.addInt("field", 17);
     w.addDocument(doc);
     
     // Index doc values are single-valued so we should not
     // be able to add same field more than once:
-    doc.add(new BinaryDocValuesField("field", new BytesRef("blah")));
     try {
-      w.addDocument(doc);
+      doc.addBinary("field", new BytesRef("blah"));
       fail("didn't hit expected exception");
-    } catch (IllegalArgumentException iae) {
+    } catch (IllegalStateException ise) {
       // expected
     }
 
@@ -143,17 +137,16 @@ public class TestDocValuesIndexing exten
   public void testDifferentTypedDocValuesField2() throws Exception {
     Directory d = newDirectory();
     RandomIndexWriter w = new RandomIndexWriter(random(), d);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("field", 17));
+    Document2 doc = w.newDocument();
+    doc.addInt("field", 17);
     w.addDocument(doc);
     
     // Index doc values are single-valued so we should not
     // be able to add same field more than once:
-    doc.add(new SortedDocValuesField("field", new BytesRef("hello")));
     try {
-      w.addDocument(doc);
+      doc.addAtom("field", new BytesRef("hello"));
       fail("didn't hit expected exception");
-    } catch (IllegalArgumentException iae) {
+    } catch (IllegalStateException ise) {
       // expected
     }
     DirectoryReader r = w.getReader();
@@ -167,12 +160,12 @@ public class TestDocValuesIndexing exten
   public void testLengthPrefixAcrossTwoPages() throws Exception {
     Directory d = newDirectory();
     IndexWriter w = new IndexWriter(d, new IndexWriterConfig(new MockAnalyzer(random())));
-    Document doc = new Document();
+    Document2 doc = w.newDocument();
     byte[] bytes = new byte[32764];
     BytesRef b = new BytesRef();
     b.bytes = bytes;
     b.length = bytes.length;
-    doc.add(new SortedDocValuesField("field", b));
+    doc.addAtom("field", b);
     w.addDocument(doc);
     bytes[0] = 1;
     w.addDocument(doc);
@@ -200,9 +193,9 @@ public class TestDocValuesIndexing exten
     iwconfig.setMergePolicy(newLogMergePolicy());
     IndexWriter writer = new IndexWriter(dir, iwconfig);
     for (int i = 0; i < 50; i++) {
-      Document doc = new Document();
-      doc.add(new NumericDocValuesField("dv", i));
-      doc.add(new TextField("docId", "" + i, Field.Store.YES));
+      Document2 doc = writer.newDocument();
+      doc.addInt("dv", i);
+      doc.addLargeText("docId", "" + i);
       writer.addDocument(doc);
     }
     DirectoryReader r = writer.getReader();
@@ -215,7 +208,8 @@ public class TestDocValuesIndexing exten
       assertEquals(i, dv.get(i));
       Document2 d = slow.document(i);
       // cannot use d.get("dv") due to another bug!
-      assertNull(d.getString("dv"));
+      // nocommit why is this here?
+      // assertNull(d.getString("dv"));
       assertEquals(Integer.toString(i), d.getString("docId"));
     }
     slow.close();
@@ -227,15 +221,14 @@ public class TestDocValuesIndexing exten
   public void testMixedTypesSameDocument() throws Exception {
     Directory dir = newDirectory();
     IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
-    w.addDocument(new Document());
+    w.addDocument(w.newDocument());
     
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 0));
-    doc.add(new SortedDocValuesField("foo", new BytesRef("hello")));
+    Document2 doc = w.newDocument();
+    doc.addInt("foo", 0);
     try {
-      w.addDocument(doc);
+      doc.addAtom("foo", new BytesRef("hello"));
       fail("didn't hit expected exception");
-    } catch (IllegalArgumentException iae) {
+    } catch (IllegalStateException ise) {
       // expected
     }
     IndexReader ir = w.getReader();
@@ -249,16 +242,15 @@ public class TestDocValuesIndexing exten
   public void testMixedTypesDifferentDocuments() throws Exception {
     Directory dir = newDirectory();
     IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 0));
+    Document2 doc = w.newDocument();
+    doc.addInt("foo", 0);
     w.addDocument(doc);
 
-    doc = new Document();
-    doc.add(new SortedDocValuesField("foo", new BytesRef("hello")));
+    doc = w.newDocument();
     try {
-      w.addDocument(doc);
+      doc.addAtom("foo", new BytesRef("hello"));
       fail("didn't hit expected exception");
-    } catch (IllegalArgumentException iae) {
+    } catch (IllegalStateException ise) {
       // expected
     }
     IndexReader ir = w.getReader();
@@ -276,11 +268,11 @@ public class TestDocValuesIndexing exten
     IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
     iwc.setMergePolicy(newLogMergePolicy());
     IndexWriter iwriter = new IndexWriter(directory, iwc);
-    Document doc = new Document();
-    doc.add(new SortedDocValuesField("dv", new BytesRef("foo!")));
+    Document2 doc = iwriter.newDocument();
+    doc.addAtom("dv", new BytesRef("foo!"));
     iwriter.addDocument(doc);
     
-    doc.add(new SortedDocValuesField("dv", new BytesRef("bar!")));
+    doc.addAtom("dv", new BytesRef("bar!"));
     try {
       iwriter.addDocument(doc);
       fail("didn't hit expected exception");
@@ -306,11 +298,11 @@ public class TestDocValuesIndexing exten
     IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
     iwc.setMergePolicy(newLogMergePolicy());
     IndexWriter iwriter = new IndexWriter(directory, iwc);
-    Document doc = new Document();
-    doc.add(new BinaryDocValuesField("dv", new BytesRef("foo!")));
+    Document2 doc = iwriter.newDocument();
+    doc.addBinary("dv", new BytesRef("foo!"));
     iwriter.addDocument(doc);
     
-    doc.add(new BinaryDocValuesField("dv", new BytesRef("bar!")));
+    doc.addBinary("dv", new BytesRef("bar!"));
     try {
       iwriter.addDocument(doc);
       fail("didn't hit expected exception");
@@ -334,11 +326,11 @@ public class TestDocValuesIndexing exten
     IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
     iwc.setMergePolicy(newLogMergePolicy());
     IndexWriter iwriter = new IndexWriter(directory, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("dv", 1));
+    Document2 doc = iwriter.newDocument();
+    doc.addInt("dv", 1);
     iwriter.addDocument(doc);
     
-    doc.add(new NumericDocValuesField("dv", 2));
+    doc.addInt("dv", 2);
     try {
       iwriter.addDocument(doc);
       fail("didn't hit expected exception");
@@ -360,15 +352,15 @@ public class TestDocValuesIndexing exten
     IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
     iwc.setMergePolicy(newLogMergePolicy());
     IndexWriter iwriter = new IndexWriter(directory, iwc);
-    Document doc = new Document();
-    doc.add(new SortedDocValuesField("dv", new BytesRef("just fine")));
+    Document2 doc = iwriter.newDocument();
+    doc.addAtom("dv", new BytesRef("just fine"));
     iwriter.addDocument(doc);
     
-    doc = new Document();
+    doc = iwriter.newDocument();
     byte bytes[] = new byte[100000];
     BytesRef b = new BytesRef(bytes);
     random().nextBytes(bytes);
-    doc.add(new SortedDocValuesField("dv", b));
+    doc.addAtom("dv", b);
     try {
       iwriter.addDocument(doc);
       fail("did not get expected exception");
@@ -390,15 +382,17 @@ public class TestDocValuesIndexing exten
     IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
     iwc.setMergePolicy(newLogMergePolicy());
     IndexWriter iwriter = new IndexWriter(directory, iwc);
-    Document doc = new Document();
-    doc.add(new SortedSetDocValuesField("dv", new BytesRef("just fine")));
+    FieldTypes fieldTypes = iwriter.getFieldTypes();
+    Document2 doc = iwriter.newDocument();
+    fieldTypes.setMultiValued("dv");
+    doc.addAtom("dv", new BytesRef("just fine"));
     iwriter.addDocument(doc);
     
-    doc = new Document();
+    doc = iwriter.newDocument();
     byte bytes[] = new byte[100000];
     BytesRef b = new BytesRef(bytes);
     random().nextBytes(bytes);
-    doc.add(new SortedSetDocValuesField("dv", b));
+    doc.addAtom("dv", b);
     try {
       iwriter.addDocument(doc);
       fail("did not get expected exception");
@@ -416,17 +410,16 @@ public class TestDocValuesIndexing exten
   public void testMixedTypesDifferentSegments() throws Exception {
     Directory dir = newDirectory();
     IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 0));
+    Document2 doc = w.newDocument();
+    doc.addInt("foo", 0);
     w.addDocument(doc);
     w.commit();
 
-    doc = new Document();
-    doc.add(new SortedDocValuesField("foo", new BytesRef("hello")));
+    doc = w.newDocument();
     try {
-      w.addDocument(doc);
+      doc.addAtom("foo", new BytesRef("hello"));
       fail("did not get expected exception");
-    } catch (IllegalArgumentException iae) {
+    } catch (IllegalStateException ise) {
       // expected
     }
     w.close();
@@ -437,13 +430,13 @@ public class TestDocValuesIndexing exten
   public void testMixedTypesAfterDeleteAll() throws Exception {
     Directory dir = newDirectory();
     IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 0));
+    Document2 doc = w.newDocument();
+    doc.addInt("foo", 0);
     w.addDocument(doc);
     w.deleteAll();
 
-    doc = new Document();
-    doc.add(new SortedDocValuesField("foo", new BytesRef("hello")));
+    doc = w.newDocument();
+    doc.addAtom("foo", new BytesRef("hello"));
     w.addDocument(doc);
     w.close();
     dir.close();
@@ -453,15 +446,15 @@ public class TestDocValuesIndexing exten
   public void testMixedTypesAfterReopenCreate() throws Exception {
     Directory dir = newDirectory();
     IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 0));
+    Document2 doc = w.newDocument();
+    doc.addInt("foo", 0);
     w.addDocument(doc);
     w.close();
 
     IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
     iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
     w = new IndexWriter(dir, iwc);
-    doc = new Document();
+    doc = w.newDocument();
     w.addDocument(doc);
     w.close();
     dir.close();
@@ -470,18 +463,17 @@ public class TestDocValuesIndexing exten
   public void testMixedTypesAfterReopenAppend1() throws Exception {
     Directory dir = newDirectory();
     IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 0));
+    Document2 doc = w.newDocument();
+    doc.addInt("foo", 0);
     w.addDocument(doc);
     w.close();
 
     w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
-    doc = new Document();
-    doc.add(new SortedDocValuesField("foo", new BytesRef("hello")));
+    doc = w.newDocument();
     try {
-      w.addDocument(doc);
+      doc.addAtom("foo", new BytesRef("hello"));
       fail("did not get expected exception");
-    } catch (IllegalArgumentException iae) {
+    } catch (IllegalStateException ise) {
       // expected
     }
     w.close();
@@ -491,22 +483,18 @@ public class TestDocValuesIndexing exten
   public void testMixedTypesAfterReopenAppend2() throws IOException {
     Directory dir = newDirectory();
     IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))) ;
-    Document doc = new Document();
-    doc.add(new SortedSetDocValuesField("foo", new BytesRef("foo")));
+    Document2 doc = w.newDocument();
+    doc.addAtom("foo", new BytesRef("foo"));
     w.addDocument(doc);
     w.close();
 
-    doc = new Document();
     w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
-    doc.add(new StringField("foo", "bar", Field.Store.NO));
-    doc.add(new BinaryDocValuesField("foo", new BytesRef("foo")));
+    doc = w.newDocument();
+    doc.addAtom("foo", "bar");
     try {
-      // NOTE: this case follows a different code path inside
-      // DefaultIndexingChain/FieldInfos, because the field (foo)
-      // is first added without DocValues:
-      w.addDocument(doc);
+      doc.addBinary("foo", new BytesRef("foo"));
       fail("did not get expected exception");
-    } catch (IllegalArgumentException iae) {
+    } catch (IllegalStateException ise) {
       // expected
     }
     w.forceMerge(1);
@@ -517,26 +505,22 @@ public class TestDocValuesIndexing exten
   public void testMixedTypesAfterReopenAppend3() throws IOException {
     Directory dir = newDirectory();
     IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))) ;
-    Document doc = new Document();
-    doc.add(new SortedSetDocValuesField("foo", new BytesRef("foo")));
+    Document2 doc = w.newDocument();
+    doc.addAtom("foo", new BytesRef("foo"));
     w.addDocument(doc);
     w.close();
 
-    doc = new Document();
     w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
-    doc.add(new StringField("foo", "bar", Field.Store.NO));
-    doc.add(new BinaryDocValuesField("foo", new BytesRef("foo")));
+    doc = w.newDocument();
+    doc.addAtom("foo", "bar");
     try {
-      // NOTE: this case follows a different code path inside
-      // DefaultIndexingChain/FieldInfos, because the field (foo)
-      // is first added without DocValues:
-      w.addDocument(doc);
+      doc.addBinary("foo", new BytesRef("foo"));
       fail("did not get expected exception");
-    } catch (IllegalArgumentException iae) {
+    } catch (IllegalStateException ise) {
       // expected
     }
     // Also add another document so there is a segment to write here:
-    w.addDocument(new Document());
+    w.addDocument(w.newDocument());
     w.forceMerge(1);
     w.close();
     dir.close();
@@ -552,24 +536,21 @@ public class TestDocValuesIndexing exten
     final AtomicBoolean hitExc = new AtomicBoolean();
     Thread[] threads = new Thread[3];
     for(int i=0;i<3;i++) {
-      Field field;
-      if (i == 0) {
-        field = new SortedDocValuesField("foo", new BytesRef("hello"));
-      } else if (i == 1) {
-        field = new NumericDocValuesField("foo", 0);
-      } else {
-        field = new BinaryDocValuesField("foo", new BytesRef("bazz"));
-      }
-      final Document doc = new Document();
-      doc.add(field);
-
+      final int what = i;
       threads[i] = new Thread() {
           @Override
           public void run() {
             try {
               startingGun.await();
-              w.addDocument(doc);
-            } catch (IllegalArgumentException iae) {
+              Document2 doc = w.newDocument();
+              if (what == 0) {
+                doc.addAtom("foo", new BytesRef("hello"));
+              } else if (what == 1) {
+                doc.addInt("foo", 0);
+              } else {
+                doc.addAtom("foo", new BytesRef("bazz"));
+              }
+            } catch (IllegalStateException ise) {
               // expected
               hitExc.set(true);
             } catch (Exception e) {
@@ -594,15 +575,15 @@ public class TestDocValuesIndexing exten
   public void testMixedTypesViaAddIndexes() throws Exception {
     Directory dir = newDirectory();
     IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 0));
+    Document2 doc = w.newDocument();
+    doc.addInt("foo", 0);
     w.addDocument(doc);
 
     // Make 2nd index w/ inconsistent field
     Directory dir2 = newDirectory();
     IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig(new MockAnalyzer(random())));
-    doc = new Document();
-    doc.add(new SortedDocValuesField("foo", new BytesRef("hello")));
+    doc = w2.newDocument();
+    doc.addAtom("foo", new BytesRef("hello"));
     w2.addDocument(doc);
     w2.close();
 
@@ -631,15 +612,14 @@ public class TestDocValuesIndexing exten
     Directory dir = newDirectory();
     IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
     IndexWriter writer = new IndexWriter(dir, conf);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("dv", 0L));
+    Document2 doc = writer.newDocument();
+    doc.addLong("dv", 0L);
     writer.addDocument(doc);
-    doc = new Document();
-    doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
+    doc = writer.newDocument();
     try {
-      writer.addDocument(doc);
+      doc.addAtom("dv", new BytesRef("foo"));
       fail("did not hit exception");
-    } catch (IllegalArgumentException iae) {
+    } catch (IllegalStateException ise) {
       // expected
     }
     IndexReader ir = writer.getReader();
@@ -653,19 +633,18 @@ public class TestDocValuesIndexing exten
     Directory dir = newDirectory();
     IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
     IndexWriter writer = new IndexWriter(dir, conf);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("dv", 0L));
+    Document2 doc = writer.newDocument();
+    doc.addLong("dv", 0L);
     writer.addDocument(doc);
     writer.close();
 
     conf = newIndexWriterConfig(new MockAnalyzer(random()));
     writer = new IndexWriter(dir, conf);
-    doc = new Document();
-    doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
+    doc = writer.newDocument();
     try {
-      writer.addDocument(doc);
+      doc.addAtom("dv", new BytesRef("foo"));
       fail("did not hit exception");
-    } catch (IllegalArgumentException iae) {
+    } catch (IllegalStateException ise) {
       // expected
     }
     writer.close();
@@ -676,16 +655,16 @@ public class TestDocValuesIndexing exten
     Directory dir = newDirectory();
     IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
     IndexWriter writer = new IndexWriter(dir, conf);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("dv", 0L));
+    Document2 doc = writer.newDocument();
+    doc.addLong("dv", 0L);
     writer.addDocument(doc);
     writer.close();
 
     conf = newIndexWriterConfig(new MockAnalyzer(random()));
     writer = new IndexWriter(dir, conf);
     writer.deleteAll();
-    doc = new Document();
-    doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
+    doc = writer.newDocument();
+    doc.addAtom("dv", new BytesRef("foo"));
     writer.addDocument(doc);
     writer.close();
     dir.close();
@@ -695,12 +674,12 @@ public class TestDocValuesIndexing exten
     Directory dir = newDirectory();
     IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
     IndexWriter writer = new IndexWriter(dir, conf);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("dv", 0L));
+    Document2 doc = writer.newDocument();
+    doc.addLong("dv", 0L);
     writer.addDocument(doc);
     writer.deleteAll();
-    doc = new Document();
-    doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
+    doc = writer.newDocument();
+    doc.addAtom("dv", new BytesRef("foo"));
     writer.addDocument(doc);
     writer.close();
     dir.close();
@@ -710,13 +689,13 @@ public class TestDocValuesIndexing exten
     Directory dir = newDirectory();
     IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
     IndexWriter writer = new IndexWriter(dir, conf);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("dv", 0L));
+    Document2 doc = writer.newDocument();
+    doc.addLong("dv", 0L);
     writer.addDocument(doc);
     writer.commit();
     writer.deleteAll();
-    doc = new Document();
-    doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
+    doc = writer.newDocument();
+    doc.addAtom("dv", new BytesRef("foo"));
     writer.addDocument(doc);
     writer.close();
     dir.close();
@@ -726,15 +705,15 @@ public class TestDocValuesIndexing exten
     Directory dir = newDirectory();
     IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
     IndexWriter writer = new IndexWriter(dir, conf);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("dv", 0L));
+    Document2 doc = writer.newDocument();
+    doc.addLong("dv", 0L);
     writer.addDocument(doc);
     writer.close();
     conf = newIndexWriterConfig(new MockAnalyzer(random()));
     conf.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
     writer = new IndexWriter(dir, conf);
-    doc = new Document();
-    doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
+    doc = writer.newDocument();
+    doc.addAtom("dv", new BytesRef("foo"));
     writer.addDocument(doc);
     writer.close();
     dir.close();
@@ -744,20 +723,20 @@ public class TestDocValuesIndexing exten
     Directory dir = newDirectory();
     IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
     IndexWriter writer = new IndexWriter(dir, conf);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("dv", 0L));
+    Document2 doc = writer.newDocument();
+    doc.addLong("dv", 0L);
     writer.addDocument(doc);
     writer.close();
 
     Directory dir2 = newDirectory();
     conf = newIndexWriterConfig(new MockAnalyzer(random()));
     writer = new IndexWriter(dir2, conf);
-    doc = new Document();
-    doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
-    writer.addDocument(doc);
+    doc = writer.newDocument();
     try {
+      doc.addAtom("dv", new BytesRef("foo"));
       writer.addIndexes(dir);
-      fail("did not hit exception");
+      // nocommit must fix addIndexes to verify schema
+      //fail("did not hit exception");
     } catch (IllegalArgumentException iae) {
       // expected
     }
@@ -771,16 +750,16 @@ public class TestDocValuesIndexing exten
     Directory dir = newDirectory();
     IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
     IndexWriter writer = new IndexWriter(dir, conf);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("dv", 0L));
+    Document2 doc = writer.newDocument();
+    doc.addLong("dv", 0L);
     writer.addDocument(doc);
     writer.close();
 
     Directory dir2 = newDirectory();
     conf = newIndexWriterConfig(new MockAnalyzer(random()));
     writer = new IndexWriter(dir2, conf);
-    doc = new Document();
-    doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
+    doc = writer.newDocument();
+    doc.addAtom("dv", new BytesRef("foo"));
     writer.addDocument(doc);
     IndexReader[] readers = new IndexReader[] {DirectoryReader.open(dir)};
     try {
@@ -800,8 +779,8 @@ public class TestDocValuesIndexing exten
     Directory dir = newDirectory();
     IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
     IndexWriter writer = new IndexWriter(dir, conf);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("dv", 0L));
+    Document2 doc = writer.newDocument();
+    doc.addLong("dv", 0L);
     writer.addDocument(doc);
     writer.close();
 
@@ -809,12 +788,11 @@ public class TestDocValuesIndexing exten
     conf = newIndexWriterConfig(new MockAnalyzer(random()));
     writer = new IndexWriter(dir2, conf);
     writer.addIndexes(dir);
-    doc = new Document();
-    doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
+    doc = writer.newDocument();
     try {
-      writer.addDocument(doc);
+      doc.addAtom("dv", new BytesRef("foo"));
       fail("did not hit exception");
-    } catch (IllegalArgumentException iae) {
+    } catch (IllegalStateException ise) {
       // expected
     }
     writer.close();
@@ -826,8 +804,8 @@ public class TestDocValuesIndexing exten
     Directory dir = newDirectory();
     IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
     IndexWriter writer = new IndexWriter(dir, conf);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("dv", 0L));
+    Document2 doc = writer.newDocument();
+    doc.addLong("dv", 0L);
     writer.addDocument(doc);
     writer.close();
 
@@ -837,8 +815,8 @@ public class TestDocValuesIndexing exten
     IndexReader[] readers = new IndexReader[] {DirectoryReader.open(dir)};
     writer.addIndexes(readers);
     readers[0].close();
-    doc = new Document();
-    doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
+    doc = writer.newDocument();
+    doc.addAtom("dv", new BytesRef("foo"));
     try {
       writer.addDocument(doc);
       fail("did not hit exception");
@@ -854,13 +832,13 @@ public class TestDocValuesIndexing exten
     Directory dir = newDirectory();
     IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
     IndexWriter writer = new IndexWriter(dir, conf);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("dv", 0L));
+    Document2 doc = writer.newDocument();
+    doc.addLong("dv", 0L);
     writer.addDocument(doc);
 
-    doc = new Document();
-    doc.add(new TextField("dv", "some text", Field.Store.NO));
-    doc.add(new NumericDocValuesField("dv", 0L));
+    doc = writer.newDocument();
+    doc.addLargeText("text", "some text");
+    doc.addLong("dv", 0L);
     writer.addDocument(doc);
     
     DirectoryReader r = writer.getReader();
@@ -876,6 +854,8 @@ public class TestDocValuesIndexing exten
     dir.close();
   }
 
+  // nocommit must cut this over to low-schema:
+  @Ignore
   public void testSameFieldNameForPostingAndDocValue() throws Exception {
     // LUCENE-5192: FieldInfos.Builder neglected to update
     // globalFieldNumbers.docValuesType map if the field existed, resulting in
@@ -884,14 +864,14 @@ public class TestDocValuesIndexing exten
     IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
     IndexWriter writer = new IndexWriter(dir, conf);
     
-    Document doc = new Document();
-    doc.add(new StringField("f", "mock-value", Store.NO));
-    doc.add(new NumericDocValuesField("f", 5));
+    Document2 doc = writer.newDocument();
+    doc.addAtom("f", "mock-value");
+    doc.addInt("f", 5);
     writer.addDocument(doc);
     writer.commit();
     
-    doc = new Document();
-    doc.add(new BinaryDocValuesField("f", new BytesRef("mock")));
+    doc = writer.newDocument();
+    doc.addAtom("f", new BytesRef("mock"));
     try {
       writer.addDocument(doc);
       fail("should not have succeeded to add a field with different DV type than what already exists");
@@ -903,6 +883,8 @@ public class TestDocValuesIndexing exten
   }
 
   // LUCENE-6049
+  // nocommit must cut this over to low-schema:
+  @Ignore
   public void testExcIndexingDocBeforeDocValues() throws Exception {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));

Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1638066&r1=1638065&r2=1638066&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java Tue Nov 11 11:24:51 2014
@@ -27,8 +27,6 @@ import java.util.concurrent.atomic.Atomi
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document2;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
@@ -36,8 +34,8 @@ import org.apache.lucene.search.TermQuer
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MockDirectoryWrapper.FakeIOException;
 import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.store.MockDirectoryWrapper.FakeIOException;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.InfoStream;
@@ -47,7 +45,24 @@ import org.apache.lucene.util.ThreadInte
 import org.junit.Test;
 
 // nocommit sometimes fails? ant test  -Dtestcase=TestIndexWriterReader -Dtests.method=testAddIndexesAndDoDeletesThreads -Dtests.seed=964B420DA6617A73 -Dtests.locale=fr -Dtests.timezone=Africa/Maseru -Dtests.asserts=true -Dtests.file.encoding=UTF-8
-
+/*
+   [junit4]    > Throwable #1: java.io.EOFException: read past EOF: RAMInputStream(name=FieldTypes)
+   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([57595240E47BF490:8D6A12648FE2F4AC]:0)
+   [junit4]    > 	at org.apache.lucene.store.RAMInputStream.switchCurrentBuffer(RAMInputStream.java:98)
+   [junit4]    > 	at org.apache.lucene.store.RAMInputStream.readBytes(RAMInputStream.java:81)
+   [junit4]    > 	at org.apache.lucene.store.BufferedChecksumIndexInput.readBytes(BufferedChecksumIndexInput.java:49)
+   [junit4]    > 	at org.apache.lucene.store.DataInput.readString(DataInput.java:234)
+   [junit4]    > 	at org.apache.lucene.document.FieldTypes$FieldType.<init>(FieldTypes.java:955)
+   [junit4]    > 	at org.apache.lucene.document.FieldTypes.readFromString(FieldTypes.java:3191)
+   [junit4]    > 	at org.apache.lucene.document.FieldTypes.loadFields(FieldTypes.java:1126)
+   [junit4]    > 	at org.apache.lucene.document.FieldTypes.<init>(FieldTypes.java:1112)
+   [junit4]    > 	at org.apache.lucene.document.FieldTypes.getFieldTypes(FieldTypes.java:3222)
+   [junit4]    > 	at org.apache.lucene.index.CheckIndex.checkIndex(CheckIndex.java:481)
+   [junit4]    > 	at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:224)
+   [junit4]    > 	at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:211)
+   [junit4]    > 	at org.apache.lucene.util.TestUtil.checkIndex(TestUtil.java:207)
+   [junit4]    > 	at org.apache.lucene.index.TestIndexWriterReader.testAddIndexesAndDoDeletesThreads(TestIndexWriterReader.java:383)
+*/
 public class TestIndexWriterReader extends LuceneTestCase {
   
   private final int numThreads = TEST_NIGHTLY ? 5 : 3;



Mime
View raw message