lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1635898 [2/6] - in /lucene/dev/branches/lucene6005/lucene: analysis/common/src/test/org/apache/lucene/collation/ analysis/icu/src/test/org/apache/lucene/collation/ analysis/uima/src/test/org/apache/lucene/analysis/uima/ backward-codecs/src...
Date Sat, 01 Nov 2014 08:39:30 GMT
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document2.java?rev=1635898&r1=1635897&r2=1635898&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document2.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document2.java Sat Nov  1 08:39:27 2014
@@ -27,11 +27,8 @@ import org.apache.lucene.analysis.Analyz
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.document.FieldTypes.FieldType;
 import org.apache.lucene.index.FieldInfo.DocValuesType;
-import org.apache.lucene.index.IndexDocument;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.IndexableFieldType;
-import org.apache.lucene.index.StorableField;
-import org.apache.lucene.index.StoredDocument;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.FilterIterator;
 
@@ -41,14 +38,16 @@ import org.apache.lucene.util.FilterIter
  *  that also tracks field properties implied by the
  *  fields being added. */
 
-public class Document2 implements IndexDocument {
+public class Document2 implements Iterable<IndexableField> {
 
   private static final float DEFAULT_BOOST = 1.0f;
 
   private final FieldTypes fieldTypes;
   private final List<FieldValue> fields = new ArrayList<>();
+  private final boolean changeSchema;
 
-  private class FieldValue implements IndexableField, StorableField {
+  // nocommit make private again and somehow deal w/ generics
+  public class FieldValue implements IndexableField {
     final String fieldName;
     final Object value;
     final float boost;
@@ -62,7 +61,21 @@ public class Document2 implements IndexD
       this.fieldName = fieldName;
       this.value = value;
       this.boost = boost;
-      this.fieldType = fieldTypes.getFieldType(fieldName);
+      FieldType curFieldType;
+      if (changeSchema == false) {
+        if (fieldTypes != null) {
+          try {
+            curFieldType = fieldTypes.getFieldType(fieldName);
+          } catch (IllegalArgumentException iae) {
+            curFieldType = null;
+          }
+        } else {
+          curFieldType = null;
+        }
+      } else {
+        curFieldType = fieldTypes.getFieldType(fieldName);
+      }
+      this.fieldType = curFieldType;
     }
     
     @Override
@@ -84,7 +97,8 @@ public class Document2 implements IndexD
       BinaryTokenStream bts;
       if (reuse != null) {
         if (reuse instanceof BinaryTokenStream == false) {
-          FieldTypes.illegalState(fieldName, "should have had BinaryTokenStream for reuse, but got " + reuse);
+          // BUG
+          FieldTypes.illegalState(fieldName, "should have received BinaryTokenStream for reuse, but got " + reuse);
         }
         bts = (BinaryTokenStream) reuse;
       } else {
@@ -94,6 +108,21 @@ public class Document2 implements IndexD
       return bts;
     }
 
+    private TokenStream getReusedStringTokenStream(String value, TokenStream reuse) {
+      StringTokenStream sts;
+      if (reuse != null) {
+        if (reuse instanceof StringTokenStream == false) {
+          // BUG
+          FieldTypes.illegalState(fieldName, "should have received StringTokenStream for reuse, but got " + reuse);
+        }
+        sts = (StringTokenStream) reuse;
+      } else {
+        sts = new StringTokenStream();
+      }
+      sts.setValue(value);
+      return sts;
+    }
+
     @Override
     public TokenStream tokenStream(Analyzer analyzerIn, TokenStream reuse) throws IOException {
       Analyzer analyzer = fieldTypes.getIndexAnalyzer();
@@ -114,17 +143,7 @@ public class Document2 implements IndexD
         return getReusedBinaryTokenStream(longToBytes(Double.doubleToLongBits(((Number) value).doubleValue())), reuse);
       case ATOM:
         if (value instanceof String) {
-          StringTokenStream sts;
-          if (reuse != null) {
-            if (reuse instanceof StringTokenStream == false) {
-              FieldTypes.illegalState(fieldName, "should have had StringTokenStream for reuse, but got " + reuse);
-            }
-            sts = (StringTokenStream) reuse;
-          } else {
-            sts = new StringTokenStream();
-          }
-          sts.setValue((String) value);
-          return sts;
+          return getReusedStringTokenStream((String) value, reuse);
         } else {
           assert value instanceof BytesRef;
           return getReusedBinaryTokenStream((BytesRef) value, reuse);
@@ -154,6 +173,13 @@ public class Document2 implements IndexD
           return analyzer.tokenStream(name(), (String) value);
         }
 
+      case BOOLEAN:
+        byte[] token = new byte[1];
+        if (value == Boolean.TRUE) {
+          token[0] = 1;
+        }
+        return getReusedBinaryTokenStream(new BytesRef(token), reuse);
+
       default:
         FieldTypes.illegalState(fieldName, "valueType=" + fieldType.valueType + " cannot be indexed");
 
@@ -164,12 +190,25 @@ public class Document2 implements IndexD
 
     @Override
     public Number numericValue() {
+      if (fieldType == null) {
+        if (value instanceof Number) {
+          return (Number) value;
+        } else {
+          return null;
+        }
+      }
       switch (fieldType.valueType) {
       case INT:
       case LONG:
       case FLOAT:
       case DOUBLE:
         return (Number) value;
+      case BOOLEAN:
+        if (value == Boolean.TRUE) {
+          return Integer.valueOf(1);
+        } else {
+          return Integer.valueOf(0);
+        }
       default:
         return null;
       }
@@ -177,6 +216,9 @@ public class Document2 implements IndexD
 
     @Override
     public Number numericDocValue() {
+      if (fieldType == null) {
+        return null;
+      }
       switch (fieldType.valueType) {
       case INT:
         return (Number) value;
@@ -186,6 +228,12 @@ public class Document2 implements IndexD
         return Integer.valueOf(Float.floatToIntBits((Float) value));
       case DOUBLE:
         return Long.valueOf(Double.doubleToLongBits((Double) value));
+      case BOOLEAN:
+        if (value == Boolean.TRUE) {
+          return Integer.valueOf(1);
+        } else {
+          return Integer.valueOf(0);
+        }
       default:
         return null;
       }
@@ -193,6 +241,14 @@ public class Document2 implements IndexD
 
     @Override
     public String stringValue() {
+      if (fieldType == null) {
+        if (value instanceof String) {
+          return (String) value;
+        } else {
+          return null;
+        }
+      }
+
       switch (fieldType.valueType) {
       case SHORT_TEXT:
       case TEXT:
@@ -214,7 +270,21 @@ public class Document2 implements IndexD
 
     @Override
     public BytesRef binaryValue() {
-      if (value instanceof BytesRef) {
+      if (fieldType == null) {
+        if (value instanceof BytesRef) {
+          return (BytesRef) value;
+        } else {
+          return null;
+        }
+      }
+
+      if (fieldType.valueType == FieldTypes.ValueType.BOOLEAN) {
+        byte[] bytes = new byte[1];
+        if (value == Boolean.TRUE) {
+          bytes[0] = 1;
+        }
+        return new BytesRef(bytes);
+      } else if (value instanceof BytesRef) {
         return (BytesRef) value;
       } else {
         return null;
@@ -235,83 +305,102 @@ public class Document2 implements IndexD
   }
 
   public Document2(FieldTypes fieldTypes) {
+    this(fieldTypes, true);
+  }
+
+  Document2(FieldTypes fieldTypes, boolean changeSchema) {
     this.fieldTypes = fieldTypes;
+    this.changeSchema = changeSchema;
   }
 
   @Override
-  public Iterable<IndexableField> indexableFields() {
-    return new Iterable<IndexableField>() {
-      @Override
-      public Iterator<IndexableField> iterator() {
-        return Document2.this.indexedFieldsIterator();
+  public Iterator<IndexableField> iterator() {
+    // nocommit how to fix generics here so I can just return fields.iterator?
+    //return fields.iterator();
+    ArrayList<IndexableField> l = new ArrayList<>();
+    l.addAll(fields);
+    return l.iterator();
+  }
+
+  public List<FieldValue> getFieldValues() {
+    return fields;
+  }
+
+  public List<IndexableField> getFields() {
+    List<IndexableField> result = new ArrayList<>();
+    result.addAll(fields);
+    return result;
+  }
+
+  public IndexableField getField(String name) {
+    List<IndexableField> result = new ArrayList<>();
+    for (FieldValue field : fields) {
+      if (field.name().equals(name)) {
+        return field;
       }
-    };
+    }
+    return null;
   }
 
-  @Override
-  public Iterable<StorableField> storableFields() {
-    return new Iterable<StorableField>() {
-      @Override
-      public Iterator<StorableField> iterator() {
-        return Document2.this.storedFieldsIterator();
-      }
-    };
-  }
-
-  private Iterator<StorableField> storedFieldsIterator() {
-    return new FilterIterator<StorableField,FieldValue>(fields.iterator()) {
-      @Override
-      protected boolean predicateFunction(FieldValue field) {
-        return field.fieldType.stored() || field.fieldType.docValueType() != null;
-      }
-    };
-  }
-  
-  private Iterator<IndexableField> indexedFieldsIterator() {
-    return new FilterIterator<IndexableField,FieldValue>(fields.iterator()) {
-      @Override
-      protected boolean predicateFunction(FieldValue field) {
-        return field.fieldType.indexOptions() != null;
+  public List<IndexableField> getFields(String name) {
+    List<IndexableField> result = new ArrayList<>();
+    for (FieldValue field : fields) {
+      if (field.name().equals(name)) {
+        result.add(field);
       }
-    };
+    }
+
+    return result;
   }
 
   /** E.g. a "country" field.  Default: indexes this value as a single token, and disables norms and freqs, and also enables sorting (indexes doc values) and stores it. */
   public void addAtom(String fieldName, String value) {
-    fieldTypes.recordValueType(fieldName, FieldTypes.ValueType.ATOM);
+    if (changeSchema) {
+      fieldTypes.recordValueType(fieldName, FieldTypes.ValueType.ATOM);
+    }
     fields.add(new FieldValue(fieldName, value));
   }
 
   /** E.g. an "id" (primary key) field.  Default: indexes this value as a single token, and disables norms and freqs. */
   public void addAtom(String fieldName, BytesRef value) {
-    fieldTypes.recordValueType(fieldName, FieldTypes.ValueType.ATOM);
+    if (changeSchema) {
+      fieldTypes.recordValueType(fieldName, FieldTypes.ValueType.ATOM);
+    }
     fields.add(new FieldValue(fieldName, value));
   }
 
   /** E.g. a "title" field.  Default: indexes this value as multiple tokens from analyzer, and disables norms and freqs, and also enables
    *  sorting (indexes sorted doc values). */
   public void addShortText(String fieldName, String value) {
-    fieldTypes.recordValueType(fieldName, FieldTypes.ValueType.SHORT_TEXT);
+    if (changeSchema) {
+      fieldTypes.recordValueType(fieldName, FieldTypes.ValueType.SHORT_TEXT);
+    }
     fields.add(new FieldValue(fieldName, value));
   }
 
   /** Default: store this value. */
   public void addStored(String fieldName, BytesRef value) {
     // nocommit akward we inferred binary here?
-    fieldTypes.recordValueType(fieldName, FieldTypes.ValueType.BINARY);
+    if (changeSchema) {
+      fieldTypes.recordValueType(fieldName, FieldTypes.ValueType.BINARY);
+    }
     fields.add(new FieldValue(fieldName, value));
   }
 
   /** Default: store this value. */
   public void addBinary(String fieldName, BytesRef value) {
-    fieldTypes.recordValueType(fieldName, FieldTypes.ValueType.BINARY);
+    if (changeSchema) {
+      fieldTypes.recordValueType(fieldName, FieldTypes.ValueType.BINARY);
+    }
     fields.add(new FieldValue(fieldName, value));
   }
 
   /** Default: store this value. */
   public void addStored(String fieldName, String value) {
     // nocommit akward we inferred large_text here?
-    fieldTypes.recordLargeTextType(fieldName, true, false);
+    if (changeSchema) {
+      fieldTypes.recordLargeTextType(fieldName, true, false);
+    }
     fields.add(new FieldValue(fieldName, value));
   }
 
@@ -322,7 +411,9 @@ public class Document2 implements IndexD
 
   /** E.g. a "body" field.  Default: indexes this value as multiple tokens from analyzer and stores the value. */
   public void addLargeText(String fieldName, String value, float boost) {
-    fieldTypes.recordLargeTextType(fieldName, true, true);
+    if (changeSchema) {
+      fieldTypes.recordLargeTextType(fieldName, true, true);
+    }
     fields.add(new FieldValue(fieldName, value, boost));
   }
 
@@ -333,7 +424,9 @@ public class Document2 implements IndexD
 
   /** E.g. a "body" field.  Default: indexes this value as multiple tokens from analyzer. */
   public void addLargeText(String fieldName, TokenStream value, float boost) {
-    fieldTypes.recordLargeTextType(fieldName, false, true);
+    if (changeSchema) {
+      fieldTypes.recordLargeTextType(fieldName, false, true);
+    }
     fields.add(new FieldValue(fieldName, value, boost));
   }
 
@@ -344,7 +437,9 @@ public class Document2 implements IndexD
 
   /** E.g. a "body" field.  Default: indexes this value as multiple tokens from analyzer. */
   public void addLargeText(String fieldName, Reader value, float boost) {
-    fieldTypes.recordLargeTextType(fieldName, false, true);
+    if (changeSchema) {
+      fieldTypes.recordLargeTextType(fieldName, false, true);
+    }
     fields.add(new FieldValue(fieldName, value, boost));
   }
 
@@ -352,46 +447,70 @@ public class Document2 implements IndexD
 
   /** Default: support for range filtering/querying and sorting (using numeric doc values). */
   public void addInt(String fieldName, int value) {
-    fieldTypes.recordValueType(fieldName, FieldTypes.ValueType.INT);
+    if (changeSchema) {
+      fieldTypes.recordValueType(fieldName, FieldTypes.ValueType.INT);
+    }
     fields.add(new FieldValue(fieldName, Integer.valueOf(value)));
   }
 
   /** Default: support for range filtering/querying and sorting (using numeric doc values). */
   public void addFloat(String fieldName, float value) {
-    fieldTypes.recordValueType(fieldName, FieldTypes.ValueType.FLOAT);
+    if (changeSchema) {
+      fieldTypes.recordValueType(fieldName, FieldTypes.ValueType.FLOAT);
+    }
     fields.add(new FieldValue(fieldName, Float.valueOf(value)));
   }
 
   /** Default: support for range filtering/querying and sorting (using numeric doc values). */
   public void addLong(String fieldName, long value) {
-    fieldTypes.recordValueType(fieldName, FieldTypes.ValueType.LONG);
+    if (changeSchema) {
+      fieldTypes.recordValueType(fieldName, FieldTypes.ValueType.LONG);
+    }
     fields.add(new FieldValue(fieldName, Long.valueOf(value)));
   }
 
   /** Default: support for range filtering/querying and sorting (using numeric doc values). */
   public void addDouble(String fieldName, double value) {
-    fieldTypes.recordValueType(fieldName, FieldTypes.ValueType.DOUBLE);
+    if (changeSchema) {
+      fieldTypes.recordValueType(fieldName, FieldTypes.ValueType.DOUBLE);
+    }
     fields.add(new FieldValue(fieldName, Double.valueOf(value)));
   }
 
-  public Object get(String fieldName) {
-    for(FieldValue fieldValue : fields) {
-      if (fieldValue.fieldName.equals(fieldName)) {
-        return fieldValue.value;
-      }
+  public void addBoolean(String fieldName, boolean value) {
+    if (changeSchema) {
+      fieldTypes.recordValueType(fieldName, FieldTypes.ValueType.BOOLEAN);
     }
+    fields.add(new FieldValue(fieldName, Boolean.valueOf(value)));
+  }
 
-    return null;
+  static {
+    // nocommit is there a cleaner/general way to detect missing enum value in case switch statically?  must we use ecj?
+    assert FieldTypes.ValueType.values().length == 9: "missing case for switch statement below";
   }
 
   /** Note: this FieldTypes must already know about all the fields in the incoming doc. */
-  public void addAll(StoredDocument storedDoc) {
-    for (StorableField field : storedDoc.getFields()) {
+  public void addAll(Document2 other) {
+    // nocommit should we insist other.fieldTypes == this.fieldTypes?  or, that they are "congruent"?
+    for (FieldValue field : other.fields) {
       String fieldName = field.name();
       FieldType fieldType = fieldTypes.getFieldType(fieldName);
       // nocommit need more checking here ... but then, we should somehow remove StoredDocument, sicne w/ FieldTypes we can now fully
       // reconstruct (as long as all fields were stored) what was indexed:
       switch (fieldType.valueType) {
+      case TEXT:
+        addLargeText(fieldName, field.stringValue());
+        break;
+      case SHORT_TEXT:
+        addShortText(fieldName, field.stringValue());
+        break;
+      case ATOM:
+        if (field.value instanceof BytesRef) {
+          addAtom(fieldName, (BytesRef) field.value);
+        } else {
+          addAtom(fieldName, (String) field.value);
+        }
+        break;
       case INT:
         addInt(fieldName, field.numericValue().intValue());
         break;
@@ -404,15 +523,15 @@ public class Document2 implements IndexD
       case DOUBLE:
         addDouble(fieldName, field.numericValue().doubleValue());
         break;
-      case TEXT:
-        addLargeText(fieldName, field.stringValue());
-        break;
-      case SHORT_TEXT:
-        addShortText(fieldName, field.stringValue());
-        break;
       case BINARY:
         addStored(fieldName, field.binaryValue());
         break;
+      case BOOLEAN:
+        addBoolean(fieldName, ((Boolean) field.value).booleanValue());
+        break;
+      default:
+        // BUG:
+        throw new AssertionError("missing valueType=" + fieldType.valueType + " in switch");
       }
     }
   }
@@ -454,4 +573,88 @@ public class Document2 implements IndexD
     }
     return token;
   }
+
+  public Boolean getBoolean(String fieldName) {
+    // nocommit can we assert this is a known field and that its type is boolean...?
+    for(FieldValue fieldValue : fields) {
+      if (fieldValue.fieldName.equals(fieldName)) {
+        return (Boolean) fieldValue.value;
+      }
+    }
+
+    return null;
+  }
+
+  public String getString(String fieldName) {
+    // nocommit can we assert this is a known field and that its type is text/short_text...?
+    for(FieldValue fieldValue : fields) {
+      if (fieldValue.fieldName.equals(fieldName)) {
+        return fieldValue.value.toString();
+      }
+    }
+
+    return null;
+  }
+
+  public String[] getStrings(String fieldName) {
+    // nocommit can we assert this is a known field and that its type is text/short_text...?
+    List<String> values = new ArrayList<>();
+    for(FieldValue fieldValue : fields) {
+      if (fieldValue.fieldName.equals(fieldName)) {
+        values.add(fieldValue.value.toString());
+      }
+    }
+
+    return values.toArray(new String[values.size()]);
+  }
+
+  public BytesRef getBinary(String fieldName) {
+    // nocommit can we assert this is a known field and that its type is text/short_text...?
+    for(FieldValue fieldValue : fields) {
+      if (fieldValue.fieldName.equals(fieldName)) {
+        return (BytesRef) fieldValue.value;
+      }
+    }
+
+    return null;
+  }
+
+  public Integer getInt(String fieldName) {
+    // nocommit can we assert this is a known field and that its type is text/short_text...?
+    for(FieldValue fieldValue : fields) {
+      if (fieldValue.fieldName.equals(fieldName)) {
+        return (Integer) fieldValue.value;
+      }
+    }
+
+    return null;
+  }
+
+  public Object get(String fieldName) {
+    for(FieldValue fieldValue : fields) {
+      if (fieldValue.fieldName.equals(fieldName)) {
+        return fieldValue.value;
+      }
+    }
+
+    return null;
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder b = new StringBuilder();
+    for(FieldValue fieldValue : fields) {
+      b.append("\n  ");
+      b.append(fieldValue.fieldName);
+      b.append(": ");
+      String s = fieldValue.value.toString();
+      if (s.length() > 20) {
+        b.append(s.substring(0, 20));
+        b.append("...");
+      } else {
+        b.append(s);
+      }
+    }
+    return b.toString();
+  }
 }

Added: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document2StoredFieldVisitor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document2StoredFieldVisitor.java?rev=1635898&view=auto
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document2StoredFieldVisitor.java (added)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Document2StoredFieldVisitor.java Sat Nov  1 08:39:27 2014
@@ -0,0 +1,132 @@
+package org.apache.lucene.document;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.util.BytesRef;
+
+/** A {@link StoredFieldVisitor} that creates a {@link
+ *  Document} containing all stored fields, or only specific
+ *  requested fields provided to {@link #DocumentStoredFieldVisitor(Set)}.
+ *  <p>
+ *  This is used by {@link IndexReader#document(int)} to load a
+ *  document.
+ *
+ * @lucene.experimental */
+
+public class Document2StoredFieldVisitor extends StoredFieldVisitor {
+  private final Document2 doc;
+  private final Set<String> fieldsToAdd;
+  private final FieldTypes fieldTypes;
+
+  /** 
+   * Load only fields named in the provided <code>Set&lt;String&gt;</code>. 
+   * @param fieldsToAdd Set of fields to load, or <code>null</code> (all fields).
+   */
+  public Document2StoredFieldVisitor(FieldTypes fieldTypes, Set<String> fieldsToAdd) {
+    doc = new Document2(fieldTypes, false);
+    this.fieldTypes = fieldTypes;
+    this.fieldsToAdd = fieldsToAdd;
+  }
+
+  /** Load only fields named in the provided fields. */
+  public Document2StoredFieldVisitor(FieldTypes fieldTypes, String... fields) {
+    doc = new Document2(fieldTypes, false);
+    this.fieldTypes = fieldTypes;
+    fieldsToAdd = new HashSet<>(fields.length);
+    for(String field : fields) {
+      fieldsToAdd.add(field);
+    }
+  }
+
+  /** Load all stored fields. */
+  public Document2StoredFieldVisitor(FieldTypes fieldTypes) {
+    doc = new Document2(fieldTypes, false);
+    this.fieldTypes = fieldTypes;
+    this.fieldsToAdd = null;
+  }
+
+  private FieldTypes.FieldType getFieldType(String fieldName) {
+    if (fieldTypes != null) {
+      try {
+        return fieldTypes.getFieldType(fieldName);
+      } catch (IllegalArgumentException iae) {
+      }
+    }
+    return null;
+  }
+
+  @Override
+  public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
+    doc.addBinary(fieldInfo.name, new BytesRef(value));
+  }
+
+  @Override
+  public void stringField(FieldInfo fieldInfo, String value) throws IOException {
+    doc.addLargeText(fieldInfo.name, value);
+  }
+
+  // nocommit it's odd that this API differentiates which number it was, vs doc values which always uses long:
+  @Override
+  public void intField(FieldInfo fieldInfo, int value) {
+    FieldTypes.FieldType fieldType = getFieldType(fieldInfo.name);
+    if (fieldType != null && fieldType.valueType == FieldTypes.ValueType.BOOLEAN) {
+      assert value == 0 || value == 1;
+      doc.addBoolean(fieldInfo.name, Boolean.valueOf(value == 1));
+    } else {
+      doc.addInt(fieldInfo.name, value);
+    }
+  }
+
+  @Override
+  public void longField(FieldInfo fieldInfo, long value) {
+    doc.addLong(fieldInfo.name, value);
+  }
+
+  @Override
+  public void floatField(FieldInfo fieldInfo, float value) {
+    doc.addFloat(fieldInfo.name, value);
+  }
+
+  @Override
+  public void doubleField(FieldInfo fieldInfo, double value) {
+    doc.addDouble(fieldInfo.name, value);
+  }
+
+  @Override
+  public Status needsField(FieldInfo fieldInfo) throws IOException {
+    return fieldsToAdd == null || fieldsToAdd.contains(fieldInfo.name) ? Status.YES : Status.NO;
+  }
+
+  /**
+   * Retrieve the visited document.
+   * @return {@link Document2} populated with stored fields. Note that only
+   *         the stored information in the field instances is valid,
+   *         data such as indexing options, term vector options,
+   *         etc is not set.
+   */
+  public Document2 getDocument() {
+    return doc;
+  }
+}

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Field.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Field.java?rev=1635898&r1=1635897&r2=1635898&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Field.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/Field.java Sat Nov  1 08:39:27 2014
@@ -29,7 +29,6 @@ import org.apache.lucene.document.FieldT
 import org.apache.lucene.index.IndexWriter; // javadocs
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.IndexableFieldType;
-import org.apache.lucene.index.StorableField;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.index.FieldInvertState; // javadocs
 
@@ -53,7 +52,7 @@ import org.apache.lucene.index.FieldInve
  * Field it is used in.  It is strongly recommended that no
  * changes be made after Field instantiation.
  */
-public class Field implements IndexableField, StorableField {
+public class Field implements IndexableField {
 
   /**
    * Field's type

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java?rev=1635898&r1=1635897&r2=1635898&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/FieldTypes.java Sat Nov  1 08:39:27 2014
@@ -97,6 +97,7 @@ import org.apache.lucene.util.Version;
 //   - no more field reuse right?
 
 // Lucene's secret schemas
+//   StoredFieldsVisitor
 //   FieldInfos/GlobalFieldNumbers
 //   SortField.type
 //   DocValuesType
@@ -106,6 +107,12 @@ import org.apache.lucene.util.Version;
 //   PerFieldAnalyzerWrapper
 //   oal.document
 
+// nocommit optimize term range query when it's really "matches all docs"
+
+// nocommit add test: make sure you can suddenly turn on DV if they were off at first
+
+// nocommit each field type should record Version when it was added
+
 // nocommit but how can we randomized IWC for tests?
 
 // nocommit maybe we should filter out our key from commit user data?  scary if apps can ... mess with it accidentally?  or just store it
@@ -167,20 +174,16 @@ import org.apache.lucene.util.Version;
 
 // nocommit how to randomize IWC?  RIW?
 
-// nocommit maybe we need change IW's setCommitData API to be "add/remove key/value from commit data"?
-
 // nocommit unique/primary key ?
 
 // nocommit must document / make sugar for creating IndexSearcher w/ sim from this class
 
 // nocommit fix all change methods to call validate / rollback
 
-// nocommit boolean, float16?
+// nocommit float16?
 
 // nocommit can we move multi-field-ness out of IW?  so IW only gets a single instance of each field
 
-// nocommit if you open a writer, just set up schema, close it, it won't save?
-
 /** Records how each field is indexed, stored, etc.  This class persists
  *  its state using {@link IndexWriter#setCommitData}, using the
  *  {@link FieldTypes#FIELD_PROPERTIES_KEY} key. */
@@ -199,8 +202,9 @@ public class FieldTypes {
     LONG,
     DOUBLE,
     BINARY, // nocommit rename to bytes?
+    BOOLEAN,
     // nocommit primary_key?
-    // nocommit boolean
+    // nocommit inet addr?
   }
 
   private final boolean readOnly;
@@ -217,8 +221,13 @@ public class FieldTypes {
 
   private final Similarity defaultSimilarity;
 
+  /** Used only in memory to record when something changed. */
+  private long changeCount;
+
   // nocommit nested docs?
 
+  // nocommit required?
+
   /** Just like current oal.document.FieldType, except for each setting it can also record "not-yet-set". */
   static class FieldType implements IndexableFieldType {
     private final String name;
@@ -286,6 +295,9 @@ public class FieldTypes {
         case FLOAT:
         case LONG:
         case DOUBLE:
+          if (highlighted == Boolean.TRUE) {
+            illegalState(name, "type " + valueType + " cannot highlight");
+          }
           if (indexAnalyzer != null) {
             illegalState(name, "type " + valueType + " cannot have an indexAnalyzer");
           }
@@ -298,6 +310,9 @@ public class FieldTypes {
           if (indexOptions != null && indexOptions.compareTo(IndexOptions.DOCS_ONLY) > 0) {
             illegalState(name, "type " + valueType + " cannot use indexOptions > DOCS_ONLY (got indexOptions " + indexOptions + ")");
           }
+          if (indexNorms == Boolean.TRUE) {
+            illegalState(name, "type " + valueType + " cannot index norms");
+          }
           break;
         case TEXT:
           if (sortable == Boolean.TRUE) {
@@ -319,6 +334,9 @@ public class FieldTypes {
           }
           break;
         case BINARY:
+          if (highlighted == Boolean.TRUE) {
+            illegalState(name, "type " + valueType + " cannot highlight");
+          }
           if (indexAnalyzer != null) {
             illegalState(name, "type " + valueType + " cannot have an indexAnalyzer");
           }
@@ -326,7 +344,7 @@ public class FieldTypes {
             illegalState(name, "type " + valueType + " cannot have a queryAnalyzer");
           }
           if (docValuesType != null && docValuesType != DocValuesType.BINARY && docValuesType != DocValuesType.SORTED && docValuesType != DocValuesType.SORTED_SET) {
-            illegalState(name, "type " + valueType + " must use BINARY docValuesType (got: " + docValuesType + ")");
+            illegalState(name, "type " + valueType + " must use BINARY, SORTED or SORTED_SET docValuesType (got: " + docValuesType + ")");
           }
           break;
         case ATOM:
@@ -336,12 +354,25 @@ public class FieldTypes {
           if (queryAnalyzer != null) {
             illegalState(name, "type " + valueType + " cannot have a queryAnalyzer");
           }
-          // nocommit make sure norms are disabled?
+          if (indexNorms == Boolean.TRUE) {
+            illegalState(name, "type " + valueType + " cannot index norms");
+          }
           if (indexOptions != null && indexOptions.compareTo(IndexOptions.DOCS_ONLY) > 0) {
             // nocommit too anal?
             illegalState(name, "type " + valueType + " can only be indexed as DOCS_ONLY; got " + indexOptions);
           }
           break;
+        case BOOLEAN:
+          if (highlighted == Boolean.TRUE) {
+            illegalState(name, "type " + valueType + " cannot highlight");
+          }
+          if (indexNorms == Boolean.TRUE) {
+            illegalState(name, "type " + valueType + " cannot index norms");
+          }
+          if (docValuesType != null && docValuesType != DocValuesType.NUMERIC && docValuesType != DocValuesType.SORTED_NUMERIC) {
+            illegalState(name, "type " + valueType + " must use NUMERIC or SORTED_NUMERIC docValuesType (got: " + docValuesType + ")");
+          }
+          break;
         default:
           throw new AssertionError("missing value type in switch");
         }
@@ -610,6 +641,9 @@ public class FieldTypes {
         case BINARY:
           out.writeByte((byte) 8);
           break;
+        case BOOLEAN:
+          out.writeByte((byte) 9);
+          break;
         default:
           throw new AssertionError("missing ValueType in switch");
         }
@@ -653,6 +687,7 @@ public class FieldTypes {
       writeNullableBoolean(out, sortReversed);
       writeNullableBoolean(out, multiValued);
       writeNullableBoolean(out, indexNorms);
+      writeNullableBoolean(out, fastRanges);
       writeNullableBoolean(out, storeTermVectors);
       writeNullableBoolean(out, storeTermVectorPositions);
       writeNullableBoolean(out, storeTermVectorOffsets);
@@ -779,6 +814,9 @@ public class FieldTypes {
       case 8:
         valueType = ValueType.BINARY;
         break;
+      case 9:
+        valueType = ValueType.BOOLEAN;
+        break;
       default:
         throw new CorruptIndexException("invalid byte for ValueType: " + b, in);
       }
@@ -828,6 +866,7 @@ public class FieldTypes {
       sortReversed = readNullableBoolean(in);
       multiValued = readNullableBoolean(in);
       indexNorms = readNullableBoolean(in);
+      fastRanges = readNullableBoolean(in);
       storeTermVectors = readNullableBoolean(in);
       storeTermVectorPositions = readNullableBoolean(in);
       storeTermVectorOffsets = readNullableBoolean(in);
@@ -901,7 +940,9 @@ public class FieldTypes {
     if (currentFieldTypes != null) {
       return readFromString(currentFieldTypes);
     } else if (isNewIndex == false) {
-      throw new CorruptIndexException("FieldTyps is missing from index", "CommitUserData");
+      // nocommit must handle back compat here :)
+      // throw new CorruptIndexException("FieldTypes is missing from this index", "CommitUserData");
+      return Version.LATEST;
     } else {
       return Version.LATEST;
     }
@@ -979,6 +1020,10 @@ public class FieldTypes {
     }
   }
 
+  public String getFieldTypeString(String fieldName) {
+    return getFieldType(fieldName).toString();
+  }
+
   synchronized FieldType getFieldType(String fieldName) {
     FieldType fieldType = fields.get(fieldName);
     if (fieldType == null) {
@@ -1960,6 +2005,7 @@ public class FieldTypes {
     }
   }
 
+  // ncommit move this method inside FildType:
   private void setDefaults(FieldType field) {
     switch (field.valueType) {
 
@@ -1970,6 +2016,9 @@ public class FieldTypes {
       if (field.highlighted == null) {
         field.highlighted = Boolean.FALSE;
       }
+      if (field.storeTermVectors == null) {
+        field.storeTermVectors = Boolean.FALSE;
+      }
       if (field.sortable == null) {
         if (field.docValuesTypeSet == false || field.docValuesType == DocValuesType.NUMERIC || field.docValuesType == DocValuesType.SORTED_NUMERIC) {
           field.sortable = Boolean.TRUE;
@@ -1983,20 +2032,19 @@ public class FieldTypes {
       if (field.stored == null) {
         field.stored = Boolean.TRUE;
       }
-      if (field.sortable == Boolean.TRUE && field.docValuesTypeSet == false) {
-        if (field.multiValued == Boolean.TRUE) {
-          field.docValuesType = DocValuesType.SORTED_NUMERIC;
-        } else {
-          field.docValuesType = DocValuesType.NUMERIC;
-        }
-        field.docValuesTypeSet = true;
-      }
       if (field.indexOptionsSet == false) {
         field.indexOptions = IndexOptions.DOCS_ONLY;
         field.indexOptionsSet = true;
       }
-      if (field.indexNorms == null) {
-        field.indexNorms = Boolean.FALSE;
+      if (field.docValuesTypeSet == false) {
+        if (field.sortable == Boolean.TRUE) {
+          if (field.multiValued == Boolean.TRUE) {
+            field.docValuesType = DocValuesType.SORTED_NUMERIC;
+          } else {
+            field.docValuesType = DocValuesType.NUMERIC;
+          }
+        }
+        field.docValuesTypeSet = true;
       }
       if (field.fastRanges == null) {
         if (field.indexOptions != null) {
@@ -2005,14 +2053,17 @@ public class FieldTypes {
           field.fastRanges = Boolean.FALSE;
         }
       }
+      if (field.indexNorms == null) {
+        field.indexNorms = Boolean.FALSE;
+      }
       break;
 
     case SHORT_TEXT:
       if (field.highlighted == null) {
         field.highlighted = Boolean.TRUE;
       }
-      if (field.fastRanges == null) {
-        field.fastRanges = Boolean.FALSE;
+      if (field.storeTermVectors == null) {
+        field.storeTermVectors = Boolean.FALSE;
       }
       if (field.sortable == null) {
         if (field.docValuesTypeSet == false || field.docValuesType == DocValuesType.SORTED || field.docValuesType == DocValuesType.SORTED_SET) {
@@ -2045,6 +2096,9 @@ public class FieldTypes {
         }
         field.indexOptionsSet = true;
       }
+      if (field.fastRanges == null) {
+        field.fastRanges = Boolean.FALSE;
+      }
       if (field.indexNorms == null) {
         field.indexNorms = Boolean.FALSE;
       }
@@ -2054,6 +2108,9 @@ public class FieldTypes {
       if (field.highlighted == null) {
         field.highlighted = Boolean.FALSE;
       }
+      if (field.storeTermVectors == null) {
+        field.storeTermVectors = Boolean.FALSE;
+      }
       if (field.sortable == null) {
         if (field.docValuesTypeSet == false || field.docValuesType == DocValuesType.SORTED || field.docValuesType == DocValuesType.SORTED_SET) {
           field.sortable = Boolean.TRUE;
@@ -2081,9 +2138,6 @@ public class FieldTypes {
         }
         field.docValuesTypeSet = true;
       }
-      if (field.indexNorms == null) {
-        field.indexNorms = Boolean.FALSE;
-      }
       if (field.fastRanges == null) {
         if (field.indexOptions != null) {
           field.fastRanges = Boolean.TRUE;
@@ -2091,14 +2145,17 @@ public class FieldTypes {
           field.fastRanges = Boolean.FALSE;
         }
       }
+      if (field.indexNorms == null) {
+        field.indexNorms = Boolean.FALSE;
+      }
       break;
 
     case BINARY:
       if (field.highlighted == null) {
         field.highlighted = Boolean.FALSE;
       }
-      if (field.fastRanges == null) {
-        field.fastRanges = Boolean.FALSE;
+      if (field.storeTermVectors == null) {
+        field.storeTermVectors = Boolean.FALSE;
       }
       if (field.sortable == null) {
         if (field.docValuesTypeSet == false || field.docValuesType == DocValuesType.SORTED || field.docValuesType == DocValuesType.SORTED_SET) {
@@ -2113,9 +2170,6 @@ public class FieldTypes {
       if (field.stored == null) {
         field.stored = Boolean.TRUE;
       }
-      if (field.indexNorms == null) {
-        field.indexNorms = Boolean.FALSE;
-      }
       if (field.indexOptionsSet == false) {
         assert field.indexOptions == null;
         field.indexOptionsSet = true;
@@ -2130,14 +2184,20 @@ public class FieldTypes {
         }
         field.docValuesTypeSet = true;
       }
+      if (field.fastRanges == null) {
+        field.fastRanges = Boolean.FALSE;
+      }
+      if (field.indexNorms == null) {
+        field.indexNorms = Boolean.FALSE;
+      }
       break;
 
     case TEXT:
       if (field.highlighted == null) {
         field.highlighted = Boolean.TRUE;
       }
-      if (field.fastRanges == null) {
-        field.fastRanges = Boolean.FALSE;
+      if (field.storeTermVectors == null) {
+        field.storeTermVectors = Boolean.FALSE;
       }
       if (field.sortable == null) {
         field.sortable = Boolean.FALSE;
@@ -2161,10 +2221,53 @@ public class FieldTypes {
       assert field.docValuesType == null;
       field.docValuesTypeSet = true;
 
+      if (field.fastRanges == null) {
+        field.fastRanges = Boolean.FALSE;
+      }
       if (field.indexNorms == null) {
         field.indexNorms = Boolean.TRUE;
       }
       break;
+    
+    case BOOLEAN:
+      if (field.highlighted == null) {
+        field.highlighted = Boolean.FALSE;
+      }
+      if (field.storeTermVectors == null) {
+        field.storeTermVectors = Boolean.FALSE;
+      }
+      if (field.sortable == null) {
+        field.sortable = Boolean.TRUE;
+      }
+      if (field.multiValued == null) {
+        field.multiValued = Boolean.FALSE;
+      }
+      if (field.stored == null) {
+        field.stored = Boolean.TRUE;
+      }
+      if (field.indexOptionsSet == false) {
+        // validate enforces this:
+        assert field.highlighted == false;
+        field.indexOptions = IndexOptions.DOCS_ONLY;
+        field.indexOptionsSet = true;
+      }
+      if (field.docValuesTypeSet == false) {
+        if (field.sortable == Boolean.TRUE) {
+          if (field.multiValued == Boolean.TRUE) {
+            field.docValuesType = DocValuesType.SORTED_NUMERIC;
+          } else {
+            field.docValuesType = DocValuesType.NUMERIC;
+          }
+        }
+        field.docValuesTypeSet = true;
+      }
+      if (field.fastRanges == null) {
+        field.fastRanges = Boolean.FALSE;
+      }
+      if (field.indexNorms == null) {
+        field.indexNorms = Boolean.FALSE;
+      }
+      break;
 
     default:
       throw new AssertionError("missing value type in switch");
@@ -2179,6 +2282,7 @@ public class FieldTypes {
 
     // nocommit assert all other settings are not null
     assert field.highlighted != null;
+    assert field.storeTermVectors != null;
     assert field.fastRanges != null;
     assert field.sortable != null;
     assert field.multiValued != null;
@@ -2192,7 +2296,7 @@ public class FieldTypes {
   } 
 
   /** Returns a query matching all documents that have this int term. */
-  public Query newTermQuery(String fieldName, int token) {
+  public Query newIntTermQuery(String fieldName, int token) {
     // nocommit should we take Number?
 
     // Field must exist:
@@ -2219,7 +2323,7 @@ public class FieldTypes {
   }
 
   /** Returns a query matching all documents that have this long term. */
-  public Query newTermQuery(String fieldName, long token) {
+  public Query newLongTermQuery(String fieldName, long token) {
 
     // Field must exist:
     FieldType fieldType = getFieldType(fieldName);
@@ -2245,7 +2349,7 @@ public class FieldTypes {
   }
 
   /** Returns a query matching all documents that have this binary token. */
-  public Query newTermQuery(String fieldName, byte[] token) {
+  public Query newBinaryTermQuery(String fieldName, byte[] token) {
 
     // Field must exist:
     FieldType fieldType = getFieldType(fieldName);
@@ -2263,7 +2367,7 @@ public class FieldTypes {
     return new TermQuery(new Term(fieldName, new BytesRef(token)));
   }
 
-  public Query newTermQuery(String fieldName, String token) {
+  public Query newStringTermQuery(String fieldName, String token) {
     // Field must exist:
     FieldType fieldType = getFieldType(fieldName);
 
@@ -2274,12 +2378,34 @@ public class FieldTypes {
 
     // Field must be text:
     if (fieldType.valueType != ValueType.TEXT && fieldType.valueType != ValueType.SHORT_TEXT && fieldType.valueType != ValueType.ATOM) {
-      illegalState(fieldName, "text term query have valueType TEXT, SHORT_TEXT or ATOM; got " + fieldType.valueType);
+      illegalState(fieldName, "string term query must have valueType TEXT, SHORT_TEXT or ATOM; got " + fieldType.valueType);
     }
 
     return new TermQuery(new Term(fieldName, token));
   }
 
+  public Query newBooleanTermQuery(String fieldName, boolean token) {
+    // Field must exist:
+    FieldType fieldType = getFieldType(fieldName);
+
+    // Field must be indexed:
+    if (fieldType.indexOptions == null) {
+      illegalState(fieldName, "cannot create term query: this field was not indexed");
+    }
+
+    // Field must be boolean:
+    if (fieldType.valueType != ValueType.BOOLEAN) {
+      illegalState(fieldName, "boolean term query must have valueType BOOLEAN; got " + fieldType.valueType);
+    }
+
+    byte[] value = new byte[1];
+    if (token) {
+      value[0] = 1;
+    }
+
+    return new TermQuery(new Term(fieldName, new BytesRef(value)));
+  }
+
   // nocommit shouldn't this be a filter?
   public Query newRangeQuery(String fieldName, Number min, boolean minInclusive, Number max, boolean maxInclusive) {
 
@@ -2432,6 +2558,7 @@ public class FieldTypes {
     case SHORT_TEXT:
     case ATOM:
     case BINARY:
+    case BOOLEAN:
       if (fieldType.multiValued == Boolean.TRUE) {
         // nocommit need to be able to set selector...
         return new SortedSetSortField(fieldName, reverse);
@@ -2450,6 +2577,11 @@ public class FieldTypes {
   private synchronized void changed() {
     ensureWritable();
     // Push to IW's commit data
+    changeCount++;
+  }
+
+  public synchronized long getChangeCount() {
+    return changeCount;
   }
 
   private synchronized void ensureWritable() {

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/StoredField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/StoredField.java?rev=1635898&r1=1635897&r2=1635898&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/StoredField.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/StoredField.java Sat Nov  1 08:39:27 2014
@@ -1,7 +1,6 @@
 package org.apache.lucene.document;
 
 import org.apache.lucene.index.IndexReader; // javadocs
-import org.apache.lucene.index.StorableField;
 import org.apache.lucene.search.IndexSearcher; // javadocs
 import org.apache.lucene.util.BytesRef;
 

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1635898&r1=1635897&r2=1635898&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Sat Nov  1 08:39:27 2014
@@ -25,15 +25,17 @@ import java.nio.file.Paths;
 import java.text.NumberFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Deque;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Deque;
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.document.Document2;
+import org.apache.lucene.document.FieldTypes;
 import org.apache.lucene.index.CheckIndex.Status.DocValuesStatus;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.search.DocIdSetIterator;
@@ -477,6 +479,8 @@ public class CheckIndex implements Close
       return result;
     }
 
+    FieldTypes fieldTypes = FieldTypes.getFieldTypes(sis.getUserData(), null, null);
+
     // find the oldest and newest segment versions
     Version oldest = null;
     Version newest = null;
@@ -632,7 +636,7 @@ public class CheckIndex implements Close
         }
         if (infoStream != null)
           infoStream.print("    test: open reader.........");
-        reader = new SegmentReader(info, IOContext.DEFAULT);
+        reader = new SegmentReader(fieldTypes, info, IOContext.DEFAULT);
         msg(infoStream, "OK");
 
         segInfoStat.openReaderPassed = true;
@@ -1687,7 +1691,7 @@ public class CheckIndex implements Close
       for (int j = 0; j < reader.maxDoc(); ++j) {
         // Intentionally pull even deleted documents to
         // make sure they too are not corrupt:
-        StoredDocument doc = reader.document(j);
+        Document2 doc = reader.document(j);
         if (liveDocs == null || liveDocs.get(j)) {
           status.docCount++;
           status.totFields += doc.getFields().size();

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java?rev=1635898&r1=1635897&r2=1635898&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java Sat Nov  1 08:39:27 2014
@@ -305,29 +305,20 @@ final class DefaultIndexingChain extends
 
     termsHash.startDocument();
 
-    // Invert indexed fields:
+    fillStoredFields(docState.docID);
+    startStoredFields();
+
     try {
-      for (IndexableField field : docState.doc.indexableFields()) {
-        IndexableFieldType fieldType = field.fieldType();
-        
-        // if the field omits norms, the boost cannot be indexed.
-        if (fieldType.omitNorms() && field.boost() != 1.0f) {
-          throw new UnsupportedOperationException("You cannot set an index-time boost: norms are omitted for field '" + field.name() + "'");
-        }
-        
-        PerField fp = getOrAddField(field.name(), fieldType, true);
-        boolean first = fp.fieldGen != fieldGen;
-        fp.invert(field, first);
-
-        if (first) {
-          fields[fieldCount++] = fp;
-          fp.fieldGen = fieldGen;
-        }
+      for (IndexableField field : docState.doc) {
+        fieldCount = processField(field, fieldGen, fieldCount);
       }
     } finally {
-      // Finish each field name seen in the document:
-      for (int i=0;i<fieldCount;i++) {
-        fields[i].finish();
+      if (docWriter.aborting == false) {
+        // Finish each indexed field name seen in the document:
+        for (int i=0;i<fieldCount;i++) {
+          fields[i].finish();
+        }
+        finishStoredFields();
       }
     }
 
@@ -342,65 +333,85 @@ final class DefaultIndexingChain extends
         docWriter.setAborting();
       }
     }
+  }
 
-    // Add stored fields:
-    fillStoredFields(docState.docID);
-    startStoredFields();
+  private int processField(IndexableField field, long fieldGen, int fieldCount) throws IOException {
+    String fieldName = field.name();
+    IndexableFieldType fieldType = field.fieldType();
 
-    // TODO: clean up this loop, it's bogus that docvalues are treated as stored fields...
-    boolean abort = false;
-    try {
-      for (StorableField field : docState.doc.storableFields()) {
-        String fieldName = field.name();
-        IndexableFieldType fieldType = field.fieldType();
+    PerField fp = null;
+
+    // Invert indexed fields:
+    if (fieldType.indexOptions() != null) {
       
-        verifyFieldType(fieldName, fieldType);
-        
-        PerField fp = getOrAddField(fieldName, fieldType, false);
-        if (fieldType.stored()) {
-          abort = true;
-          storedFieldsWriter.writeField(fp.fieldInfo, field);
-          abort = false;
-        }
+      // if the field omits norms, the boost cannot be indexed.
+      if (fieldType.omitNorms() && field.boost() != 1.0f) {
+        throw new UnsupportedOperationException("You cannot set an index-time boost: norms are omitted for field '" + field.name() + "'");
+      }
+      
+      fp = getOrAddField(fieldName, fieldType, true);
+      boolean first = fp.fieldGen != fieldGen;
+      fp.invert(field, first);
 
-        DocValuesType dvType = fieldType.docValueType();
-        if (dvType != null) {
-          indexDocValue(fp, dvType, field);
+      if (first) {
+        fields[fieldCount++] = fp;
+        fp.fieldGen = fieldGen;
+      }
+    } else {
+      verifyUnIndexedFieldType(fieldName, fieldType);
+    }
+
+    // Add stored fields:
+    if (fieldType.stored()) {
+      if (fp == null) {
+        fp = getOrAddField(fieldName, fieldType, false);
+      }
+      if (fieldType.stored()) {
+        boolean success = false;
+        try {
+          storedFieldsWriter.writeField(fp.fieldInfo, field);
+          success = true;
+        } finally {
+          if (!success) {
+            docWriter.setAborting();
+          }
         }
       }
-    } finally {
-      if (abort) {
-        docWriter.setAborting();
-      } else {
-        finishStoredFields();
+    }
+
+    DocValuesType dvType = fieldType.docValueType();
+    if (dvType != null) {
+      if (fp == null) {
+        fp = getOrAddField(fieldName, fieldType, false);
       }
+      indexDocValue(fp, dvType, field);
     }
+    
+    return fieldCount;
   }
 
-  private static void verifyFieldType(String name, IndexableFieldType ft) {
-    if (ft.indexOptions() == null) {
-      if (ft.storeTermVectors()) {
-        throw new IllegalArgumentException("cannot store term vectors "
-                                           + "for a field that is not indexed (field=\"" + name + "\")");
-      }
-      if (ft.storeTermVectorPositions()) {
-        throw new IllegalArgumentException("cannot store term vector positions "
-                                           + "for a field that is not indexed (field=\"" + name + "\")");
-      }
-      if (ft.storeTermVectorOffsets()) {
-        throw new IllegalArgumentException("cannot store term vector offsets "
-                                           + "for a field that is not indexed (field=\"" + name + "\")");
-      }
-      if (ft.storeTermVectorPayloads()) {
-        throw new IllegalArgumentException("cannot store term vector payloads "
-                                           + "for a field that is not indexed (field=\"" + name + "\")");
-      }
+  private static void verifyUnIndexedFieldType(String name, IndexableFieldType ft) {
+    if (ft.storeTermVectors()) {
+      throw new IllegalArgumentException("cannot store term vectors "
+                                         + "for a field that is not indexed (field=\"" + name + "\")");
+    }
+    if (ft.storeTermVectorPositions()) {
+      throw new IllegalArgumentException("cannot store term vector positions "
+                                         + "for a field that is not indexed (field=\"" + name + "\")");
+    }
+    if (ft.storeTermVectorOffsets()) {
+      throw new IllegalArgumentException("cannot store term vector offsets "
+                                         + "for a field that is not indexed (field=\"" + name + "\")");
+    }
+    if (ft.storeTermVectorPayloads()) {
+      throw new IllegalArgumentException("cannot store term vector payloads "
+                                         + "for a field that is not indexed (field=\"" + name + "\")");
     }
   }
 
   /** Called from processDocument to index one field's doc
    *  value */
-  private void indexDocValue(PerField fp, DocValuesType dvType, StorableField field) throws IOException {
+  private void indexDocValue(PerField fp, DocValuesType dvType, IndexableField field) throws IOException {
 
     boolean hasDocValues = fp.fieldInfo.hasDocValues();
 

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1635898&r1=1635897&r2=1635898&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java Sat Nov  1 08:39:27 2014
@@ -394,7 +394,7 @@ final class DocumentsWriter implements C
     }
   }
 
-  boolean updateDocuments(final Iterable<? extends IndexDocument> docs, final Analyzer analyzer,
+  boolean updateDocuments(final Iterable<? extends Iterable<? extends IndexableField>> docs, final Analyzer analyzer,
                           final Term delTerm) throws IOException {
     boolean hasEvents = preUpdate();
 
@@ -434,7 +434,7 @@ final class DocumentsWriter implements C
     return postUpdate(flushingDWPT, hasEvents);
   }
 
-  boolean updateDocument(final IndexDocument doc, final Analyzer analyzer,
+  boolean updateDocument(final Iterable<? extends IndexableField> doc, final Analyzer analyzer,
       final Term delTerm) throws IOException {
 
     boolean hasEvents = preUpdate();

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1635898&r1=1635897&r2=1635898&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Sat Nov  1 08:39:27 2014
@@ -71,7 +71,7 @@ class DocumentsWriterPerThread {
     InfoStream infoStream;
     Similarity similarity;
     int docID;
-    IndexDocument doc;
+    Iterable<? extends IndexableField> doc;
 
     DocState(DocumentsWriterPerThread docWriter, InfoStream infoStream) {
       this.docWriter = docWriter;
@@ -219,7 +219,7 @@ class DocumentsWriterPerThread {
     }
   }
 
-  public void updateDocument(IndexDocument doc, Analyzer analyzer, Term delTerm) throws IOException {
+  public void updateDocument(Iterable<? extends IndexableField> doc, Analyzer analyzer, Term delTerm) throws IOException {
     testPoint("DocumentsWriterPerThread addDocument start");
     assert deleteQueue != null;
     docState.doc = doc;
@@ -257,7 +257,7 @@ class DocumentsWriterPerThread {
     finishDocument(delTerm);
   }
 
-  public int updateDocuments(Iterable<? extends IndexDocument> docs, Analyzer analyzer, Term delTerm) throws IOException {
+  public int updateDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs, Analyzer analyzer, Term delTerm) throws IOException {
     testPoint("DocumentsWriterPerThread addDocuments start");
     assert deleteQueue != null;
     docState.analyzer = analyzer;
@@ -268,7 +268,7 @@ class DocumentsWriterPerThread {
     boolean allDocsIndexed = false;
     try {
       
-      for(IndexDocument doc : docs) {
+      for(Iterable<? extends IndexableField> doc : docs) {
         // Even on exception, the document is still added (but marked
         // deleted), so we don't need to un-reserve at that point.
         // Aborting exceptions will actually "lose" more than one

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexReader.java?rev=1635898&r1=1635897&r2=1635898&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexReader.java Sat Nov  1 08:39:27 2014
@@ -26,7 +26,9 @@ import java.util.Set;
 import java.util.WeakHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import org.apache.lucene.document.DocumentStoredFieldVisitor;
+import org.apache.lucene.document.Document2;
+import org.apache.lucene.document.Document2StoredFieldVisitor;
+import org.apache.lucene.document.FieldTypes;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.util.Bits; // javadocs
 import org.apache.lucene.util.IOUtils;
@@ -103,6 +105,11 @@ public abstract class IndexReader implem
 
   // nocommit need getFieldTypes; how should MultiReader impl?
 
+  // nocommit make abstract
+  public FieldTypes getFieldTypes() {
+    return null;
+  }
+
   private final Set<ReaderClosedListener> readerClosedListeners = 
       Collections.synchronizedSet(new LinkedHashSet<ReaderClosedListener>());
 
@@ -341,13 +348,13 @@ public abstract class IndexReader implem
    *  custom processing/loading of each field.  If you
    *  simply want to load all fields, use {@link
    *  #document(int)}.  If you want to load a subset, use
-   *  {@link DocumentStoredFieldVisitor}.  */
+   *  {@link Document2StoredFieldVisitor}.  */
   public abstract void document(int docID, StoredFieldVisitor visitor) throws IOException;
-  
+
   /**
    * Returns the stored fields of the <code>n</code><sup>th</sup>
    * <code>Document</code> in this index.  This is just
-   * sugar for using {@link DocumentStoredFieldVisitor}.
+   * sugar for using {@link Document2StoredFieldVisitor}.
    * <p>
    * <b>NOTE:</b> for performance reasons, this method does not check if the
    * requested document is deleted, and therefore asking for a deleted document
@@ -366,8 +373,8 @@ public abstract class IndexReader implem
   // TODO: we need a separate StoredField, so that the
   // Document returned here contains that class not
   // IndexableField
-  public final StoredDocument document(int docID) throws IOException {
-    final DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor();
+  public final Document2 document(int docID) throws IOException {
+    final Document2StoredFieldVisitor visitor = new Document2StoredFieldVisitor(getFieldTypes());
     document(docID, visitor);
     return visitor.getDocument();
   }
@@ -375,12 +382,11 @@ public abstract class IndexReader implem
   /**
    * Like {@link #document(int)} but only loads the specified
    * fields.  Note that this is simply sugar for {@link
-   * DocumentStoredFieldVisitor#DocumentStoredFieldVisitor(Set)}.
+   * Document2StoredFieldVisitor#Document2StoredFieldVisitor(Set)}.
    */
-  public final StoredDocument document(int docID, Set<String> fieldsToLoad)
+  public final Document2 document(int docID, Set<String> fieldsToLoad)
       throws IOException {
-    final DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor(
-        fieldsToLoad);
+    final Document2StoredFieldVisitor visitor = new Document2StoredFieldVisitor(getFieldTypes(), fieldsToLoad);
     document(docID, visitor);
     return visitor.getDocument();
   }

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1635898&r1=1635897&r2=1635898&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Sat Nov  1 08:39:27 2014
@@ -253,11 +253,13 @@ public class IndexWriter implements Clos
 
   private volatile long changeCount; // increments every time a change is completed
   private volatile long lastCommitChangeCount; // last changeCount that was committed
+  private volatile long lastCommitFieldTypesChangeCount; // last FieldTypes.getChangeCount()
 
   private List<SegmentCommitInfo> rollbackSegments;      // list of segmentInfo we will fallback to if the commit fails
 
   volatile SegmentInfos pendingCommit;            // set when a commit is pending (after prepareCommit() & before commit())
   volatile long pendingCommitChangeCount;
+  volatile long pendingCommitFieldTypesChangeCount;
 
   private Collection<String> filesToCommit;
 
@@ -422,6 +424,7 @@ public class IndexWriter implements Clos
           // just like we do when loading segments_N
           synchronized(this) {
             maybeApplyDeletes(applyAllDeletes);
+            // nocommit must serialize field types into index commit data here?
             r = StandardDirectoryReader.open(this, segmentInfos, applyAllDeletes);
             if (infoStream.isEnabled("IW")) {
               infoStream.message("IW", "return reader version=" + r.getVersion() + " reader=" + r);
@@ -1137,7 +1140,7 @@ public class IndexWriter implements Clos
    * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
    */
-  public void addDocument(IndexDocument doc) throws IOException {
+  public void addDocument(Iterable<? extends IndexableField> doc) throws IOException {
     addDocument(doc, analyzer);
   }
 
@@ -1152,7 +1155,7 @@ public class IndexWriter implements Clos
    * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
    */
-  public void addDocument(IndexDocument doc, Analyzer analyzer) throws IOException {
+  public void addDocument(Iterable<? extends IndexableField> doc, Analyzer analyzer) throws IOException {
     updateDocument(null, doc, analyzer);
   }
 
@@ -1177,7 +1180,7 @@ public class IndexWriter implements Clos
    * perhaps to obtain better index compression), in which case
    * you may need to fully re-index your documents at that time.
    *
-   * <p>See {@link #addDocument(IndexDocument)} for details on
+   * <p>See {@link #addDocument(Iterable)} for details on
    * index and IndexWriter state after an Exception, and
    * flushing/merging temporary free space requirements.</p>
    *
@@ -1193,7 +1196,7 @@ public class IndexWriter implements Clos
    *
    * @lucene.experimental
    */
-  public void addDocuments(Iterable<? extends IndexDocument> docs) throws IOException {
+  public void addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
     addDocuments(docs, analyzer);
   }
 
@@ -1208,7 +1211,7 @@ public class IndexWriter implements Clos
    *
    * @lucene.experimental
    */
-  public void addDocuments(Iterable<? extends IndexDocument> docs, Analyzer analyzer) throws IOException {
+  public void addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs, Analyzer analyzer) throws IOException {
     updateDocuments(null, docs, analyzer);
   }
 
@@ -1225,7 +1228,7 @@ public class IndexWriter implements Clos
    *
    * @lucene.experimental
    */
-  public void updateDocuments(Term delTerm, Iterable<? extends IndexDocument> docs) throws IOException {
+  public void updateDocuments(Term delTerm, Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
     updateDocuments(delTerm, docs, analyzer);
   }
 
@@ -1243,7 +1246,7 @@ public class IndexWriter implements Clos
    *
    * @lucene.experimental
    */
-  public void updateDocuments(Term delTerm, Iterable<? extends IndexDocument> docs, Analyzer analyzer) throws IOException {
+  public void updateDocuments(Term delTerm, Iterable<? extends Iterable<? extends IndexableField>> docs, Analyzer analyzer) throws IOException {
     ensureOpen();
     try {
       boolean success = false;
@@ -1393,7 +1396,7 @@ public class IndexWriter implements Clos
    * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
    */
-  public void updateDocument(Term term, IndexDocument doc) throws IOException {
+  public void updateDocument(Term term, Iterable<? extends IndexableField> doc) throws IOException {
     ensureOpen();
     updateDocument(term, doc, analyzer);
   }
@@ -1412,7 +1415,7 @@ public class IndexWriter implements Clos
    * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
    */
-  public void updateDocument(Term term, IndexDocument doc, Analyzer analyzer)
+  public void updateDocument(Term term, Iterable<? extends IndexableField> doc, Analyzer analyzer)
       throws IOException {
     ensureOpen();
     try {
@@ -2566,7 +2569,7 @@ public class IndexWriter implements Clos
       SegmentInfo info = new SegmentInfo(directory, Version.LATEST, mergedName, -1,
                                          false, codec, null, StringHelper.randomId());
 
-      SegmentMerger merger = new SegmentMerger(mergeReaders, info, infoStream, trackingDir,
+      SegmentMerger merger = new SegmentMerger(fieldTypes, mergeReaders, info, infoStream, trackingDir,
                                                MergeState.CheckAbort.NONE, globalFieldNumberMap, 
                                                context);
       
@@ -2776,6 +2779,7 @@ public class IndexWriter implements Clos
               toCommit.getUserData().put(FieldTypes.FIELD_TYPES_KEY, fieldTypes.writeToString());
 
               pendingCommitChangeCount = changeCount;
+              pendingCommitFieldTypesChangeCount = fieldTypes.getChangeCount();
 
               // This protects the segmentInfos we are now going
               // to commit.  This is important in case, eg, while
@@ -2889,7 +2893,10 @@ public class IndexWriter implements Clos
    *  merged finished, this method may return true right
    *  after you had just called {@link #commit}. */
   public final boolean hasUncommittedChanges() {
-    return changeCount != lastCommitChangeCount || docWriter.anyChanges() || bufferedUpdatesStream.any();
+    return fieldTypes.getChangeCount() != lastCommitFieldTypesChangeCount ||
+      changeCount != lastCommitChangeCount ||
+      docWriter.anyChanges() ||
+      bufferedUpdatesStream.any();
   }
 
   private final void commitInternal(MergePolicy mergePolicy) throws IOException {
@@ -2945,6 +2952,7 @@ public class IndexWriter implements Clos
             deleter.checkpoint(pendingCommit, true);
 
             lastCommitChangeCount = pendingCommitChangeCount;
+            lastCommitFieldTypesChangeCount = pendingCommitFieldTypesChangeCount;
             rollbackSegments = pendingCommit.createBackupSegmentInfos();
 
             finished = true;
@@ -3936,7 +3944,8 @@ public class IndexWriter implements Clos
           // fix the reader's live docs and del count
           assert delCount > reader.numDeletedDocs(); // beware of zombies
 
-          SegmentReader newReader = new SegmentReader(info, reader, liveDocs, info.info.getDocCount() - delCount);
+          // nocommit we are passing our fieldTypes here:
+          SegmentReader newReader = new SegmentReader(fieldTypes, info, reader, liveDocs, info.info.getDocCount() - delCount);
           boolean released = false;
           try {
             rld.release(reader);
@@ -3959,7 +3968,7 @@ public class IndexWriter implements Clos
       
       // we pass merge.getMergeReaders() instead of merge.readers to allow the
       // OneMerge to return a view over the actual segments to merge
-      final SegmentMerger merger = new SegmentMerger(merge.getMergeReaders(),
+      final SegmentMerger merger = new SegmentMerger(fieldTypes, merge.getMergeReaders(),
                                                      merge.info.info, infoStream, dirWrapper,
                                                      checkAbort, globalFieldNumberMap, 
                                                      context);
@@ -4275,10 +4284,12 @@ public class IndexWriter implements Clos
       }
 
       synchronized(this) {
+        long fieldTypesChangeCount = fieldTypes.getChangeCount();
 
         assert lastCommitChangeCount <= changeCount: "lastCommitChangeCount=" + lastCommitChangeCount + " changeCount=" + changeCount;
+        assert lastCommitFieldTypesChangeCount <= fieldTypesChangeCount: "lastCommitFieldTypesChangeCount=" + lastCommitFieldTypesChangeCount + " fieldTypesChangeCount=" + fieldTypesChangeCount;
 
-        if (pendingCommitChangeCount == lastCommitChangeCount) {
+        if (pendingCommitChangeCount == lastCommitChangeCount && pendingCommitFieldTypesChangeCount == lastCommitFieldTypesChangeCount) {
           if (infoStream.isEnabled("IW")) {
             infoStream.message("IW", "  skip startCommit(): no changes pending");
           }
@@ -4291,7 +4302,7 @@ public class IndexWriter implements Clos
         }
 
         if (infoStream.isEnabled("IW")) {
-          infoStream.message("IW", "startCommit index=" + segString(toLiveInfos(toSync)) + " changeCount=" + changeCount);
+          infoStream.message("IW", "startCommit index=" + segString(toLiveInfos(toSync)) + " changeCount=" + changeCount + " fieldTypesChangeCount=" + fieldTypesChangeCount);
         }
 
         assert filesExist(toSync);

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableField.java?rev=1635898&r1=1635897&r2=1635898&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableField.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableField.java Sat Nov  1 08:39:27 2014
@@ -23,6 +23,7 @@ import org.apache.lucene.analysis.Analyz
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.search.similarities.DefaultSimilarity; // javadocs
 import org.apache.lucene.search.similarities.Similarity; // javadocs
+import org.apache.lucene.util.BytesRef;
 
 // TODO: how to handle versioning here...?
 
@@ -31,7 +32,15 @@ import org.apache.lucene.search.similari
  *
  *  @lucene.experimental */
 
-public interface IndexableField extends GeneralField {
+public interface IndexableField {
+
+
+  /** Field name */
+  public String name();
+
+  /** {@link IndexableFieldType} describing the properties
+   * of this field. */
+  public IndexableFieldType fieldType();
 
   /**
    * Creates the TokenStream used for indexing this field.  If appropriate,
@@ -71,4 +80,19 @@ public interface IndexableField extends 
    * @see DefaultSimilarity#encodeNormValue(float)
    */
   public float boost();
+
+  /** Non-null if this field has a stored binary value */
+  public BytesRef binaryValue();
+
+  /** Non-null if this field has a binary doc value */
+  public BytesRef binaryDocValue();
+
+  /** Non-null if this field has a string value */
+  public String stringValue();
+
+  /** Non-null if this field has a numeric value */
+  public Number numericValue(); 
+
+  /** Non-null if this field has a numeric doc value */
+  public Number numericDocValue(); 
 }

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/MergeState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/MergeState.java?rev=1635898&r1=1635897&r2=1635898&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/MergeState.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/MergeState.java Sat Nov  1 08:39:27 2014
@@ -27,6 +27,7 @@ import org.apache.lucene.codecs.FieldsPr
 import org.apache.lucene.codecs.NormsProducer;
 import org.apache.lucene.codecs.StoredFieldsReader;
 import org.apache.lucene.codecs.TermVectorsReader;
+import org.apache.lucene.document.FieldTypes;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.Bits;
@@ -82,13 +83,16 @@ public class MergeState {
   /** InfoStream for debugging messages. */
   public final InfoStream infoStream;
 
+  public final FieldTypes fieldTypes;
+
   /** Counter used for periodic calls to checkAbort
    * @lucene.internal */
   public int checkAbortCount;
 
   /** Sole constructor. */
-  MergeState(List<LeafReader> readers, SegmentInfo segmentInfo, InfoStream infoStream, CheckAbort checkAbort) throws IOException {
+  MergeState(FieldTypes fieldTypes, List<LeafReader> readers, SegmentInfo segmentInfo, InfoStream infoStream, CheckAbort checkAbort) throws IOException {
 
+    this.fieldTypes = fieldTypes;
     int numReaders = readers.size();
     docMaps = new DocMap[numReaders];
     docBase = new int[numReaders];

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java?rev=1635898&r1=1635897&r2=1635898&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java Sat Nov  1 08:39:27 2014
@@ -131,7 +131,8 @@ class ReadersAndUpdates {
   public SegmentReader getReader(IOContext context) throws IOException {
     if (reader == null) {
       // We steal returned ref:
-      reader = new SegmentReader(info, context);
+      // nocommit clone the field types?
+      reader = new SegmentReader(writer.getFieldTypes(), info, context);
       if (liveDocs == null) {
         liveDocs = reader.getLiveDocs();
       }
@@ -189,7 +190,7 @@ class ReadersAndUpdates {
     // force new liveDocs in initWritableLiveDocs even if it's null
     liveDocsShared = true;
     if (liveDocs != null) {
-      return new SegmentReader(reader.getSegmentInfo(), reader, liveDocs, info.info.getDocCount() - info.getDelCount() - pendingDeleteCount);
+      return new SegmentReader(writer.fieldTypes, reader.getSegmentInfo(), reader, liveDocs, info.info.getDocCount() - info.getDelCount() - pendingDeleteCount);
     } else {
       // liveDocs == null and reader != null. That can only be if there are no deletes
       assert reader.getLiveDocs() == null;
@@ -477,7 +478,7 @@ class ReadersAndUpdates {
 
       // reader could be null e.g. for a just merged segment (from
       // IndexWriter.commitMergedDeletes).
-      final SegmentReader reader = this.reader == null ? new SegmentReader(info, IOContext.READONCE) : this.reader;
+      final SegmentReader reader = this.reader == null ? new SegmentReader(writer.getFieldTypes(), info, IOContext.READONCE) : this.reader;
       try {
         // clone FieldInfos so that we can update their dvGen separately from
         // the reader's infos and write them to a new fieldInfos_gen file
@@ -577,7 +578,7 @@ class ReadersAndUpdates {
 
     // if there is a reader open, reopen it to reflect the updates
     if (reader != null) {
-      SegmentReader newReader = new SegmentReader(info, reader, liveDocs, info.info.getDocCount() - info.getDelCount() - pendingDeleteCount);
+      SegmentReader newReader = new SegmentReader(writer.fieldTypes, info, reader, liveDocs, info.info.getDocCount() - info.getDelCount() - pendingDeleteCount);
       boolean reopened = false;
       try {
         reader.decRef();

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1635898&r1=1635897&r2=1635898&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java Sat Nov  1 08:39:27 2014
@@ -26,6 +26,7 @@ import org.apache.lucene.codecs.FieldsCo
 import org.apache.lucene.codecs.NormsConsumer;
 import org.apache.lucene.codecs.StoredFieldsWriter;
 import org.apache.lucene.codecs.TermVectorsWriter;
+import org.apache.lucene.document.FieldTypes;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.util.IOUtils;
@@ -49,7 +50,7 @@ final class SegmentMerger {
   private final FieldInfos.Builder fieldInfosBuilder;
 
   // note, just like in codec apis Directory 'dir' is NOT the same as segmentInfo.dir!!
-  SegmentMerger(List<LeafReader> readers, SegmentInfo segmentInfo, InfoStream infoStream, Directory dir,
+  SegmentMerger(FieldTypes fieldTypes, List<LeafReader> readers, SegmentInfo segmentInfo, InfoStream infoStream, Directory dir,
                 MergeState.CheckAbort checkAbort, FieldInfos.FieldNumbers fieldNumbers, IOContext context) throws IOException {
     // validate incoming readers
     for (LeafReader reader : readers) {
@@ -60,7 +61,8 @@ final class SegmentMerger {
       }
     }
 
-    mergeState = new MergeState(readers, segmentInfo, infoStream, checkAbort);
+    // nocommit should we refuse to merge if fieldTypes differs from "ours" and from all our incoming readers?
+    mergeState = new MergeState(fieldTypes, readers, segmentInfo, infoStream, checkAbort);
     directory = dir;
     this.codec = segmentInfo.getCodec();
     this.context = context;

Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java?rev=1635898&r1=1635897&r2=1635898&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java Sat Nov  1 08:39:27 2014
@@ -32,6 +32,7 @@ import org.apache.lucene.codecs.FieldsPr
 import org.apache.lucene.codecs.NormsProducer;
 import org.apache.lucene.codecs.StoredFieldsReader;
 import org.apache.lucene.codecs.TermVectorsReader;
+import org.apache.lucene.document.FieldTypes;
 import org.apache.lucene.index.FieldInfo.DocValuesType;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
@@ -65,6 +66,7 @@ public final class SegmentReader extends
 
   final SegmentCoreReaders core;
   final SegmentDocValues segDocValues;
+  final FieldTypes fieldTypes;
   
   final CloseableThreadLocal<Map<String,Object>> docValuesLocal = new CloseableThreadLocal<Map<String,Object>>() {
     @Override
@@ -89,8 +91,9 @@ public final class SegmentReader extends
    * @throws IOException if there is a low-level IO error
    */
   // TODO: why is this public?
-  public SegmentReader(SegmentCommitInfo si, IOContext context) throws IOException {
+  public SegmentReader(FieldTypes fieldTypes, SegmentCommitInfo si, IOContext context) throws IOException {
     this.si = si;
+    this.fieldTypes = fieldTypes;
     core = new SegmentCoreReaders(this, si.info.dir, si, context);
     segDocValues = new SegmentDocValues();
     
@@ -125,8 +128,8 @@ public final class SegmentReader extends
   /** Create new SegmentReader sharing core from a previous
    *  SegmentReader and loading new live docs from a new
    *  deletes file.  Used by openIfChanged. */
-  SegmentReader(SegmentCommitInfo si, SegmentReader sr) throws IOException {
-    this(si, sr,
+  SegmentReader(FieldTypes fieldTypes, SegmentCommitInfo si, SegmentReader sr) throws IOException {
+    this(fieldTypes, si, sr,
          si.info.getCodec().liveDocsFormat().readLiveDocs(si.info.dir, si, IOContext.READONCE),
          si.info.getDocCount() - si.getDelCount());
   }
@@ -135,7 +138,8 @@ public final class SegmentReader extends
    *  SegmentReader and using the provided in-memory
    *  liveDocs.  Used by IndexWriter to provide a new NRT
    *  reader */
-  SegmentReader(SegmentCommitInfo si, SegmentReader sr, Bits liveDocs, int numDocs) throws IOException {
+  SegmentReader(FieldTypes fieldTypes, SegmentCommitInfo si, SegmentReader sr, Bits liveDocs, int numDocs) throws IOException {
+    this.fieldTypes = fieldTypes;
     this.si = si;
     this.liveDocs = liveDocs;
     this.numDocs = numDocs;
@@ -171,6 +175,11 @@ public final class SegmentReader extends
       return segDocValues.getDocValuesProducer(-1L, si, IOContext.READ, dir, dvFormat, fieldInfos);
     }
   }
+
+  @Override
+  public FieldTypes getFieldTypes() {
+    return fieldTypes;
+  }
   
   /**
    * init most recent FieldInfos for the current commit



Mime
View raw message