lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1632133 - in /lucene/dev/branches/branch_5x: ./ lucene/ lucene/backward-codecs/ lucene/backward-codecs/src/java/org/apache/lucene/codecs/ lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/ lucene/backward-codecs/src/java/or...
Date Wed, 15 Oct 2014 17:32:36 GMT
Author: mikemccand
Date: Wed Oct 15 17:32:35 2014
New Revision: 1632133

URL: http://svn.apache.org/r1632133
Log:
LUCENE-6006: remove unnecessary FieldInfo.normType

Added:
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/UndeadNormsProducer.java
      - copied unchanged from r1632120, lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/UndeadNormsProducer.java
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/package.html
      - copied unchanged from r1632120, lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/package.html
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/index.40.undeadnorms.zip
      - copied unchanged from r1632120, lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/index.40.undeadnorms.zip
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/index.42.undeadnorms.zip
      - copied unchanged from r1632120, lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/index.42.undeadnorms.zip
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/TestLucene46UndeadNorms.java
      - copied unchanged from r1632120, lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/TestLucene46UndeadNorms.java
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/index.46.undeadnorms.zip
      - copied unchanged from r1632120, lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/index.46.undeadnorms.zip
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/index.49.undeadnorms.zip
      - copied unchanged from r1632120, lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/index.49.undeadnorms.zip
Modified:
    lucene/dev/branches/branch_5x/   (props changed)
    lucene/dev/branches/branch_5x/lucene/   (props changed)
    lucene/dev/branches/branch_5x/lucene/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/branch_5x/lucene/backward-codecs/   (props changed)
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsReader.java
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsFormat.java
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsProducer.java
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWFieldInfosFormat.java
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40FieldInfoFormat.java
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40NormsFormat.java
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWFieldInfosFormat.java
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/TestLucene42NormsFormat.java
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java
    lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
    lucene/dev/branches/branch_5x/lucene/codecs/   (props changed)
    lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java
    lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
    lucene/dev/branches/branch_5x/lucene/core/   (props changed)
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
    lucene/dev/branches/branch_5x/lucene/memory/   (props changed)
    lucene/dev/branches/branch_5x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
    lucene/dev/branches/branch_5x/lucene/misc/   (props changed)
    lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java
    lucene/dev/branches/branch_5x/lucene/test-framework/   (props changed)
    lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
    lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java
    lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
    lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
    lucene/dev/branches/branch_5x/solr/   (props changed)
    lucene/dev/branches/branch_5x/solr/core/   (props changed)
    lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/Insanity.java

Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Wed Oct 15 17:32:35 2014
@@ -143,6 +143,10 @@ API Changes
   their jflex impl directly.
   (Ryan Ernst)
 
+* LUCENE-6006: Removed FieldInfo.normType since it's redundant: it
+  will be DocValuesType.NUMERIC if the field indexed and does not omit
+  norms, else null.  (Robert Muir, Mike McCandless)
+
 Bug Fixes
 
 * LUCENE-5650: Enforce read-only access to any path outside the temporary

Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java Wed Oct 15 17:32:35 2014
@@ -23,13 +23,14 @@ import java.util.Map;
 
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.UndeadNormsProducer;
 import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
@@ -91,7 +92,7 @@ public class Lucene40FieldInfosFormat ex
         byte val = input.readByte();
         final LegacyDocValuesType oldValuesType = getDocValuesType((byte) (val & 0x0F));
         final LegacyDocValuesType oldNormsType = getDocValuesType((byte) ((val >>> 4) & 0x0F));
-        final Map<String,String> attributes = input.readStringStringMap();;
+        final Map<String,String> attributes = input.readStringStringMap();
         if (oldValuesType.mapping != null) {
           attributes.put(LEGACY_DV_TYPE_KEY, oldValuesType.name());
         }
@@ -101,8 +102,12 @@ public class Lucene40FieldInfosFormat ex
           }
           attributes.put(LEGACY_NORM_TYPE_KEY, oldNormsType.name());
         }
+        if (isIndexed && omitNorms == false && oldNormsType.mapping == null) {
+          // Undead norms!  Lucene40NormsReader will check this and bring norms back from the dead:
+          UndeadNormsProducer.setUndead(attributes);
+        }
         infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, 
-          omitNorms, storePayloads, indexOptions, oldValuesType.mapping, oldNormsType.mapping, -1, Collections.unmodifiableMap(attributes));
+          omitNorms, storePayloads, indexOptions, oldValuesType.mapping, -1, Collections.unmodifiableMap(attributes));
       }
 
       CodecUtil.checkEOF(input);

Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java Wed Oct 15 17:32:35 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
 import org.apache.lucene.codecs.NormsConsumer;
 import org.apache.lucene.codecs.NormsFormat;
 import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.UndeadNormsProducer;
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SegmentWriteState;
@@ -46,6 +47,10 @@ public class Lucene40NormsFormat extends
     String filename = IndexFileNames.segmentFileName(state.segmentInfo.name, 
                                                      "nrm", 
                                                      Lucene40CompoundFormat.COMPOUND_FILE_EXTENSION);
-    return new Lucene40NormsReader(state, filename);
+    if (UndeadNormsProducer.isUndeadArmy(state.fieldInfos)) {
+      return UndeadNormsProducer.INSTANCE;
+    } else {
+      return new Lucene40NormsReader(state, filename);
+    }
   }
 }

Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsReader.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsReader.java Wed Oct 15 17:32:35 2014
@@ -21,6 +21,8 @@ import java.io.IOException;
 
 import org.apache.lucene.codecs.DocValuesProducer;
 import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.UndeadNormsProducer;
+import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.SegmentReadState;
@@ -45,6 +47,10 @@ final class Lucene40NormsReader extends 
   
   @Override
   public NumericDocValues getNorms(FieldInfo field) throws IOException {
+    if (UndeadNormsProducer.isUndead(field)) {
+      // Bring undead norms back to life; this is set in Lucene40FieldInfosFormat, to emulate pre-5.0 undead norms
+      return DocValues.emptyNumeric();
+    }
     return impl.getNumeric(field);
   }
   

Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java Wed Oct 15 17:32:35 2014
@@ -23,13 +23,14 @@ import java.util.Map;
 
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.UndeadNormsProducer;
 import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
@@ -41,7 +42,7 @@ import org.apache.lucene.util.IOUtils;
  */
 @Deprecated
 public class Lucene42FieldInfosFormat extends FieldInfosFormat {
-  
+
   /** Sole constructor. */
   public Lucene42FieldInfosFormat() {
   }
@@ -86,8 +87,14 @@ public class Lucene42FieldInfosFormat ex
         final DocValuesType docValuesType = getDocValuesType(input, (byte) (val & 0x0F));
         final DocValuesType normsType = getDocValuesType(input, (byte) ((val >>> 4) & 0x0F));
         final Map<String,String> attributes = input.readStringStringMap();
+
+        if (isIndexed && omitNorms == false && normsType == null) {
+          // Undead norms!  Lucene42NormsProducer will check this and bring norms back from the dead:
+          UndeadNormsProducer.setUndead(attributes);
+        }
+
         infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, 
-          omitNorms, storePayloads, indexOptions, docValuesType, normsType, -1, Collections.unmodifiableMap(attributes));
+          omitNorms, storePayloads, indexOptions, docValuesType, -1, Collections.unmodifiableMap(attributes));
       }
 
       CodecUtil.checkEOF(input);

Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsFormat.java Wed Oct 15 17:32:35 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
 import org.apache.lucene.codecs.NormsConsumer;
 import org.apache.lucene.codecs.NormsFormat;
 import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.UndeadNormsProducer;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.util.packed.PackedInts;
@@ -62,7 +63,11 @@ public class Lucene42NormsFormat extends
   
   @Override
   public final NormsProducer normsProducer(SegmentReadState state) throws IOException {
-    return new Lucene42NormsProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
+    if (UndeadNormsProducer.isUndeadArmy(state.fieldInfos)) {
+      return UndeadNormsProducer.INSTANCE;
+    } else {
+      return new Lucene42NormsProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
+    }
   }
   
   static final String DATA_CODEC = "Lucene41NormsData";

Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsProducer.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsProducer.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsProducer.java Wed Oct 15 17:32:35 2014
@@ -21,6 +21,8 @@ import java.io.IOException;
 
 import org.apache.lucene.codecs.DocValuesProducer;
 import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.UndeadNormsProducer;
+import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.SegmentReadState;
@@ -45,6 +47,10 @@ final class Lucene42NormsProducer extend
 
   @Override
   public NumericDocValues getNorms(FieldInfo field) throws IOException {
+    if (UndeadNormsProducer.isUndead(field)) {
+      // Bring undead norms back to life; this is set in Lucene42FieldInfosFormat, to emulate pre-5.0 undead norms
+      return DocValues.emptyNumeric();
+    }
     return impl.getNumeric(field);
   }
 

Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java Wed Oct 15 17:32:35 2014
@@ -23,13 +23,14 @@ import java.util.Map;
 
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.UndeadNormsProducer;
 import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.store.ChecksumIndexInput;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
@@ -42,7 +43,7 @@ import org.apache.lucene.store.IndexOutp
  */
 @Deprecated
 public final class Lucene46FieldInfosFormat extends FieldInfosFormat {
-  
+
   /** Sole constructor. */
   public Lucene46FieldInfosFormat() {
   }
@@ -88,8 +89,14 @@ public final class Lucene46FieldInfosFor
         final DocValuesType normsType = getDocValuesType(input, (byte) ((val >>> 4) & 0x0F));
         final long dvGen = input.readLong();
         final Map<String,String> attributes = input.readStringStringMap();
+
+        if (isIndexed && omitNorms == false && normsType == null) {
+          // Undead norms!  Lucene42NormsProducer will check this and bring norms back from the dead:
+          UndeadNormsProducer.setUndead(attributes);
+        }
+
         infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, 
-          omitNorms, storePayloads, indexOptions, docValuesType, normsType, dvGen, Collections.unmodifiableMap(attributes));
+          omitNorms, storePayloads, indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(attributes));
       }
       
       if (codecVersion >= Lucene46FieldInfosFormat.FORMAT_CHECKSUM) {
@@ -148,7 +155,7 @@ public final class Lucene46FieldInfosFor
 
         // pack the DV types in one byte
         final byte dv = docValuesByte(fi.getDocValuesType());
-        final byte nrm = docValuesByte(fi.getNormType());
+        final byte nrm = docValuesByte(fi.hasNorms() ? DocValuesType.NUMERIC : null);
         assert (dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0;
         byte val = (byte) (0xff & ((nrm << 4) | dv));
         output.writeByte(val);

Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java Wed Oct 15 17:32:35 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
 import org.apache.lucene.codecs.NormsConsumer;
 import org.apache.lucene.codecs.NormsFormat;
 import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.UndeadNormsProducer;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SegmentWriteState;
 
@@ -42,7 +43,11 @@ public class Lucene49NormsFormat extends
 
   @Override
   public final NormsProducer normsProducer(SegmentReadState state) throws IOException {
-    return new Lucene49NormsProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
+    if (UndeadNormsProducer.isUndeadArmy(state.fieldInfos)) {
+      return UndeadNormsProducer.INSTANCE;
+    } else {
+      return new Lucene49NormsProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
+    }
   }
   
   static final String DATA_CODEC = "Lucene49NormsData";

Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java Wed Oct 15 17:32:35 2014
@@ -25,7 +25,9 @@ import java.util.concurrent.atomic.Atomi
 
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.UndeadNormsProducer;
 import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexFileNames;
@@ -40,8 +42,8 @@ import org.apache.lucene.util.RamUsageEs
 import org.apache.lucene.util.packed.BlockPackedReader;
 import org.apache.lucene.util.packed.PackedInts;
 
-import static org.apache.lucene.codecs.lucene49.Lucene49NormsFormat.VERSION_START;
 import static org.apache.lucene.codecs.lucene49.Lucene49NormsFormat.VERSION_CURRENT;
+import static org.apache.lucene.codecs.lucene49.Lucene49NormsFormat.VERSION_START;
 
 /**
  * Reader for 4.9 norms
@@ -153,6 +155,10 @@ final class Lucene49NormsProducer extend
 
   @Override
   public synchronized NumericDocValues getNorms(FieldInfo field) throws IOException {
+    if (UndeadNormsProducer.isUndead(field)) {
+      // Bring undead norms back to life; this is set in Lucene46FieldInfosFormat, to emulate pre-5.0 undead norms
+      return DocValues.emptyNumeric();
+    }
     NumericDocValues instance = instances.get(field.name);
     if (instance == null) {
       instance = loadNorms(field);

Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWFieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWFieldInfosFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWFieldInfosFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWFieldInfosFormat.java Wed Oct 15 17:32:35 2014
@@ -75,7 +75,7 @@ public final class Lucene40RWFieldInfosF
 
         // pack the DV types in one byte
         final byte dv = docValuesByte(fi.getDocValuesType(), fi.getAttribute(LEGACY_DV_TYPE_KEY));
-        final byte nrm = docValuesByte(fi.getNormType(), fi.getAttribute(LEGACY_NORM_TYPE_KEY));
+        final byte nrm = docValuesByte(fi.hasNorms() ? DocValuesType.NUMERIC : null, fi.getAttribute(LEGACY_NORM_TYPE_KEY));
         assert (dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0;
         byte val = (byte) (0xff & ((nrm << 4) | dv));
         output.writeByte(val);

Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40FieldInfoFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40FieldInfoFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40FieldInfoFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40FieldInfoFormat.java Wed Oct 15 17:32:35 2014
@@ -64,7 +64,7 @@ public class TestLucene40FieldInfoFormat
       }
     }
     
-    if (fi.getNormType() != null) {
+    if (fi.hasNorms()) {
       fi.putAttribute(Lucene40FieldInfosFormat.LEGACY_NORM_TYPE_KEY, LegacyDocValuesType.FIXED_INTS_8.name());
     }
   }

Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40NormsFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40NormsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40NormsFormat.java Wed Oct 15 17:32:35 2014
@@ -17,8 +17,18 @@ package org.apache.lucene.codecs.lucene4
  * limitations under the License.
  */
 
+import java.io.InputStream;
+import java.nio.file.Path;
+
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.index.BaseNormsFormatTestCase;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.TestUtil;
 
 /** Tests Lucene40's norms format */
 public class TestLucene40NormsFormat extends BaseNormsFormatTestCase {
@@ -28,4 +38,101 @@ public class TestLucene40NormsFormat ext
   protected Codec getCodec() {
     return codec;
   }
+
+  /** Copy this back to /l/400/lucene/CreateUndeadNorms.java, then:
+   *   - ant clean
+   *   - pushd analysis/common; ant jar; popd
+   *   - pushd core; ant jar; popd
+   *   - javac -cp build/analysis/common/lucene-analyzers-common-4.0-SNAPSHOT.jar:build/core/lucene-core-4.0-SNAPSHOT.jar CreateUndeadNorms.java
+   *   - java -cp .:build/analysis/common/lucene-analyzers-common-4.0-SNAPSHOT.jar:build/core/lucene-core-4.0-SNAPSHOT.jar CreateUndeadNorms
+   *   - cd /tmp/undeadnorms  ; zip index.40.undeadnorms.zip *
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.Version;
+
+public class CreateUndeadNorms {
+  public static void main(String[] args) throws Exception {
+    File file = new File("/tmp/undeadnorms");
+    if (file.exists()) {
+      throw new RuntimeException("please remove /tmp/undeadnorms first");
+    }
+    Directory dir = FSDirectory.open(new File("/tmp/undeadnorms"));
+    IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_40, new WhitespaceAnalyzer(Version.LUCENE_40)));
+    Document doc = new Document();
+    doc.add(new StringField("id", "0", Field.Store.NO));
+    w.addDocument(doc);
+    doc = new Document();
+    doc.add(new StringField("id", "1", Field.Store.NO));
+    Field content = new TextField("content", "some content", Field.Store.NO);
+    content.setTokenStream(new TokenStream() {
+        @Override
+        public boolean incrementToken() throws IOException {
+          throw new IOException("brains brains!");
+        }
+      });
+
+    doc.add(content);
+    try {
+      w.addDocument(doc);
+      throw new RuntimeException("didn't hit exception");
+    } catch (IOException ioe) {
+      // perfect
+    }
+    w.close();
+    dir.close();
+  }
+}
+*/
+
+  /** 
+   * LUCENE-6006: Test undead norms.
+   *                                 .....            
+   *                             C C  /            
+   *                            /<   /             
+   *             ___ __________/_#__=o             
+   *            /(- /(\_\________   \              
+   *            \ ) \ )_      \o     \             
+   *            /|\ /|\       |'     |             
+   *                          |     _|             
+   *                          /o   __\             
+   *                         / '     |             
+   *                        / /      |             
+   *                       /_/\______|             
+   *                      (   _(    <              
+   *                       \    \    \             
+   *                        \    \    |            
+   *                         \____\____\           
+   *                         ____\_\__\_\          
+   *                       /`   /`     o\          
+   *                       |___ |_______|
+   *
+   */
+  public void testReadUndeadNorms() throws Exception {
+    InputStream resource = TestLucene40NormsFormat.class.getResourceAsStream("index.40.undeadnorms.zip");
+    assertNotNull(resource);
+    Path path = createTempDir("undeadnorms");
+    TestUtil.unzip(resource, path);
+    Directory dir = FSDirectory.open(path);
+    IndexReader r = DirectoryReader.open(dir);
+    NumericDocValues undeadNorms = MultiDocValues.getNormValues(r, "content");
+    assertNotNull(undeadNorms);
+    assertEquals(2, r.maxDoc());
+    assertEquals(0, undeadNorms.get(0));
+    assertEquals(0, undeadNorms.get(1));
+    dir.close();
+    r.close();
+  }
 }

Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWFieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWFieldInfosFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWFieldInfosFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWFieldInfosFormat.java Wed Oct 15 17:32:35 2014
@@ -76,7 +76,7 @@ public final class Lucene42RWFieldInfosF
 
         // pack the DV types in one byte
         final byte dv = docValuesByte(fi.getDocValuesType());
-        final byte nrm = docValuesByte(fi.getNormType());
+        final byte nrm = docValuesByte(fi.hasNorms() ? DocValuesType.NUMERIC : null);
         assert (dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0;
         byte val = (byte) (0xff & ((nrm << 4) | dv));
         output.writeByte(val);

Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/TestLucene42NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/TestLucene42NormsFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/TestLucene42NormsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/TestLucene42NormsFormat.java Wed Oct 15 17:32:35 2014
@@ -17,10 +17,18 @@ package org.apache.lucene.codecs.lucene4
  * limitations under the License.
  */
 
+import java.io.InputStream;
+import java.nio.file.Path;
+
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.index.BaseNormsFormatTestCase;
-import org.junit.BeforeClass;
-
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.TestUtil;
 
 /** Tests Lucene42's norms format */
 public class TestLucene42NormsFormat extends BaseNormsFormatTestCase {
@@ -30,4 +38,100 @@ public class TestLucene42NormsFormat ext
   protected Codec getCodec() {
     return codec;
   }
+
+  /** Copy this back to /l/421/lucene/CreateUndeadNorms.java, then:
+   *   - ant clean
+   *   - pushd analysis/common; ant jar; popd
+   *   - pushd core; ant jar; popd
+   *   - javac -cp build/analysis/common/lucene-analyzers-common-4.2.1-SNAPSHOT.jar:build/core/lucene-core-4.2.1-SNAPSHOT.jar CreateUndeadNorms.java
+   *   - java -cp .:build/analysis/common/lucene-analyzers-common-4.2.1-SNAPSHOT.jar:build/core/lucene-core-4.2.1-SNAPSHOT.jar CreateUndeadNorms
+   *   - cd /tmp/undeadnorms  ; zip index.42.undeadnorms.zip *
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.Version;
+
+public class CreateUndeadNorms {
+  public static void main(String[] args) throws Exception {
+    File file = new File("/tmp/undeadnorms");
+    if (file.exists()) {
+      throw new RuntimeException("please remove /tmp/undeadnorms first");
+    }
+    Directory dir = FSDirectory.open(file);
+    IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_42, new WhitespaceAnalyzer(Version.LUCENE_42)));
+    Document doc = new Document();
+    doc.add(new StringField("id", "0", Field.Store.NO));
+    w.addDocument(doc);
+    doc = new Document();
+    doc.add(new StringField("id", "1", Field.Store.NO));
+    Field content = new TextField("content", "some content", Field.Store.NO);
+    content.setTokenStream(new TokenStream() {
+        @Override
+        public boolean incrementToken() throws IOException {
+          throw new IOException("brains brains!");
+        }
+      });
+
+    doc.add(content);
+    try {
+      w.addDocument(doc);
+      throw new RuntimeException("didn't hit exception");
+    } catch (IOException ioe) {
+      // perfect
+    }
+    w.close();
+    dir.close();
+  }
+}
+*/
+  /** 
+   * LUCENE-6006: Test undead norms.
+   *                                 .....            
+   *                             C C  /            
+   *                            /<   /             
+   *             ___ __________/_#__=o             
+   *            /(- /(\_\________   \              
+   *            \ ) \ )_      \o     \             
+   *            /|\ /|\       |'     |             
+   *                          |     _|             
+   *                          /o   __\             
+   *                         / '     |             
+   *                        / /      |             
+   *                       /_/\______|             
+   *                      (   _(    <              
+   *                       \    \    \             
+   *                        \    \    |            
+   *                         \____\____\           
+   *                         ____\_\__\_\          
+   *                       /`   /`     o\          
+   *                       |___ |_______|
+   *
+   */
+  public void testReadUndeadNorms() throws Exception {
+    InputStream resource = TestLucene42NormsFormat.class.getResourceAsStream("index.42.undeadnorms.zip");
+    assertNotNull(resource);
+    Path path = createTempDir("undeadnorms");
+    TestUtil.unzip(resource, path);
+    Directory dir = FSDirectory.open(path);
+    IndexReader r = DirectoryReader.open(dir);
+    NumericDocValues undeadNorms = MultiDocValues.getNormValues(r, "content");
+    assertNotNull(undeadNorms);
+    assertEquals(2, r.maxDoc());
+    assertEquals(0, undeadNorms.get(0));
+    assertEquals(0, undeadNorms.get(1));
+    dir.close();
+    r.close();
+  }
 }

Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java Wed Oct 15 17:32:35 2014
@@ -17,8 +17,18 @@ package org.apache.lucene.codecs.lucene4
  * limitations under the License.
  */
 
+import java.io.InputStream;
+import java.nio.file.Path;
+
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.index.BaseNormsFormatTestCase;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.TestUtil;
 
 /**
  * Tests Lucene49NormsFormat
@@ -30,4 +40,101 @@ public class TestLucene49NormsFormat ext
   protected Codec getCodec() {
     return codec;
   } 
+
+  /** Copy this back to /l/491/lucene/CreateUndeadNorms.java, then:
+   *   - ant clean
+   *   - pushd analysis/common; ant jar; popd
+   *   - pushd core; ant jar; popd
+   *   - javac -cp build/analysis/common/lucene-analyzers-common-4.9-SNAPSHOT.jar:build/core/lucene-core-4.9-SNAPSHOT.jar CreateUndeadNorms.java
+   *   - java -cp .:build/analysis/common/lucene-analyzers-common-4.9-SNAPSHOT.jar:build/core/lucene-core-4.9-SNAPSHOT.jar CreateUndeadNorms
+   *   - cd /tmp/undeadnorms  ; zip index.49.undeadnorms.zip *
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.Version;
+
+public class CreateUndeadNorms {
+  public static void main(String[] args) throws Exception {
+    File file = new File("/tmp/undeadnorms");
+    if (file.exists()) {
+      throw new RuntimeException("please remove /tmp/undeadnorms first");
+    }
+    Directory dir = FSDirectory.open(file);
+    IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_4_9, new WhitespaceAnalyzer(Version.LUCENE_4_9)));
+    Document doc = new Document();
+    doc.add(new StringField("id", "0", Field.Store.NO));
+    w.addDocument(doc);
+    doc = new Document();
+    doc.add(new StringField("id", "1", Field.Store.NO));
+    Field content = new TextField("content", "some content", Field.Store.NO);
+    content.setTokenStream(new TokenStream() {
+        @Override
+        public boolean incrementToken() throws IOException {
+          throw new IOException("brains brains!");
+        }
+      });
+
+    doc.add(content);
+    try {
+      w.addDocument(doc);
+      throw new RuntimeException("didn't hit exception");
+    } catch (IOException ioe) {
+      // perfect
+    }
+    w.close();
+    dir.close();
+  }
+}
+*/
+
+  /** 
+   * LUCENE-6006: Test undead norms.
+   *                                 .....            
+   *                             C C  /            
+   *                            /<   /             
+   *             ___ __________/_#__=o             
+   *            /(- /(\_\________   \              
+   *            \ ) \ )_      \o     \             
+   *            /|\ /|\       |'     |             
+   *                          |     _|             
+   *                          /o   __\             
+   *                         / '     |             
+   *                        / /      |             
+   *                       /_/\______|             
+   *                      (   _(    <              
+   *                       \    \    \             
+   *                        \    \    |            
+   *                         \____\____\           
+   *                         ____\_\__\_\          
+   *                       /`   /`     o\          
+   *                       |___ |_______|
+   *
+   */
+  public void testReadUndeadNorms() throws Exception {
+    InputStream resource = TestLucene49NormsFormat.class.getResourceAsStream("index.49.undeadnorms.zip");
+    assertNotNull(resource);
+    Path path = createTempDir("undeadnorms");
+    TestUtil.unzip(resource, path);
+    Directory dir = FSDirectory.open(path);
+    IndexReader r = DirectoryReader.open(dir);
+    NumericDocValues undeadNorms = MultiDocValues.getNormValues(r, "content");
+    assertNotNull(undeadNorms);
+    assertEquals(2, r.maxDoc());
+    assertEquals(0, undeadNorms.get(0));
+    assertEquals(0, undeadNorms.get(1));
+    dir.close();
+    r.close();
+  }
 }

Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Wed Oct 15 17:32:35 2014
@@ -22,7 +22,6 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.PrintStream;
 import java.lang.reflect.Modifier;
-import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -121,7 +120,7 @@ public class TestBackwardsCompatibility 
 
   private Path getIndexDir() {
     String path = System.getProperty("tests.bwcdir");
-    assumeTrue("backcompat creation tests must be run with -Dtests,bwcdir=/path/to/write/indexes", path != null);
+    assumeTrue("backcompat creation tests must be run with -Dtests.bwcdir=/path/to/write/indexes", path != null);
     return Paths.get(path);
   }
   

Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java Wed Oct 15 17:32:35 2014
@@ -71,8 +71,7 @@ class SimpleTextDocValuesWriter extends 
   @Override
   public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
     assert fieldSeen(field.name);
-    assert (field.getDocValuesType() == FieldInfo.DocValuesType.NUMERIC ||
-            field.getNormType() == FieldInfo.DocValuesType.NUMERIC);
+    assert field.getDocValuesType() == FieldInfo.DocValuesType.NUMERIC || field.hasNorms();
     writeFieldEntry(field, FieldInfo.DocValuesType.NUMERIC);
 
     // first pass to find min/max

Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java Wed Oct 15 17:32:35 2014
@@ -59,7 +59,6 @@ public class SimpleTextFieldInfosFormat 
   static final BytesRef STORETVOFF      =  new BytesRef("  term vector offsets ");
   static final BytesRef PAYLOADS        =  new BytesRef("  payloads ");
   static final BytesRef NORMS           =  new BytesRef("  norms ");
-  static final BytesRef NORMS_TYPE      =  new BytesRef("  norms type ");
   static final BytesRef DOCVALUES       =  new BytesRef("  doc values ");
   static final BytesRef DOCVALUES_GEN   =  new BytesRef("  doc values gen ");
   static final BytesRef INDEXOPTIONS    =  new BytesRef("  index options ");
@@ -116,11 +115,6 @@ public class SimpleTextFieldInfosFormat 
         boolean omitNorms = !Boolean.parseBoolean(readString(NORMS.length, scratch));
         
         SimpleTextUtil.readLine(input, scratch);
-        assert StringHelper.startsWith(scratch.get(), NORMS_TYPE);
-        String nrmType = readString(NORMS_TYPE.length, scratch);
-        final DocValuesType normsType = docValuesType(nrmType);
-        
-        SimpleTextUtil.readLine(input, scratch);
         assert StringHelper.startsWith(scratch.get(), DOCVALUES);
         String dvType = readString(DOCVALUES.length, scratch);
         final DocValuesType docValuesType = docValuesType(dvType);
@@ -146,7 +140,7 @@ public class SimpleTextFieldInfosFormat 
         }
 
         infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, 
-          omitNorms, storePayloads, indexOptions, docValuesType, normsType, dvGen, Collections.unmodifiableMap(atts));
+          omitNorms, storePayloads, indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(atts));
       }
 
       SimpleTextUtil.checkFooter(input);
@@ -217,11 +211,7 @@ public class SimpleTextFieldInfosFormat 
         SimpleTextUtil.write(out, NORMS);
         SimpleTextUtil.write(out, Boolean.toString(!fi.omitsNorms()), scratch);
         SimpleTextUtil.writeNewline(out);
-        
-        SimpleTextUtil.write(out, NORMS_TYPE);
-        SimpleTextUtil.write(out, getDocValuesType(fi.getNormType()), scratch);
-        SimpleTextUtil.writeNewline(out);
-        
+
         SimpleTextUtil.write(out, DOCVALUES);
         SimpleTextUtil.write(out, getDocValuesType(fi.getDocValuesType()), scratch);
         SimpleTextUtil.writeNewline(out);

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java Wed Oct 15 17:32:35 2014
@@ -102,7 +102,7 @@ import org.apache.lucene.store.IndexOutp
  * @lucene.experimental
  */
 public final class Lucene50FieldInfosFormat extends FieldInfosFormat {
-  
+
   /** Sole constructor. */
   public Lucene50FieldInfosFormat() {
   }
@@ -149,12 +149,11 @@ public final class Lucene50FieldInfosFor
           // DV Types are packed in one byte
           byte val = input.readByte();
           final DocValuesType docValuesType = getDocValuesType(input, (byte) (val & 0x0F));
-          final DocValuesType normsType = getDocValuesType(input, (byte) ((val >>> 4) & 0x0F));
           final long dvGen = input.readLong();
           final Map<String,String> attributes = input.readStringStringMap();
           try {
             infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, 
-                                     indexOptions, docValuesType, normsType, dvGen, Collections.unmodifiableMap(attributes));
+                                     indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(attributes));
             infos[i].checkConsistency();
           } catch (IllegalStateException e) {
             throw new CorruptIndexException("invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e);
@@ -215,12 +214,10 @@ public final class Lucene50FieldInfosFor
         output.writeVInt(fi.number);
         output.writeByte(bits);
 
-        // pack the DV types in one byte
+        // pack the DV type and hasNorms in one byte
         final byte dv = docValuesByte(fi.getDocValuesType());
-        final byte nrm = docValuesByte(fi.getNormType());
-        assert (dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0;
-        byte val = (byte) (0xff & ((nrm << 4) | dv));
-        output.writeByte(val);
+        assert (dv & (~0xF)) == 0;
+        output.writeByte(dv);
         output.writeLong(fi.getDocValuesGen());
         output.writeStringStringMap(fi.attributes());
       }

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Wed Oct 15 17:32:35 2014
@@ -1791,12 +1791,8 @@ public class CheckIndex implements Close
   }
   
   private static void checkNorms(FieldInfo fi, LeafReader reader, PrintStream infoStream) throws IOException {
-    switch(fi.getNormType()) {
-      case NUMERIC:
-        checkNumericDocValues(fi.name, reader, reader.getNormValues(fi.name), new Bits.MatchAllBits(reader.maxDoc()));
-        break;
-      default:
-        throw new AssertionError("wtf: " + fi.getNormType());
+    if (fi.hasNorms()) {
+      checkNumericDocValues(fi.name, reader, reader.getNormValues(fi.name), new Bits.MatchAllBits(reader.maxDoc()));
     }
   }
 

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java Wed Oct 15 17:32:35 2014
@@ -183,14 +183,10 @@ final class DefaultIndexingChain extends
 
           // we must check the final value of omitNorms for the fieldinfo: it could have 
           // changed for this field since the first time we added it.
-          if (fi.omitsNorms() == false) {
-            if (perField.norms != null) {
-              perField.norms.finish(state.segmentInfo.getDocCount());
-              perField.norms.flush(state, normsConsumer);
-              assert fi.getNormType() == DocValuesType.NUMERIC;
-            } else if (fi.isIndexed()) {
-              assert fi.getNormType() == null: "got " + fi.getNormType() + "; field=" + fi.name;
-            }
+          if (fi.omitsNorms() == false && fi.isIndexed()) {
+            assert perField.norms != null: "field=" + fi.name;
+            perField.norms.finish(state.segmentInfo.getDocCount());
+            perField.norms.flush(state, normsConsumer);
           }
         }
       }
@@ -546,6 +542,11 @@ final class DefaultIndexingChain extends
     void setInvertState() {
       invertState = new FieldInvertState(fieldInfo.name);
       termsHashPerField = termsHash.addField(invertState, fieldInfo);
+      if (fieldInfo.omitsNorms() == false) {
+        assert norms == null;
+        // Even if no documents actually succeed in setting a norm, we still write norms for this segment:
+        norms = new NormValuesWriter(fieldInfo, docState.docWriter.bytesUsed);
+      }
     }
 
     @Override
@@ -554,14 +555,8 @@ final class DefaultIndexingChain extends
     }
 
     public void finish() throws IOException {
-      if (fieldInfo.omitsNorms() == false) {
-        if (norms == null) {
-          fieldInfo.setNormValueType(FieldInfo.DocValuesType.NUMERIC);
-          norms = new NormValuesWriter(fieldInfo, docState.docWriter.bytesUsed);
-        }
-        if (invertState.length != 0) {
-          norms.addValue(docState.docID, similarity.computeNorm(invertState));
-        }
+      if (fieldInfo.omitsNorms() == false && invertState.length != 0) {
+        norms.addValue(docState.docID, similarity.computeNorm(invertState));
       }
 
       termsHashPerField.finish();

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java Wed Oct 15 17:32:35 2014
@@ -40,15 +40,15 @@ public final class FieldInfo {
   // True if any document indexed term vectors
   private boolean storeTermVector;
 
-  private DocValuesType normType;
   private boolean omitNorms; // omit norms associated with indexed fields  
+
   private IndexOptions indexOptions;
   private boolean storePayloads; // whether this field stores payloads together with term positions
 
   private Map<String,String> attributes;
 
   private long dvGen;
-  
+
   /**
    * Controls how much information is stored in the postings lists.
    * @lucene.experimental
@@ -120,12 +120,12 @@ public final class FieldInfo {
   }
 
   /**
-   * Sole Constructor.
+   * Sole constructor.
    *
    * @lucene.experimental
    */
   public FieldInfo(String name, boolean indexed, int number, boolean storeTermVector, boolean omitNorms, 
-      boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues, DocValuesType normsType, 
+      boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues,
       long dvGen, Map<String,String> attributes) {
     this.name = name;
     this.indexed = indexed;
@@ -136,13 +136,11 @@ public final class FieldInfo {
       this.storePayloads = storePayloads;
       this.omitNorms = omitNorms;
       this.indexOptions = indexOptions;
-      this.normType = !omitNorms ? normsType : null;
     } else { // for non-indexed fields, leave defaults
       this.storeTermVector = false;
       this.storePayloads = false;
       this.omitNorms = false;
       this.indexOptions = null;
-      this.normType = null;
     }
     this.dvGen = dvGen;
     this.attributes = attributes;
@@ -158,11 +156,6 @@ public final class FieldInfo {
       if (indexOptions == null) {
         throw new IllegalStateException("indexed field '" + name + "' must have index options");
       }
-      if (omitNorms) {
-        if (normType != null) {
-          throw new IllegalStateException("indexed field '" + name + "' cannot both omit norms and have norms");
-        }
-      }
       // Cannot store payloads unless positions are indexed:
       if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0 && storePayloads) {
         throw new IllegalStateException("indexed field '" + name + "' cannot have payloads without positions");
@@ -177,12 +170,8 @@ public final class FieldInfo {
       if (omitNorms) {
         throw new IllegalStateException("non-indexed field '" + name + "' cannot omit norms");
       }
-      if (normType != null) {
-        throw new IllegalStateException("non-indexed field '" + name + "' cannot have norms");
-      }
       if (indexOptions != null) {
         throw new IllegalStateException("non-indexed field '" + name + "' cannot have index options");
-
       }
     }
     
@@ -206,7 +195,6 @@ public final class FieldInfo {
       this.storePayloads |= storePayloads;
       if (this.omitNorms != omitNorms) {
         this.omitNorms = true;                // if one require omitNorms at least once, it remains off for life
-        this.normType = null;
       }
       if (this.indexOptions != indexOptions) {
         if (this.indexOptions == null) {
@@ -265,13 +253,6 @@ public final class FieldInfo {
     return dvGen;
   }
   
-  /**
-   * Returns {@link DocValuesType} of the norm. this may be null if the field has no norms.
-   */
-  public DocValuesType getNormType() {
-    return normType;
-  }
-
   void setStoreTermVectors() {
     storeTermVector = true;
     assert checkConsistency();
@@ -284,14 +265,6 @@ public final class FieldInfo {
     assert checkConsistency();
   }
 
-  void setNormValueType(DocValuesType type) {
-    if (normType != null && normType != type) {
-      throw new IllegalArgumentException("cannot change Norm type from " + normType + " to " + type + " for field \"" + name + "\"");
-    }
-    normType = type;
-    assert checkConsistency();
-  }
-  
   /**
    * Returns true if norms are explicitly omitted for this field
    */
@@ -303,7 +276,7 @@ public final class FieldInfo {
    * Returns true if this field actually has any norms.
    */
   public boolean hasNorms() {
-    return normType != null;
+    return indexed && omitNorms == false;
   }
   
   /**

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java Wed Oct 15 17:32:35 2014
@@ -276,10 +276,8 @@ public class FieldInfos implements Itera
     }
    
     /** NOTE: this method does not carry over termVector
-     *  booleans nor docValuesType; the indexer chain
-     *  (TermVectorsConsumerPerField, DocFieldProcessor) must
-     *  set these fields when they succeed in consuming
-     *  the document */
+     *  the indexer chain must set these fields when they
+     *  succeed in consuming the document */
     public FieldInfo addOrUpdate(String name, IndexableFieldType fieldType) {
       // TODO: really, indexer shouldn't even call this
       // method (it's only called from DocFieldProcessor);
@@ -288,12 +286,12 @@ public class FieldInfos implements Itera
       // be updated by maybe FreqProxTermsWriterPerField:
       return addOrUpdateInternal(name, -1, fieldType.indexed(), false,
                                  fieldType.omitNorms(), false,
-                                 fieldType.indexOptions(), fieldType.docValueType(), null);
+                                 fieldType.indexOptions(), fieldType.docValueType());
     }
 
     private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber, boolean isIndexed,
         boolean storeTermVector,
-        boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues, DocValuesType normType) {
+        boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues) {
       FieldInfo fi = fieldInfo(name);
       if (fi == null) {
         // This field wasn't yet added to this in-RAM
@@ -302,7 +300,7 @@ public class FieldInfos implements Itera
         // before then we'll get the same name and number,
         // else we'll allocate a new one:
         final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber, docValues);
-        fi = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, normType, -1, null);
+        fi = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, -1, null);
         assert !byName.containsKey(fi.name);
         assert globalFieldNumbers.containsConsistent(Integer.valueOf(fi.number), fi.name, fi.getDocValuesType());
         byName.put(fi.name, fi);
@@ -319,26 +317,22 @@ public class FieldInfos implements Itera
             globalFieldNumbers.setDocValuesType(fi.number, name, docValues);
           }
         }
-
-        if (!fi.omitsNorms() && normType != null) {
-          fi.setNormValueType(normType);
-        }
       }
       return fi;
     }
-    
+
     public FieldInfo add(FieldInfo fi) {
       // IMPORTANT - reuse the field number if possible for consistent field numbers across segments
       return addOrUpdateInternal(fi.name, fi.number, fi.isIndexed(), fi.hasVectors(),
                  fi.omitsNorms(), fi.hasPayloads(),
-                 fi.getIndexOptions(), fi.getDocValuesType(), fi.getNormType());
+                 fi.getIndexOptions(), fi.getDocValuesType());
     }
     
     public FieldInfo fieldInfo(String fieldName) {
       return byName.get(fieldName);
     }
     
-    final FieldInfos finish() {
+    FieldInfos finish() {
       return new FieldInfos(byName.values().toArray(new FieldInfo[byName.size()]));
     }
   }

Modified: lucene/dev/branches/branch_5x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/branch_5x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Wed Oct 15 17:32:35 2014
@@ -445,7 +445,7 @@ public class MemoryIndex {
 
       if (!fieldInfos.containsKey(fieldName)) {
         fieldInfos.put(fieldName, 
-            new FieldInfo(fieldName, true, fieldInfos.size(), false, false, false, this.storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS , null, null, -1, null));
+            new FieldInfo(fieldName, true, fieldInfos.size(), false, false, false, this.storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS , null, -1, null));
       }
       TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class);
       PositionIncrementAttribute posIncrAttribute = stream.addAttribute(PositionIncrementAttribute.class);

Modified: lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java Wed Oct 15 17:32:35 2014
@@ -21,22 +21,22 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Map;
 
+import org.apache.lucene.document.BinaryDocValuesField; // javadocs
+import org.apache.lucene.document.DoubleField; // javadocs
+import org.apache.lucene.document.FloatField; // javadocs
 import org.apache.lucene.document.IntField; // javadocs
 import org.apache.lucene.document.LongField; // javadocs
-import org.apache.lucene.document.FloatField; // javadocs
-import org.apache.lucene.document.DoubleField; // javadocs
-import org.apache.lucene.document.BinaryDocValuesField; // javadocs
 import org.apache.lucene.document.NumericDocValuesField; // javadocs
 import org.apache.lucene.document.SortedDocValuesField; // javadocs
 import org.apache.lucene.document.SortedSetDocValuesField; // javadocs
 import org.apache.lucene.document.StringField; // javadocs
-import org.apache.lucene.index.FilterLeafReader;
-import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.FilterDirectoryReader;
+import org.apache.lucene.index.FilterLeafReader;
+import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
@@ -213,7 +213,7 @@ public class UninvertingReader extends F
         }
       }
       filteredInfos.add(new FieldInfo(fi.name, fi.isIndexed(), fi.number, fi.hasVectors(), fi.omitsNorms(),
-                                      fi.hasPayloads(), fi.getIndexOptions(), type, fi.getNormType(), -1, null));
+                                      fi.hasPayloads(), fi.getIndexOptions(), type, -1, null));
     }
     fieldInfos = new FieldInfos(filteredInfos.toArray(new FieldInfo[filteredInfos.size()]));
   }

Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java Wed Oct 15 17:32:35 2014
@@ -93,7 +93,7 @@ public class AssertingNormsFormat extend
 
     @Override
     public NumericDocValues getNorms(FieldInfo field) throws IOException {
-      assert field.getNormType() == FieldInfo.DocValuesType.NUMERIC;
+      assert field.hasNorms();
       NumericDocValues values = in.getNorms(field);
       assert values != null;
       return new AssertingLeafReader.AssertingNumericDocValues(values, maxDoc);

Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java Wed Oct 15 17:32:35 2014
@@ -89,11 +89,6 @@ public abstract class BaseFieldInfoForma
           fi.setStorePayloads();
         }
       }
-      if (fi.isIndexed() && !fi.omitsNorms()) {
-        if (random().nextBoolean()) {
-          fi.setNormValueType(DocValuesType.NUMERIC);
-        }
-      }
       addAttributes(fi);
     }
     FieldInfos infos = builder.finish();
@@ -165,7 +160,6 @@ public abstract class BaseFieldInfoForma
     assertEquals(expected.name, actual.name);
     assertEquals(expected.getDocValuesType(), actual.getDocValuesType());
     assertEquals(expected.getIndexOptions(), actual.getIndexOptions());
-    assertEquals(expected.getNormType(), actual.getNormType());
     assertEquals(expected.hasDocValues(), actual.hasDocValues());
     assertEquals(expected.hasNorms(), actual.hasNorms());
     assertEquals(expected.hasPayloads(), actual.hasPayloads());

Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java Wed Oct 15 17:32:35 2014
@@ -18,6 +18,8 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Random;
 
 import org.apache.lucene.analysis.Analyzer;
@@ -128,7 +130,6 @@ public abstract class BaseNormsFormatTes
   
   public void testAllZeros() throws Exception {
     int iterations = atLeast(1);
-    final Random r = random();
     for (int i = 0; i < iterations; i++) {
       doTestNormsVersusStoredFields(new LongProducer() {
         @Override
@@ -264,4 +265,60 @@ public abstract class BaseNormsFormatTes
   }
   
   // TODO: test thread safety (e.g. across different fields) explicitly here
+
+  /** 
+   * LUCENE-6006: Tests undead norms.
+   *                                 .....            
+   *                             C C  /            
+   *                            /<   /             
+   *             ___ __________/_#__=o             
+   *            /(- /(\_\________   \              
+   *            \ ) \ )_      \o     \             
+   *            /|\ /|\       |'     |             
+   *                          |     _|             
+   *                          /o   __\             
+   *                         / '     |             
+   *                        / /      |             
+   *                       /_/\______|             
+   *                      (   _(    <              
+   *                       \    \    \             
+   *                        \    \    |            
+   *                         \____\____\           
+   *                         ____\_\__\_\          
+   *                       /`   /`     o\          
+   *                       |___ |_______|
+   *
+   */
+  public void testUndeadNorms() throws Exception {
+    Directory dir = newDirectory();
+    RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+    int numDocs = atLeast(1000);
+    List<Integer> toDelete = new ArrayList<>();
+    for(int i=0;i<numDocs;i++) {
+      Document doc = new Document();
+      doc.add(new StringField("id", ""+i, Field.Store.NO));
+      if (random().nextInt(5) == 1) {
+        toDelete.add(i);
+        doc.add(new TextField("content", "some content", Field.Store.NO));
+      }
+      w.addDocument(doc);
+    }
+    for(Integer id : toDelete) {
+      w.deleteDocuments(new Term("id", ""+id));
+    }
+    w.forceMerge(1);
+    IndexReader r = w.getReader();
+
+    // Confusingly, norms should exist, and should all be 0, even though we deleted all docs that had the field "content".  They should not
+    // be undead:
+    NumericDocValues norms = MultiDocValues.getNormValues(r, "content");
+    assertNotNull(norms);
+    for(int i=0;i<r.maxDoc();i++) {
+      assertEquals(0, norms.get(i));
+    }
+
+    r.close();
+    w.close();
+    dir.close();
+  }
 }

Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java Wed Oct 15 17:32:35 2014
@@ -45,7 +45,6 @@ import org.apache.lucene.codecs.perfield
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.index.TermsEnum.SeekStatus;
 import org.apache.lucene.store.Directory;
@@ -373,7 +372,7 @@ public abstract class BasePostingsFormat
 
       fieldInfoArray[fieldUpto] = new FieldInfo(field, true, fieldUpto, false, false, true,
                                                 IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
-                                                null, DocValuesType.NUMERIC, -1, null);
+                                                null, -1, null);
       fieldUpto++;
 
       SortedMap<BytesRef,SeedAndOrd> postings = new TreeMap<>();
@@ -702,7 +701,6 @@ public abstract class BasePostingsFormat
                                                    doPayloads,
                                                    indexOptions,
                                                    null,
-                                                   DocValuesType.NUMERIC,
                                                    -1,
                                                    null);
     }

Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/Insanity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/Insanity.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/Insanity.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/Insanity.java Wed Oct 15 17:32:35 2014
@@ -65,7 +65,7 @@ public class Insanity {
       for (FieldInfo fi : in.getFieldInfos()) {
         if (fi.name.equals(insaneField)) {
           filteredInfos.add(new FieldInfo(fi.name, fi.isIndexed(), fi.number, fi.hasVectors(), fi.omitsNorms(),
-                                          fi.hasPayloads(), fi.getIndexOptions(), null, fi.getNormType(), -1, null));
+                                          fi.hasPayloads(), fi.getIndexOptions(), null, -1, null));
         } else {
           filteredInfos.add(fi);
         }



Mime
View raw message