lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rm...@apache.org
Subject svn commit: r1210185 - in /lucene/dev/branches/lucene3606/lucene: contrib/misc/src/java/org/apache/lucene/index/codecs/appending/ src/java/org/apache/lucene/index/ src/java/org/apache/lucene/index/codecs/ src/java/org/apache/lucene/index/codecs/lucene3...
Date Sun, 04 Dec 2011 19:24:09 GMT
Author: rmuir
Date: Sun Dec  4 19:24:08 2011
New Revision: 1210185

URL: http://svn.apache.org/viewvc?rev=1210185&view=rev
Log:
LUCENE-3606: move IW's norms writing to codec

Added:
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/NormsConsumer.java
      - copied, changed from r1210104, lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/NormsWriter.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/NormsConsumerPerField.java
      - copied, changed from r1210104, lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/NormsFormat.java
  (with props)
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/NormsWriter.java
  (with props)
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsFormat.java
  (with props)
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsWriter.java
  (with props)
Removed:
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/NormsWriter.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java
Modified:
    lucene/dev/branches/lucene3606/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/SegmentNorms.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/SegmentReader.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/Codec.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java
    lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java
    lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java

Modified: lucene/dev/branches/lucene3606/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java?rev=1210185&r1=1210184&r2=1210185&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java
(original)
+++ lucene/dev/branches/lucene3606/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java
Sun Dec  4 19:24:08 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs.a
 import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.index.codecs.DocValuesFormat;
 import org.apache.lucene.index.codecs.FieldInfosFormat;
+import org.apache.lucene.index.codecs.NormsFormat;
 import org.apache.lucene.index.codecs.StoredFieldsFormat;
 import org.apache.lucene.index.codecs.PostingsFormat;
 import org.apache.lucene.index.codecs.SegmentInfosFormat;
@@ -27,6 +28,7 @@ import org.apache.lucene.index.codecs.Te
 import org.apache.lucene.index.codecs.lucene40.Lucene40Codec;
 import org.apache.lucene.index.codecs.lucene40.Lucene40FieldInfosFormat;
 import org.apache.lucene.index.codecs.lucene40.Lucene40DocValuesFormat;
+import org.apache.lucene.index.codecs.lucene40.Lucene40NormsFormat;
 import org.apache.lucene.index.codecs.lucene40.Lucene40StoredFieldsFormat;
 import org.apache.lucene.index.codecs.lucene40.Lucene40TermVectorsFormat;
 
@@ -47,6 +49,7 @@ public class AppendingCodec extends Code
   private final FieldInfosFormat fieldInfos = new Lucene40FieldInfosFormat();
   private final TermVectorsFormat vectors = new Lucene40TermVectorsFormat();
   private final DocValuesFormat docValues = new Lucene40DocValuesFormat();
+  private final NormsFormat norms = new Lucene40NormsFormat();
   
   @Override
   public PostingsFormat postingsFormat() {
@@ -77,4 +80,9 @@ public class AppendingCodec extends Code
   public FieldInfosFormat fieldInfosFormat() {
     return fieldInfos;
   }
+
+  @Override
+  public NormsFormat normsFormat() {
+    return norms;
+  }
 }

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1210185&r1=1210184&r2=1210185&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
(original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
Sun Dec  4 19:24:08 2011
@@ -52,7 +52,7 @@ import org.apache.lucene.util.InfoStream
  * are processing the document).
  *
  * Other consumers, eg {@link FreqProxTermsWriter} and
- * {@link NormsWriter}, buffer bytes in RAM and flush only
+ * {@link NormsConsumer}, buffer bytes in RAM and flush only
  * when a new segment is produced.
 
  * Once we have used our allowed RAM buffer, or the number

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1210185&r1=1210184&r2=1210185&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
(original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
Sun Dec  4 19:24:08 2011
@@ -79,7 +79,7 @@ public class DocumentsWriterPerThread {
 
       final InvertedDocConsumer  termsHash = new TermsHash(documentsWriterPerThread, freqProxWriter,
true,
                                                            new TermsHash(documentsWriterPerThread,
termVectorsWriter, false, null));
-      final NormsWriter normsWriter = new NormsWriter();
+      final NormsConsumer normsWriter = new NormsConsumer(documentsWriterPerThread);
       final DocInverter docInverter = new DocInverter(documentsWriterPerThread.docState,
termsHash, normsWriter);
       return new DocFieldProcessor(documentsWriterPerThread, docInverter);
     }

Copied: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/NormsConsumer.java
(from r1210104, lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/NormsWriter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/NormsConsumer.java?p2=lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/NormsConsumer.java&p1=lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/NormsWriter.java&r1=1210104&r2=1210185&rev=1210185&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/NormsWriter.java
(original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/NormsConsumer.java
Sun Dec  4 19:24:08 2011
@@ -21,8 +21,8 @@ import java.io.IOException;
 import java.util.Collection;
 import java.util.Map;
 
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.IOContext.Context;
+import org.apache.lucene.index.codecs.NormsFormat;
+import org.apache.lucene.index.codecs.NormsWriter;
 import org.apache.lucene.util.IOUtils;
 
 // TODO FI: norms could actually be stored as doc store
@@ -32,8 +32,12 @@ import org.apache.lucene.util.IOUtils;
  *  merges all of these together into a single _X.nrm file.
  */
 
-final class NormsWriter extends InvertedDocEndConsumer {
-
+final class NormsConsumer extends InvertedDocEndConsumer {
+  final NormsFormat normsFormat;
+  
+  public NormsConsumer(DocumentsWriterPerThread dwpt) {
+    normsFormat = dwpt.codec.normsFormat();
+  }
 
   @Override
   public void abort() {}
@@ -49,27 +53,23 @@ final class NormsWriter extends Inverted
       return;
     }
 
-    final String normsFileName = IndexFileNames.segmentFileName(state.segmentName, "", IndexFileNames.NORMS_EXTENSION);
-    IndexOutput normsOut = state.directory.createOutput(normsFileName, state.context);
+    NormsWriter normsOut = null;
     boolean success = false;
     try {
-      normsOut.writeBytes(SegmentNorms.NORMS_HEADER, 0, SegmentNorms.NORMS_HEADER.length);
-
-      int normCount = 0;
+      normsOut = normsFormat.normsWriter(state);
 
       for (FieldInfo fi : state.fieldInfos) {
-        final NormsWriterPerField toWrite = (NormsWriterPerField) fieldsToFlush.get(fi);
+        final NormsConsumerPerField toWrite = (NormsConsumerPerField) fieldsToFlush.get(fi);
         int upto = 0;
-        if (toWrite != null && toWrite.upto > 0) {
-          normCount++;
-
+        if (!fi.omitNorms && toWrite != null && toWrite.upto > 0) {
+          normsOut.startField(fi);
           int docID = 0;
           for (; docID < state.numDocs; docID++) {
             if (upto < toWrite.upto && toWrite.docIDs[upto] == docID) {
-              normsOut.writeByte(toWrite.norms[upto]);
+              normsOut.writeNorm(toWrite.norms[upto]);
               upto++;
             } else {
-              normsOut.writeByte((byte) 0);
+              normsOut.writeNorm((byte) 0);
             }
           }
 
@@ -78,14 +78,13 @@ final class NormsWriter extends Inverted
 
           toWrite.reset();
         } else if (fi.isIndexed && !fi.omitNorms) {
-          normCount++;
           // Fill entire field with default norm:
+          normsOut.startField(fi);
           for(;upto<state.numDocs;upto++)
-            normsOut.writeByte((byte) 0);
+            normsOut.writeNorm((byte) 0);
         }
-
-        assert 4+normCount*state.numDocs == normsOut.getFilePointer() : ".nrm file size mismatch:
expected=" + (4+normCount*state.numDocs) + " actual=" + normsOut.getFilePointer();
       }
+      normsOut.finish(state.numDocs);
       success = true;
     } finally {
       if (success) {
@@ -105,6 +104,6 @@ final class NormsWriter extends Inverted
   @Override
   InvertedDocEndConsumerPerField addField(DocInverterPerField docInverterPerField,
       FieldInfo fieldInfo) {
-    return new NormsWriterPerField(docInverterPerField, fieldInfo);
+    return new NormsConsumerPerField(docInverterPerField, fieldInfo);
   }
 }

Copied: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/NormsConsumerPerField.java
(from r1210104, lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/NormsConsumerPerField.java?p2=lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/NormsConsumerPerField.java&p1=lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java&r1=1210104&r2=1210185&rev=1210185&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java
(original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/NormsConsumerPerField.java
Sun Dec  4 19:24:08 2011
@@ -25,7 +25,7 @@ import org.apache.lucene.util.ArrayUtil;
  *  just look at the length for the field (docState.length)
  *  and record the norm. */
 
-final class NormsWriterPerField extends InvertedDocEndConsumerPerField implements Comparable<NormsWriterPerField>
{
+final class NormsConsumerPerField extends InvertedDocEndConsumerPerField implements Comparable<NormsConsumerPerField>
{
 
   final FieldInfo fieldInfo;
   final DocumentsWriterPerThread.DocState docState;
@@ -45,7 +45,7 @@ final class NormsWriterPerField extends 
     upto = 0;
   }
 
-  public NormsWriterPerField(final DocInverterPerField docInverterPerField, final FieldInfo
fieldInfo) {
+  public NormsConsumerPerField(final DocInverterPerField docInverterPerField, final FieldInfo
fieldInfo) {
     this.fieldInfo = fieldInfo;
     docState = docInverterPerField.docState;
     fieldState = docInverterPerField.fieldState;
@@ -57,7 +57,7 @@ final class NormsWriterPerField extends 
     upto = 0;
   }
 
-  public int compareTo(NormsWriterPerField other) {
+  public int compareTo(NormsConsumerPerField other) {
     return fieldInfo.name.compareTo(other.fieldInfo.name);
   }
   

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1210185&r1=1210184&r2=1210185&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
(original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
Sun Dec  4 19:24:08 2011
@@ -31,6 +31,7 @@ import org.apache.lucene.index.codecs.Fi
 import org.apache.lucene.index.codecs.StoredFieldsWriter;
 import org.apache.lucene.index.codecs.PerDocConsumer;
 import org.apache.lucene.index.codecs.TermVectorsWriter;
+import org.apache.lucene.index.codecs.lucene40.Lucene40NormsWriter;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexOutput;
@@ -324,6 +325,7 @@ final class SegmentMerger {
       }
   }
 
+  // TODO: implement merge in normsformat instead.
   private void mergeNorms() throws IOException {
     IndexOutput output = null;
     boolean success = false;
@@ -332,7 +334,7 @@ final class SegmentMerger {
         if (fi.isIndexed && !fi.omitNorms) {
           if (output == null) {
             output = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.NORMS_EXTENSION),
context);
-            output.writeBytes(SegmentNorms.NORMS_HEADER, SegmentNorms.NORMS_HEADER.length);
+            output.writeBytes(Lucene40NormsWriter.NORMS_HEADER, Lucene40NormsWriter.NORMS_HEADER.length);
           }
           for (MergeState.IndexReaderAndLiveDocs reader : mergeState.readers) {
             final int maxDoc = reader.reader.maxDoc();

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/SegmentNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/SegmentNorms.java?rev=1210185&r1=1210184&r2=1210185&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/SegmentNorms.java
(original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/SegmentNorms.java
Sun Dec  4 19:24:08 2011
@@ -35,9 +35,6 @@ import org.apache.lucene.store.IndexOutp
 
 final class SegmentNorms implements Cloneable {
 
-  /** norms header placeholder */
-  static final byte[] NORMS_HEADER = new byte[]{'N','R','M',-1};
-
   int refCount = 1;
 
   // If this instance is a clone, the originalNorm

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1210185&r1=1210184&r2=1210185&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/SegmentReader.java
(original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/SegmentReader.java
Sun Dec  4 19:24:08 2011
@@ -33,6 +33,7 @@ import org.apache.lucene.index.FieldInfo
 import org.apache.lucene.index.codecs.StoredFieldsReader;
 import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.codecs.TermVectorsReader;
+import org.apache.lucene.index.codecs.lucene40.Lucene40NormsWriter;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.util.BitVector;
 import org.apache.lucene.util.Bits;
@@ -540,7 +541,8 @@ public class SegmentReader extends Index
   }
 
   private void openNorms(Directory cfsDir, IOContext context) throws IOException {
-    long nextNormSeek = SegmentNorms.NORMS_HEADER.length; //skip header (header unused for
now)
+    boolean normsInitiallyEmpty = norms.isEmpty(); // only used for assert
+    long nextNormSeek = Lucene40NormsWriter.NORMS_HEADER.length; //skip header (header unused
for now)
     int maxDoc = maxDoc();
     for (FieldInfo fi : core.fieldInfos) {
       if (norms.containsKey(fi.name)) {
@@ -585,7 +587,7 @@ public class SegmentReader extends Index
           if (isUnversioned) {
             normSeek = 0;
           } else {
-            normSeek = SegmentNorms.NORMS_HEADER.length;
+            normSeek = Lucene40NormsWriter.NORMS_HEADER.length;
           }
         }
 
@@ -593,6 +595,8 @@ public class SegmentReader extends Index
         nextNormSeek += maxDoc; // increment also if some norms are separate
       }
     }
+    // nocommit: change to a real check? see LUCENE-3619
+    assert singleNormStream == null || !normsInitiallyEmpty || nextNormSeek == singleNormStream.length();
   }
 
   // for testing only

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/Codec.java?rev=1210185&r1=1210184&r2=1210185&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/Codec.java
(original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/Codec.java
Sun Dec  4 19:24:08 2011
@@ -51,6 +51,8 @@ public abstract class Codec implements N
     // TODO: segmentInfosFormat should be allowed to declare additional files
     // if it wants, in addition to segments_N
     docValuesFormat().files(dir, info, files);
+    // nocommit: not yet
+    //normsFormat().files(dir, info, files);
   }
   
   /** Encodes/decodes postings */
@@ -71,6 +73,9 @@ public abstract class Codec implements N
   /** Encodes/decodes segments file */
   public abstract SegmentInfosFormat segmentInfosFormat();
   
+  /** Encodes/decodes document normalization values */
+  public abstract NormsFormat normsFormat();
+  
   /** looks up a codec by name */
   public static Codec forName(String name) {
     return loader.lookup(name);

Added: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/NormsFormat.java?rev=1210185&view=auto
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/NormsFormat.java
(added)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/NormsFormat.java
Sun Dec  4 19:24:08 2011
@@ -0,0 +1,33 @@
+package org.apache.lucene.index.codecs;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.store.Directory;
+
+/**
+ * format for normalization factors
+ */
+public abstract class NormsFormat {
+  public abstract NormsWriter normsWriter(SegmentWriteState state) throws IOException;
+  public abstract void files(Directory dir, SegmentInfo info, Set<String> files) throws
IOException;
+}

Added: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/NormsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/NormsWriter.java?rev=1210185&view=auto
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/NormsWriter.java
(added)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/NormsWriter.java
Sun Dec  4 19:24:08 2011
@@ -0,0 +1,34 @@
+package org.apache.lucene.index.codecs;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.lucene.index.FieldInfo;
+
+// simple api just for now before switching to docvalues apis
+public abstract class NormsWriter implements Closeable {
+
+  // TODO: I think IW should set info.normValueType from Similarity,
+  // and then this method just returns DocValuesConsumer
+  public abstract void startField(FieldInfo info) throws IOException;
+  public abstract void writeNorm(byte norm) throws IOException;
+  public abstract void finish(int numDocs) throws IOException;
+  
+}

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java?rev=1210185&r1=1210184&r2=1210185&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java
(original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java
Sun Dec  4 19:24:08 2011
@@ -26,6 +26,7 @@ import org.apache.lucene.index.SegmentRe
 import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.index.codecs.DocValuesFormat;
 import org.apache.lucene.index.codecs.FieldInfosFormat;
+import org.apache.lucene.index.codecs.NormsFormat;
 import org.apache.lucene.index.codecs.StoredFieldsFormat;
 import org.apache.lucene.index.codecs.PerDocConsumer;
 import org.apache.lucene.index.codecs.PerDocValues;
@@ -33,6 +34,7 @@ import org.apache.lucene.index.codecs.Po
 import org.apache.lucene.index.codecs.SegmentInfosFormat;
 import org.apache.lucene.index.codecs.TermVectorsFormat;
 import org.apache.lucene.index.codecs.lucene40.Lucene40FieldInfosFormat;
+import org.apache.lucene.index.codecs.lucene40.Lucene40NormsFormat;
 import org.apache.lucene.index.codecs.lucene40.Lucene40SegmentInfosFormat;
 import org.apache.lucene.index.codecs.lucene40.Lucene40StoredFieldsFormat;
 import org.apache.lucene.index.codecs.lucene40.Lucene40TermVectorsFormat;
@@ -62,6 +64,9 @@ public class Lucene3xCodec extends Codec
   // this way IR.commit fails on delete/undelete/setNorm/etc ?
   private final SegmentInfosFormat infosFormat = new Lucene40SegmentInfosFormat();
   
+  // TODO: this should really be a different impl
+  private final NormsFormat normsFormat = new Lucene40NormsFormat();
+  
   // 3.x doesn't support docvalues
   private final DocValuesFormat docValuesFormat = new DocValuesFormat() {
     @Override
@@ -107,4 +112,9 @@ public class Lucene3xCodec extends Codec
   public SegmentInfosFormat segmentInfosFormat() {
     return infosFormat;
   }
+
+  @Override
+  public NormsFormat normsFormat() {
+    return normsFormat;
+  }
 }

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java?rev=1210185&r1=1210184&r2=1210185&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java
(original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java
Sun Dec  4 19:24:08 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs.l
 import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.index.codecs.DocValuesFormat;
 import org.apache.lucene.index.codecs.FieldInfosFormat;
+import org.apache.lucene.index.codecs.NormsFormat;
 import org.apache.lucene.index.codecs.StoredFieldsFormat;
 import org.apache.lucene.index.codecs.PostingsFormat;
 import org.apache.lucene.index.codecs.SegmentInfosFormat;
@@ -40,6 +41,7 @@ public class Lucene40Codec extends Codec
   private final FieldInfosFormat fieldInfosFormat = new Lucene40FieldInfosFormat();
   private final DocValuesFormat docValuesFormat = new Lucene40DocValuesFormat();
   private final SegmentInfosFormat infosFormat = new Lucene40SegmentInfosFormat();
+  private final NormsFormat normsFormat = new Lucene40NormsFormat();
   private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
     @Override
     public PostingsFormat getPostingsFormatForField(String field) {
@@ -81,6 +83,11 @@ public class Lucene40Codec extends Codec
     return infosFormat;
   }
 
+  @Override
+  public NormsFormat normsFormat() {
+    return normsFormat;
+  }
+
   /** Returns the postings format that should be used for writing 
    *  new segments of <code>field</code>.
    *  

Added: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsFormat.java?rev=1210185&view=auto
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsFormat.java
(added)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsFormat.java
Sun Dec  4 19:24:08 2011
@@ -0,0 +1,41 @@
+package org.apache.lucene.index.codecs.lucene40;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.codecs.NormsFormat;
+import org.apache.lucene.index.codecs.NormsWriter;
+import org.apache.lucene.store.Directory;
+
+public class Lucene40NormsFormat extends NormsFormat {
+
+  @Override
+  public NormsWriter normsWriter(SegmentWriteState state) throws IOException {
+    return new Lucene40NormsWriter(state.directory, state.segmentName, state.context);
+  }
+
+  @Override
+  public void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException
{
+    // nocommit: hairy calculations involving .s files, etc
+  }
+  
+}

Added: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsWriter.java?rev=1210185&view=auto
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsWriter.java
(added)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40NormsWriter.java
Sun Dec  4 19:24:08 2011
@@ -0,0 +1,78 @@
+package org.apache.lucene.index.codecs.lucene40;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.codecs.NormsWriter;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.IOUtils;
+
+public class Lucene40NormsWriter extends NormsWriter {
+  private IndexOutput out;
+  private int normCount = 0;
+  
+  /** norms header placeholder */
+  // nocommit: not public
+  public static final byte[] NORMS_HEADER = new byte[]{'N','R','M',-1};
+  
+  public Lucene40NormsWriter(Directory directory, String segment, IOContext context) throws
IOException {
+    final String normsFileName = IndexFileNames.segmentFileName(segment, "", IndexFileNames.NORMS_EXTENSION);
+    boolean success = false;
+    try {
+      out = directory.createOutput(normsFileName, context);
+      out.writeBytes(NORMS_HEADER, 0, NORMS_HEADER.length);
+      success = true;
+    } finally {
+      if (!success) {
+        IOUtils.closeWhileHandlingException(out);
+      }
+    }
+  }
+
+  @Override
+  public void startField(FieldInfo info) throws IOException {
+    assert info.omitNorms == false;
+    normCount++;
+  }
+  
+  @Override
+  public void writeNorm(byte norm) throws IOException {
+    out.writeByte(norm);
+  }
+  
+  @Override
+  public void finish(int numDocs) throws IOException {
+    if (4+normCount*numDocs != out.getFilePointer()) {
+      throw new RuntimeException(".nrm file size mismatch: expected=" + (4+normCount*numDocs)
+ " actual=" + out.getFilePointer());
+    }
+  }
+
+  @Override
+  public void close() throws IOException {
+    try {
+      IOUtils.close(out);
+    } finally {
+      out = null;
+    }
+  }
+}

Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java?rev=1210185&r1=1210184&r2=1210185&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java
(original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java
Sun Dec  4 19:24:08 2011
@@ -20,11 +20,13 @@ package org.apache.lucene.index.codecs.s
 import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.index.codecs.DocValuesFormat;
 import org.apache.lucene.index.codecs.FieldInfosFormat;
+import org.apache.lucene.index.codecs.NormsFormat;
 import org.apache.lucene.index.codecs.PostingsFormat;
 import org.apache.lucene.index.codecs.SegmentInfosFormat;
 import org.apache.lucene.index.codecs.StoredFieldsFormat;
 import org.apache.lucene.index.codecs.TermVectorsFormat;
 import org.apache.lucene.index.codecs.lucene40.Lucene40DocValuesFormat;
+import org.apache.lucene.index.codecs.lucene40.Lucene40NormsFormat;
 
 /**
  * plain text index format.
@@ -40,6 +42,8 @@ public final class SimpleTextCodec exten
   private final TermVectorsFormat vectorsFormat = new SimpleTextTermVectorsFormat();
   // TODO: need a plain-text impl
   private final DocValuesFormat docValues = new Lucene40DocValuesFormat();
+  // TODO: need a plain-text impl (using the above)
+  private final NormsFormat normsFormat = new Lucene40NormsFormat();
   
   public SimpleTextCodec() {
     super("SimpleText");
@@ -74,4 +78,9 @@ public final class SimpleTextCodec exten
   public SegmentInfosFormat segmentInfosFormat() {
     return segmentInfos;
   }
+
+  @Override
+  public NormsFormat normsFormat() {
+    return normsFormat;
+  }
 }

Modified: lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1210185&r1=1210184&r2=1210185&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
(original)
+++ lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
Sun Dec  4 19:24:08 2011
@@ -33,6 +33,7 @@ import org.apache.lucene.index.IndexWrit
 import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.index.codecs.DocValuesFormat;
 import org.apache.lucene.index.codecs.FieldInfosFormat;
+import org.apache.lucene.index.codecs.NormsFormat;
 import org.apache.lucene.index.codecs.StoredFieldsFormat;
 import org.apache.lucene.index.codecs.PostingsFormat;
 import org.apache.lucene.index.codecs.SegmentInfosFormat;
@@ -40,6 +41,7 @@ import org.apache.lucene.index.codecs.Te
 import org.apache.lucene.index.codecs.lucene40.Lucene40Codec;
 import org.apache.lucene.index.codecs.lucene40.Lucene40FieldInfosFormat;
 import org.apache.lucene.index.codecs.lucene40.Lucene40DocValuesFormat;
+import org.apache.lucene.index.codecs.lucene40.Lucene40NormsFormat;
 import org.apache.lucene.index.codecs.lucene40.Lucene40SegmentInfosFormat;
 import org.apache.lucene.index.codecs.lucene40.Lucene40StoredFieldsFormat;
 import org.apache.lucene.index.codecs.lucene40.Lucene40TermVectorsFormat;
@@ -1122,6 +1124,11 @@ public class TestAddIndexes extends Luce
     public SegmentInfosFormat segmentInfosFormat() {
       return new Lucene40SegmentInfosFormat();
     }
+
+    @Override
+    public NormsFormat normsFormat() {
+      return new Lucene40NormsFormat();
+    }
   }
   
   /*



Mime
View raw message