lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sar...@apache.org
Subject svn commit: r1129205 [3/7] - in /lucene/dev/branches/solr2452: ./ dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/spellchecker/ dev-tools/idea/modules/suggest/ dev-tools/maven/lucene/contrib/ dev-tools/maven/lucene/contrib/spellc...
Date Mon, 30 May 2011 14:51:37 GMT
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Mon May 30 14:51:25 2011
@@ -27,13 +27,13 @@ import org.apache.lucene.document.Docume
 import org.apache.lucene.index.IndexReader.FieldOption;
 import org.apache.lucene.index.MergePolicy.MergeAbortedException;
 import org.apache.lucene.index.codecs.Codec;
-import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.MergeState;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.MultiBits;
 import org.apache.lucene.util.ReaderUtil;
 
@@ -46,10 +46,6 @@ import org.apache.lucene.util.ReaderUtil
  * @see #add
  */
 final class SegmentMerger {
-
-  /** norms header placeholder */
-  static final byte[] NORMS_HEADER = new byte[]{'N','R','M',-1};
-
   private Directory directory;
   private String segment;
   private int termIndexInterval = IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL;
@@ -124,6 +120,12 @@ final class SegmentMerger {
     return mergedDocs;
   }
 
+  /**
+   * NOTE: this method creates a compound file for all files returned by
+   * info.files(). While, generally, this may include separate norms and
+   * deletion files, this SegmentInfo must not reference such files when this
+   * method is called, because they are not allowed within a compound file.
+   */
   final Collection<String> createCompoundFile(String fileName, final SegmentInfo info)
           throws IOException {
 
@@ -131,6 +133,10 @@ final class SegmentMerger {
     Collection<String> files = info.files();
     CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, fileName, checkAbort);
     for (String file : files) {
+      assert !IndexFileNames.matchesExtension(file, IndexFileNames.DELETES_EXTENSION) 
+                : ".del file is not allowed in .cfs: " + file;
+      assert !IndexFileNames.isSeparateNormsFile(file) 
+                : "separate norms file (.s[0-9]+) is not allowed in .cfs: " + file;
       cfsWriter.addFile(file);
     }
 
@@ -139,7 +145,7 @@ final class SegmentMerger {
 
     return files;
   }
-
+  
   private static void addIndexed(IndexReader reader, FieldInfos fInfos,
       Collection<String> names, boolean storeTermVectors,
       boolean storePositionWithTermVector, boolean storeOffsetWithTermVector,
@@ -540,14 +546,13 @@ final class SegmentMerger {
     }
     codec = segmentWriteState.segmentCodecs.codec();
     final FieldsConsumer consumer = codec.fieldsConsumer(segmentWriteState);
-
-    // NOTE: this is silly, yet, necessary -- we create a
-    // MultiBits as our skip docs only to have it broken
-    // apart when we step through the docs enums in
-    // MultiDocsEnum.
-    mergeState.multiDeletedDocs = new MultiBits(bits, bitsStarts);
-
     try {
+      // NOTE: this is silly, yet, necessary -- we create a
+      // MultiBits as our skip docs only to have it broken
+      // apart when we step through the docs enums in
+      // MultiDocsEnum.
+      mergeState.multiDeletedDocs = new MultiBits(bits, bitsStarts);
+      
       consumer.merge(mergeState,
                      new MultiFields(fields.toArray(Fields.EMPTY_ARRAY),
                                      slices.toArray(ReaderUtil.Slice.EMPTY_ARRAY)));
@@ -573,12 +578,13 @@ final class SegmentMerger {
 
   private void mergeNorms() throws IOException {
     IndexOutput output = null;
+    boolean success = false;
     try {
       for (FieldInfo fi : fieldInfos) {
         if (fi.isIndexed && !fi.omitNorms) {
           if (output == null) {
             output = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.NORMS_EXTENSION));
-            output.writeBytes(NORMS_HEADER,NORMS_HEADER.length);
+            output.writeBytes(SegmentNorms.NORMS_HEADER, SegmentNorms.NORMS_HEADER.length);
           }
           for (IndexReader reader : readers) {
             final int maxDoc = reader.maxDoc();
@@ -606,10 +612,9 @@ final class SegmentMerger {
           }
         }
       }
+      success = true;
     } finally {
-      if (output != null) {
-        output.close();
-      }
+      IOUtils.closeSafely(!success, output);
     }
   }
 }

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentNorms.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentNorms.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentNorms.java Mon May 30 14:51:25 2011
@@ -33,6 +33,10 @@ import org.apache.lucene.store.IndexOutp
  */
 
 final class SegmentNorms implements Cloneable {
+
+  /** norms header placeholder */
+  static final byte[] NORMS_HEADER = new byte[]{'N','R','M',-1};
+
   int refCount = 1;
 
   // If this instance is a clone, the originalNorm
@@ -219,7 +223,7 @@ final class SegmentNorms implements Clon
     boolean success = false;
     try {
       try {
-        out.writeBytes(SegmentMerger.NORMS_HEADER, 0, SegmentMerger.NORMS_HEADER.length);
+        out.writeBytes(SegmentNorms.NORMS_HEADER, 0, SegmentNorms.NORMS_HEADER.length);
         out.writeBytes(bytes, owner.maxDoc());
       } finally {
         out.close();

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentReader.java Mon May 30 14:51:25 2011
@@ -574,7 +574,7 @@ public class SegmentReader extends Index
   }
 
   private void openNorms(Directory cfsDir, int readBufferSize) throws IOException {
-    long nextNormSeek = SegmentMerger.NORMS_HEADER.length; //skip header (header unused for now)
+    long nextNormSeek = SegmentNorms.NORMS_HEADER.length; //skip header (header unused for now)
     int maxDoc = maxDoc();
     for (FieldInfo fi : core.fieldInfos) {
       if (norms.containsKey(fi.name)) {
@@ -619,7 +619,7 @@ public class SegmentReader extends Index
           if (isUnversioned) {
             normSeek = 0;
           } else {
-            normSeek = SegmentMerger.NORMS_HEADER.length;
+            normSeek = SegmentNorms.NORMS_HEADER.length;
           }
         }
 

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java Mon May 30 14:51:25 2011
@@ -54,9 +54,7 @@ final class TermVectorsTermsWriter exten
       fill(state.numDocs);
       assert state.segmentName != null;
       String idxName = IndexFileNames.segmentFileName(state.segmentName, "", IndexFileNames.VECTORS_INDEX_EXTENSION);
-      tvx.close();
-      tvf.close();
-      tvd.close();
+      IOUtils.closeSafely(false, tvx, tvf, tvd);
       tvx = tvd = tvf = null;
       if (4+((long) state.numDocs)*16 != state.directory.fileLength(idxName)) {
         throw new RuntimeException("after flush: tvx size mismatch: " + state.numDocs + " docs vs " + state.directory.fileLength(idxName) + " length in bytes of " + idxName + " file exists?=" + state.directory.fileExists(idxName));
@@ -89,18 +87,25 @@ final class TermVectorsTermsWriter exten
 
   private final void initTermVectorsWriter() throws IOException {
     if (tvx == null) {
-
-      // If we hit an exception while init'ing the term
-      // vector output files, we must abort this segment
-      // because those files will be in an unknown
-      // state:
-      tvx = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_INDEX_EXTENSION));
-      tvd = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION));
-      tvf = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_FIELDS_EXTENSION));
-
-      tvx.writeInt(TermVectorsReader.FORMAT_CURRENT);
-      tvd.writeInt(TermVectorsReader.FORMAT_CURRENT);
-      tvf.writeInt(TermVectorsReader.FORMAT_CURRENT);
+      boolean success = false;
+      try {
+        // If we hit an exception while init'ing the term
+        // vector output files, we must abort this segment
+        // because those files will be in an unknown
+        // state:
+        tvx = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_INDEX_EXTENSION));
+        tvd = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION));
+        tvf = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_FIELDS_EXTENSION));
+
+        tvx.writeInt(TermVectorsReader.FORMAT_CURRENT);
+        tvd.writeInt(TermVectorsReader.FORMAT_CURRENT);
+        tvf.writeInt(TermVectorsReader.FORMAT_CURRENT);
+        success = true;
+      } finally {
+        if (!success) {
+          IOUtils.closeSafely(true, tvx, tvd, tvf);
+        }
+      }
 
       lastDocID = 0;
     }
@@ -139,7 +144,7 @@ final class TermVectorsTermsWriter exten
       }
     }
 
-    assert lastDocID == docState.docID;
+    assert lastDocID == docState.docID: "lastDocID=" + lastDocID + " docState.docID=" + docState.docID;
 
     lastDocID++;
 
@@ -152,21 +157,27 @@ final class TermVectorsTermsWriter exten
   public void abort() {
     hasVectors = false;
     try {
-      IOUtils.closeSafely(tvx, tvd, tvf);
-    } catch (IOException ignored) {
+      IOUtils.closeSafely(true, tvx, tvd, tvf);
+    } catch (IOException e) {
+      // cannot happen since we suppress exceptions
+      throw new RuntimeException(e);
     }
+    
     try {
       docWriter.directory.deleteFile(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_INDEX_EXTENSION));
     } catch (IOException ignored) {
     }
+    
     try {
       docWriter.directory.deleteFile(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION));
     } catch (IOException ignored) {
     }
+    
     try {
       docWriter.directory.deleteFile(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_FIELDS_EXTENSION));
     } catch (IOException ignored) {
     }
+    
     tvx = tvd = tvf = null;
     lastDocID = 0;
 

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermVectorsWriter.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermVectorsWriter.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermVectorsWriter.java Mon May 30 14:51:25 2011
@@ -31,15 +31,22 @@ final class TermVectorsWriter {
   private FieldInfos fieldInfos;
 
   public TermVectorsWriter(Directory directory, String segment,
-                           FieldInfos fieldInfos)
-    throws IOException {
-    // Open files for TermVector storage
-    tvx = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_INDEX_EXTENSION));
-    tvx.writeInt(TermVectorsReader.FORMAT_CURRENT);
-    tvd = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION));
-    tvd.writeInt(TermVectorsReader.FORMAT_CURRENT);
-    tvf = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_FIELDS_EXTENSION));
-    tvf.writeInt(TermVectorsReader.FORMAT_CURRENT);
+                           FieldInfos fieldInfos) throws IOException {
+    boolean success = false;
+    try {
+      // Open files for TermVector storage
+      tvx = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_INDEX_EXTENSION));
+      tvx.writeInt(TermVectorsReader.FORMAT_CURRENT);
+      tvd = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION));
+      tvd.writeInt(TermVectorsReader.FORMAT_CURRENT);
+      tvf = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_FIELDS_EXTENSION));
+      tvf.writeInt(TermVectorsReader.FORMAT_CURRENT);
+      success = true;
+    } finally {
+      if (!success) {
+        IOUtils.closeSafely(true, tvx, tvd, tvf);
+      }
+    }
 
     this.fieldInfos = fieldInfos;
   }
@@ -51,8 +58,7 @@ final class TermVectorsWriter {
    * @param vectors
    * @throws IOException
    */
-  public final void addAllDocVectors(TermFreqVector[] vectors)
-      throws IOException {
+  public final void addAllDocVectors(TermFreqVector[] vectors) throws IOException {
 
     tvx.writeLong(tvd.getFilePointer());
     tvx.writeLong(tvf.getFilePointer());
@@ -187,6 +193,6 @@ final class TermVectorsWriter {
   final void close() throws IOException {
     // make an effort to close all streams we can but remember and re-throw
     // the first exception encountered in this process
-    IOUtils.closeSafely(tvx, tvd, tvf);
+    IOUtils.closeSafely(false, tvx, tvd, tvf);
   }
 }

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermsHash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermsHash.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermsHash.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermsHash.java Mon May 30 14:51:25 2011
@@ -54,7 +54,6 @@ final class TermsHash extends InvertedDo
 
   final boolean trackAllocations;
 
-
   public TermsHash(final DocumentsWriterPerThread docWriter, final TermsHashConsumer consumer, boolean trackAllocations, final TermsHash nextTermsHash) {
     this.docState = docWriter.docState;
     this.docWriter = docWriter;
@@ -108,11 +107,11 @@ final class TermsHash extends InvertedDo
     }
 
     for (final Map.Entry<FieldInfo,InvertedDocConsumerPerField> entry : fieldsToFlush.entrySet()) {
-        TermsHashPerField perField = (TermsHashPerField) entry.getValue();
-        childFields.put(entry.getKey(), perField.consumer);
-        if (nextTermsHash != null) {
-          nextChildFields.put(entry.getKey(), perField.nextPerField);
-        }
+      TermsHashPerField perField = (TermsHashPerField) entry.getValue();
+      childFields.put(entry.getKey(), perField.consumer);
+      if (nextTermsHash != null) {
+        nextChildFields.put(entry.getKey(), perField.nextPerField);
+      }
     }
 
     consumer.flush(childFields, state);
@@ -134,12 +133,9 @@ final class TermsHash extends InvertedDo
 
   @Override
   void finishDocument() throws IOException {
-    try {
-      consumer.finishDocument(this);
-    } finally {
-      if (nextTermsHash != null) {
-        nextTermsHash.consumer.finishDocument(nextTermsHash);
-      }
+    consumer.finishDocument(this);
+    if (nextTermsHash != null) {
+      nextTermsHash.consumer.finishDocument(nextTermsHash);
     }
   }
 

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/ThreadAffinityDocumentsWriterThreadPool.java Mon May 30 14:51:25 2011
@@ -18,7 +18,6 @@ package org.apache.lucene.index;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
-import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState; //javadoc
 
 /**
@@ -48,12 +47,10 @@ public class ThreadAffinityDocumentsWrit
   }
 
   @Override
-  public ThreadState getAndLock(Thread requestingThread, DocumentsWriter documentsWriter, Document doc) {
+  public ThreadState getAndLock(Thread requestingThread, DocumentsWriter documentsWriter) {
     ThreadState threadState = threadBindings.get(requestingThread);
-    if (threadState != null) {
-      if (threadState.tryLock()) {
-        return threadState;
-      }
+    if (threadState != null && threadState.tryLock()) {
+      return threadState;
     }
     ThreadState minThreadState = null;
 

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java Mon May 30 14:51:25 2011
@@ -31,6 +31,7 @@ import org.apache.lucene.store.RAMOutput
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CodecUtil;
+import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.RamUsageEstimator;
 
 // TODO: currently we encode all terms between two indexed
@@ -66,24 +67,29 @@ public class BlockTermsWriter extends Fi
 
   //private final String segment;
 
-  public BlockTermsWriter(
-      TermsIndexWriterBase termsIndexWriter,
-      SegmentWriteState state,
-      PostingsWriterBase postingsWriter)
-    throws IOException
-  {
+  public BlockTermsWriter(TermsIndexWriterBase termsIndexWriter,
+      SegmentWriteState state, PostingsWriterBase postingsWriter)
+      throws IOException {
     final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_EXTENSION);
     this.termsIndexWriter = termsIndexWriter;
     out = state.directory.createOutput(termsFileName);
-    fieldInfos = state.fieldInfos;
-    writeHeader(out);
-    currentField = null;
-    this.postingsWriter = postingsWriter;
-    //segment = state.segmentName;
-
-    //System.out.println("BTW.init seg=" + state.segmentName);
-
-    postingsWriter.start(out);                          // have consumer write its format/header
+    boolean success = false;
+    try {
+      fieldInfos = state.fieldInfos;
+      writeHeader(out);
+      currentField = null;
+      this.postingsWriter = postingsWriter;
+      //segment = state.segmentName;
+      
+      //System.out.println("BTW.init seg=" + state.segmentName);
+      
+      postingsWriter.start(out); // have consumer write its format/header
+      success = true;
+    } finally {
+      if (!success) {
+        IOUtils.closeSafely(true, out);
+      }
+    }
   }
   
   protected void writeHeader(IndexOutput out) throws IOException {
@@ -130,20 +136,11 @@ public class BlockTermsWriter extends Fi
       }
       writeTrailer(dirStart);
     } finally {
-      try {
-        out.close();
-      } finally {
-        try {
-          postingsWriter.close();
-        } finally {
-          termsIndexWriter.close();
-        }
-      }
+      IOUtils.closeSafely(false, out, postingsWriter, termsIndexWriter);
     }
   }
 
   protected void writeTrailer(long dirStart) throws IOException {
-    // TODO Auto-generated method stub
     out.seek(CodecUtil.headerLength(CODEC_NAME));
     out.writeLong(dirStart);    
   }

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java Mon May 30 14:51:25 2011
@@ -88,6 +88,15 @@ public class CodecProvider {
     return codec;
   }
 
+  /**
+   * Returns <code>true</code> iff a codec with the given name is registered
+   * @param name codec name
+   * @return <code>true</code> iff a codec with the given name is registered, otherwise <code>false</code>.
+   */
+  public synchronized boolean isCodecRegistered(String name) {
+    return codecs.containsKey(name);
+  }
+
   public SegmentInfosWriter getSegmentInfosWriter() {
     return infosWriter;
   }
@@ -146,6 +155,14 @@ public class CodecProvider {
   }
 
   /**
+   * Returns <code>true</code> if this provider has a Codec registered for this
+   * field.
+   */
+  public synchronized boolean hasFieldCodec(String name) {
+    return perFieldMap.containsKey(name);
+  }
+
+  /**
    * Returns the default {@link Codec} for this {@link CodecProvider}
    * 
    * @return the default {@link Codec} for this {@link CodecProvider}

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java Mon May 30 14:51:25 2011
@@ -24,6 +24,7 @@ import org.apache.lucene.index.SegmentIn
 import org.apache.lucene.store.ChecksumIndexOutput;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.IOUtils;
 
 /**
  * Default implementation of {@link SegmentInfosWriter}.
@@ -56,16 +57,24 @@ public class DefaultSegmentInfosWriter e
   public IndexOutput writeInfos(Directory dir, String segmentFileName, SegmentInfos infos)
           throws IOException {
     IndexOutput out = createOutput(dir, segmentFileName);
-    out.writeInt(FORMAT_CURRENT); // write FORMAT
-    out.writeLong(infos.version);
-    out.writeInt(infos.counter); // write counter
-    out.writeLong(infos.getGlobalFieldMapVersion());
-    out.writeInt(infos.size()); // write infos
-    for (SegmentInfo si : infos) {
-      si.write(out);
+    boolean success = false;
+    try {
+      out.writeInt(FORMAT_CURRENT); // write FORMAT
+      out.writeLong(infos.version);
+      out.writeInt(infos.counter); // write counter
+      out.writeLong(infos.getGlobalFieldMapVersion());
+      out.writeInt(infos.size()); // write infos
+      for (SegmentInfo si : infos) {
+        si.write(out);
+      }
+      out.writeStringStringMap(infos.getUserData());
+      success = true;
+      return out;
+    } finally {
+      if (!success) {
+        IOUtils.closeSafely(true, out);
+      }
     }
-    out.writeStringStringMap(infos.getUserData());
-    return out;
   }
   
   protected IndexOutput createOutput(Directory dir, String segmentFileName)

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java Mon May 30 14:51:25 2011
@@ -24,6 +24,7 @@ import org.apache.lucene.index.FieldInfo
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CodecUtil;
+import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.PagedBytes;
 import org.apache.lucene.util.packed.PackedInts;
 
@@ -108,6 +109,7 @@ public class FixedGapTermsIndexReader ex
       }
       success = true;
     } finally {
+      if (!success) IOUtils.closeSafely(true, in);
       if (indexDivisor > 0) {
         in.close();
         in = null;

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java Mon May 30 14:51:25 2011
@@ -25,6 +25,7 @@ import org.apache.lucene.index.SegmentWr
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CodecUtil;
 import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.packed.PackedInts;
 
 import java.util.List;
@@ -58,9 +59,17 @@ public class FixedGapTermsIndexWriter ex
     final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_INDEX_EXTENSION);
     termIndexInterval = state.termIndexInterval;
     out = state.directory.createOutput(indexFileName);
-    fieldInfos = state.fieldInfos;
-    writeHeader(out);
-    out.writeInt(termIndexInterval);
+    boolean success = false;
+    try {
+      fieldInfos = state.fieldInfos;
+      writeHeader(out);
+      out.writeInt(termIndexInterval);
+      success = true;
+    } finally {
+      if (!success) {
+        IOUtils.closeSafely(true, out);
+      }
+    }
   }
   
   protected void writeHeader(IndexOutput out) throws IOException {
@@ -202,33 +211,37 @@ public class FixedGapTermsIndexWriter ex
     }
   }
 
-  @Override
   public void close() throws IOException {
-    final long dirStart = out.getFilePointer();
-    final int fieldCount = fields.size();
-
-    int nonNullFieldCount = 0;
-    for(int i=0;i<fieldCount;i++) {
-      SimpleFieldWriter field = fields.get(i);
-      if (field.numIndexTerms > 0) {
-        nonNullFieldCount++;
+    boolean success = false;
+    try {
+      final long dirStart = out.getFilePointer();
+      final int fieldCount = fields.size();
+      
+      int nonNullFieldCount = 0;
+      for(int i=0;i<fieldCount;i++) {
+        SimpleFieldWriter field = fields.get(i);
+        if (field.numIndexTerms > 0) {
+          nonNullFieldCount++;
+        }
       }
-    }
-
-    out.writeVInt(nonNullFieldCount);
-    for(int i=0;i<fieldCount;i++) {
-      SimpleFieldWriter field = fields.get(i);
-      if (field.numIndexTerms > 0) {
-        out.writeVInt(field.fieldInfo.number);
-        out.writeVInt(field.numIndexTerms);
-        out.writeVLong(field.termsStart);
-        out.writeVLong(field.indexStart);
-        out.writeVLong(field.packedIndexStart);
-        out.writeVLong(field.packedOffsetsStart);
+      
+      out.writeVInt(nonNullFieldCount);
+      for(int i=0;i<fieldCount;i++) {
+        SimpleFieldWriter field = fields.get(i);
+        if (field.numIndexTerms > 0) {
+          out.writeVInt(field.fieldInfo.number);
+          out.writeVInt(field.numIndexTerms);
+          out.writeVLong(field.termsStart);
+          out.writeVLong(field.indexStart);
+          out.writeVLong(field.packedIndexStart);
+          out.writeVLong(field.packedOffsetsStart);
+        }
       }
+      writeTrailer(dirStart);
+      success = true;
+    } finally {
+      IOUtils.closeSafely(!success, out);
     }
-    writeTrailer(dirStart);
-    out.close();
   }
 
   protected void writeTrailer(long dirStart) throws IOException {

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/TermsIndexWriterBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/TermsIndexWriterBase.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/TermsIndexWriterBase.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/TermsIndexWriterBase.java Mon May 30 14:51:25 2011
@@ -19,10 +19,12 @@ package org.apache.lucene.index.codecs;
 
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.util.BytesRef;
+
+import java.io.Closeable;
 import java.io.IOException;
 
 /** @lucene.experimental */
-public abstract class TermsIndexWriterBase {
+public abstract class TermsIndexWriterBase implements Closeable {
 
   public abstract class FieldWriter {
     public abstract boolean checkIndexTerm(BytesRef text, TermStats stats) throws IOException;
@@ -31,6 +33,4 @@ public abstract class TermsIndexWriterBa
   }
 
   public abstract FieldWriter addField(FieldInfo fieldInfo, long termsFilePointer) throws IOException;
-
-  public abstract void close() throws IOException;
 }

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java Mon May 30 14:51:25 2011
@@ -33,11 +33,11 @@ import org.apache.lucene.store.Directory
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CodecUtil;
-import org.apache.lucene.util.automaton.fst.Builder;
-import org.apache.lucene.util.automaton.fst.BytesRefFSTEnum;
-import org.apache.lucene.util.automaton.fst.FST;
-import org.apache.lucene.util.automaton.fst.PositiveIntOutputs;
-import org.apache.lucene.util.automaton.fst.Util; // for toDot
+import org.apache.lucene.util.fst.Builder;
+import org.apache.lucene.util.fst.BytesRefFSTEnum;
+import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.PositiveIntOutputs;
+import org.apache.lucene.util.fst.Util; // for toDot
 
 /** See {@link VariableGapTermsIndexWriter}
  * 

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java Mon May 30 14:51:25 2011
@@ -28,9 +28,10 @@ import org.apache.lucene.index.SegmentWr
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CodecUtil;
-import org.apache.lucene.util.automaton.fst.Builder;
-import org.apache.lucene.util.automaton.fst.FST;
-import org.apache.lucene.util.automaton.fst.PositiveIntOutputs;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.fst.Builder;
+import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.PositiveIntOutputs;
 
 /**
  * Selects index terms according to provided pluggable
@@ -159,9 +160,17 @@ public class VariableGapTermsIndexWriter
   public VariableGapTermsIndexWriter(SegmentWriteState state, IndexTermSelector policy) throws IOException {
     final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_INDEX_EXTENSION);
     out = state.directory.createOutput(indexFileName);
-    fieldInfos = state.fieldInfos;
-    this.policy = policy;
-    writeHeader(out);
+    boolean success = false;
+    try {
+      fieldInfos = state.fieldInfos;
+      this.policy = policy;
+      writeHeader(out);
+      success = true;
+    } finally {
+      if (!success) {
+        IOUtils.closeSafely(true, out);
+      }
+    }
   }
   
   protected void writeHeader(IndexOutput out) throws IOException {
@@ -265,8 +274,8 @@ public class VariableGapTermsIndexWriter
     }
   }
 
-  @Override
   public void close() throws IOException {
+    try {
     final long dirStart = out.getFilePointer();
     final int fieldCount = fields.size();
 
@@ -287,8 +296,10 @@ public class VariableGapTermsIndexWriter
       }
     }
     writeTrailer(dirStart);
+    } finally {
     out.close();
   }
+  }
 
   protected void writeTrailer(long dirStart) throws IOException {
     out.seek(CodecUtil.headerLength(CODEC_NAME));

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/intblock/VariableIntBlockIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/intblock/VariableIntBlockIndexOutput.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/intblock/VariableIntBlockIndexOutput.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/intblock/VariableIntBlockIndexOutput.java Mon May 30 14:51:25 2011
@@ -41,6 +41,7 @@ public abstract class VariableIntBlockIn
   protected final IndexOutput out;
 
   private int upto;
+  private boolean hitExcDuringWrite;
 
   // TODO what Var-Var codecs exist in practice... and what are there blocksizes like?
   // if its less than 128 we should set that as max and use byte?
@@ -105,19 +106,23 @@ public abstract class VariableIntBlockIn
 
   @Override
   public void write(int v) throws IOException {
+    hitExcDuringWrite = true;
     upto -= add(v)-1;
+    hitExcDuringWrite = false;
     assert upto >= 0;
   }
 
   @Override
   public void close() throws IOException {
     try {
-      // stuff 0s in until the "real" data is flushed:
-      int stuffed = 0;
-      while(upto > stuffed) {
-        upto -= add(0)-1;
-        assert upto >= 0;
-        stuffed += 1;
+      if (!hitExcDuringWrite) {
+        // stuff 0s in until the "real" data is flushed:
+        int stuffed = 0;
+        while(upto > stuffed) {
+          upto -= add(0)-1;
+          assert upto >= 0;
+          stuffed += 1;
+        }
       }
     } finally {
       out.close();

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java Mon May 30 14:51:25 2011
@@ -38,6 +38,7 @@ import org.apache.lucene.index.codecs.Te
 import org.apache.lucene.index.codecs.TermsIndexWriterBase;
 import org.apache.lucene.index.codecs.standard.StandardCodec;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
 
 /** This codec "inlines" the postings for terms that have
  *  low docFreq.  It wraps another codec, which is used for
@@ -81,7 +82,7 @@ public class PulsingCodec extends Codec 
       success = true;
     } finally {
       if (!success) {
-        pulsingWriter.close();
+        IOUtils.closeSafely(true, pulsingWriter);
       }
     }
 
@@ -93,11 +94,7 @@ public class PulsingCodec extends Codec 
       return ret;
     } finally {
       if (!success) {
-        try {
-          pulsingWriter.close();
-        } finally {
-          indexWriter.close();
-        }
+        IOUtils.closeSafely(true, pulsingWriter, indexWriter);
       }
     }
   }

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsWriterImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsWriterImpl.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsWriterImpl.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsWriterImpl.java Mon May 30 14:51:25 2011
@@ -71,8 +71,6 @@ public final class PulsingPostingsWriter
    *  for this term) is <= maxPositions, then the postings are
    *  inlined into terms dict */
   public PulsingPostingsWriterImpl(int maxPositions, PostingsWriterBase wrappedPostingsWriter) throws IOException {
-    super();
-
     pending = new Position[maxPositions];
     for(int i=0;i<maxPositions;i++) {
       pending[i] = new Position();

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriterImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriterImpl.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriterImpl.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriterImpl.java Mon May 30 14:51:25 2011
@@ -31,6 +31,7 @@ import org.apache.lucene.store.IndexOutp
 import org.apache.lucene.store.RAMOutputStream;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CodecUtil;
+import org.apache.lucene.util.IOUtils;
 
 /** Writes frq to .frq, docs to .doc, pos to .pos, payloads
  *  to .pyl, skip data to .skp
@@ -49,18 +50,18 @@ public final class SepPostingsWriterImpl
   final static int VERSION_START = 0;
   final static int VERSION_CURRENT = VERSION_START;
 
-  final IntIndexOutput freqOut;
-  final IntIndexOutput.Index freqIndex;
+  IntIndexOutput freqOut;
+  IntIndexOutput.Index freqIndex;
 
-  final IntIndexOutput posOut;
-  final IntIndexOutput.Index posIndex;
+  IntIndexOutput posOut;
+  IntIndexOutput.Index posIndex;
 
-  final IntIndexOutput docOut;
-  final IntIndexOutput.Index docIndex;
+  IntIndexOutput docOut;
+  IntIndexOutput.Index docIndex;
 
-  final IndexOutput payloadOut;
+  IndexOutput payloadOut;
 
-  final IndexOutput skipOut;
+  IndexOutput skipOut;
   IndexOutput termsOut;
 
   final SepSkipListWriter skipListWriter;
@@ -107,44 +108,51 @@ public final class SepPostingsWriterImpl
   }
 
   public SepPostingsWriterImpl(SegmentWriteState state, IntStreamFactory factory, int skipInterval) throws IOException {
-    super();
-    this.skipInterval = skipInterval;
-    this.skipMinimum = skipInterval; /* set to the same for now */
-    final String docFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, DOC_EXTENSION);
-    docOut = factory.createOutput(state.directory, docFileName);
-    docIndex = docOut.index();
-
-    if (state.fieldInfos.hasProx()) {
-      final String frqFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, FREQ_EXTENSION);
-      freqOut = factory.createOutput(state.directory, frqFileName);
-      freqIndex = freqOut.index();
-
-      final String posFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, POS_EXTENSION);
-      posOut = factory.createOutput(state.directory, posFileName);
-      posIndex = posOut.index();
-
-      // TODO: -- only if at least one field stores payloads?
-      final String payloadFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, PAYLOAD_EXTENSION);
-      payloadOut = state.directory.createOutput(payloadFileName);
+    freqOut = null;
+    freqIndex = null;
+    posOut = null;
+    posIndex = null;
+    payloadOut = null;
+    boolean success = false;
+    try {
+      this.skipInterval = skipInterval;
+      this.skipMinimum = skipInterval; /* set to the same for now */
+      final String docFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, DOC_EXTENSION);
+      docOut = factory.createOutput(state.directory, docFileName);
+      docIndex = docOut.index();
+      
+      if (state.fieldInfos.hasProx()) {
+        final String frqFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, FREQ_EXTENSION);
+        freqOut = factory.createOutput(state.directory, frqFileName);
+        freqIndex = freqOut.index();
+        
+        final String posFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, POS_EXTENSION);
+        posOut = factory.createOutput(state.directory, posFileName);
+        posIndex = posOut.index();
+        
+        // TODO: -- only if at least one field stores payloads?
+        final String payloadFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, PAYLOAD_EXTENSION);
+        payloadOut = state.directory.createOutput(payloadFileName);
+      }
+      
+      final String skipFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, SKIP_EXTENSION);
+      skipOut = state.directory.createOutput(skipFileName);
+      
+      totalNumDocs = state.numDocs;
+      
+      skipListWriter = new SepSkipListWriter(skipInterval,
+          maxSkipLevels,
+          state.numDocs,
+          freqOut, docOut,
+          posOut, payloadOut);
+      
+      success = true;
+    } finally {
+      if (!success) {
+        IOUtils.closeSafely(true, docOut, skipOut, freqOut, posOut, payloadOut);
+      }
 
-    } else {
-      freqOut = null;
-      freqIndex = null;
-      posOut = null;
-      posIndex = null;
-      payloadOut = null;
     }
-
-    final String skipFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, SKIP_EXTENSION);
-    skipOut = state.directory.createOutput(skipFileName);
-
-    totalNumDocs = state.numDocs;
-
-    skipListWriter = new SepSkipListWriter(skipInterval,
-                                           maxSkipLevels,
-                                           state.numDocs,
-                                           freqOut, docOut,
-                                           posOut, payloadOut);
   }
 
   @Override
@@ -306,25 +314,7 @@ public final class SepPostingsWriterImpl
 
   @Override
   public void close() throws IOException {
-    try {
-      docOut.close();
-    } finally {
-      try {
-        skipOut.close();
-      } finally {
-        if (freqOut != null) {
-          try {
-            freqOut.close();
-          } finally {
-            try {
-              posOut.close();
-            } finally {
-              payloadOut.close();
-            }
-          }
-        }
-      }
-    }
+    IOUtils.closeSafely(false, docOut, skipOut, freqOut, posOut, payloadOut);
   }
 
   public static void getExtensions(Set<String> extensions) {

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java Mon May 30 14:51:25 2011
@@ -29,13 +29,14 @@ import org.apache.lucene.index.FieldInfo
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.CharsRef;
 import org.apache.lucene.util.StringHelper;
 import org.apache.lucene.util.UnicodeUtil;
-import org.apache.lucene.util.automaton.fst.Builder;
-import org.apache.lucene.util.automaton.fst.BytesRefFSTEnum;
-import org.apache.lucene.util.automaton.fst.FST;
-import org.apache.lucene.util.automaton.fst.PositiveIntOutputs;
-import org.apache.lucene.util.automaton.fst.PairOutputs;
+import org.apache.lucene.util.fst.Builder;
+import org.apache.lucene.util.fst.BytesRefFSTEnum;
+import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.PositiveIntOutputs;
+import org.apache.lucene.util.fst.PairOutputs;
 
 import java.io.IOException;
 import java.util.Comparator;
@@ -236,7 +237,7 @@ class SimpleTextFieldsReader extends Fie
     private int tf;
     private Bits skipDocs;
     private final BytesRef scratch = new BytesRef(10);
-    private final UnicodeUtil.UTF16Result scratchUTF16 = new UnicodeUtil.UTF16Result();
+    private final CharsRef scratchUTF16 = new CharsRef(10);
     
     public SimpleTextDocsEnum() {
       this.inStart = SimpleTextFieldsReader.this.in;
@@ -286,7 +287,7 @@ class SimpleTextFieldsReader extends Fie
             return docID;
           }
           UnicodeUtil.UTF8toUTF16(scratch.bytes, scratch.offset+DOC.length, scratch.length-DOC.length, scratchUTF16);
-          docID = ArrayUtil.parseInt(scratchUTF16.result, 0, scratchUTF16.length);
+          docID = ArrayUtil.parseInt(scratchUTF16.chars, 0, scratchUTF16.length);
           termFreq = 0;
           first = false;
         } else if (scratch.startsWith(POS)) {
@@ -323,8 +324,8 @@ class SimpleTextFieldsReader extends Fie
     private Bits skipDocs;
     private final BytesRef scratch = new BytesRef(10);
     private final BytesRef scratch2 = new BytesRef(10);
-    private final UnicodeUtil.UTF16Result scratchUTF16 = new UnicodeUtil.UTF16Result();
-    private final UnicodeUtil.UTF16Result scratchUTF16_2 = new UnicodeUtil.UTF16Result();
+    private final CharsRef scratchUTF16 = new CharsRef(10);
+    private final CharsRef scratchUTF16_2 = new CharsRef(10);
     private BytesRef payload;
     private long nextDocStart;
 
@@ -368,7 +369,7 @@ class SimpleTextFieldsReader extends Fie
             return docID;
           }
           UnicodeUtil.UTF8toUTF16(scratch.bytes, scratch.offset+DOC.length, scratch.length-DOC.length, scratchUTF16);
-          docID = ArrayUtil.parseInt(scratchUTF16.result, 0, scratchUTF16.length);
+          docID = ArrayUtil.parseInt(scratchUTF16.chars, 0, scratchUTF16.length);
           tf = 0;
           posStart = in.getFilePointer();
           first = false;
@@ -400,7 +401,7 @@ class SimpleTextFieldsReader extends Fie
       readLine(in, scratch);
       assert scratch.startsWith(POS): "got line=" + scratch.utf8ToString();
       UnicodeUtil.UTF8toUTF16(scratch.bytes, scratch.offset+POS.length, scratch.length-POS.length, scratchUTF16_2);
-      final int pos = ArrayUtil.parseInt(scratchUTF16_2.result, 0, scratchUTF16_2.length);
+      final int pos = ArrayUtil.parseInt(scratchUTF16_2.chars, 0, scratchUTF16_2.length);
       final long fp = in.getFilePointer();
       readLine(in, scratch);
       if (scratch.startsWith(PAYLOAD)) {

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java Mon May 30 14:51:25 2011
@@ -143,8 +143,11 @@ class SimpleTextFieldsWriter extends Fie
 
   @Override
   public void close() throws IOException {
-    write(END);
-    newline();
-    out.close();
+    try {
+      write(END);
+      newline();
+    } finally {
+      out.close();
+    }
   }
 }

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java Mon May 30 14:51:25 2011
@@ -33,6 +33,7 @@ import org.apache.lucene.store.IndexOutp
 import org.apache.lucene.store.RAMOutputStream;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CodecUtil;
+import org.apache.lucene.util.IOUtils;
 
 /** @lucene.experimental */
 public final class StandardPostingsWriter extends PostingsWriterBase {
@@ -42,8 +43,8 @@ public final class StandardPostingsWrite
   final static int VERSION_START = 0;
   final static int VERSION_CURRENT = VERSION_START;
 
-  final IndexOutput freqOut;
-  final IndexOutput proxOut;
+  IndexOutput freqOut;
+  IndexOutput proxOut;
   final DefaultSkipListWriter skipListWriter;
   /** Expert: The fraction of TermDocs entries stored in skip tables,
    * used to accelerate {@link DocsEnum#advance(int)}.  Larger values result in
@@ -85,31 +86,35 @@ public final class StandardPostingsWrite
   public StandardPostingsWriter(SegmentWriteState state) throws IOException {
     this(state, DEFAULT_SKIP_INTERVAL);
   }
+  
   public StandardPostingsWriter(SegmentWriteState state, int skipInterval) throws IOException {
-    super();
     this.skipInterval = skipInterval;
     this.skipMinimum = skipInterval; /* set to the same for now */
     //this.segment = state.segmentName;
     String fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.FREQ_EXTENSION);
     freqOut = state.directory.createOutput(fileName);
-
-    if (state.fieldInfos.hasProx()) {
-      // At least one field does not omit TF, so create the
-      // prox file
-      fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.PROX_EXTENSION);
-      proxOut = state.directory.createOutput(fileName);
-    } else {
-      // Every field omits TF so we will write no prox file
-      proxOut = null;
+    boolean success = false;
+    try {
+      if (state.fieldInfos.hasProx()) {
+        // At least one field does not omit TF, so create the
+        // prox file
+        fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.PROX_EXTENSION);
+        proxOut = state.directory.createOutput(fileName);
+      } else {
+        // Every field omits TF so we will write no prox file
+        proxOut = null;
+      }
+      
+      totalNumDocs = state.numDocs;
+      
+      skipListWriter = new DefaultSkipListWriter(skipInterval, maxSkipLevels,
+          state.numDocs, freqOut, proxOut);
+      success = true;
+    } finally {
+      if (!success) {
+        IOUtils.closeSafely(true, freqOut, proxOut);
+      }
     }
-
-    totalNumDocs = state.numDocs;
-
-    skipListWriter = new DefaultSkipListWriter(skipInterval,
-                                               maxSkipLevels,
-                                               state.numDocs,
-                                               freqOut,
-                                               proxOut);
   }
 
   @Override
@@ -267,12 +272,6 @@ public final class StandardPostingsWrite
 
   @Override
   public void close() throws IOException {
-    try {
-      freqOut.close();
-    } finally {
-      if (proxOut != null) {
-        proxOut.close();
-      }
-    }
+    IOUtils.closeSafely(false, freqOut, proxOut);
   }
 }

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/CachingCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/CachingCollector.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/CachingCollector.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/CachingCollector.java Mon May 30 14:51:25 2011
@@ -181,6 +181,7 @@ public abstract class CachingCollector e
             curUpto = 0;
           }
           cachedScorer.score = curScores[curUpto];
+          cachedScorer.doc = curDocs[curUpto];
           other.collect(curDocs[curUpto++]);
         }
       }

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/FSDirectory.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/store/FSDirectory.java Mon May 30 14:51:25 2011
@@ -22,8 +22,6 @@ import java.io.FileNotFoundException;
 import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.RandomAccessFile;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
 
 import java.util.Collection;
 import static java.util.Collections.synchronizedSet;
@@ -111,15 +109,6 @@ import org.apache.lucene.util.Constants;
  * @see Directory
  */
 public abstract class FSDirectory extends Directory {
-  private final static MessageDigest DIGESTER;
-
-  static {
-    try {
-      DIGESTER = MessageDigest.getInstance("MD5");
-    } catch (NoSuchAlgorithmException e) {
-        throw new RuntimeException(e.toString(), e);
-    }
-  }
 
   /**
    * Default read chunk size.  This is a conditional default: on 32bit JVMs, it defaults to 100 MB.  On 64bit JVMs, it's
@@ -337,12 +326,6 @@ public abstract class FSDirectory extend
     return openInput(name, BufferedIndexInput.BUFFER_SIZE);
   }
 
-  /**
-   * So we can do some byte-to-hexchar conversion below
-   */
-  private static final char[] HEX_DIGITS =
-  {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'};
-  
   @Override
   public String getLockID() {
     ensureOpen();
@@ -353,19 +336,12 @@ public abstract class FSDirectory extend
       throw new RuntimeException(e.toString(), e);
     }
 
-    byte digest[];
-    synchronized (DIGESTER) {
-      digest = DIGESTER.digest(dirName.getBytes());
-    }
-    StringBuilder buf = new StringBuilder();
-    buf.append("lucene-");
-    for (int i = 0; i < digest.length; i++) {
-      int b = digest[i];
-      buf.append(HEX_DIGITS[(b >> 4) & 0xf]);
-      buf.append(HEX_DIGITS[b & 0xf]);
+    int digest = 0;
+    for(int charIDX=0;charIDX<dirName.length();charIDX++) {
+      final char ch = dirName.charAt(charIDX);
+      digest = 31 * digest + ch;
     }
-
-    return buf.toString();
+    return "lucene-" + Integer.toHexString(digest);
   }
 
   /** Closes the store to future operations. */

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/BytesRef.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/BytesRef.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/BytesRef.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/BytesRef.java Mon May 30 14:51:25 2011
@@ -18,7 +18,6 @@ package org.apache.lucene.util;
  */
 
 import java.util.Comparator;
-import java.io.UnsupportedEncodingException;
 
 /** Represents byte[], as a slice (offset + length) into an
  *  existing byte[].
@@ -122,6 +121,7 @@ public final class BytesRef implements C
   public void copy(char text[], int offset, int length) {
     UnicodeUtil.UTF16toUTF8(text, offset, length, this);
   }
+  
   public boolean bytesEquals(BytesRef other) {
     if (length == other.length) {
       int otherUpto = other.offset;
@@ -198,13 +198,15 @@ public final class BytesRef implements C
   /** Interprets stored bytes as UTF8 bytes, returning the
    *  resulting string */
   public String utf8ToString() {
-    try {
-      return new String(bytes, offset, length, "UTF-8");
-    } catch (UnsupportedEncodingException uee) {
-      // should not happen -- UTF8 is presumably supported
-      // by all JREs
-      throw new RuntimeException(uee);
-    }
+    final CharsRef ref = new CharsRef(length);
+    UnicodeUtil.UTF8toUTF16(bytes, offset, length, ref);
+    return ref.toString(); 
+  }
+  
+  /** Interprets stored bytes as UTF8 bytes into the given {@link CharsRef} */
+  public CharsRef utf8ToChars(CharsRef ref) {
+    UnicodeUtil.UTF8toUTF16(bytes, offset, length, ref);
+    return ref;
   }
 
   /** Returns hex encoded bytes, eg [0x6c 0x75 0x63 0x65 0x6e 0x65] */

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/Constants.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/Constants.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/Constants.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/Constants.java Mon May 30 14:51:25 2011
@@ -43,6 +43,8 @@ public final class Constants {
   public static final boolean WINDOWS = OS_NAME.startsWith("Windows");
   /** True iff running on SunOS. */
   public static final boolean SUN_OS = OS_NAME.startsWith("SunOS");
+  /** True iff running on Mac OS X */
+  public static final boolean MAC_OS_X = OS_NAME.startsWith("Mac OS X");
 
   public static final String OS_ARCH = System.getProperty("os.arch");
   public static final String OS_VERSION = System.getProperty("os.version");

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/IOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/IOUtils.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/IOUtils.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/IOUtils.java Mon May 30 14:51:25 2011
@@ -47,44 +47,113 @@ public final class IOUtils {
    * @param objects         objects to call <tt>close()</tt> on
    */
   public static <E extends Exception> void closeSafely(E priorException, Closeable... objects) throws E, IOException {
-    IOException firstIOE = null;
+    Throwable th = null;
 
     for (Closeable object : objects) {
       try {
-        if (object != null)
+        if (object != null) {
           object.close();
-      } catch (IOException ioe) {
-        if (firstIOE == null)
-          firstIOE = ioe;
+        }
+      } catch (Throwable t) {
+        if (th == null) {
+          th = t;
+        }
       }
     }
 
-    if (priorException != null)
+    if (priorException != null) {
       throw priorException;
-    else if (firstIOE != null)
-      throw firstIOE;
+    } else if (th != null) {
+      if (th instanceof IOException) throw (IOException) th;
+      if (th instanceof RuntimeException) throw (RuntimeException) th;
+      if (th instanceof Error) throw (Error) th;
+      throw new RuntimeException(th);
+    }
+  }
+
+  /** @see #closeSafely(Exception, Closeable...) */
+  public static <E extends Exception> void closeSafely(E priorException, Iterable<Closeable> objects) throws E, IOException {
+    Throwable th = null;
+
+    for (Closeable object : objects) {
+      try {
+        if (object != null) {
+          object.close();
+        }
+      } catch (Throwable t) {
+        if (th == null) {
+          th = t;
+        }
+      }
+    }
+
+    if (priorException != null) {
+      throw priorException;
+    } else if (th != null) {
+      if (th instanceof IOException) throw (IOException) th;
+      if (th instanceof RuntimeException) throw (RuntimeException) th;
+      if (th instanceof Error) throw (Error) th;
+      throw new RuntimeException(th);
+    }
   }
 
   /**
-   * <p>Closes all given <tt>Closeable</tt>s, suppressing all thrown exceptions. Some of the <tt>Closeable</tt>s
-   * may be null, they are ignored. After everything is closed, method either throws the first of suppressed exceptions,
-   * or completes normally.</p>
-   * @param objects         objects to call <tt>close()</tt> on
+   * Closes all given <tt>Closeable</tt>s, suppressing all thrown exceptions.
+   * Some of the <tt>Closeable</tt>s may be null, they are ignored. After
+   * everything is closed, and if {@code suppressExceptions} is {@code false},
+   * method either throws the first of suppressed exceptions, or completes
+   * normally.
+   * 
+   * @param suppressExceptions
+   *          if true then exceptions that occur during close() are suppressed
+   * @param objects
+   *          objects to call <tt>close()</tt> on
    */
-  public static void closeSafely(Closeable... objects) throws IOException {
-    IOException firstIOE = null;
+  public static void closeSafely(boolean suppressExceptions, Closeable... objects) throws IOException {
+    Throwable th = null;
 
     for (Closeable object : objects) {
       try {
-        if (object != null)
+        if (object != null) {
           object.close();
-      } catch (IOException ioe) {
-        if (firstIOE == null)
-          firstIOE = ioe;
+        }
+      } catch (Throwable t) {
+        if (th == null)
+          th = t;
       }
     }
 
-    if (firstIOE != null)
-      throw firstIOE;
+    if (th != null && !suppressExceptions) {
+      if (th instanceof IOException) throw (IOException) th;
+      if (th instanceof RuntimeException) throw (RuntimeException) th;
+      if (th instanceof Error) throw (Error) th;
+      throw new RuntimeException(th);
+    }
   }
+  
+  /**
+   * @see #closeSafely(boolean, Closeable...)
+   */
+  public static void closeSafely(boolean suppressExceptions, Iterable<? extends Closeable> objects) throws IOException {
+    Throwable th = null;
+
+    for (Closeable object : objects) {
+      try {
+        if (object != null) {
+          object.close();
+        }
+      } catch (Throwable t) {
+        if (th == null)
+          th = t;
+      }
+    }
+
+    if (th != null && !suppressExceptions) {
+      if (th instanceof IOException) throw (IOException) th;
+      if (th instanceof RuntimeException) throw (RuntimeException) th;
+      if (th instanceof Error) throw (Error) th;
+      throw new RuntimeException(th);
+    }
+  }
+
 }

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/OpenBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/OpenBitSet.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/OpenBitSet.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/OpenBitSet.java Mon May 30 14:51:25 2011
@@ -78,11 +78,15 @@ public class OpenBitSet extends DocIdSet
   protected long[] bits;
   protected int wlen;   // number of words (elements) used in the array
 
+  // Used only for assert:
+  private long numBits;
+
   /** Constructs an OpenBitSet large enough to hold numBits.
    *
    * @param numBits
    */
   public OpenBitSet(long numBits) {
+    this.numBits = numBits;
     bits = new long[bits2words(numBits)];
     wlen = bits.length;
   }
@@ -107,6 +111,7 @@ public class OpenBitSet extends DocIdSet
   public OpenBitSet(long[] bits, int numWords) {
     this.bits = bits;
     this.wlen = numWords;
+    this.numBits = wlen * 64;
   }
   
   @Override
@@ -170,6 +175,7 @@ public class OpenBitSet extends DocIdSet
    * The index should be less than the OpenBitSet size
    */
   public boolean fastGet(int index) {
+    assert index >= 0 && index < numBits;
     int i = index >> 6;               // div 64
     // signed shift will keep a negative index and force an
     // array-index-out-of-bounds-exception, removing the need for an explicit check.
@@ -194,6 +200,7 @@ public class OpenBitSet extends DocIdSet
    * The index should be less than the OpenBitSet size.
    */
   public boolean fastGet(long index) {
+    assert index >= 0 && index < numBits;
     int i = (int)(index >> 6);               // div 64
     int bit = (int)index & 0x3f;           // mod 64
     long bitmask = 1L << bit;
@@ -217,6 +224,7 @@ public class OpenBitSet extends DocIdSet
    * The index should be less than the OpenBitSet size
    */
   public int getBit(int index) {
+    assert index >= 0 && index < numBits;
     int i = index >> 6;                // div 64
     int bit = index & 0x3f;            // mod 64
     return ((int)(bits[i]>>>bit)) & 0x01;
@@ -245,6 +253,7 @@ public class OpenBitSet extends DocIdSet
   * The index should be less than the OpenBitSet size.
   */
   public void fastSet(int index) {
+    assert index >= 0 && index < numBits;
     int wordNum = index >> 6;      // div 64
     int bit = index & 0x3f;     // mod 64
     long bitmask = 1L << bit;
@@ -255,6 +264,7 @@ public class OpenBitSet extends DocIdSet
   * The index should be less than the OpenBitSet size.
   */
   public void fastSet(long index) {
+    assert index >= 0 && index < numBits;
     int wordNum = (int)(index >> 6);
     int bit = (int)index & 0x3f;
     long bitmask = 1L << bit;
@@ -296,6 +306,7 @@ public class OpenBitSet extends DocIdSet
       ensureCapacity(index+1);
       wlen = wordNum+1;
     }
+    assert (numBits = Math.max(numBits, index+1)) >= 0;
     return wordNum;
   }
 
@@ -304,6 +315,7 @@ public class OpenBitSet extends DocIdSet
    * The index should be less than the OpenBitSet size.
    */
   public void fastClear(int index) {
+    assert index >= 0 && index < numBits;
     int wordNum = index >> 6;
     int bit = index & 0x03f;
     long bitmask = 1L << bit;
@@ -321,6 +333,7 @@ public class OpenBitSet extends DocIdSet
    * The index should be less than the OpenBitSet size.
    */
   public void fastClear(long index) {
+    assert index >= 0 && index < numBits;
     int wordNum = (int)(index >> 6); // div 64
     int bit = (int)index & 0x3f;     // mod 64
     long bitmask = 1L << bit;
@@ -415,6 +428,7 @@ public class OpenBitSet extends DocIdSet
    * The index should be less than the OpenBitSet size.
    */
   public boolean getAndSet(int index) {
+    assert index >= 0 && index < numBits;
     int wordNum = index >> 6;      // div 64
     int bit = index & 0x3f;     // mod 64
     long bitmask = 1L << bit;
@@ -427,6 +441,7 @@ public class OpenBitSet extends DocIdSet
    * The index should be less than the OpenBitSet size.
    */
   public boolean getAndSet(long index) {
+    assert index >= 0 && index < numBits;
     int wordNum = (int)(index >> 6);      // div 64
     int bit = (int)index & 0x3f;     // mod 64
     long bitmask = 1L << bit;
@@ -439,6 +454,7 @@ public class OpenBitSet extends DocIdSet
    * The index should be less than the OpenBitSet size.
    */
   public void fastFlip(int index) {
+    assert index >= 0 && index < numBits;
     int wordNum = index >> 6;      // div 64
     int bit = index & 0x3f;     // mod 64
     long bitmask = 1L << bit;
@@ -449,6 +465,7 @@ public class OpenBitSet extends DocIdSet
    * The index should be less than the OpenBitSet size.
    */
   public void fastFlip(long index) {
+    assert index >= 0 && index < numBits;
     int wordNum = (int)(index >> 6);   // div 64
     int bit = (int)index & 0x3f;       // mod 64
     long bitmask = 1L << bit;
@@ -467,6 +484,7 @@ public class OpenBitSet extends DocIdSet
    * The index should be less than the OpenBitSet size.
    */
   public boolean flipAndGet(int index) {
+    assert index >= 0 && index < numBits;
     int wordNum = index >> 6;      // div 64
     int bit = index & 0x3f;     // mod 64
     long bitmask = 1L << bit;
@@ -478,6 +496,7 @@ public class OpenBitSet extends DocIdSet
    * The index should be less than the OpenBitSet size.
    */
   public boolean flipAndGet(long index) {
+    assert index >= 0 && index < numBits;
     int wordNum = (int)(index >> 6);   // div 64
     int bit = (int)index & 0x3f;       // mod 64
     long bitmask = 1L << bit;
@@ -674,6 +693,7 @@ public class OpenBitSet extends DocIdSet
   public void union(OpenBitSet other) {
     int newLen = Math.max(wlen,other.wlen);
     ensureCapacityWords(newLen);
+    assert (numBits = Math.max(other.numBits, numBits)) >= 0;
 
     long[] thisArr = this.bits;
     long[] otherArr = other.bits;
@@ -702,6 +722,7 @@ public class OpenBitSet extends DocIdSet
   public void xor(OpenBitSet other) {
     int newLen = Math.max(wlen,other.wlen);
     ensureCapacityWords(newLen);
+    assert (numBits = Math.max(other.numBits, numBits)) >= 0;
 
     long[] thisArr = this.bits;
     long[] otherArr = other.bits;

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/UnicodeUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/UnicodeUtil.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/UnicodeUtil.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/UnicodeUtil.java Mon May 30 14:51:25 2011
@@ -94,6 +94,19 @@ package org.apache.lucene.util;
  */
 
 public final class UnicodeUtil {
+  
+  /** A binary term consisting of a number of 0xff bytes, likely to be bigger than other terms
+   *  one would normally encounter, and definitely bigger than any UTF-8 terms.
+   *  <p>
+   *  WARNING: This is not a valid UTF8 Term  
+   **/
+  public static final BytesRef BIG_TERM = new BytesRef(
+      new byte[] {-1,-1,-1,-1,-1,-1,-1,-1,-1,-1}
+  ); // TODO this is unrelated here find a better place for it
+  
+  public static void main(String[] args) {
+    System.out.println(Character.toChars(0x10FFFF + 1));
+  }
 
   private UnicodeUtil() {} // no instance
 
@@ -112,33 +125,6 @@ public final class UnicodeUtil {
     Character.MIN_SUPPLEMENTARY_CODE_POINT - 
     (UNI_SUR_HIGH_START << HALF_SHIFT) - UNI_SUR_LOW_START;
 
-  /**
-   * @lucene.internal
-   */
-  public static final class UTF16Result {
-    public char[] result = new char[10];
-    public int[] offsets = new int[10];
-    public int length;
-
-    public void setLength(int newLength) {
-      if (result.length < newLength)
-        result = ArrayUtil.grow(result, newLength);
-      length = newLength;
-    }
-
-    public void copyText(UTF16Result other) {
-      setLength(other.length);
-      System.arraycopy(other.result, 0, result, 0, length);
-    }
-
-    public void copyText(String other) {
-      final int otherLength = other.length();
-      setLength(otherLength);
-      other.getChars(0, otherLength, result, 0);
-      length = otherLength;
-    }
-  }
-
   /** Encode characters from a char[] source, starting at
    *  offset for length chars.  Returns a hash of the resulting bytes.  After encoding, result.offset will always be 0. */
   public static int UTF16toUTF8WithHash(final char[] source, final int offset, final int length, BytesRef result) {
@@ -302,135 +288,6 @@ public final class UnicodeUtil {
     result.length = upto;
   }
 
-  /** Convert UTF8 bytes into UTF16 characters.  If offset
-   *  is non-zero, conversion starts at that starting point
-   *  in utf8, re-using the results from the previous call
-   *  up until offset. */
-  public static void UTF8toUTF16(final byte[] utf8, final int offset, final int length, final UTF16Result result) {
-
-    final int end = offset + length;
-    char[] out = result.result;
-    if (result.offsets.length <= end) {
-      result.offsets = ArrayUtil.grow(result.offsets, end+1);
-    }
-    final int[] offsets = result.offsets;
-
-    // If incremental decoding fell in the middle of a
-    // single unicode character, rollback to its start:
-    int upto = offset;
-    while(offsets[upto] == -1)
-      upto--;
-
-    int outUpto = offsets[upto];
-
-    // Pre-allocate for worst case 1-for-1
-    if (outUpto+length >= out.length) {
-      out = result.result = ArrayUtil.grow(out, outUpto+length+1);
-    }
-
-    while (upto < end) {
-
-      final int b = utf8[upto]&0xff;
-      final int ch;
-
-      offsets[upto++] = outUpto;
-
-      if (b < 0xc0) {
-        assert b < 0x80;
-        ch = b;
-      } else if (b < 0xe0) {
-        ch = ((b&0x1f)<<6) + (utf8[upto]&0x3f);
-        offsets[upto++] = -1;
-      } else if (b < 0xf0) {
-        ch = ((b&0xf)<<12) + ((utf8[upto]&0x3f)<<6) + (utf8[upto+1]&0x3f);
-        offsets[upto++] = -1;
-        offsets[upto++] = -1;
-      } else {
-        assert b < 0xf8;
-        ch = ((b&0x7)<<18) + ((utf8[upto]&0x3f)<<12) + ((utf8[upto+1]&0x3f)<<6) + (utf8[upto+2]&0x3f);
-        offsets[upto++] = -1;
-        offsets[upto++] = -1;
-        offsets[upto++] = -1;
-      }
-
-      if (ch <= UNI_MAX_BMP) {
-        // target is a character <= 0xFFFF
-        out[outUpto++] = (char) ch;
-      } else {
-        // target is a character in range 0xFFFF - 0x10FFFF
-        out[outUpto++] = (char) ((ch >> HALF_SHIFT) + 0xD7C0 /* UNI_SUR_HIGH_START - 64 */);
-        out[outUpto++] = (char) ((ch & HALF_MASK) + UNI_SUR_LOW_START);
-      }
-    }
-    offsets[upto] = outUpto;
-    result.length = outUpto;
-  }
-
-  /**
-   * Get the next valid UTF-16 String in UTF-16 order.
-   * <p>
-   * If the input String is already valid, it is returned.
-   * Otherwise the next String in code unit order is returned.
-   * </p>
-   * @param s input String (possibly with unpaired surrogates)
-   * @return next valid UTF-16 String in UTF-16 order
-   */
-  public static String nextValidUTF16String(String s) {
-    if (validUTF16String(s))
-        return s;
-    else {
-      UTF16Result chars = new UTF16Result();
-      chars.copyText(s);
-      nextValidUTF16String(chars);
-      return new String(chars.result, 0, chars.length);
-    }
-  }
-  
-  public static void nextValidUTF16String(UTF16Result s) {
-    final int size = s.length;
-    for (int i = 0; i < size; i++) {
-      char ch = s.result[i];
-      if (ch >= UnicodeUtil.UNI_SUR_HIGH_START
-          && ch <= UnicodeUtil.UNI_SUR_HIGH_END) {
-        if (i < size - 1) {
-          i++;
-          char nextCH = s.result[i];
-          if (nextCH >= UnicodeUtil.UNI_SUR_LOW_START
-              && nextCH <= UnicodeUtil.UNI_SUR_LOW_END) {
-            // Valid surrogate pair
-          } else
-          // Unmatched high surrogate
-            if (nextCH < UnicodeUtil.UNI_SUR_LOW_START) { // SMP not enumerated
-              s.setLength(i + 1);
-              s.result[i] = (char) UnicodeUtil.UNI_SUR_LOW_START;             
-              return;
-            } else { // SMP already enumerated
-              if (s.result[i - 1] == UnicodeUtil.UNI_SUR_HIGH_END) {
-                s.result[i - 1] = (char) (UnicodeUtil.UNI_SUR_LOW_END + 1);
-                s.setLength(i);               
-              } else {
-                s.result[i - 1]++;
-                s.result[i] = (char) UnicodeUtil.UNI_SUR_LOW_START;
-                s.setLength(i + 1);
-              }            
-              return;
-            }
-        } else {
-        // Unmatched high surrogate in final position, SMP not yet enumerated
-          s.setLength(i + 2);
-          s.result[i + 1] = (char) UnicodeUtil.UNI_SUR_LOW_START;
-          return;
-        }
-      } else if (ch >= UnicodeUtil.UNI_SUR_LOW_START
-          && ch <= UnicodeUtil.UNI_SUR_LOW_END) {
-      // Unmatched low surrogate, SMP already enumerated
-        s.setLength(i + 1);
-        s.result[i] = (char) (UnicodeUtil.UNI_SUR_LOW_END + 1);
-        return;
-      }
-    }
-  }
-  
   // Only called from assert
   /*
   private static boolean matches(char[] source, int offset, int length, byte[] result, int upto) {
@@ -705,4 +562,51 @@ public final class UnicodeUtil {
     }
     return sb.toString();
   }
+  
+  /**
+   * Interprets the given byte array as UTF-8 and converts to UTF-16. The {@link CharsRef} will be extended if 
+   * it doesn't provide enough space to hold the worst case of each byte becoming a UTF-16 codepoint.
+   * <p>
+   * NOTE: Full characters are read, even if this reads past the length passed (and
+   * can result in an ArrayOutOfBoundsException if invalid UTF-8 is passed).
+   * Explicit checks for valid UTF-8 are not performed. 
+   */
+  public static void UTF8toUTF16(byte[] utf8, int offset, int length, CharsRef chars) {
+    int out_offset = chars.offset = 0;
+    final char[] out = chars.chars =  ArrayUtil.grow(chars.chars, length);
+    final int limit = offset + length;
+    while (offset < limit) {
+      int b = utf8[offset++]&0xff;
+      if (b < 0xc0) {
+        assert b < 0x80;
+        out[out_offset++] = (char)b;
+      } else if (b < 0xe0) {
+        out[out_offset++] = (char)(((b&0x1f)<<6) + (utf8[offset++]&0x3f));
+      } else if (b < 0xf0) {
+        out[out_offset++] = (char)(((b&0xf)<<12) + ((utf8[offset]&0x3f)<<6) + (utf8[offset+1]&0x3f));
+        offset += 2;
+      } else {
+        assert b < 0xf8;
+        int ch = ((b&0x7)<<18) + ((utf8[offset]&0x3f)<<12) + ((utf8[offset+1]&0x3f)<<6) + (utf8[offset+2]&0x3f);
+        offset += 3;
+        if (ch < UNI_MAX_BMP) {
+          out[out_offset++] = (char)ch;
+        } else {
+          int chHalf = ch - 0x0010000;
+          out[out_offset++] = (char) ((chHalf >> 10) + 0xD800);
+          out[out_offset++] = (char) ((chHalf & HALF_MASK) + 0xDC00);          
+        }
+      }
+    }
+    chars.length = out_offset - chars.offset;
+  }
+  
+  /**
+   * Utility method for {@link #UTF8toUTF16(byte[], int, int, CharsRef)}
+   * @see #UTF8toUTF16(byte[], int, int, CharsRef)
+   */
+  public static void UTF8toUTF16(BytesRef bytesRef, CharsRef chars) {
+    UTF8toUTF16(bytesRef.bytes, bytesRef.offset, bytesRef.length, chars);
+  }
+
 }

Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/Version.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/Version.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/Version.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/util/Version.java Mon May 30 14:51:25 2011
@@ -48,6 +48,13 @@ public enum Version {
    */
   @Deprecated
   LUCENE_32,
+  
+  /**
+   * Match settings and bugs in Lucene's 3.3 release.
+   * @deprecated (4.0) Use latest
+   */
+  @Deprecated
+  LUCENE_33,
 
   /** Match settings and bugs in Lucene's 4.0 release. 
    *  <p>

Modified: lucene/dev/branches/solr2452/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java Mon May 30 14:51:25 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.Random;
 
 import org.apache.lucene.analysis.Analyzer;
@@ -97,8 +98,43 @@ public class RandomIndexWriter implement
    * Adds a Document.
    * @see IndexWriter#addDocument(Document)
    */
-  public void addDocument(Document doc) throws IOException {
-    w.addDocument(doc);
+  public void addDocument(final Document doc) throws IOException {
+    if (r.nextInt(5) == 3) {
+      // TODO: maybe, we should simply buffer up added docs
+      // (but we need to clone them), and only when
+      // getReader, commit, etc. are called, we do an
+      // addDocuments?  Would be better testing.
+      w.addDocuments(new Iterable<Document>() {
+
+        // @Override -- not until Java 1.6
+        public Iterator<Document> iterator() {
+          return new Iterator<Document>() {
+            boolean done;
+            
+            // @Override -- not until Java 1.6
+            public boolean hasNext() {
+              return !done;
+            }
+
+            // @Override -- not until Java 1.6
+            public void remove() {
+              throw new UnsupportedOperationException();
+            }
+
+            // @Override -- not until Java 1.6
+            public Document next() {
+              if (done) {
+                throw new IllegalStateException();
+              }
+              done = true;
+              return doc;
+            }
+          };
+        }
+        });
+    } else {
+      w.addDocument(doc);
+    }
     maybeCommit();
   }
 
@@ -116,12 +152,53 @@ public class RandomIndexWriter implement
     }
   }
   
+  public void addDocuments(Iterable<Document> docs) throws IOException {
+    w.addDocuments(docs);
+    maybeCommit();
+  }
+
+  public void updateDocuments(Term delTerm, Iterable<Document> docs) throws IOException {
+    w.updateDocuments(delTerm, docs);
+    maybeCommit();
+  }
+
   /**
    * Updates a document.
    * @see IndexWriter#updateDocument(Term, Document)
    */
-  public void updateDocument(Term t, Document doc) throws IOException {
-    w.updateDocument(t, doc);
+  public void updateDocument(Term t, final Document doc) throws IOException {
+    if (r.nextInt(5) == 3) {
+      w.updateDocuments(t, new Iterable<Document>() {
+
+        // @Override -- not until Java 1.6
+        public Iterator<Document> iterator() {
+          return new Iterator<Document>() {
+            boolean done;
+            
+            // @Override -- not until Java 1.6
+            public boolean hasNext() {
+              return !done;
+            }
+
+            // @Override -- not until Java 1.6
+            public void remove() {
+              throw new UnsupportedOperationException();
+            }
+
+            // @Override -- not until Java 1.6
+            public Document next() {
+              if (done) {
+                throw new IllegalStateException();
+              }
+              done = true;
+              return doc;
+            }
+          };
+        }
+        });
+    } else {
+      w.updateDocument(t, doc);
+    }
     maybeCommit();
   }
   

Modified: lucene/dev/branches/solr2452/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java?rev=1129205&r1=1129204&r2=1129205&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java Mon May 30 14:51:25 2011
@@ -44,6 +44,7 @@ import org.apache.lucene.index.codecs.Te
 import org.apache.lucene.index.codecs.standard.StandardCodec;
 import org.apache.lucene.store.*;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
 
 /**
  * A silly test codec to verify core support for fixed
@@ -97,15 +98,25 @@ public class MockFixedIntBlockCodec exte
 
     @Override
     public IntIndexOutput createOutput(Directory dir, String fileName) throws IOException {
-      return new FixedIntBlockIndexOutput(dir.createOutput(fileName), blockSize) {
-        @Override
-        protected void flushBlock() throws IOException {
-          for(int i=0;i<buffer.length;i++) {
-            assert buffer[i] >= 0;
-            out.writeVInt(buffer[i]);
+      IndexOutput out = dir.createOutput(fileName);
+      boolean success = false;
+      try {
+        FixedIntBlockIndexOutput ret = new FixedIntBlockIndexOutput(out, blockSize) {
+          @Override
+          protected void flushBlock() throws IOException {
+            for(int i=0;i<buffer.length;i++) {
+              assert buffer[i] >= 0;
+              out.writeVInt(buffer[i]);
+            }
           }
+        };
+        success = true;
+        return ret;
+      } finally {
+        if (!success) {
+          IOUtils.closeSafely(true, out);
         }
-      };
+      }
     }
   }
 



Mime
View raw message