lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yo...@apache.org
Subject svn commit: r1197949 [2/7] - in /lucene/dev/branches/solrcloud: ./ dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/ dev-tools/idea/lucene/contrib/ dev-tools/idea/lucene/contrib/misc/ dev-tools/maven/ dev-tools/maven/lucene/contrib/misc/ ...
Date Sat, 05 Nov 2011 12:47:05 GMT
Modified: lucene/dev/branches/solrcloud/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java Sat Nov  5 12:46:53 2011
@@ -31,9 +31,9 @@ import org.apache.lucene.util.CodecUtil;
 public class AppendingTermsIndexReader extends FixedGapTermsIndexReader {
 
   public AppendingTermsIndexReader(Directory dir, FieldInfos fieldInfos,
-          String segment, int indexDivisor, Comparator<BytesRef> termComp, int codecId, IOContext context)
+          String segment, int indexDivisor, Comparator<BytesRef> termComp, String segmentSuffix, IOContext context)
           throws IOException {
-    super(dir, fieldInfos, segment, indexDivisor, termComp, codecId, context);
+    super(dir, fieldInfos, segment, indexDivisor, termComp, segmentSuffix, context);
   }
   
   @Override

Modified: lucene/dev/branches/solrcloud/lucene/contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java (original)
+++ lucene/dev/branches/solrcloud/lucene/contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java Sat Nov  5 12:46:53 2011
@@ -245,6 +245,7 @@ public class DirectIOLinuxDirectory exte
 
     public DirectIOLinuxIndexInput(File path, int bufferSize) throws IOException {
       // TODO make use of IOContext
+      super("DirectIOLinuxIndexInput(path=\"" + path.getPath() + "\")");
       FileDescriptor fd = NativePosixUtil.open_direct(path.toString(), true);
       fis = new FileInputStream(fd);
       channel = fis.getChannel();
@@ -259,6 +260,7 @@ public class DirectIOLinuxDirectory exte
 
     // for clone
     public DirectIOLinuxIndexInput(DirectIOLinuxIndexInput other) throws IOException {
+      super(other.toString());
       this.fis = null;
       channel = other.channel;
       this.bufferSize = other.bufferSize;
@@ -308,7 +310,7 @@ public class DirectIOLinuxDirectory exte
       try {
         return channel.size();
       } catch (IOException ioe) {
-        throw new RuntimeException(ioe);
+        throw new RuntimeException("IOException during length(): " + this, ioe);
       }
     }
 
@@ -331,9 +333,14 @@ public class DirectIOLinuxDirectory exte
       bufferPos = 0;
       assert (filePos & ALIGN_NOT_MASK) == filePos : "filePos=" + filePos + " anded=" + (filePos & ALIGN_NOT_MASK);
       //System.out.println("X refill filePos=" + filePos);
-      int n = channel.read(buffer, filePos);
+      int n;
+      try {
+        n = channel.read(buffer, filePos);
+      } catch (IOException ioe) {
+        throw new IOException(ioe.getMessage() + ": " + this, ioe);
+      }
       if (n < 0) {
-        throw new IOException("eof");
+        throw new IOException("eof: " + this);
       }
       buffer.rewind();
     }
@@ -365,7 +372,7 @@ public class DirectIOLinuxDirectory exte
       try {
         return new DirectIOLinuxIndexInput(this);
       } catch (IOException ioe) {
-        throw new RuntimeException(ioe);
+        throw new RuntimeException("IOException during clone: " + this, ioe);
       }
     }
   }

Modified: lucene/dev/branches/solrcloud/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java (original)
+++ lucene/dev/branches/solrcloud/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java Sat Nov  5 12:46:53 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.store;
 
 import java.io.File;
 import java.io.IOException;
+import java.io.EOFException;
 
 import org.apache.lucene.store.Directory; // javadoc
 import org.apache.lucene.store.NativeFSLockFactory; // javadoc
@@ -80,7 +81,7 @@ public class WindowsDirectory extends FS
     boolean isOpen;
     
     public WindowsIndexInput(File file, int bufferSize) throws IOException {
-      super(bufferSize);
+      super("WindowsIndexInput(path=\"" + file.getPath() + "\")", bufferSize);
       fd = WindowsDirectory.open(file.getPath());
       length = WindowsDirectory.length(fd);
       isOpen = true;
@@ -88,8 +89,16 @@ public class WindowsDirectory extends FS
     
     @Override
     protected void readInternal(byte[] b, int offset, int length) throws IOException {
-      if (WindowsDirectory.read(fd, b, offset, length, getFilePointer()) != length)
-        throw new IOException("Read past EOF");
+      int bytesRead;
+      try {
+        bytesRead = WindowsDirectory.read(fd, b, offset, length, getFilePointer());
+      } catch (IOException ioe) {
+        throw new IOException(ioe.getMessage() + ": " + this, ioe);
+      }
+
+      if (bytesRead != length) {
+        throw new EOFException("Read past EOF: " + this);
+      }
     }
 
     @Override

Modified: lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java (original)
+++ lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java Sat Nov  5 12:46:53 2011
@@ -27,7 +27,9 @@ import org.apache.lucene.search.Searcher
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.NRTCachingDirectory;
+import org.apache.lucene.util.LuceneTestCase.UseNoMemoryExpensiveCodec;
 
+@UseNoMemoryExpensiveCodec
 public class TestNRTManager extends ThreadedIndexingAndSearchingTestCase {
 
   private final ThreadLocal<Long> lastGens = new ThreadLocal<Long>();

Modified: lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java Sat Nov  5 12:46:53 2011
@@ -34,11 +34,6 @@ import org.apache.lucene.index.MultiFiel
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.TermsEnum.SeekStatus;
-import org.apache.lucene.index.codecs.Codec;
-import org.apache.lucene.index.codecs.CodecProvider;
-import org.apache.lucene.index.codecs.DefaultSegmentInfosReader;
-import org.apache.lucene.index.codecs.SegmentInfosReader;
-import org.apache.lucene.index.codecs.SegmentInfosWriter;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexOutput;
@@ -50,30 +45,7 @@ import org.apache.lucene.util.Version;
 
 public class TestAppendingCodec extends LuceneTestCase {
   
-  static class AppendingCodecProvider extends CodecProvider {
-    Codec appending = new AppendingCodec();
-    SegmentInfosWriter infosWriter = new AppendingSegmentInfosWriter();
-    SegmentInfosReader infosReader = new DefaultSegmentInfosReader();
-    public AppendingCodecProvider() {
-      setDefaultFieldCodec(appending.name);
-    }
-    @Override
-    public Codec lookup(String name) {
-      return appending;
-    }
-   
-    @Override
-    public SegmentInfosReader getSegmentInfosReader() {
-      return infosReader;
-    }
-    @Override
-    public SegmentInfosWriter getSegmentInfosWriter() {
-      return infosWriter;
-    }
-    
-  }
-  
-  private static class AppendingIndexOutputWrapper extends IndexOutput {
+    private static class AppendingIndexOutputWrapper extends IndexOutput {
     IndexOutput wrapped;
     
     public AppendingIndexOutputWrapper(IndexOutput wrapped) {
@@ -137,7 +109,7 @@ public class TestAppendingCodec extends 
     Directory dir = new AppendingRAMDirectory(random, new RAMDirectory());
     IndexWriterConfig cfg = new IndexWriterConfig(Version.LUCENE_40, new MockAnalyzer(random));
     
-    cfg.setCodecProvider(new AppendingCodecProvider());
+    cfg.setCodec(new AppendingCodec());
     ((TieredMergePolicy)cfg.getMergePolicy()).setUseCompoundFile(false);
     IndexWriter writer = new IndexWriter(dir, cfg);
     Document doc = new Document();
@@ -151,7 +123,7 @@ public class TestAppendingCodec extends 
     writer.addDocument(doc);
     writer.optimize();
     writer.close();
-    IndexReader reader = IndexReader.open(dir, null, true, 1, new AppendingCodecProvider());
+    IndexReader reader = IndexReader.open(dir, null, true, 1);
     assertEquals(2, reader.numDocs());
     Document doc2 = reader.document(0);
     assertEquals(text, doc2.get("f"));

Modified: lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java (original)
+++ lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java Sat Nov  5 12:46:53 2011
@@ -34,9 +34,11 @@ import org.apache.lucene.index.Term;
 import org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase.UseNoMemoryExpensiveCodec;
 import org.apache.lucene.util.NamedThreadFactory;
 import org.apache.lucene.util._TestUtil;
 
+@UseNoMemoryExpensiveCodec
 public class TestSearcherManager extends ThreadedIndexingAndSearchingTestCase {
 
   boolean warmCalled;

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/CheckIndex.java Sat Nov  5 12:46:53 2011
@@ -25,7 +25,7 @@ import org.apache.lucene.store.Directory
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter;
 import java.io.File;
 import java.io.IOException;
@@ -143,8 +143,8 @@ public class CheckIndex {
       /** Name of the segment. */
       public String name;
 
-      /** CodecInfo used to read this segment. */
-      public SegmentCodecs codec;
+      /** Codec used to read this segment. */
+      public Codec codec;
 
       /** Document count (does not take deletions into account). */
       public int docCount;
@@ -322,10 +322,6 @@ public class CheckIndex {
   public Status checkIndex() throws IOException {
     return checkIndex(null);
   }
-
-  public Status checkIndex(List<String> onlySegments) throws IOException {
-    return checkIndex(onlySegments, CodecProvider.getDefault());
-  }
   
   /** Returns a {@link Status} instance detailing
    *  the state of the index.
@@ -339,13 +335,13 @@ public class CheckIndex {
    *  <p><b>WARNING</b>: make sure
    *  you only call this when the index is not opened by any
    *  writer. */
-  public Status checkIndex(List<String> onlySegments, CodecProvider codecs) throws IOException {
+  public Status checkIndex(List<String> onlySegments) throws IOException {
     NumberFormat nf = NumberFormat.getInstance();
-    SegmentInfos sis = new SegmentInfos(codecs);
+    SegmentInfos sis = new SegmentInfos();
     Status result = new Status();
     result.dir = dir;
     try {
-      sis.read(dir, codecs);
+      sis.read(dir);
     } catch (Throwable t) {
       msg("ERROR: could not read any segments file in directory");
       result.missingSegments = true;
@@ -377,6 +373,7 @@ public class CheckIndex {
 
     final int numSegments = sis.size();
     final String segmentsFileName = sis.getCurrentSegmentFileName();
+    // note: we only read the format byte (required preamble) here!
     IndexInput input = null;
     try {
       input = dir.openInput(segmentsFileName, IOContext.DEFAULT);
@@ -489,7 +486,7 @@ public class CheckIndex {
       SegmentReader reader = null;
 
       try {
-        final SegmentCodecs codec = info.getSegmentCodecs();
+        final Codec codec = info.getCodec();
         msg("    codec=" + codec);
         segInfoStat.codec = codec;
         msg("    compound=" + info.getUseCompoundFile());
@@ -1182,11 +1179,11 @@ public class CheckIndex {
    *
    * <p><b>WARNING</b>: Make sure you only call this when the
    *  index is not opened  by any writer. */
-  public void fixIndex(Status result) throws IOException {
+  public void fixIndex(Status result, Codec codec) throws IOException {
     if (result.partial)
       throw new IllegalArgumentException("can only fix an index that was fully checked (this status checked a subset of segments)");
     result.newSegments.changed();
-    result.newSegments.commit(result.dir);
+    result.newSegments.commit(result.dir, codec);
   }
 
   private static boolean assertsOn;
@@ -1236,6 +1233,7 @@ public class CheckIndex {
   public static void main(String[] args) throws IOException, InterruptedException {
 
     boolean doFix = false;
+    Codec codec = Codec.getDefault(); // only used when fixing
     boolean verbose = false;
     List<String> onlySegments = new ArrayList<String>();
     String indexPath = null;
@@ -1244,6 +1242,13 @@ public class CheckIndex {
       if (args[i].equals("-fix")) {
         doFix = true;
         i++;
+      } else if (args[i].equals("-codec")) {
+        if (i == args.length-1) {
+          System.out.println("ERROR: missing name for -codec option");
+          System.exit(1);
+        }
+        codec = Codec.forName(args[i+1]);
+        i+=2;
       } else if (args[i].equals("-verbose")) {
         verbose = true;
         i++;
@@ -1269,6 +1274,7 @@ public class CheckIndex {
       System.out.println("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" +
                          "\n" +
                          "  -fix: actually write a new segments_N file, removing any problematic segments\n" +
+                         "  -codec X: when fixing, codec to write the new segments_N file with\n" +
                          "  -verbose: print additional details\n" +
                          "  -segment X: only check the specified segments.  This can be specified multiple\n" + 
                          "              times, to check more than one segment, eg '-segment _2 -segment _a'.\n" +
@@ -1329,7 +1335,7 @@ public class CheckIndex {
           System.out.println("  " + (5-s) + "...");
         }
         System.out.println("Writing...");
-        checker.fixIndex(result);
+        checker.fixIndex(result, codec);
         System.out.println("OK");
         System.out.println("Wrote new segments file \"" + result.newSegments.getCurrentSegmentFileName() + "\"");
       }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DirectoryReader.java Sat Nov  5 12:46:53 2011
@@ -33,7 +33,6 @@ import org.apache.lucene.store.Directory
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
@@ -45,8 +44,6 @@ import org.apache.lucene.util.MapBackedS
 class DirectoryReader extends IndexReader implements Cloneable {
   protected Directory directory;
   protected boolean readOnly;
-  
-  protected CodecProvider codecs;
 
   IndexWriter writer;
 
@@ -78,15 +75,13 @@ class DirectoryReader extends IndexReade
 //  }
   
   static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly,
-                          final int termInfosIndexDivisor, CodecProvider codecs) throws CorruptIndexException, IOException {
-    final CodecProvider codecProvider = codecs == null ? CodecProvider.getDefault()
-        : codecs;
+                          final int termInfosIndexDivisor) throws CorruptIndexException, IOException {
     return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) {
       @Override
       protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
-        SegmentInfos infos = new SegmentInfos(codecProvider);
-        infos.read(directory, segmentFileName, codecProvider);
-        return new DirectoryReader(directory, infos, deletionPolicy, readOnly, termInfosIndexDivisor, codecProvider);
+        SegmentInfos infos = new SegmentInfos();
+        infos.read(directory, segmentFileName);
+        return new DirectoryReader(directory, infos, deletionPolicy, readOnly, termInfosIndexDivisor);
       }
     }.run(commit);
   }
@@ -97,17 +92,12 @@ class DirectoryReader extends IndexReade
 //  }
   
   /** Construct reading the named set of readers. */
-  DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, CodecProvider codecs) throws IOException {
+  DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws IOException {
     this.directory = directory;
     this.readOnly = readOnly;
     this.segmentInfos = sis;
     this.deletionPolicy = deletionPolicy;
     this.termInfosIndexDivisor = termInfosIndexDivisor;
-    if (codecs == null) {
-      this.codecs = CodecProvider.getDefault();
-    } else {
-      this.codecs = codecs;
-    }
     readerFinishedListeners = new MapBackedSet<ReaderFinishedListener>(new ConcurrentHashMap<ReaderFinishedListener,Boolean>());
     applyAllDeletes = false;
 
@@ -141,17 +131,12 @@ class DirectoryReader extends IndexReade
   }
 
   // Used by near real-time search
-  DirectoryReader(IndexWriter writer, SegmentInfos infos, CodecProvider codecs, boolean applyAllDeletes) throws IOException {
+  DirectoryReader(IndexWriter writer, SegmentInfos infos, boolean applyAllDeletes) throws IOException {
     this.directory = writer.getDirectory();
     this.readOnly = true;
     this.applyAllDeletes = applyAllDeletes;       // saved for reopen
 
     this.termInfosIndexDivisor = writer.getConfig().getReaderTermsIndexDivisor();
-    if (codecs == null) {
-      this.codecs = CodecProvider.getDefault();
-    } else {
-      this.codecs = codecs;
-    }
     readerFinishedListeners = writer.getReaderFinishedListeners();
 
     // IndexWriter synchronizes externally before calling
@@ -200,8 +185,7 @@ class DirectoryReader extends IndexReade
 
   /** This constructor is only used for {@link #doOpenIfChanged()} */
   DirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders,
-                  boolean readOnly, boolean doClone, int termInfosIndexDivisor, CodecProvider codecs,
-                  Collection<ReaderFinishedListener> readerFinishedListeners) throws IOException {
+                  boolean readOnly, boolean doClone, int termInfosIndexDivisor, Collection<ReaderFinishedListener> readerFinishedListeners) throws IOException {
     this.directory = directory;
     this.readOnly = readOnly;
     this.segmentInfos = infos;
@@ -209,13 +193,6 @@ class DirectoryReader extends IndexReade
     this.readerFinishedListeners = readerFinishedListeners;
     applyAllDeletes = false;
 
-    if (codecs == null) {
-      this.codecs = CodecProvider.getDefault();
-    } else {
-      this.codecs = codecs;
-    }
-    
-
     // we put the old SegmentReaders in a map, that allows us
     // to lookup a reader using its segment name
     Map<String,Integer> segmentReaders = new HashMap<String,Integer>();
@@ -347,7 +324,7 @@ class DirectoryReader extends IndexReade
     starts[subReaders.length] = maxDoc;
 
     if (!readOnly) {
-      maxIndexVersion = SegmentInfos.readCurrentVersion(directory, codecs);
+      maxIndexVersion = SegmentInfos.readCurrentVersion(directory);
     }
   }
 
@@ -498,15 +475,15 @@ class DirectoryReader extends IndexReade
     return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) {
       @Override
       protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
-        final SegmentInfos infos = new SegmentInfos(codecs);
-        infos.read(directory, segmentFileName, codecs);
+        final SegmentInfos infos = new SegmentInfos();
+        infos.read(directory, segmentFileName);
         return doOpenIfChanged(infos, false, openReadOnly);
       }
     }.run(commit);
   }
 
   private synchronized DirectoryReader doOpenIfChanged(SegmentInfos infos, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException {
-    return new DirectoryReader(directory, infos, subReaders, openReadOnly, doClone, termInfosIndexDivisor, codecs, readerFinishedListeners);
+    return new DirectoryReader(directory, infos, subReaders, openReadOnly, doClone, termInfosIndexDivisor, readerFinishedListeners);
   }
 
   /** Version number when this IndexReader was opened. */
@@ -712,7 +689,7 @@ class DirectoryReader extends IndexReade
 
         // we have to check whether index has changed since this reader was opened.
         // if so, this reader is no longer valid for deletion
-        if (SegmentInfos.readCurrentVersion(directory, codecs) > maxIndexVersion) {
+        if (SegmentInfos.readCurrentVersion(directory) > maxIndexVersion) {
           stale = true;
           this.writeLock.release();
           this.writeLock = null;
@@ -743,7 +720,7 @@ class DirectoryReader extends IndexReade
       // KeepOnlyLastCommitDeleter:
       IndexFileDeleter deleter = new IndexFileDeleter(directory,
                                                       deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy,
-                                                      segmentInfos, null, codecs, null);
+                                                      segmentInfos, null, null);
       segmentInfos.updateGeneration(deleter.getLastSegmentInfos());
       segmentInfos.changed();
 
@@ -764,7 +741,7 @@ class DirectoryReader extends IndexReade
 
         // Sync all files we just wrote
         directory.sync(segmentInfos.files(directory, false));
-        segmentInfos.commit(directory);
+        segmentInfos.commit(directory, segmentInfos.codecFormat());
         success = true;
       } finally {
 
@@ -842,7 +819,7 @@ class DirectoryReader extends IndexReade
     ensureOpen();
     if (writer == null || writer.isClosed()) {
       // we loaded SegmentInfos from the directory
-      return SegmentInfos.readCurrentVersion(directory, codecs) == segmentInfos.getVersion();
+      return SegmentInfos.readCurrentVersion(directory) == segmentInfos.getVersion();
     } else {
       return writer.nrtIsCurrent(segmentInfos);
     }
@@ -925,17 +902,12 @@ class DirectoryReader extends IndexReade
 
   /** @see org.apache.lucene.index.IndexReader#listCommits */
   public static List<IndexCommit> listCommits(Directory dir) throws IOException {
-    return listCommits(dir, CodecProvider.getDefault());
-  }
-
-  /** @see org.apache.lucene.index.IndexReader#listCommits */
-  public static List<IndexCommit> listCommits(Directory dir, CodecProvider codecs) throws IOException {
     final String[] files = dir.listAll();
 
     List<IndexCommit> commits = new ArrayList<IndexCommit>();
 
-    SegmentInfos latest = new SegmentInfos(codecs);
-    latest.read(dir, codecs);
+    SegmentInfos latest = new SegmentInfos();
+    latest.read(dir);
     final long currentGen = latest.getGeneration();
 
     commits.add(new ReaderCommit(latest, dir));
@@ -948,11 +920,11 @@ class DirectoryReader extends IndexReade
           !fileName.equals(IndexFileNames.SEGMENTS_GEN) &&
           SegmentInfos.generationFromSegmentsFileName(fileName) < currentGen) {
 
-        SegmentInfos sis = new SegmentInfos(codecs);
+        SegmentInfos sis = new SegmentInfos();
         try {
           // IOException allowed to throw there, in case
           // segments_N is corrupt
-          sis.read(dir, fileName, codecs);
+          sis.read(dir, fileName);
         } catch (FileNotFoundException fnfe) {
           // LUCENE-948: on NFS (and maybe others), if
           // you have writers switching back and forth

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java Sat Nov  5 12:46:53 2011
@@ -25,7 +25,7 @@ import java.util.HashSet;
 import java.util.Map;
 
 import org.apache.lucene.index.DocumentsWriterPerThread.DocState;
-import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.DocValuesFormat;
 import org.apache.lucene.index.codecs.DocValuesConsumer;
 import org.apache.lucene.index.codecs.PerDocConsumer;
 import org.apache.lucene.index.values.PerDocFieldValues;
@@ -320,14 +320,13 @@ final class DocFieldProcessor extends Do
       docValuesConsumerAndDocID.docID = docState.docID;
       return docValuesConsumerAndDocID.docValuesConsumer;
     }
-    PerDocConsumer perDocConsumer = perDocConsumers.get(fieldInfo.getCodecId());
+
+    PerDocConsumer perDocConsumer = perDocConsumers.get(0);
     if (perDocConsumer == null) {
-      PerDocWriteState perDocWriteState = docState.docWriter.newPerDocWriteState(fieldInfo.getCodecId());
-      SegmentCodecs codecs = perDocWriteState.segmentCodecs;
-      assert codecs.codecs.length > fieldInfo.getCodecId();
-      Codec codec = codecs.codecs[fieldInfo.getCodecId()];
-      perDocConsumer = codec.docsConsumer(perDocWriteState);
-      perDocConsumers.put(Integer.valueOf(fieldInfo.getCodecId()), perDocConsumer);
+      PerDocWriteState perDocWriteState = docState.docWriter.newPerDocWriteState("");
+      DocValuesFormat dvFormat = docState.docWriter.codec.docValuesFormat();
+      perDocConsumer = dvFormat.docsConsumer(perDocWriteState);
+      perDocConsumers.put(0, perDocConsumer);
     }
     boolean success = false;
     DocValuesConsumer docValuesConsumer = null;

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java Sat Nov  5 12:46:53 2011
@@ -31,6 +31,7 @@ import org.apache.lucene.index.Documents
 import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain;
 import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState;
 import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
+import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.similarities.SimilarityProvider;
 import org.apache.lucene.store.AlreadyClosedException;
@@ -116,7 +117,14 @@ final class DocumentsWriter {
 
   // TODO: cut over to BytesRefHash in BufferedDeletes
   volatile DocumentsWriterDeleteQueue deleteQueue = new DocumentsWriterDeleteQueue();
-  private final Queue<FlushTicket> ticketQueue = new LinkedList<DocumentsWriter.FlushTicket>();
+  private final TicketQueue ticketQueue = new TicketQueue();
+  /*
+   * we preserve changes during a full flush since IW might not checkout before
+   * we release all changes. NRT Readers otherwise suddenly return true from
+   * isCurrent while there are actually changes currently committed. See also
+   * #anyChanges() & #flushAllThreads
+   */
+  private volatile boolean pendingChangesInCurrentFullFlush;
 
   private Collection<String> abortedFiles;               // List of files that were written before last abort()
 
@@ -125,8 +133,11 @@ final class DocumentsWriter {
   final DocumentsWriterPerThreadPool perThreadPool;
   final FlushPolicy flushPolicy;
   final DocumentsWriterFlushControl flushControl;
-  DocumentsWriter(IndexWriterConfig config, Directory directory, IndexWriter writer, FieldNumberBiMap globalFieldNumbers,
+  
+  final Codec codec;
+  DocumentsWriter(Codec codec, IndexWriterConfig config, Directory directory, IndexWriter writer, FieldNumberBiMap globalFieldNumbers,
       BufferedDeletesStream bufferedDeletesStream) throws IOException {
+    this.codec = codec;
     this.directory = directory;
     this.indexWriter = writer;
     this.similarityProvider = config.getSimilarityProvider();
@@ -166,6 +177,7 @@ final class DocumentsWriter {
   private void applyAllDeletes(DocumentsWriterDeleteQueue deleteQueue) throws IOException {
     if (deleteQueue != null && !flushControl.isFullFlush()) {
       synchronized (ticketQueue) {
+        ticketQueue.incTicketCount();// first inc the ticket count - freeze opens a window for #anyChanges to fail
         // Freeze and insert the delete flush ticket in the queue
         ticketQueue.add(new FlushTicket(deleteQueue.freezeGlobalBuffer(null), false));
         applyFlushTickets();
@@ -252,9 +264,22 @@ final class DocumentsWriter {
   }
 
   boolean anyChanges() {
-    return numDocsInRAM.get() != 0 || anyDeletions();
+    if (infoStream != null) {
+      message("docWriter: anyChanges? numDocsInRam=" + numDocsInRAM.get()
+          + " deletes=" + anyDeletions() + " hasTickets:"
+          + ticketQueue.hasTickets() + " pendingChangesInFullFlush: "
+          + pendingChangesInCurrentFullFlush);
+    }
+    /*
+     * changes are either in a DWPT or in the deleteQueue.
+     * yet if we currently flush deletes and / or dwpt there
+     * could be a window where all changes are in the ticket queue
+     * before they are published to the IW. ie we need to check if the 
+     * ticket queue has any tickets.
+     */
+    return numDocsInRAM.get() != 0 || anyDeletions() || ticketQueue.hasTickets() || pendingChangesInCurrentFullFlush;
   }
-
+  
   public int getBufferedDeleteTermsSize() {
     return deleteQueue.getBufferedDeleteTermsSize();
   }
@@ -279,7 +304,7 @@ final class DocumentsWriter {
     if (flushControl.anyStalledThreads() || flushControl.numQueuedFlushes() > 0) {
       // Help out flushing any queued DWPTs so we can un-stall:
       if (infoStream != null) {
-        message("DocumentsWriter has queued dwpt; will hijack this thread to flush pending segment(s)");
+        message("docWriter: DocumentsWriter has queued dwpt; will hijack this thread to flush pending segment(s)");
       }
       do {
         // Try pick up pending threads here if possible
@@ -413,7 +438,7 @@ final class DocumentsWriter {
           synchronized (ticketQueue) {
             // Each flush is assigned a ticket in the order they acquire the ticketQueue lock
             ticket =  new FlushTicket(flushingDWPT.prepareFlush(), true);
-            ticketQueue.add(ticket);
+            ticketQueue.incrementAndAdd(ticket);
           }
   
           // flush concurrently without locking
@@ -470,8 +495,11 @@ final class DocumentsWriter {
         // Keep publishing eligible flushed segments:
         final FlushTicket head = ticketQueue.peek();
         if (head != null && head.canPublish()) {
-          ticketQueue.poll();
-          finishFlush(head.segment, head.frozenDeletes);
+          try {
+            finishFlush(head.segment, head.frozenDeletes);
+          } finally {
+            ticketQueue.poll();
+          }
         } else {
           break;
         }
@@ -485,7 +513,7 @@ final class DocumentsWriter {
     if (newSegment == null) {
       assert bufferedDeletes != null;
       if (bufferedDeletes != null && bufferedDeletes.any()) {
-        indexWriter.bufferedDeletesStream.push(bufferedDeletes);
+        indexWriter.publishFrozenDeletes(bufferedDeletes);
         if (infoStream != null) {
           message("flush: push buffered deletes: " + bufferedDeletes);
         }
@@ -531,6 +559,7 @@ final class DocumentsWriter {
   
   // for asserts
   private volatile DocumentsWriterDeleteQueue currentFullFlushDelQueue = null;
+
   // for asserts
   private synchronized boolean setFlushingDeleteQueue(DocumentsWriterDeleteQueue session) {
     currentFullFlushDelQueue = session;
@@ -550,6 +579,7 @@ final class DocumentsWriter {
     }
     
     synchronized (this) {
+      pendingChangesInCurrentFullFlush = anyChanges();
       flushingDeleteQueue = deleteQueue;
       /* Cutover to a new delete queue.  This must be synced on the flush control
        * otherwise a new DWPT could sneak into the loop with an already flushing
@@ -569,11 +599,12 @@ final class DocumentsWriter {
       }
       // If a concurrent flush is still in flight wait for it
       flushControl.waitForFlush();  
-      if (!anythingFlushed) { // apply deletes if we did not flush any document
+      if (!anythingFlushed && flushingDeleteQueue.anyChanges()) { // apply deletes if we did not flush any document
         if (infoStream != null) {
          message(Thread.currentThread().getName() + ": flush naked frozen global deletes");
         }
         synchronized (ticketQueue) {
+          ticketQueue.incTicketCount(); // first inc the ticket count - freeze opens a window for #anyChanges to fail
           ticketQueue.add(new FlushTicket(flushingDeleteQueue.freezeGlobalBuffer(null), false));
         }
         applyFlushTickets();
@@ -586,16 +617,21 @@ final class DocumentsWriter {
   }
   
   final void finishFullFlush(boolean success) {
-    if (infoStream != null) {
-      message(Thread.currentThread().getName() + " finishFullFlush success=" + success);
-    }
-    assert setFlushingDeleteQueue(null);
-    if (success) {
-      // Release the flush lock
-      flushControl.finishFullFlush();
-    } else {
-      flushControl.abortFullFlushes();
+    try {
+      if (infoStream != null) {
+        message(Thread.currentThread().getName() + " finishFullFlush success=" + success);
+      }
+      assert setFlushingDeleteQueue(null);
+      if (success) {
+        // Release the flush lock
+        flushControl.finishFullFlush();
+      } else {
+        flushControl.abortFullFlushes();
+      }
+    } finally {
+      pendingChangesInCurrentFullFlush = false;
     }
+    
   }
 
   static final class FlushTicket {
@@ -614,6 +650,46 @@ final class DocumentsWriter {
     }
   }
   
+  static final class TicketQueue {
+    private final Queue<FlushTicket> queue = new LinkedList<FlushTicket>();
+    final AtomicInteger ticketCount = new AtomicInteger();
+    
+    void incTicketCount() {
+      ticketCount.incrementAndGet();
+    }
+    
+    public boolean hasTickets() {
+      assert ticketCount.get() >= 0;
+      return ticketCount.get() != 0;
+    }
+
+    void incrementAndAdd(FlushTicket ticket) {
+      incTicketCount();
+      add(ticket);
+    }
+    
+    void add(FlushTicket ticket) {
+      queue.add(ticket);
+    }
+    
+    FlushTicket peek() {
+      return queue.peek();
+    }
+    
+    FlushTicket poll() {
+      try {
+        return queue.poll();
+      } finally {
+        ticketCount.decrementAndGet();
+      }
+    }
+    
+    void clear() {
+      queue.clear();
+      ticketCount.set(0);
+    }
+  }
+  
   // use by IW during close to assert all DWPT are inactive after final flush
   boolean assertNoActiveDWPT() {
     Iterator<ThreadState> activePerThreadsIterator = perThreadPool.getAllPerThreadsIterator();

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Sat Nov  5 12:46:53 2011
@@ -26,7 +26,7 @@ import java.text.NumberFormat;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.index.DocumentsWriterDeleteQueue.DeleteSlice;
-import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.search.similarities.SimilarityProvider;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FlushInfo;
@@ -152,7 +152,7 @@ public class DocumentsWriterPerThread {
   }
   private final static boolean INFO_VERBOSE = false;
   final DocumentsWriter parent;
-  final CodecProvider codecProvider;
+  final Codec codec;
   final IndexWriter writer;
   final Directory directory;
   final DocState docState;
@@ -183,7 +183,7 @@ public class DocumentsWriterPerThread {
     this.fieldInfos = fieldInfos;
     this.writer = parent.indexWriter;
     this.infoStream = parent.infoStream;
-    this.codecProvider = this.writer.codecs;
+    this.codec = parent.codec;
     this.docState = new DocState(this);
     this.docState.similarityProvider = parent.indexWriter.getConfig()
         .getSimilarityProvider();
@@ -405,8 +405,8 @@ public class DocumentsWriterPerThread {
     return numDocsInRAM;
   }
 
-  SegmentCodecs getCodec() {
-    return flushState.segmentCodecs;
+  Codec getCodec() {
+    return flushState.codec;
   }
 
   /** Reset after a flush */
@@ -443,7 +443,7 @@ public class DocumentsWriterPerThread {
     assert deleteSlice == null : "all deletes must be applied in prepareFlush";
     flushState = new SegmentWriteState(infoStream, directory, segment, fieldInfos,
         numDocsInRAM, writer.getConfig().getTermIndexInterval(),
-        fieldInfos.buildSegmentCodecs(true), pendingDeletes, new IOContext(new FlushInfo(numDocsInRAM, bytesUsed())));
+        codec, pendingDeletes, new IOContext(new FlushInfo(numDocsInRAM, bytesUsed())));
     final double startMBUsed = parent.flushControl.netBytes() / 1024. / 1024.;
     // Apply delete-by-docID now (delete-byDocID only
     // happens when an exception is hit processing that
@@ -474,12 +474,12 @@ public class DocumentsWriterPerThread {
     try {
       consumer.flush(flushState);
       pendingDeletes.terms.clear();
-      final SegmentInfo newSegment = new SegmentInfo(segment, flushState.numDocs, directory, false, flushState.segmentCodecs, fieldInfos.asReadOnly());
+      final SegmentInfo newSegment = new SegmentInfo(segment, flushState.numDocs, directory, false, flushState.codec, fieldInfos.asReadOnly());
       if (infoStream != null) {
         message("new segment has " + (flushState.liveDocs == null ? 0 : (flushState.numDocs - flushState.liveDocs.count())) + " deleted docs");
         message("new segment has " + (newSegment.getHasVectors() ? "vectors" : "no vectors"));
         message("flushedFiles=" + newSegment.files());
-        message("flushed codecs=" + newSegment.getSegmentCodecs());
+        message("flushed codec=" + newSegment.getCodec());
       }
       flushedDocCount += flushState.numDocs;
 
@@ -556,9 +556,9 @@ public class DocumentsWriterPerThread {
     bytesUsed.addAndGet(-(length *(INT_BLOCK_SIZE*RamUsageEstimator.NUM_BYTES_INT)));
   }
 
-  PerDocWriteState newPerDocWriteState(int codecId) {
+  PerDocWriteState newPerDocWriteState(String segmentSuffix) {
     assert segment != null;
-    return new PerDocWriteState(infoStream, directory, segment, fieldInfos, bytesUsed, codecId, IOContext.DEFAULT);
+    return new PerDocWriteState(infoStream, directory, segment, fieldInfos, bytesUsed, segmentSuffix, IOContext.DEFAULT);
   }
   
   void setInfoStream(PrintStream infoStream) {

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java Sat Nov  5 12:46:53 2011
@@ -20,8 +20,6 @@ import java.util.Iterator;
 import java.util.concurrent.locks.ReentrantLock;
 
 import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
-import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder;
-import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.util.SetOnce;
 
 /**
@@ -128,7 +126,6 @@ public abstract class DocumentsWriterPer
 
   private final ThreadState[] perThreads;
   private volatile int numThreadStatesActive;
-  private CodecProvider codecProvider;
   private FieldNumberBiMap globalFieldMap;
   private final SetOnce<DocumentsWriter> documentsWriter = new SetOnce<DocumentsWriter>();
   
@@ -148,11 +145,9 @@ public abstract class DocumentsWriterPer
 
   public void initialize(DocumentsWriter documentsWriter, FieldNumberBiMap globalFieldMap, IndexWriterConfig config) {
     this.documentsWriter.set(documentsWriter); // thread pool is bound to DW
-    final CodecProvider codecs = config.getCodecProvider();
-    this.codecProvider = codecs;
     this.globalFieldMap = globalFieldMap;
     for (int i = 0; i < perThreads.length; i++) {
-      final FieldInfos infos = globalFieldMap.newFieldInfos(SegmentCodecsBuilder.create(codecs));
+      final FieldInfos infos = new FieldInfos(globalFieldMap);
       perThreads[i] = new ThreadState(new DocumentsWriterPerThread(documentsWriter.directory, documentsWriter, infos, documentsWriter.chain));
     }
   }
@@ -240,7 +235,7 @@ public abstract class DocumentsWriterPer
     assert threadState.isHeldByCurrentThread();
     final DocumentsWriterPerThread dwpt = threadState.perThread;
     if (!closed) {
-      final FieldInfos infos = globalFieldMap.newFieldInfos(SegmentCodecsBuilder.create(codecProvider));
+      final FieldInfos infos = new FieldInfos(globalFieldMap);
       final DocumentsWriterPerThread newDwpt = new DocumentsWriterPerThread(dwpt, infos);
       newDwpt.initialize();
       threadState.resetWriter(newDwpt);

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfo.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfo.java Sat Nov  5 12:46:53 2011
@@ -21,7 +21,6 @@ import org.apache.lucene.index.values.Va
 
 /** @lucene.experimental */
 public final class FieldInfo {
-  public static final int UNASSIGNED_CODEC_ID = -1;
   public final String name;
   public final int number;
 
@@ -38,7 +37,6 @@ public final class FieldInfo {
   public IndexOptions indexOptions;
 
   public boolean storePayloads; // whether this field stores payloads together with term positions
-  private int codecId = UNASSIGNED_CODEC_ID; // set inside SegmentCodecs#build() during segment flush - this is used to identify the codec used to write this field
 
   /**
    * Controls how much information is stored in the postings lists.
@@ -77,21 +75,11 @@ public final class FieldInfo {
     }
     assert indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !storePayloads;
   }
-
-  void setCodecId(int codecId) {
-    assert this.codecId == UNASSIGNED_CODEC_ID : "CodecId can only be set once.";
-    this.codecId = codecId;
-  }
-
-  public int getCodecId() {
-    return codecId;
-  }
   
   @Override
   public Object clone() {
     FieldInfo clone = new FieldInfo(name, isIndexed, number, storeTermVector, storePositionWithTermVector,
                          storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValues);
-    clone.codecId = this.codecId;
     return clone;
   }
 

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfos.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfos.java Sat Nov  5 12:46:53 2011
@@ -29,9 +29,6 @@ import java.util.TreeMap;
 import java.util.Map.Entry;
 
 import org.apache.lucene.index.FieldInfo.IndexOptions;
-import org.apache.lucene.index.SegmentCodecs; // Required for Java 1.5 javadocs
-import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder;
-import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.index.values.ValueType;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
@@ -146,17 +143,6 @@ public final class FieldInfos implements
     }
     
     /**
-     * Returns a new {@link FieldInfos} instance with this as the global field
-     * map
-     * 
-     * @return a new {@link FieldInfos} instance with this as the global field
-     *         map
-     */
-    public FieldInfos newFieldInfos(SegmentCodecsBuilder segmentCodecsBuilder) {
-      return new FieldInfos(this, segmentCodecsBuilder);
-    }
-
-    /**
      * Returns <code>true</code> iff the last committed version differs from the
      * current version, otherwise <code>false</code>
      * 
@@ -198,7 +184,6 @@ public final class FieldInfos implements
   private final SortedMap<Integer,FieldInfo> byNumber = new TreeMap<Integer,FieldInfo>();
   private final HashMap<String,FieldInfo> byName = new HashMap<String,FieldInfo>();
   private final FieldNumberBiMap globalFieldNumbers;
-  private final SegmentCodecsBuilder segmentCodecsBuilder;
   
   // First used in 2.9; prior to 2.9 there was no format header
   public static final int FORMAT_START = -2;
@@ -230,16 +215,15 @@ public final class FieldInfos implements
 
   /**
    * Creates a new {@link FieldInfos} instance with a private
-   * {@link org.apache.lucene.index.FieldInfos.FieldNumberBiMap} and a default {@link SegmentCodecsBuilder}
-   * initialized with {@link CodecProvider#getDefault()}.
+   * {@link org.apache.lucene.index.FieldInfos.FieldNumberBiMap} 
    * <p>
    * Note: this ctor should not be used during indexing use
    * {@link FieldInfos#FieldInfos(FieldInfos)} or
-   * {@link FieldInfos#FieldInfos(FieldNumberBiMap,org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder)}
+   * {@link FieldInfos#FieldInfos(FieldNumberBiMap)}
    * instead.
    */
   public FieldInfos() {
-    this(new FieldNumberBiMap(), SegmentCodecsBuilder.create(CodecProvider.getDefault()));
+    this(new FieldNumberBiMap());
   }
   
   /**
@@ -249,7 +233,7 @@ public final class FieldInfos implements
    * @see #isReadOnly()
    */
   FieldInfos(FieldInfos other) {
-    this(other.globalFieldNumbers, other.segmentCodecsBuilder);
+    this(other.globalFieldNumbers);
   }
   
   /**
@@ -257,9 +241,8 @@ public final class FieldInfos implements
    * If the {@link FieldNumberBiMap} is <code>null</code> this instance will be read-only.
    * @see #isReadOnly()
    */
-  FieldInfos(FieldNumberBiMap globalFieldNumbers, SegmentCodecsBuilder segmentCodecsBuilder) {
+  FieldInfos(FieldNumberBiMap globalFieldNumbers) {
     this.globalFieldNumbers = globalFieldNumbers;
-    this.segmentCodecsBuilder = segmentCodecsBuilder;
   }
 
   /**
@@ -273,7 +256,7 @@ public final class FieldInfos implements
    * @throws IOException
    */
   public FieldInfos(Directory d, String name) throws IOException {
-    this((FieldNumberBiMap)null, null); // use null here to make this FIs Read-Only
+    this((FieldNumberBiMap)null); // use null here to make this FIs Read-Only
     final IndexInput input = d.openInput(name, IOContext.READONCE);
     try {
       read(input, name);
@@ -309,7 +292,7 @@ public final class FieldInfos implements
    */
   @Override
   synchronized public Object clone() {
-    FieldInfos fis = new FieldInfos(globalFieldNumbers, segmentCodecsBuilder);
+    FieldInfos fis = new FieldInfos(globalFieldNumbers);
     fis.format = format;
     fis.hasFreq = hasFreq;
     fis.hasProx = hasProx;
@@ -468,7 +451,6 @@ public final class FieldInfos implements
     if (globalFieldNumbers == null) {
       throw new IllegalStateException("FieldInfos are read-only, create a new instance with a global field map to make modifications to FieldInfos");
     }
-    assert segmentCodecsBuilder != null : "SegmentCodecsBuilder is set to null but FieldInfos is not read-only";
     FieldInfo fi = fieldInfo(name);
     if (fi == null) {
       final int fieldNumber = nextFieldNumber(name, preferredFieldNumber);
@@ -477,9 +459,6 @@ public final class FieldInfos implements
       fi.update(isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions);
       fi.setDocValues(docValues);
     }
-    if ((fi.isIndexed || fi.hasDocValues()) && fi.getCodecId() == FieldInfo.UNASSIGNED_CODEC_ID) {
-      segmentCodecsBuilder.tryAddAndSet(fi);
-    }
     version++;
     return fi;
   }
@@ -569,22 +548,6 @@ public final class FieldInfos implements
     }
     return false;
   }
-  
-  /**
-   * Builds the {@link SegmentCodecs} mapping for this {@link FieldInfos} instance.
-   * @param clearBuilder <code>true</code> iff the internal {@link SegmentCodecsBuilder} must be cleared otherwise <code>false</code>
-   */
-  public SegmentCodecs buildSegmentCodecs(boolean clearBuilder) {
-    if (globalFieldNumbers == null) {
-      throw new IllegalStateException("FieldInfos are read-only no SegmentCodecs available");
-    }
-    assert segmentCodecsBuilder != null;
-    final SegmentCodecs segmentCodecs = segmentCodecsBuilder.build();
-    if (clearBuilder) {
-      segmentCodecsBuilder.clear();
-    }
-    return segmentCodecs;
-  }
 
   public void write(Directory d, String name) throws IOException {
     IndexOutput output = d.createOutput(name, IOContext.READONCE);
@@ -628,7 +591,6 @@ public final class FieldInfos implements
         bits |= OMIT_POSITIONS;
       output.writeString(fi.name);
       output.writeInt(fi.number);
-      output.writeInt(fi.getCodecId());
       output.writeByte(bits);
 
       final byte b;
@@ -688,19 +650,17 @@ public final class FieldInfos implements
     format = input.readVInt();
 
     if (format > FORMAT_MINIMUM) {
-      throw new IndexFormatTooOldException(fileName, format, FORMAT_MINIMUM, FORMAT_CURRENT);
+      throw new IndexFormatTooOldException(input, format, FORMAT_MINIMUM, FORMAT_CURRENT);
     }
     if (format < FORMAT_CURRENT) {
-      throw new IndexFormatTooNewException(fileName, format, FORMAT_MINIMUM, FORMAT_CURRENT);
+      throw new IndexFormatTooNewException(input, format, FORMAT_MINIMUM, FORMAT_CURRENT);
     }
 
     final int size = input.readVInt(); //read in the size
 
     for (int i = 0; i < size; i++) {
       String name = input.readString();
-      // if this is a previous format codec 0 will be preflex!
       final int fieldNumber = format <= FORMAT_FLEX? input.readInt():i;
-      final int codecId = format <= FORMAT_FLEX? input.readInt():0;
       byte bits = input.readByte();
       boolean isIndexed = (bits & IS_INDEXED) != 0;
       boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0;
@@ -715,7 +675,7 @@ public final class FieldInfos implements
         if (format <= FORMAT_OMIT_POSITIONS) {
           indexOptions = IndexOptions.DOCS_AND_FREQS;
         } else {
-          throw new CorruptIndexException("Corrupt fieldinfos, OMIT_POSITIONS set but format=" + format);
+          throw new CorruptIndexException("Corrupt fieldinfos, OMIT_POSITIONS set but format=" + format + " (resource: " + input + ")");
         }
       } else {
         indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
@@ -781,12 +741,11 @@ public final class FieldInfos implements
           throw new IllegalStateException("unhandled indexValues type " + b);
         }
       }
-      final FieldInfo addInternal = addInternal(name, fieldNumber, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValuesType);
-      addInternal.setCodecId(codecId);
+      addInternal(name, fieldNumber, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValuesType);
     }
 
     if (input.getFilePointer() != input.length()) {
-      throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length());
+      throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")");
     }    
   }
   
@@ -804,7 +763,7 @@ public final class FieldInfos implements
     if (isReadOnly()) {
       return this;
     }
-    final FieldInfos roFis = new FieldInfos((FieldNumberBiMap)null, null);
+    final FieldInfos roFis = new FieldInfos((FieldNumberBiMap)null);
     for (FieldInfo fieldInfo : this) {
       FieldInfo clone = (FieldInfo) (fieldInfo).clone();
       roFis.putInternal(clone);
@@ -814,5 +773,14 @@ public final class FieldInfos implements
     }
     return roFis;
   }
-  
+
+  public boolean anyDocValuesFields() {
+    for (FieldInfo fi : this) {
+      if (fi.hasDocValues()) { 
+        return true;
+      }
+    }
+
+    return false;
+  }
 }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java Sat Nov  5 12:46:53 2011
@@ -57,7 +57,7 @@ final class FreqProxTermsWriter extends 
     // Sort by field name
     CollectionUtil.quickSort(allFields);
 
-    final FieldsConsumer consumer = state.segmentCodecs.codec().fieldsConsumer(state);
+    final FieldsConsumer consumer = state.codec.postingsFormat().fieldsConsumer(state);
 
     boolean success = false;
 

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java Sat Nov  5 12:46:53 2011
@@ -18,7 +18,6 @@ package org.apache.lucene.index;
  */
 
 import java.io.FileNotFoundException;
-import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.PrintStream;
 import java.util.ArrayList;
@@ -29,7 +28,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.NoSuchDirectoryException;
 import org.apache.lucene.util.CollectionUtil;
@@ -122,8 +120,6 @@ final class IndexFileDeleter {
     infoStream.println("IFD [" + new Date() + "; " + Thread.currentThread().getName() + "]: " + message);
   }
 
-  private final FilenameFilter indexFilenameFilter;
-
   // called only from assert
   private boolean locked() {
     return writer == null || Thread.holdsLock(writer);
@@ -138,7 +134,7 @@ final class IndexFileDeleter {
    * @throws IOException if there is a low-level IO error
    */
   public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos,
-                          PrintStream infoStream, CodecProvider codecs, IndexWriter writer) throws CorruptIndexException, IOException {
+                          PrintStream infoStream, IndexWriter writer) throws CorruptIndexException, IOException {
     this.infoStream = infoStream;
     this.writer = writer;
 
@@ -154,7 +150,6 @@ final class IndexFileDeleter {
     // First pass: walk the files and initialize our ref
     // counts:
     long currentGen = segmentInfos.getGeneration();
-    indexFilenameFilter = new IndexFileNameFilter(codecs);
 
     CommitPoint currentCommitPoint = null;
     String[] files = null;
@@ -167,7 +162,7 @@ final class IndexFileDeleter {
 
     for (String fileName : files) {
 
-      if ((indexFilenameFilter.accept(null, fileName)) && !fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
+      if ((IndexFileNameFilter.INSTANCE.accept(null, fileName)) && !fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
 
         // Add this file to refCounts with initial count 0:
         getRefCount(fileName);
@@ -180,9 +175,9 @@ final class IndexFileDeleter {
           if (infoStream != null) {
             message("init: load commit \"" + fileName + "\"");
           }
-          SegmentInfos sis = new SegmentInfos(codecs);
+          SegmentInfos sis = new SegmentInfos();
           try {
-            sis.read(directory, fileName, codecs);
+            sis.read(directory, fileName);
           } catch (FileNotFoundException e) {
             // LUCENE-948: on NFS (and maybe others), if
             // you have writers switching back and forth
@@ -253,9 +248,9 @@ final class IndexFileDeleter {
       // listing was stale (eg when index accessed via NFS
       // client with stale directory listing cache).  So we
       // try now to explicitly open this commit point:
-      SegmentInfos sis = new SegmentInfos(codecs);
+      SegmentInfos sis = new SegmentInfos();
       try {
-        sis.read(directory, currentSegmentsFile, codecs);
+        sis.read(directory, currentSegmentsFile);
       } catch (IOException e) {
         throw new CorruptIndexException("failed to locate current segments_N file");
       }
@@ -373,7 +368,7 @@ final class IndexFileDeleter {
     for(int i=0;i<files.length;i++) {
       String fileName = files[i];
       if ((segmentName == null || fileName.startsWith(segmentPrefix1) || fileName.startsWith(segmentPrefix2)) &&
-          indexFilenameFilter.accept(null, fileName) &&
+          IndexFileNameFilter.INSTANCE.accept(null, fileName) &&
           !refCounts.containsKey(fileName) &&
           !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
         // Unreferenced file, so remove it

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexFileNameFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexFileNameFilter.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexFileNameFilter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexFileNameFilter.java Sat Nov  5 12:46:53 2011
@@ -20,50 +20,43 @@ package org.apache.lucene.index;
 import java.io.File;
 import java.io.FilenameFilter;
 import java.util.HashSet;
-import org.apache.lucene.index.codecs.CodecProvider;
+import java.util.regex.Pattern;
 
 /**
- * Filename filter that accept filenames and extensions only
- * created by Lucene.
+ * Filename filter that attempts to accept only filenames
+ * created by Lucene.  Note that this is a "best effort"
+ * process.  If a file is used in a Lucene index, it will
+ * always match the file; but if a file is not used in a
+ * Lucene index but is named in a similar way to Lucene's
+ * files then this filter may accept the file.
+ *
+ * <p>This does not accept <code>*-write.lock</code> files.
  *
  * @lucene.internal
  */
 
 public class IndexFileNameFilter implements FilenameFilter {
 
-  private final HashSet<String> extensions;
-
-  public IndexFileNameFilter(CodecProvider codecs) {
-    extensions = new HashSet<String>();
-    for (String ext : IndexFileNames.INDEX_EXTENSIONS) {
-      extensions.add(ext);
-    }
-    if (codecs != null) {
-      for(String ext : codecs.getAllExtensions()) {
-        extensions.add(ext);
-      }
-    }
+  public static final FilenameFilter INSTANCE = new IndexFileNameFilter();
+  
+  private IndexFileNameFilter() {
   }
 
+  // Approximate match for files that seem to be Lucene
+  // index files.  This can easily over-match, ie if some
+  // app names a file _foo_bar.go:
+  private final Pattern luceneFilePattern = Pattern.compile("^_[a-z0-9]+(_[a-z0-9]+)?\\.[a-z0-9]+$");
+
   /* (non-Javadoc)
    * @see java.io.FilenameFilter#accept(java.io.File, java.lang.String)
    */
   public boolean accept(File dir, String name) {
-    int i = name.lastIndexOf('.');
-    if (i != -1) {
-      String extension = name.substring(1+i);
-      if (extensions.contains(extension)) {
-        return true;
-      } else if (extension.startsWith("f") &&
-                 extension.matches("f\\d+")) {
-        return true;
-      } else if (extension.startsWith("s") &&
-                 extension.matches("s\\d+")) {
-        return true;
-      }
+    if (name.lastIndexOf('.') != -1) {
+      // Has an extension
+      return luceneFilePattern.matcher(name).matches();
     } else {
-      if (name.startsWith(IndexFileNames.SEGMENTS)) return true;
+      // No extension -- only segments_N file;
+      return name.startsWith(IndexFileNames.SEGMENTS);
     }
-    return false;
   }
 }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexFileNames.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexFileNames.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexFileNames.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexFileNames.java Sat Nov  5 12:46:53 2011
@@ -19,7 +19,9 @@ package org.apache.lucene.index;
 
 import java.util.regex.Pattern;
 
-import org.apache.lucene.index.codecs.Codec;  // for javadocs
+import org.apache.lucene.index.codecs.PostingsFormat;  // for javadocs
+
+// TODO: put all files under codec and remove all the static extensions here
 
 /**
  * This class contains useful constants representing filenames and extensions
@@ -31,7 +33,7 @@ import org.apache.lucene.index.codecs.Co
  * {@link #segmentFileName(String, String, String) segmentFileName}).
  *
  * <p><b>NOTE</b>: extensions used by codecs are not
- * listed here.  You must interact with the {@link Codec}
+ * listed here.  You must interact with the {@link PostingsFormat}
  * directly.
  *
  * @lucene.internal
@@ -188,20 +190,20 @@ public final class IndexFileNames {
    * <b>NOTE:</b> .&lt;ext&gt; is added to the result file name only if
    * <code>ext</code> is not empty.
    * <p>
-   * <b>NOTE:</b> _&lt;name&gt; is added to the result file name only if
-   * <code>name</code> is not empty.
+   * <b>NOTE:</b> _&lt;segmentSuffix&gt; is added to the result file name only if
+   * it's not the empty string
    * <p>
    * <b>NOTE:</b> all custom files should be named using this method, or
    * otherwise some structures may fail to handle them properly (such as if they
    * are added to compound files).
    */
-  public static String segmentFileName(String segmentName, String name, String ext) {
-    if (ext.length() > 0 || name.length() > 0) {
+  public static String segmentFileName(String segmentName, String segmentSuffix, String ext) {
+    if (ext.length() > 0 || segmentSuffix.length() > 0) {
       assert !ext.startsWith(".");
-      StringBuilder sb = new StringBuilder(segmentName.length() + 2 + name.length() + ext.length());
+      StringBuilder sb = new StringBuilder(segmentName.length() + 2 + segmentSuffix.length() + ext.length());
       sb.append(segmentName);
-      if (name.length() > 0) {
-        sb.append('_').append(name);
+      if (segmentSuffix.length() > 0) {
+        sb.append('_').append(segmentSuffix);
       }
       if (ext.length() > 0) {
         sb.append('.').append(ext);
@@ -212,11 +214,6 @@ public final class IndexFileNames {
     }
   }
 
-  /** Sugar for passing "" + name instead */
-  public static String segmentFileName(String segmentName, int name, String ext) {
-    return segmentFileName(segmentName, ""+name, ext);
-  }
-
   /**
    * Returns true if the given filename ends with the given extension. One
    * should provide a <i>pure</i> extension, without '.'.

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexFormatTooNewException.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexFormatTooNewException.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexFormatTooNewException.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexFormatTooNewException.java Sat Nov  5 12:46:53 2011
@@ -17,15 +17,24 @@
 
 package org.apache.lucene.index;
 
+import org.apache.lucene.store.DataInput;
+
 /**
  * This exception is thrown when Lucene detects
  * an index that is newer than this Lucene version.
  */
 public class IndexFormatTooNewException extends CorruptIndexException {
 
-  public IndexFormatTooNewException(String filename, int version, int minVersion, int maxVersion) {
-    super("Format version is not supported" + (filename!=null ? (" in file '" + filename + "'") : "") +
-      ": " + version + " (needs to be between " + minVersion + " and " + maxVersion + ")");
+  /** @lucene.internal */
+  public IndexFormatTooNewException(String resourceDesc, int version, int minVersion, int maxVersion) {
+    super("Format version is not supported (resource: " + resourceDesc + "): "
+      + version + " (needs to be between " + minVersion + " and " + maxVersion + ")");
+    assert resourceDesc != null;
+  }
+
+  /** @lucene.internal */
+  public IndexFormatTooNewException(DataInput in, int version, int minVersion, int maxVersion) {
+    this(in.toString(), version, minVersion, maxVersion);
   }
 
 }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexFormatTooOldException.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexFormatTooOldException.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexFormatTooOldException.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexFormatTooOldException.java Sat Nov  5 12:46:53 2011
@@ -17,21 +17,36 @@
 
 package org.apache.lucene.index;
 
+import org.apache.lucene.store.DataInput;
+
 /**
  * This exception is thrown when Lucene detects
  * an index that is too old for this Lucene version
  */
 public class IndexFormatTooOldException extends CorruptIndexException {
 
-  public IndexFormatTooOldException(String filename, String version) {
-    super("Format version is not supported" + (filename!=null ? (" in file '" + filename + "'") : "") +
-        ": " + version + ". This version of Lucene only supports indexes created with release 3.0 and later.");
+  /** @lucene.internal */
+  public IndexFormatTooOldException(String resourceDesc, String version) {
+    super("Format version is not supported (resource: " + resourceDesc + "): " +
+        version + ". This version of Lucene only supports indexes created with release 3.0 and later.");
+    assert resourceDesc != null;
+  }
+
+  /** @lucene.internal */
+  public IndexFormatTooOldException(DataInput in, String version) {
+    this(in.toString(), version);
   }
   
-  public IndexFormatTooOldException(String filename, int version, int minVersion, int maxVersion) {
-    super("Format version is not supported" + (filename!=null ? (" in file '" + filename + "'") : "") +
-        ": " + version + " (needs to be between " + minVersion + " and " + maxVersion +
+  /** @lucene.internal */
+  public IndexFormatTooOldException(String resourceDesc, int version, int minVersion, int maxVersion) {
+    super("Format version is not supported (resource: " + resourceDesc + "): " +
+        version + " (needs to be between " + minVersion + " and " + maxVersion +
     "). This version of Lucene only supports indexes created with release 3.0 and later.");
+    assert resourceDesc != null;
   }
 
+  /** @lucene.internal */
+  public IndexFormatTooOldException(DataInput in, int version, int minVersion, int maxVersion) {
+    this(in.toString(), version, minVersion, maxVersion);
+  }
 }

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java Sat Nov  5 12:46:53 2011
@@ -28,8 +28,7 @@ import java.util.concurrent.atomic.Atomi
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.DocumentStoredFieldVisitor;
-import org.apache.lucene.index.codecs.Codec;
-import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.index.codecs.PostingsFormat;
 import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.values.IndexDocValues;
 import org.apache.lucene.search.FieldCache; // javadocs
@@ -312,7 +311,7 @@ public abstract class IndexReader implem
    * @throws IOException if there is a low-level IO error
    */
   public static IndexReader open(final Directory directory) throws CorruptIndexException, IOException {
-    return open(directory, null, null, true, DEFAULT_TERMS_INDEX_DIVISOR, null);
+    return open(directory, null, null, true, DEFAULT_TERMS_INDEX_DIVISOR);
   }
 
   /** Returns an IndexReader reading the index in the given
@@ -326,9 +325,9 @@ public abstract class IndexReader implem
    * @throws IOException if there is a low-level IO error
    */
   public static IndexReader open(final Directory directory, boolean readOnly) throws CorruptIndexException, IOException {
-    return open(directory, null, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null);
+    return open(directory, null, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR);
   }
-
+  
   /**
    * Open a near real time IndexReader from the {@link org.apache.lucene.index.IndexWriter}.
    *
@@ -363,7 +362,7 @@ public abstract class IndexReader implem
    * @throws IOException if there is a low-level IO error
    */
   public static IndexReader open(final IndexCommit commit, boolean readOnly) throws CorruptIndexException, IOException {
-    return open(commit.getDirectory(), null, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null);
+    return open(commit.getDirectory(), null, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR);
   }
 
   /** Expert: returns an IndexReader reading the index in
@@ -381,7 +380,7 @@ public abstract class IndexReader implem
    * @throws IOException if there is a low-level IO error
    */
   public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly) throws CorruptIndexException, IOException {
-    return open(directory, deletionPolicy, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null);
+    return open(directory, deletionPolicy, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR);
   }
 
   /** Expert: returns an IndexReader reading the index in
@@ -409,7 +408,7 @@ public abstract class IndexReader implem
    * @throws IOException if there is a low-level IO error
    */
   public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException {
-    return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor, null);
+    return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor);
   }
 
   /** Expert: returns an IndexReader reading the index in
@@ -429,7 +428,7 @@ public abstract class IndexReader implem
    * @throws IOException if there is a low-level IO error
    */
   public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly) throws CorruptIndexException, IOException {
-    return open(commit.getDirectory(), deletionPolicy, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null);
+    return open(commit.getDirectory(), deletionPolicy, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR);
   }
 
   /** Expert: returns an IndexReader reading the index in
@@ -462,78 +461,11 @@ public abstract class IndexReader implem
    * @throws IOException if there is a low-level IO error
    */
   public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException {
-    return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor, null);
-  }
-
-  /** Expert: returns an IndexReader reading the index in
-   *  the given Directory, with a custom {@link
-   *  IndexDeletionPolicy}, and specified {@link CodecProvider}.
-   *  You should pass readOnly=true, since it gives much
-   *  better concurrent performance, unless you intend to do
-   *  write operations (delete documents or change norms)
-   *  with the reader.
-   * @param directory the index directory
-   * @param deletionPolicy a custom deletion policy (only used
-   *  if you use this reader to perform deletes or to set
-   *  norms); see {@link IndexWriter} for details.
-   * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader
-   * @param termInfosIndexDivisor Subsamples which indexed
-   *  terms are loaded into RAM. This has the same effect as {@link
-   *  IndexWriterConfig#setTermIndexInterval} except that setting
-   *  must be done at indexing time while this setting can be
-   *  set per reader.  When set to N, then one in every
-   *  N*termIndexInterval terms in the index is loaded into
-   *  memory.  By setting this to a value > 1 you can reduce
-   *  memory usage, at the expense of higher latency when
-   *  loading a TermInfo.  The default value is 1.  Set this
-   *  to -1 to skip loading the terms index entirely.
-   * @param codecs CodecProvider to use when opening index
-   * @throws CorruptIndexException if the index is corrupt
-   * @throws IOException if there is a low-level IO error
-   */
-  public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, CodecProvider codecs) throws CorruptIndexException, IOException {
-    return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor, codecs);
+    return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor);
   }
 
-  /** Expert: returns an IndexReader reading the index in
-   *  the given Directory, using a specific commit and with
-   *  a custom {@link IndexDeletionPolicy} and specified
-   *  {@link CodecProvider}.  You should pass readOnly=true, since
-   *  it gives much better concurrent performance, unless
-   *  you intend to do write operations (delete documents or
-   *  change norms) with the reader.
-
-   * @param commit the specific {@link IndexCommit} to open;
-   * see {@link IndexReader#listCommits} to list all commits
-   * in a directory
-   * @param deletionPolicy a custom deletion policy (only used
-   *  if you use this reader to perform deletes or to set
-   *  norms); see {@link IndexWriter} for details.
-   * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader
-   * @param termInfosIndexDivisor Subsamples which indexed
-   *  terms are loaded into RAM. This has the same effect as {@link
-   *  IndexWriterConfig#setTermIndexInterval} except that setting
-   *  must be done at indexing time while this setting can be
-   *  set per reader.  When set to N, then one in every
-   *  N*termIndexInterval terms in the index is loaded into
-   *  memory.  By setting this to a value > 1 you can reduce
-   *  memory usage, at the expense of higher latency when
-   *  loading a TermInfo.  The default value is 1.  Set this
-   *  to -1 to skip loading the terms index entirely.
-   * @param codecs CodecProvider to use when opening index
-   * @throws CorruptIndexException if the index is corrupt
-   * @throws IOException if there is a low-level IO error
-   */
-  public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, CodecProvider codecs) throws CorruptIndexException, IOException {
-    return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor, codecs);
-  }
-
-  private static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, int termInfosIndexDivisor,
-      CodecProvider codecs) throws CorruptIndexException, IOException {
-    if (codecs == null) {
-      codecs = CodecProvider.getDefault();
-    }
-    return DirectoryReader.open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor, codecs);
+  private static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException {
+    return DirectoryReader.open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor);
   }
 
   /**
@@ -767,20 +699,6 @@ public abstract class IndexReader implem
         }
       }.run()).longValue();
   }
-
-  /**
-   * Reads version number from segments files. The version number is
-   * initialized with a timestamp and then increased by one for each change of
-   * the index.
-   * 
-   * @param directory where the index resides.
-   * @return version number.
-   * @throws CorruptIndexException if the index is corrupt
-   * @throws IOException if there is a low-level IO error
-   */
-  public static long getCurrentVersion(Directory directory) throws CorruptIndexException, IOException {
-    return getCurrentVersion(directory, CodecProvider.getDefault());
-  }
   
   /**
    * Reads version number from segments files. The version number is
@@ -788,34 +706,14 @@ public abstract class IndexReader implem
    * the index.
    * 
    * @param directory where the index resides.
-   * @param codecs the {@link CodecProvider} holding all {@link Codec}s required to open the index
    * @return version number.
    * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
    */
-  public static long getCurrentVersion(Directory directory, CodecProvider codecs) throws CorruptIndexException, IOException {
-    return SegmentInfos.readCurrentVersion(directory, codecs);
-  }
-
-  /**
-   * Reads commitUserData, previously passed to {@link
-   * IndexWriter#commit(Map)}, from current index
-   * segments file.  This will return null if {@link
-   * IndexWriter#commit(Map)} has never been called for
-   * this index.
-   * 
-   * @param directory where the index resides.
-   * @return commit userData.
-   * @throws CorruptIndexException if the index is corrupt
-   * @throws IOException if there is a low-level IO error
-   *
-   * @see #getCommitUserData()
-   */
-  public static Map<String,String> getCommitUserData(Directory directory) throws CorruptIndexException, IOException {
-    return getCommitUserData(directory,  CodecProvider.getDefault());
+  public static long getCurrentVersion(Directory directory) throws CorruptIndexException, IOException {
+    return SegmentInfos.readCurrentVersion(directory);
   }
   
-  
   /**
    * Reads commitUserData, previously passed to {@link
    * IndexWriter#commit(Map)}, from current index
@@ -824,15 +722,14 @@ public abstract class IndexReader implem
    * this index.
    * 
    * @param directory where the index resides.
-   * @param codecs the {@link CodecProvider} provider holding all {@link Codec}s required to open the index
    * @return commit userData.
    * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
    *
    * @see #getCommitUserData()
    */
-  public static Map<String, String> getCommitUserData(Directory directory, CodecProvider codecs) throws CorruptIndexException, IOException {
-    return SegmentInfos.readCurrentUserData(directory, codecs);
+  public static Map<String, String> getCommitUserData(Directory directory) throws CorruptIndexException, IOException {
+    return SegmentInfos.readCurrentUserData(directory);
   }
 
   /**
@@ -985,22 +882,6 @@ public abstract class IndexReader implem
     }
   }
 
-  /**
-   * Returns <code>true</code> if an index exists at the specified directory.
-   * @param  directory the directory to check for an index
-   * @param  codecProvider provides a CodecProvider in case the index uses non-core codecs
-   * @return <code>true</code> if an index exists; <code>false</code> otherwise
-   * @throws IOException if there is a problem with accessing the index
-   */
-  public static boolean indexExists(Directory directory, CodecProvider codecProvider) throws IOException {
-    try {
-      new SegmentInfos().read(directory, codecProvider);
-      return true;
-    } catch (IOException ioe) {
-      return false;
-    }
-  }
-
   /** Returns the number of documents in this index. */
   public abstract int numDocs();
 

Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java?rev=1197949&r1=1197948&r2=1197949&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java Sat Nov  5 12:46:53 2011
@@ -114,12 +114,12 @@ public final class IndexUpgrader {
   }
   
   public void upgrade() throws IOException {
-    if (!IndexReader.indexExists(dir, iwc.getCodecProvider())) {
+    if (!IndexReader.indexExists(dir)) {
       throw new IndexNotFoundException(dir.toString());
     }
   
     if (!deletePriorCommits) {
-      final Collection<IndexCommit> commits = DirectoryReader.listCommits(dir, iwc.getCodecProvider());
+      final Collection<IndexCommit> commits = DirectoryReader.listCommits(dir);
       if (commits.size() > 1) {
         throw new IllegalArgumentException("This tool was invoked to not delete prior commit points, but the following commits were found: " + commits);
       }



Mime
View raw message