lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sim...@apache.org
Subject svn commit: r1144196 [1/5] - in /lucene/dev/trunk: ./ dev-tools/idea/lucene/contrib/ lucene/ lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/ lucene/contrib/misc/src/java/org/apache/lucene/store/ lucene/contrib/misc/src/test/org/a...
Date Fri, 08 Jul 2011 08:52:45 GMT
Author: simonw
Date: Fri Jul  8 08:52:40 2011
New Revision: 1144196

URL: http://svn.apache.org/viewvc?rev=1144196&view=rev
Log:
LUCENE-2793: Directory#createOutput & Directory#openInput now accept an IOContext instead of a buffer size to allow low level optimizations for different usecases like merging, flushing and reading.

Added:
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/FlushInfo.java
      - copied unchanged from r1144189, lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/store/FlushInfo.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/IOContext.java
      - copied unchanged from r1144189, lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/store/IOContext.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/MergeInfo.java
      - copied unchanged from r1144189, lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/store/MergeInfo.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/RateLimiter.java
      - copied unchanged from r1144189, lucene/dev/branches/LUCENE2793/lucene/src/java/org/apache/lucene/store/RateLimiter.java
Modified:
    lucene/dev/trunk/   (props changed)
    lucene/dev/trunk/dev-tools/idea/lucene/contrib/   (props changed)
    lucene/dev/trunk/lucene/   (props changed)
    lucene/dev/trunk/lucene/CHANGES.txt
    lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java
    lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosReader.java
    lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java
    lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java
    lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java
    lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java
    lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java
    lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java
    lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java
    lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java
    lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java
    lucene/dev/trunk/lucene/contrib/queries/src/test/org/apache/lucene/search/regex/TestSpanRegexQuery.java   (props changed)
    lucene/dev/trunk/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/builders/TestNumericRangeFilterBuilder.java   (props changed)
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CheckIndex.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldsReader.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldsWriter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexReader.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MergePolicy.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/NormsWriter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentNorms.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentReadState.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentReader.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsWriter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/IntStreamFactory.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriterImpl.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Bytes.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Floats.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Ints.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/IntsImpl.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Writer.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/DefaultCompoundFileDirectory.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/Directory.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/FSDirectory.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/MMapDirectory.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/RAMDirectory.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/BitVector.java
    lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java
    lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java
    lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java
    lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java
    lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java
    lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java
    lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java
    lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java
    lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java
    lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java
    lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/store/MockCompoundFileDirectoryWrapper.java
    lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java
    lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestCodecs.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestCompoundFile.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDoc.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexInput.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMultiReader.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/values/TestDocValues.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestBoolean2.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/store/TestCopyBytes.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/store/TestDirectory.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/store/TestMultiMMap.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/store/TestRAMDirectory.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestBitVector.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestByteBlockPool.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java
    lucene/dev/trunk/modules/facet/src/test/org/apache/lucene/util/SlowRAMDirectory.java
    lucene/dev/trunk/solr/   (props changed)
    lucene/dev/trunk/solr/CHANGES.txt   (props changed)
    lucene/dev/trunk/solr/LICENSE.txt   (props changed)
    lucene/dev/trunk/solr/NOTICE.txt   (props changed)
    lucene/dev/trunk/solr/README.txt   (props changed)
    lucene/dev/trunk/solr/build.xml   (props changed)
    lucene/dev/trunk/solr/client/   (props changed)
    lucene/dev/trunk/solr/common-build.xml   (props changed)
    lucene/dev/trunk/solr/contrib/   (props changed)
    lucene/dev/trunk/solr/example/   (props changed)
    lucene/dev/trunk/solr/lib/   (props changed)
    lucene/dev/trunk/solr/site/   (props changed)
    lucene/dev/trunk/solr/src/   (props changed)
    lucene/dev/trunk/solr/src/java/org/apache/solr/core/RefCntRamDirectory.java
    lucene/dev/trunk/solr/testlogging.properties   (props changed)

Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Fri Jul  8 08:52:40 2011
@@ -462,6 +462,15 @@ New features
   IndexSearcher.  SortFields can have SortField.REWRITEABLE type which
   requires they are rewritten before they are used. (Chris Male)
 
+* LUCENE-3203: FSDirectory can now limit the max allowed write rate
+  (MB/sec) of all running merges, to reduce impact ongoing merging has
+  on searching, NRT reopen time, etc.  (Mike McCandless)
+
+* LUCENE-2793: Directory#createOutput & Directory#openInput now accept an
+  IOContext instead of a buffer size to allow low level optimizations for 
+  different usecases like merging, flushing and reading.
+  (Simon Willnauer, Mike McCandless, Varun Thacker)
+
 Optimizations
 
 * LUCENE-2588: Don't store unnecessary suffixes when writing the terms

Modified: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java Fri Jul  8 08:52:40 2011
@@ -93,7 +93,7 @@ public class AppendingCodec extends Code
   @Override
   public FieldsProducer fieldsProducer(SegmentReadState state)
           throws IOException {
-    PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize, state.codecId);
+    PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId);
     TermsIndexReaderBase indexReader;
 
     boolean success = false;
@@ -103,7 +103,7 @@ public class AppendingCodec extends Code
               state.segmentInfo.name,
               state.termsIndexDivisor,
               BytesRef.getUTF8SortedAsUnicodeComparator(),
-              state.codecId);
+              state.codecId, state.context);
       success = true;
     } finally {
       if (!success) {
@@ -115,7 +115,7 @@ public class AppendingCodec extends Code
       FieldsProducer ret = new AppendingTermsDictReader(indexReader,
               state.dir, state.fieldInfos, state.segmentInfo.name,
               docsReader,
-              state.readBufferSize,
+              state.context,
               StandardCodec.TERMS_CACHE_SIZE,
               state.codecId);
       success = true;
@@ -153,6 +153,6 @@ public class AppendingCodec extends Code
 
   @Override
   public PerDocValues docsProducer(SegmentReadState state) throws IOException {
-    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator());
+    return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
   }
 }

Modified: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosReader.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosReader.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosReader.java Fri Jul  8 08:52:40 2011
@@ -22,6 +22,7 @@ import java.io.IOException;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.codecs.DefaultSegmentInfosReader;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 
 public class AppendingSegmentInfosReader extends DefaultSegmentInfosReader {
@@ -33,9 +34,9 @@ public class AppendingSegmentInfosReader
   }
 
   @Override
-  public IndexInput openInput(Directory dir, String segmentsFileName)
+  public IndexInput openInput(Directory dir, String segmentsFileName, IOContext context)
           throws IOException {
-    return dir.openInput(segmentsFileName);
+    return dir.openInput(segmentsFileName, context);
   }
 
 }

Modified: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java Fri Jul  8 08:52:40 2011
@@ -21,14 +21,15 @@ import java.io.IOException;
 
 import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexOutput;
 
 public class AppendingSegmentInfosWriter extends DefaultSegmentInfosWriter {
 
   @Override
-  protected IndexOutput createOutput(Directory dir, String segmentsFileName)
+  protected IndexOutput createOutput(Directory dir, String segmentsFileName, IOContext context)
           throws IOException {
-    return dir.createOutput(segmentsFileName);
+    return dir.createOutput(segmentsFileName, context);
   }
 
   @Override

Modified: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java Fri Jul  8 08:52:40 2011
@@ -25,6 +25,7 @@ import org.apache.lucene.index.codecs.Bl
 import org.apache.lucene.index.codecs.BlockTermsWriter;
 import org.apache.lucene.index.codecs.TermsIndexReaderBase;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.CodecUtil;
 
@@ -32,9 +33,9 @@ public class AppendingTermsDictReader ex
 
   public AppendingTermsDictReader(TermsIndexReaderBase indexReader,
           Directory dir, FieldInfos fieldInfos, String segment,
-          PostingsReaderBase postingsReader, int readBufferSize,
+          PostingsReaderBase postingsReader, IOContext context,
           int termsCacheSize, int codecId) throws IOException {
-    super(indexReader, dir, fieldInfos, segment, postingsReader, readBufferSize,
+    super(indexReader, dir, fieldInfos, segment, postingsReader, context,
           termsCacheSize, codecId);
   }
   

Modified: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java Fri Jul  8 08:52:40 2011
@@ -23,6 +23,7 @@ import java.util.Comparator;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CodecUtil;
@@ -30,9 +31,9 @@ import org.apache.lucene.util.CodecUtil;
 public class AppendingTermsIndexReader extends FixedGapTermsIndexReader {
 
   public AppendingTermsIndexReader(Directory dir, FieldInfos fieldInfos,
-          String segment, int indexDivisor, Comparator<BytesRef> termComp, int codecId)
+          String segment, int indexDivisor, Comparator<BytesRef> termComp, int codecId, IOContext context)
           throws IOException {
-    super(dir, fieldInfos, segment, indexDivisor, termComp, codecId);
+    super(dir, fieldInfos, segment, indexDivisor, termComp, codecId, context);
   }
   
   @Override

Modified: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java Fri Jul  8 08:52:40 2011
@@ -69,16 +69,22 @@ public class DirectIOLinuxDirectory exte
   }
 
   @Override
-  public IndexInput openInput(String name, int bufferSize) throws IOException {
+  public IndexInput openInput(String name, IOContext context) throws IOException {
     ensureOpen();
-    return new DirectIOLinuxIndexInput(new File(getDirectory(), name), forcedBufferSize == 0 ? bufferSize : forcedBufferSize);
+    return new DirectIOLinuxIndexInput(new File(getDirectory(), name),
+        bufferSize(context));
   }
 
   @Override
-  public IndexOutput createOutput(String name) throws IOException {
+  public IndexOutput createOutput(String name, IOContext context) throws IOException {
     ensureOpen();
     ensureCanWrite(name);
-    return new DirectIOLinuxIndexOutput(new File(getDirectory(), name), forcedBufferSize == 0 ? BufferedIndexOutput.BUFFER_SIZE : forcedBufferSize);
+    return new DirectIOLinuxIndexOutput(new File(getDirectory(), name), bufferSize(context));
+  }
+  
+  private int bufferSize(IOContext context) {
+    return forcedBufferSize != 0 ? forcedBufferSize : BufferedIndexInput
+        .bufferSize(context);
   }
 
   private final static class DirectIOLinuxIndexOutput extends IndexOutput {
@@ -238,6 +244,7 @@ public class DirectIOLinuxDirectory exte
     private int bufferPos;
 
     public DirectIOLinuxIndexInput(File path, int bufferSize) throws IOException {
+      // TODO make use of IOContext
       FileDescriptor fd = NativePosixUtil.open_direct(path.toString(), true);
       fis = new FileInputStream(fd);
       channel = fis.getChannel();

Modified: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java Fri Jul  8 08:52:40 2011
@@ -21,13 +21,8 @@ import java.io.IOException;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
 
-import org.apache.lucene.index.ConcurrentMergeScheduler;
 import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.IndexWriter;       // javadocs
-import org.apache.lucene.index.MergePolicy;
-import org.apache.lucene.index.MergeScheduler;
 import org.apache.lucene.store.RAMDirectory;      // javadocs
 import org.apache.lucene.util.IOUtils;
 
@@ -38,11 +33,7 @@ import org.apache.lucene.util.IOUtils;
 /**
  * Wraps a {@link RAMDirectory}
  * around any provided delegate directory, to
- * be used during NRT search.  Make sure you pull the merge
- * scheduler using {@link #getMergeScheduler} and pass that to your
- * {@link IndexWriter}; this class uses that to keep track of which
- * merges are being done by which threads, to decide when to
- * cache each written file.
+ * be used during NRT search.
  *
  * <p>This class is likely only useful in a near-real-time
  * context, where indexing rate is lowish but reopen
@@ -54,20 +45,12 @@ import org.apache.lucene.util.IOUtils;
  * <p>This is safe to use: when your app calls {IndexWriter#commit},
  * all cached files will be flushed from the cached and sync'd.</p>
  *
- * <p><b>NOTE</b>: this class is somewhat sneaky in its
- * approach for spying on merges to determine the size of a
- * merge: it records which threads are running which merges
- * by watching ConcurrentMergeScheduler's doMerge method.
- * While this works correctly, likely future versions of
- * this class will take a more general approach.
- *
  * <p>Here's a simple example usage:
  *
  * <pre>
  *   Directory fsDir = FSDirectory.open(new File("/path/to/index"));
  *   NRTCachingDirectory cachedFSDir = new NRTCachingDirectory(fsDir, 5.0, 60.0);
  *   IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_32, analyzer);
- *   conf.setMergeScheduler(cachedFSDir.getMergeScheduler());
  *   IndexWriter writer = new IndexWriter(cachedFSDir, conf);
  * </pre>
  *
@@ -193,17 +176,17 @@ public class NRTCachingDirectory extends
   }
 
   @Override
-  public IndexOutput createOutput(String name) throws IOException {
+  public IndexOutput createOutput(String name, IOContext context) throws IOException {
     if (VERBOSE) {
       System.out.println("nrtdir.createOutput name=" + name);
     }
-    if (doCacheWrite(name)) {
+    if (doCacheWrite(name, context)) {
       if (VERBOSE) {
         System.out.println("  to cache");
       }
-      return cache.createOutput(name);
+      return cache.createOutput(name, context);
     } else {
-      return delegate.createOutput(name);
+      return delegate.createOutput(name, context);
     }
   }
 
@@ -219,7 +202,7 @@ public class NRTCachingDirectory extends
   }
 
   @Override
-  public synchronized IndexInput openInput(String name) throws IOException {
+  public synchronized IndexInput openInput(String name, IOContext context) throws IOException {
     if (VERBOSE) {
       System.out.println("nrtdir.openInput name=" + name);
     }
@@ -227,39 +210,31 @@ public class NRTCachingDirectory extends
       if (VERBOSE) {
         System.out.println("  from cache");
       }
-      return cache.openInput(name);
+      return cache.openInput(name, context);
     } else {
-      return delegate.openInput(name);
+      return delegate.openInput(name, context);
     }
   }
 
   @Override
-  public synchronized CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
+  public synchronized CompoundFileDirectory openCompoundInput(String name, IOContext context) throws IOException {
     if (cache.fileExists(name)) {
-      return cache.openCompoundInput(name, bufferSize);
+      return cache.openCompoundInput(name, context);
     } else {
-      return delegate.openCompoundInput(name, bufferSize);
+      return delegate.openCompoundInput(name, context);
     }
   }
   
   @Override
-  public synchronized CompoundFileDirectory createCompoundOutput(String name)
+  public synchronized CompoundFileDirectory createCompoundOutput(String name, IOContext context)
       throws IOException {
     if (cache.fileExists(name)) {
       throw new IOException("File " + name + "already exists");
     } else {
-      return delegate.createCompoundOutput(name);
+      return delegate.createCompoundOutput(name, context);
     }
   }
 
-  @Override
-  public synchronized IndexInput openInput(String name, int bufferSize) throws IOException {
-    if (cache.fileExists(name)) {
-      return cache.openInput(name, bufferSize);
-    } else {
-      return delegate.openInput(name, bufferSize);
-    }
-  }
 
   /** Close this directory, which flushes any cached files
    *  to the delegate and then closes the delegate. */
@@ -272,36 +247,21 @@ public class NRTCachingDirectory extends
     delegate.close();
   }
 
-  private final ConcurrentHashMap<Thread,MergePolicy.OneMerge> merges = new ConcurrentHashMap<Thread,MergePolicy.OneMerge>();
-
-  public MergeScheduler getMergeScheduler() {
-    return new ConcurrentMergeScheduler() {
-      @Override
-      protected void doMerge(MergePolicy.OneMerge merge) throws IOException {
-        try {
-          merges.put(Thread.currentThread(), merge);
-          super.doMerge(merge);
-        } finally {
-          merges.remove(Thread.currentThread());
-        }
-      }
-    };
-  }
-
   /** Subclass can override this to customize logic; return
    *  true if this file should be written to the RAMDirectory. */
-  protected boolean doCacheWrite(String name) {
-    final MergePolicy.OneMerge merge = merges.get(Thread.currentThread());
+  protected boolean doCacheWrite(String name, IOContext context) {
+    final MergeInfo merge = context.mergeInfo;
     //System.out.println(Thread.currentThread().getName() + ": CACHE check merge=" + merge + " size=" + (merge==null ? 0 : merge.estimatedMergeBytes));
     return !name.equals(IndexFileNames.SEGMENTS_GEN) && (merge == null || merge.estimatedMergeBytes <= maxMergeSizeBytes) && cache.sizeInBytes() <= maxCachedBytes;
   }
 
   private void unCache(String fileName) throws IOException {
     final IndexOutput out;
+    IOContext context = IOContext.DEFAULT;
     synchronized(this) {
       if (!delegate.fileExists(fileName)) {
         assert cache.fileExists(fileName);
-        out = delegate.createOutput(fileName);
+        out = delegate.createOutput(fileName, context);
       } else {
         out = null;
       }
@@ -310,7 +270,7 @@ public class NRTCachingDirectory extends
     if (out != null) {
       IndexInput in = null;
       try {
-        in = cache.openInput(fileName);
+        in = cache.openInput(fileName, context);
         in.copyBytes(out, in.length());
       } finally {
         IOUtils.closeSafely(false, in, out);

Modified: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java Fri Jul  8 08:52:40 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.store;
 
 import java.io.File;
 import java.io.IOException;
+
 import org.apache.lucene.store.Directory; // javadoc
 import org.apache.lucene.store.NativeFSLockFactory; // javadoc
 
@@ -67,9 +68,9 @@ public class WindowsDirectory extends FS
   }
 
   @Override
-  public IndexInput openInput(String name, int bufferSize) throws IOException {
+  public IndexInput openInput(String name, IOContext context) throws IOException {
     ensureOpen();
-    return new WindowsIndexInput(new File(getDirectory(), name), Math.max(bufferSize, DEFAULT_BUFFERSIZE));
+    return new WindowsIndexInput(new File(getDirectory(), name), Math.max(BufferedIndexInput.bufferSize(context), DEFAULT_BUFFERSIZE));
   }
   
   protected static class WindowsIndexInput extends BufferedIndexInput {

Modified: lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java Fri Jul  8 08:52:40 2011
@@ -159,9 +159,7 @@ public class TestNRTManager extends Luce
         System.out.println("TEST: wrap NRTCachingDir");
       }
 
-      NRTCachingDirectory nrtDir = new NRTCachingDirectory(dir, 5.0, 60.0);
-      conf.setMergeScheduler(nrtDir.getMergeScheduler());
-      dir = nrtDir;
+      dir = new NRTCachingDirectory(dir, 5.0, 60.0);
     }
     
     final IndexWriter writer = new IndexWriter(dir, conf);

Modified: lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java Fri Jul  8 08:52:40 2011
@@ -40,6 +40,7 @@ import org.apache.lucene.index.codecs.Co
 import org.apache.lucene.index.codecs.SegmentInfosReader;
 import org.apache.lucene.index.codecs.SegmentInfosWriter;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.store.RAMDirectory;
@@ -124,8 +125,8 @@ public class TestAppendingCodec extends 
     }
 
     @Override
-    public IndexOutput createOutput(String name) throws IOException {
-      return new AppendingIndexOutputWrapper(super.createOutput(name));
+    public IndexOutput createOutput(String name, IOContext context) throws IOException {
+      return new AppendingIndexOutputWrapper(super.createOutput(name, context));
     }
     
   }

Modified: lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java Fri Jul  8 08:52:40 2011
@@ -44,7 +44,6 @@ public class TestNRTCachingDirectory ext
     Directory dir = newDirectory();
     NRTCachingDirectory cachedDir = new NRTCachingDirectory(dir, 2.0, 25.0);
     IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
-    conf.setMergeScheduler(cachedDir.getMergeScheduler());
     RandomIndexWriter w = new RandomIndexWriter(random, cachedDir, conf);
     w.w.setInfoStream(VERBOSE ? System.out : null);
     final LineFileDocs docs = new LineFileDocs(random);    
@@ -108,13 +107,12 @@ public class TestNRTCachingDirectory ext
     Directory fsDir = FSDirectory.open(new File("/path/to/index"));
     NRTCachingDirectory cachedFSDir = new NRTCachingDirectory(fsDir, 2.0, 25.0);
     IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_32, analyzer);
-    conf.setMergeScheduler(cachedFSDir.getMergeScheduler());
     IndexWriter writer = new IndexWriter(cachedFSDir, conf);
   }
 
   public void testDeleteFile() throws Exception {
     Directory dir = new NRTCachingDirectory(newDirectory(), 2.0, 25.0);
-    dir.createOutput("foo.txt").close();
+    dir.createOutput("foo.txt", IOContext.DEFAULT).close();
     dir.deleteFile("foo.txt");
     assertEquals(0, dir.listAll().length);
     dir.close();

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java Fri Jul  8 08:52:40 2011
@@ -32,6 +32,7 @@ import org.apache.lucene.search.DocIdSet
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryWrapperFilter;
+import org.apache.lucene.store.IOContext;
 
 /* Tracks the stream of {@link BufferedDeletes}.
  * When DocumentsWriterPerThread flushes, its buffered
@@ -224,7 +225,7 @@ class BufferedDeletesStream {
 
         // Lock order: IW -> BD -> RP
         assert readerPool.infoIsLive(info);
-        final SegmentReader reader = readerPool.get(info, false);
+        final SegmentReader reader = readerPool.get(info, false, IOContext.READ);
         int delCount = 0;
         final boolean segAllDeletes;
         try {
@@ -273,7 +274,7 @@ class BufferedDeletesStream {
         if (coalescedDeletes != null) {
           // Lock order: IW -> BD -> RP
           assert readerPool.infoIsLive(info);
-          SegmentReader reader = readerPool.get(info, false);
+          SegmentReader reader = readerPool.get(info, false, IOContext.READ);
           int delCount = 0;
           final boolean segAllDeletes;
           try {

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CheckIndex.java Fri Jul  8 08:52:40 2011
@@ -17,6 +17,13 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintStream;
@@ -34,12 +41,6 @@ import org.apache.lucene.index.codecs.De
 import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.index.values.IndexDocValues;
 import org.apache.lucene.index.values.ValuesEnum;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.StringHelper;
@@ -364,7 +365,7 @@ public class CheckIndex {
     final String segmentsFileName = sis.getCurrentSegmentFileName();
     IndexInput input = null;
     try {
-      input = dir.openInput(segmentsFileName);
+      input = dir.openInput(segmentsFileName, IOContext.DEFAULT);
     } catch (Throwable t) {
       msg("ERROR: could not open segments file in directory");
       if (infoStream != null)
@@ -513,7 +514,7 @@ public class CheckIndex {
         }
         if (infoStream != null)
           infoStream.print("    test: open reader.........");
-        reader = SegmentReader.get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
+        reader = SegmentReader.get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, IOContext.DEFAULT);
 
         segInfoStat.openReaderPassed = true;
 

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java Fri Jul  8 08:52:40 2011
@@ -32,6 +32,7 @@ import java.util.concurrent.ConcurrentHa
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.FieldSelector;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.index.codecs.CodecProvider;
@@ -121,7 +122,7 @@ class DirectoryReader extends IndexReade
     for (int i = sis.size()-1; i >= 0; i--) {
       boolean success = false;
       try {
-        readers[i] = SegmentReader.get(readOnly, sis.info(i), termInfosIndexDivisor);
+        readers[i] = SegmentReader.get(readOnly, sis.info(i), termInfosIndexDivisor, IOContext.READ);
         readers[i].readerFinishedListeners = readerFinishedListeners;
         success = true;
       } finally {
@@ -170,7 +171,8 @@ class DirectoryReader extends IndexReade
       try {
         final SegmentInfo info = infos.info(i);
         assert info.dir == dir;
-        final SegmentReader reader = writer.readerPool.getReadOnlyClone(info, true, termInfosIndexDivisor);
+        final SegmentReader reader = writer.readerPool.getReadOnlyClone(info, true, termInfosIndexDivisor,
+                                                                        IOContext.READ);
         if (reader.numDocs() > 0 || writer.getKeepFullyDeletedSegments()) {
           reader.readerFinishedListeners = readerFinishedListeners;
           readers.add(reader);
@@ -254,7 +256,7 @@ class DirectoryReader extends IndexReade
           assert !doClone;
 
           // this is a new reader; in case we hit an exception we can close it safely
-          newReader = SegmentReader.get(readOnly, infos.info(i), termInfosIndexDivisor);
+          newReader = SegmentReader.get(readOnly, infos.info(i), termInfosIndexDivisor, IOContext.READ);
           newReader.readerFinishedListeners = readerFinishedListeners;
         } else {
           newReader = newReaders[i].reopenSegment(infos.info(i), doClone, readOnly);

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Fri Jul  8 08:52:40 2011
@@ -30,6 +30,8 @@ import org.apache.lucene.document.Docume
 import org.apache.lucene.index.DocumentsWriterDeleteQueue.DeleteSlice;
 import org.apache.lucene.search.SimilarityProvider;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FlushInfo;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.util.BitVector;
 import org.apache.lucene.util.ByteBlockPool.Allocator;
 import org.apache.lucene.util.ByteBlockPool.DirectTrackingAllocator;
@@ -428,7 +430,7 @@ public class DocumentsWriterPerThread {
     assert deleteSlice == null : "all deletes must be applied in prepareFlush";
     flushState = new SegmentWriteState(infoStream, directory, segment, fieldInfos,
         numDocsInRAM, writer.getConfig().getTermIndexInterval(),
-        fieldInfos.buildSegmentCodecs(true), pendingDeletes);
+        fieldInfos.buildSegmentCodecs(true), pendingDeletes, new IOContext(new FlushInfo(numDocsInRAM, bytesUsed())));
     final double startMBUsed = parent.flushControl.netBytes() / 1024. / 1024.;
     // Apply delete-by-docID now (delete-byDocID only
     // happens when an exception is hit processing that
@@ -543,7 +545,7 @@ public class DocumentsWriterPerThread {
 
   PerDocWriteState newPerDocWriteState(int codecId) {
     assert segment != null;
-    return new PerDocWriteState(infoStream, directory, segment, fieldInfos, bytesUsed, codecId);
+    return new PerDocWriteState(infoStream, directory, segment, fieldInfos, bytesUsed, codecId, IOContext.DEFAULT);
   }
   
   void setInfoStream(PrintStream infoStream) {

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java Fri Jul  8 08:52:40 2011
@@ -33,6 +33,7 @@ import org.apache.lucene.index.SegmentCo
 import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.index.values.ValueType;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.util.CodecUtil;
@@ -270,7 +271,7 @@ public final class FieldInfos implements
    */
   public FieldInfos(Directory d, String name) throws IOException {
     this((FieldNumberBiMap)null, null); // use null here to make this FIs Read-Only
-    final IndexInput input = d.openInput(name);
+    final IndexInput input = d.openInput(name, IOContext.READONCE);
     try {
       read(input, name);
     } finally {
@@ -562,7 +563,7 @@ public final class FieldInfos implements
   }
 
   public void write(Directory d, String name) throws IOException {
-    IndexOutput output = d.createOutput(name);
+    IndexOutput output = d.createOutput(name, IOContext.READONCE);
     try {
       write(output);
     } finally {

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldsReader.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldsReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldsReader.java Fri Jul  8 08:52:40 2011
@@ -28,6 +28,7 @@ import org.apache.lucene.document.Numeri
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.BufferedIndexInput;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.CloseableThreadLocal;
 import org.apache.lucene.util.IOUtils;
@@ -84,7 +85,7 @@ public final class FieldsReader implemen
   /** Verifies that the code version which wrote the segment is supported. */
   public static void checkCodeVersion(Directory dir, String segment) throws IOException {
     final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION);
-    IndexInput idxStream = dir.openInput(indexStreamFN, 1024);
+    IndexInput idxStream = dir.openInput(indexStreamFN, IOContext.DEFAULT);
     
     try {
       int format = idxStream.readInt();
@@ -113,18 +114,18 @@ public final class FieldsReader implemen
   }
   
   public FieldsReader(Directory d, String segment, FieldInfos fn) throws IOException {
-    this(d, segment, fn, BufferedIndexInput.BUFFER_SIZE, -1, 0);
+    this(d, segment, fn, IOContext.DEFAULT, -1, 0);
   }
 
-  public FieldsReader(Directory d, String segment, FieldInfos fn, int readBufferSize, int docStoreOffset, int size) throws IOException {
+  public FieldsReader(Directory d, String segment, FieldInfos fn, IOContext context, int docStoreOffset, int size) throws IOException {
     boolean success = false;
     isOriginal = true;
     try {
       fieldInfos = fn;
 
-      cloneableFieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION), readBufferSize);
+      cloneableFieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION), context);
       final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION);
-      cloneableIndexStream = d.openInput(indexStreamFN, readBufferSize);
+      cloneableIndexStream = d.openInput(indexStreamFN, context);
       
       format = cloneableIndexStream.readInt();
 

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldsWriter.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldsWriter.java Fri Jul  8 08:52:40 2011
@@ -23,6 +23,7 @@ import org.apache.lucene.document.Docume
 import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.document.NumericField;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.util.IOUtils;
@@ -65,14 +66,14 @@ final class FieldsWriter {
   private IndexOutput fieldsStream;
   private IndexOutput indexStream;
 
-  FieldsWriter(Directory directory, String segment) throws IOException {
+  FieldsWriter(Directory directory, String segment, IOContext context) throws IOException {
     this.directory = directory;
     this.segment = segment;
 
     boolean success = false;
     try {
-      fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION));
-      indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION));
+      fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION), context);
+      indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION), context);
 
       fieldsStream.writeInt(FORMAT_CURRENT);
       indexStream.writeInt(FORMAT_CURRENT);

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexReader.java Fri Jul  8 08:52:40 2011
@@ -1436,13 +1436,14 @@ public abstract class IndexReader implem
 
     Directory dir = null;
     CompoundFileDirectory cfr = null;
+    IOContext context = IOContext.READ;
 
     try {
       File file = new File(filename);
       String dirname = file.getAbsoluteFile().getParent();
       filename = file.getName();
       dir = FSDirectory.open(new File(dirname));
-      cfr = dir.openCompoundInput(filename, BufferedIndexInput.BUFFER_SIZE);
+      cfr = dir.openCompoundInput(filename, IOContext.DEFAULT);
 
       String [] files = cfr.listAll();
       ArrayUtil.mergeSort(files);   // sort the array of filename so that the output is more readable
@@ -1452,7 +1453,7 @@ public abstract class IndexReader implem
 
         if (extract) {
           System.out.println("extract " + files[i] + " with " + len + " bytes to local directory...");
-          IndexInput ii = cfr.openInput(files[i]);
+          IndexInput ii = cfr.openInput(files[i], context);
 
           FileOutputStream f = new FileOutputStream(files[i]);
 

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java Fri Jul  8 08:52:40 2011
@@ -45,11 +45,13 @@ import org.apache.lucene.index.SegmentCo
 import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.AlreadyClosedException;
-import org.apache.lucene.store.BufferedIndexInput;
 import org.apache.lucene.store.CompoundFileDirectory;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FlushInfo;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.store.MergeInfo;
 import org.apache.lucene.util.BitVector;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.Constants;
@@ -207,15 +209,6 @@ public class IndexWriter implements Clos
    * #setInfoStream}).
    */
   public final static int MAX_TERM_LENGTH = DocumentsWriterPerThread.MAX_TERM_LENGTH_UTF8;
-
-  // The normal read buffer size defaults to 1024, but
-  // increasing this during merging seems to yield
-  // performance gains.  However we don't want to increase
-  // it too much because there are quite a few
-  // BufferedIndexInputs created during merging.  See
-  // LUCENE-888 for details.
-  private final static int MERGE_READ_BUFFER_SIZE = 4096;
-
   // Used for printing messages
   private static final AtomicInteger MESSAGE_ID = new AtomicInteger();
   private int messageID = MESSAGE_ID.getAndIncrement();
@@ -594,8 +587,8 @@ public class IndexWriter implements Clos
      * enrolled in the pool, so you should simply close()
      * it when you're done (ie, do not call release()).
      */
-    public synchronized SegmentReader getReadOnlyClone(SegmentInfo info, boolean doOpenStores, int termInfosIndexDivisor) throws IOException {
-      SegmentReader sr = get(info, doOpenStores, BufferedIndexInput.BUFFER_SIZE, termInfosIndexDivisor);
+    public synchronized SegmentReader getReadOnlyClone(SegmentInfo info, boolean doOpenStores, int termInfosIndexDivisor, IOContext context) throws IOException {
+      SegmentReader sr = get(info, doOpenStores, context, termInfosIndexDivisor);
       try {
         return (SegmentReader) sr.clone(true);
       } finally {
@@ -611,8 +604,8 @@ public class IndexWriter implements Clos
      * @param doOpenStores
      * @throws IOException
      */
-    public synchronized SegmentReader get(SegmentInfo info, boolean doOpenStores) throws IOException {
-      return get(info, doOpenStores, BufferedIndexInput.BUFFER_SIZE, config.getReaderTermsIndexDivisor());
+    public synchronized SegmentReader get(SegmentInfo info, boolean doOpenStores, IOContext context) throws IOException {
+      return get(info, doOpenStores, context, config.getReaderTermsIndexDivisor());
     }
 
     /**
@@ -626,18 +619,20 @@ public class IndexWriter implements Clos
      * @param termsIndexDivisor
      * @throws IOException
      */
-    public synchronized SegmentReader get(SegmentInfo info, boolean doOpenStores, int readBufferSize, int termsIndexDivisor) throws IOException {
+    public synchronized SegmentReader get(SegmentInfo info, boolean doOpenStores, IOContext context, int termsIndexDivisor) throws IOException {
 
-      if (poolReaders) {
-        readBufferSize = BufferedIndexInput.BUFFER_SIZE;
-      }
+      //      if (poolReaders) {
+      //        readBufferSize = BufferedIndexInput.BUFFER_SIZE;
+      //      }
+      
+      // TODO: context should be part of the key used to cache that reader in the pool.
 
       SegmentReader sr = readerMap.get(info);
       if (sr == null) {
         // TODO: we may want to avoid doing this while
         // synchronized
         // Returns a ref, which we xfer to readerMap:
-        sr = SegmentReader.get(false, info.dir, info, readBufferSize, doOpenStores, termsIndexDivisor);
+        sr = SegmentReader.get(false, info.dir, info, doOpenStores, termsIndexDivisor, context);
         sr.readerFinishedListeners = readerFinishedListeners;
 
         if (info.dir == directory) {
@@ -2185,6 +2180,8 @@ public class IndexWriter implements Clos
     SegmentInfo newSegment = flushedSegment.segmentInfo;
 
     setDiagnostics(newSegment, "flush");
+    
+    IOContext context = new IOContext(new FlushInfo(newSegment.docCount, newSegment.sizeInBytes(true)));
 
     boolean success = false;
     try {
@@ -2192,11 +2189,11 @@ public class IndexWriter implements Clos
         String compoundFileName = IndexFileNames.segmentFileName(newSegment.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION);
         message("creating compound file " + compoundFileName);
         // Now build compound file
-        final Directory cfsDir = directory.createCompoundOutput(compoundFileName);
+        final Directory cfsDir = directory.createCompoundOutput(compoundFileName, context);
         IOException prior = null;
         try {
           for(String fileName : newSegment.files()) {
-            directory.copy(cfsDir, fileName, fileName);
+            directory.copy(cfsDir, fileName, fileName, context);
           }
         } catch(IOException ex) {
           prior = ex;
@@ -2230,7 +2227,7 @@ public class IndexWriter implements Clos
           // shortly-to-be-opened SegmentReader and let it
           // carry the changes; there's no reason to use
           // filesystem as intermediary here.
-          flushedSegment.liveDocs.write(directory, delFileName);
+          flushedSegment.liveDocs.write(directory, delFileName, context);
           success2 = true;
         } finally {
           if (!success2) {
@@ -2399,11 +2396,13 @@ public class IndexWriter implements Clos
                 // optimize case only for segments that don't share doc stores
                 && versionComparator.compare(info.getVersion(), "3.1") >= 0;
           }
-
+          
+          IOContext context = new IOContext(new MergeInfo(info.docCount, info.sizeInBytes(true), true, false));
+          
           if (createCFS) {
-            copySegmentIntoCFS(info, newSegName);
+            copySegmentIntoCFS(info, newSegName, context);
           } else {
-            copySegmentAsIs(info, newSegName, dsNames, dsFilesCopied);
+            copySegmentAsIs(info, newSegName, dsNames, dsFilesCopied, context);
           }
 
           infos.add(info);
@@ -2447,6 +2446,7 @@ public class IndexWriter implements Clos
    */
   public void addIndexes(IndexReader... readers) throws CorruptIndexException, IOException {
     ensureOpen();
+    int numDocs = 0;
 
     try {
       if (infoStream != null)
@@ -2454,15 +2454,19 @@ public class IndexWriter implements Clos
       flush(false, true);
 
       String mergedName = newSegmentName();
+      for (IndexReader indexReader : readers) {
+        numDocs += indexReader.numDocs();
+       }
+       final IOContext context = new IOContext(new MergeInfo(numDocs, -1, true, false));      
+
       // TODO: somehow we should fix this merge so it's
       // abortable so that IW.close(false) is able to stop it
       SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(),
                                                mergedName, null, payloadProcessorProvider,
-                                               globalFieldNumberMap.newFieldInfos(SegmentCodecsBuilder.create(codecs)));
+                                               globalFieldNumberMap.newFieldInfos(SegmentCodecsBuilder.create(codecs)), context);
 
       for (IndexReader reader : readers)      // add new indexes
         merger.add(reader);
-
       int docCount = merger.merge();                // merge 'em
 
       final FieldInfos fieldInfos = merger.fieldInfos();
@@ -2483,7 +2487,7 @@ public class IndexWriter implements Clos
 
       // Now create the compound file if needed
       if (useCompoundFile) {
-        merger.createCompoundFile(mergedName + ".cfs", info);
+        merger.createCompoundFile(mergedName + ".cfs", info, context);
 
         // delete new non cfs files directly: they were never
         // registered with IFD
@@ -2507,19 +2511,19 @@ public class IndexWriter implements Clos
   }
 
   /** Copies the segment into the IndexWriter's directory, as a compound segment. */
-  private void copySegmentIntoCFS(SegmentInfo info, String segName) throws IOException {
+  private void copySegmentIntoCFS(SegmentInfo info, String segName, IOContext context) throws IOException {
     String segFileName = IndexFileNames.segmentFileName(segName, "", IndexFileNames.COMPOUND_FILE_EXTENSION);
     Collection<String> files = info.files();
-    final CompoundFileDirectory cfsdir = directory.createCompoundOutput(segFileName);
+    final CompoundFileDirectory cfsdir = directory.createCompoundOutput(segFileName, context);
     try {
       for (String file : files) {
         String newFileName = segName + IndexFileNames.stripSegmentName(file);
         if (!IndexFileNames.matchesExtension(file, IndexFileNames.DELETES_EXTENSION)
             && !IndexFileNames.isSeparateNormsFile(file)) {
-          info.dir.copy(cfsdir, file, file);
+          info.dir.copy(cfsdir, file, file, context);
         } else {
           assert !directory.fileExists(newFileName): "file \"" + newFileName + "\" already exists";
-          info.dir.copy(directory, file, newFileName);
+          info.dir.copy(directory, file, newFileName, context);
         }
       }
     } finally {
@@ -2533,7 +2537,7 @@ public class IndexWriter implements Clos
   
   /** Copies the segment files as-is into the IndexWriter's directory. */
   private void copySegmentAsIs(SegmentInfo info, String segName,
-      Map<String, String> dsNames, Set<String> dsFilesCopied)
+      Map<String, String> dsNames, Set<String> dsFilesCopied, IOContext context)
       throws IOException {
     // Determine if the doc store of this segment needs to be copied. It's
     // only relevant for segments that share doc store with others,
@@ -2569,7 +2573,7 @@ public class IndexWriter implements Clos
       }
       
       assert !directory.fileExists(newFileName): "file \"" + newFileName + "\" already exists";
-      info.dir.copy(directory, file, newFileName);
+      info.dir.copy(directory, file, newFileName, context);
     }
     
     info.setDocStore(info.getDocStoreOffset(), newDsName, info.getDocStoreIsCompoundFile());
@@ -3425,9 +3429,11 @@ public class IndexWriter implements Clos
     int mergedDocCount = 0;
 
     List<SegmentInfo> sourceSegments = merge.segments;
+    
+    IOContext context = new IOContext(merge.getMergeInfo());
 
     SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(), mergedName, merge,
-                                             payloadProcessorProvider, merge.info.getFieldInfos());
+                                             payloadProcessorProvider, merge.info.getFieldInfos(), context);
 
     if (infoStream != null) {
       message("merging " + merge.segString(directory) + " mergeVectors=" + merge.info.getFieldInfos().hasVectors());
@@ -3448,7 +3454,7 @@ public class IndexWriter implements Clos
         // Hold onto the "live" reader; we will use this to
         // commit merged deletes
         final SegmentReader reader = readerPool.get(info, true,
-                                                    MERGE_READ_BUFFER_SIZE,
+                                                    context,
                                                     -config.getReaderTermsIndexDivisor());
         merge.readers.add(reader);
 
@@ -3502,7 +3508,7 @@ public class IndexWriter implements Clos
           if (infoStream != null) {
             message("create compound file " + compoundFileName);
           }
-          merger.createCompoundFile(compoundFileName, merge.info);
+          merger.createCompoundFile(compoundFileName, merge.info, new IOContext(merge.getMergeInfo()));
           success = true;
         } catch (IOException ioe) {
           synchronized(this) {
@@ -3574,7 +3580,7 @@ public class IndexWriter implements Clos
       // keep deletes (it's costly to open entire reader
       // when we just need deletes)
 
-      final SegmentReader mergedReader = readerPool.get(merge.info, loadDocStores, BufferedIndexInput.BUFFER_SIZE, termsIndexDivisor);
+      final SegmentReader mergedReader = readerPool.get(merge.info, loadDocStores, context, termsIndexDivisor);
       try {
         if (poolReaders && mergedSegmentWarmer != null) {
           mergedSegmentWarmer.warm(mergedReader);

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MergePolicy.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MergePolicy.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MergePolicy.java Fri Jul  8 08:52:40 2011
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
  */
 
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MergeInfo;
 import org.apache.lucene.util.SetOnce;
 import org.apache.lucene.util.SetOnce.AlreadySetException;
 
@@ -189,6 +190,10 @@ public abstract class MergePolicy implem
       }
       return total;
     }
+    
+    public MergeInfo getMergeInfo() {
+      return new MergeInfo(totalDocCount, estimatedMergeBytes, isExternal, optimize);
+    }    
   }
 
   /**

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/NormsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/NormsWriter.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/NormsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/NormsWriter.java Fri Jul  8 08:52:40 2011
@@ -22,6 +22,7 @@ import java.util.Collection;
 import java.util.Map;
 
 import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.IOContext.Context;
 import org.apache.lucene.util.IOUtils;
 
 // TODO FI: norms could actually be stored as doc store
@@ -49,7 +50,7 @@ final class NormsWriter extends Inverted
     }
 
     final String normsFileName = IndexFileNames.segmentFileName(state.segmentName, "", IndexFileNames.NORMS_EXTENSION);
-    IndexOutput normsOut = state.directory.createOutput(normsFileName);
+    IndexOutput normsOut = state.directory.createOutput(normsFileName, state.context);
     boolean success = false;
     try {
       normsOut.writeBytes(SegmentNorms.NORMS_HEADER, 0, SegmentNorms.NORMS_HEADER.length);

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java Fri Jul  8 08:52:40 2011
@@ -20,6 +20,7 @@ import java.util.concurrent.atomic.Atomi
 
 import org.apache.lucene.index.codecs.PerDocConsumer;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
 
 /**
  * Encapsulates all necessary state to initiate a {@link PerDocConsumer} and
@@ -35,10 +36,11 @@ public class PerDocWriteState {
   public final AtomicLong bytesUsed;
   public final SegmentCodecs segmentCodecs;
   public final int codecId;
+  public final IOContext context;
 
   PerDocWriteState(PrintStream infoStream, Directory directory,
       String segmentName, FieldInfos fieldInfos, AtomicLong bytesUsed,
-      int codecId) {
+      int codecId, IOContext context) {
     this.infoStream = infoStream;
     this.directory = directory;
     this.segmentName = segmentName;
@@ -46,6 +48,7 @@ public class PerDocWriteState {
     this.segmentCodecs = fieldInfos.buildSegmentCodecs(false);
     this.codecId = codecId;
     this.bytesUsed = bytesUsed;
+    this.context = context;
   }
 
   PerDocWriteState(SegmentWriteState state) {
@@ -56,6 +59,7 @@ public class PerDocWriteState {
     fieldInfos = state.fieldInfos;
     codecId = state.codecId;
     bytesUsed = new AtomicLong(0);
+    context = state.context;
   }
 
   PerDocWriteState(PerDocWriteState state, int codecId) {
@@ -66,5 +70,6 @@ public class PerDocWriteState {
     this.segmentCodecs = state.segmentCodecs;
     this.codecId = codecId;
     this.bytesUsed = state.bytesUsed;
+    this.context = state.context;
   }
 }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java Fri Jul  8 08:52:40 2011
@@ -36,6 +36,7 @@ import org.apache.lucene.index.codecs.Te
 import org.apache.lucene.index.codecs.DocValuesConsumer;
 import org.apache.lucene.index.values.IndexDocValues;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.util.IOUtils;
 
 /**
@@ -99,7 +100,7 @@ final class PerFieldCodecWrapper extends
     private final Map<String, FieldsProducer> codecs = new HashMap<String, FieldsProducer>();
 
     public FieldsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo si,
-        int readBufferSize, int indexDivisor) throws IOException {
+        IOContext context, int indexDivisor) throws IOException {
 
       final Map<Codec, FieldsProducer> producers = new HashMap<Codec, FieldsProducer>();
       boolean success = false;
@@ -111,7 +112,7 @@ final class PerFieldCodecWrapper extends
             Codec codec = segmentCodecs.codecs[fi.getCodecId()];
             if (!producers.containsKey(codec)) {
               producers.put(codec, codec.fieldsProducer(new SegmentReadState(dir,
-                                                                             si, fieldInfos, readBufferSize, indexDivisor, fi.getCodecId())));
+                                                                             si, fieldInfos, context, indexDivisor, fi.getCodecId())));
             }
             codecs.put(fi.name, producers.get(codec));
           }
@@ -187,7 +188,7 @@ final class PerFieldCodecWrapper extends
   public FieldsProducer fieldsProducer(SegmentReadState state)
       throws IOException {
     return new FieldsReader(state.dir, state.fieldInfos, state.segmentInfo,
-        state.readBufferSize, state.termsIndexDivisor);
+        state.context, state.termsIndexDivisor);
   }
 
   @Override
@@ -212,14 +213,14 @@ final class PerFieldCodecWrapper extends
   @Override
   public PerDocValues docsProducer(SegmentReadState state) throws IOException {
     return new PerDocProducers(state.dir, state.fieldInfos, state.segmentInfo,
-    state.readBufferSize, state.termsIndexDivisor);
+    state.context, state.termsIndexDivisor);
   }
   
   private final class PerDocProducers extends PerDocValues {
     private final TreeMap<String, PerDocValues> codecs = new TreeMap<String, PerDocValues>();
 
     public PerDocProducers(Directory dir, FieldInfos fieldInfos, SegmentInfo si,
-        int readBufferSize, int indexDivisor) throws IOException {
+        IOContext context, int indexDivisor) throws IOException {
       final Map<Codec, PerDocValues> producers = new HashMap<Codec, PerDocValues>();
       boolean success = false;
       try {
@@ -229,7 +230,7 @@ final class PerFieldCodecWrapper extends
             Codec codec = segmentCodecs.codecs[fi.getCodecId()];
             if (!producers.containsKey(codec)) {
               producers.put(codec, codec.docsProducer(new SegmentReadState(dir,
-                si, fieldInfos, readBufferSize, indexDivisor, fi.getCodecId())));
+                si, fieldInfos, context, indexDivisor, fi.getCodecId())));
             }
             codecs.put(fi.name, producers.get(codec));
           }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java Fri Jul  8 08:52:40 2011
@@ -25,6 +25,7 @@ import org.apache.lucene.index.codecs.Fi
 import org.apache.lucene.index.codecs.PerDocValues;
 import org.apache.lucene.store.CompoundFileDirectory;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.util.IOUtils;
 
 /** Holds core readers that are shared (unchanged) when
@@ -47,7 +48,7 @@ final class SegmentCoreReaders {
 
   final Directory dir;
   final Directory cfsDir;
-  final int readBufferSize;
+  final IOContext context;
   final int termsIndexDivisor;
   
   private final SegmentReader owner;
@@ -59,7 +60,7 @@ final class SegmentCoreReaders {
 
   
   
-  SegmentCoreReaders(SegmentReader owner, Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor) throws IOException {
+  SegmentCoreReaders(SegmentReader owner, Directory dir, SegmentInfo si, IOContext context, int termsIndexDivisor) throws IOException {
     
     if (termsIndexDivisor == 0) {
       throw new IllegalArgumentException("indexDivisor must be < 0 (don't load terms index) or greater than 0 (got 0)");
@@ -67,7 +68,7 @@ final class SegmentCoreReaders {
     
     segment = si.name;
     final SegmentCodecs segmentCodecs = si.getSegmentCodecs();
-    this.readBufferSize = readBufferSize;
+    this.context = context;
     this.dir = dir;
     
     boolean success = false;
@@ -75,7 +76,7 @@ final class SegmentCoreReaders {
     try {
       Directory dir0 = dir;
       if (si.getUseCompoundFile()) {
-        cfsReader = dir.openCompoundInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize);
+        cfsReader = dir.openCompoundInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context);
         dir0 = cfsReader;
       }
       cfsDir = dir0;
@@ -84,7 +85,7 @@ final class SegmentCoreReaders {
       
       this.termsIndexDivisor = termsIndexDivisor;
       final Codec codec = segmentCodecs.codec();
-      final SegmentReadState segmentReadState = new SegmentReadState(cfsDir, si, fieldInfos, readBufferSize, termsIndexDivisor);
+      final SegmentReadState segmentReadState = new SegmentReadState(cfsDir, si, fieldInfos, context, termsIndexDivisor);
       // Ask codec for its Fields
       fields = codec.fieldsProducer(segmentReadState);
       assert fields != null;
@@ -141,7 +142,7 @@ final class SegmentCoreReaders {
           assert storeCFSReader == null;
           storeCFSReader = dir.openCompoundInput(
               IndexFileNames.segmentFileName(si.getDocStoreSegment(), "", IndexFileNames.COMPOUND_FILE_STORE_EXTENSION),
-              readBufferSize);
+              context);
           storeDir = storeCFSReader;
           assert storeDir != null;
         } else {
@@ -153,7 +154,7 @@ final class SegmentCoreReaders {
         // was not used, but then we are asked to open doc
         // stores after the segment has switched to CFS
         if (cfsReader == null) {
-          cfsReader = dir.openCompoundInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize);
+          cfsReader = dir.openCompoundInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context);
         }
         storeDir = cfsReader;
         assert storeDir != null;
@@ -163,7 +164,7 @@ final class SegmentCoreReaders {
       }
       
       final String storesSegment = si.getDocStoreSegment();
-      fieldsReaderOrig = new FieldsReader(storeDir, storesSegment, fieldInfos, readBufferSize,
+      fieldsReaderOrig = new FieldsReader(storeDir, storesSegment, fieldInfos, context,
           si.getDocStoreOffset(), si.docCount);
       
       // Verify two sources of "maxDoc" agree:
@@ -172,7 +173,7 @@ final class SegmentCoreReaders {
       }
       
       if (si.getHasVectors()) { // open term vector files only as needed
-        termVectorsReaderOrig = new TermVectorsReader(storeDir, storesSegment, fieldInfos, readBufferSize, si.getDocStoreOffset(), si.docCount);
+        termVectorsReaderOrig = new TermVectorsReader(storeDir, storesSegment, fieldInfos, context, si.getDocStoreOffset(), si.docCount);
       }
     }
   }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Fri Jul  8 08:52:40 2011
@@ -30,8 +30,8 @@ import java.util.Set;
 import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter;
-import org.apache.lucene.store.BufferedIndexInput;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.util.Constants;
@@ -247,7 +247,7 @@ public final class SegmentInfo implement
       }
       final Directory dirToTest;
       if (isCompoundFile) {
-        dirToTest = dir.openCompoundInput(IndexFileNames.segmentFileName(storesSegment, "", ext), BufferedIndexInput.BUFFER_SIZE);
+        dirToTest = dir.openCompoundInput(IndexFileNames.segmentFileName(storesSegment, "", ext), IOContext.READONCE);
       } else {
         dirToTest = dir;
       }
@@ -266,7 +266,7 @@ public final class SegmentInfo implement
       Directory dir0 = dir;
       if (isCompoundFile && checkCompoundFile) {
         dir0 = dir.openCompoundInput(IndexFileNames.segmentFileName(name,
-            "", IndexFileNames.COMPOUND_FILE_EXTENSION), BufferedIndexInput.BUFFER_SIZE);
+            "", IndexFileNames.COMPOUND_FILE_EXTENSION), IOContext.READONCE);
       }
       try {
         fieldInfos = new FieldInfos(dir0, IndexFileNames.segmentFileName(name,

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1144196&r1=1144195&r2=1144196&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentInfos.java Fri Jul  8 08:52:40 2011
@@ -37,6 +37,7 @@ import org.apache.lucene.index.codecs.De
 import org.apache.lucene.index.codecs.SegmentInfosReader;
 import org.apache.lucene.index.codecs.SegmentInfosWriter;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.NoSuchDirectoryException;
@@ -254,7 +255,7 @@ public final class SegmentInfos implemen
 
     try {
       SegmentInfosReader infosReader = codecs.getSegmentInfosReader();
-      infosReader.read(directory, segmentFileName, codecs, this);
+      infosReader.read(directory, segmentFileName, codecs, this, IOContext.READ);
       success = true;
     }
     finally {
@@ -322,7 +323,7 @@ public final class SegmentInfos implemen
 
     try {
       SegmentInfosWriter infosWriter = codecs.getSegmentInfosWriter();
-      segnOutput = infosWriter.writeInfos(directory, segmentFileName, this);
+      segnOutput = infosWriter.writeInfos(directory, segmentFileName, this, IOContext.DEFAULT);
       infosWriter.prepareCommit(segnOutput);
       pendingSegnOutput = segnOutput;
       success = true;
@@ -597,7 +598,7 @@ public final class SegmentInfos implemen
           for(int i=0;i<defaultGenFileRetryCount;i++) {
             IndexInput genInput = null;
             try {
-              genInput = directory.openInput(IndexFileNames.SEGMENTS_GEN);
+              genInput = directory.openInput(IndexFileNames.SEGMENTS_GEN, IOContext.READONCE);
             } catch (FileNotFoundException e) {
               if (infoStream != null) {
                 message("segments.gen open: FileNotFoundException " + e);
@@ -814,7 +815,7 @@ public final class SegmentInfos implemen
   }
   
   private final long writeGlobalFieldMap(FieldNumberBiMap map, Directory dir, String name) throws IOException {
-    final IndexOutput output = dir.createOutput(name);
+    final IndexOutput output = dir.createOutput(name, IOContext.READONCE);
     boolean success = false;
     long version;
     try {
@@ -843,7 +844,7 @@ public final class SegmentInfos implemen
   
   private void readGlobalFieldMap(FieldNumberBiMap map, Directory dir) throws IOException {
     final String name = getGlobalFieldNumberName(lastGlobalFieldMapVersion);
-    final IndexInput input = dir.openInput(name);
+    final IndexInput input = dir.openInput(name, IOContext.READONCE);
     try {
       map.read(input);
     } finally {
@@ -934,7 +935,7 @@ public final class SegmentInfos implemen
     }
 
     try {
-      IndexOutput genOutput = dir.createOutput(IndexFileNames.SEGMENTS_GEN);
+      IndexOutput genOutput = dir.createOutput(IndexFileNames.SEGMENTS_GEN, IOContext.READONCE);
       try {
         genOutput.writeInt(FORMAT_SEGMENTS_GEN_CURRENT);
         genOutput.writeLong(generation);



Mime
View raw message