lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rm...@apache.org
Subject svn commit: r1050738 [3/3] - in /lucene/dev/branches/bulkpostings: ./ lucene/ lucene/contrib/ant/src/java/org/apache/lucene/ant/ lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/ lucene/contrib/wordnet/src/java/org/apache/lucene/wo...
Date Sun, 19 Dec 2010 00:24:06 GMT
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/store/MMapDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/store/MMapDirectory.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/store/MMapDirectory.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/store/MMapDirectory.java Sun Dec 19 00:24:04 2010
@@ -248,8 +248,35 @@ public class MMapDirectory extends FSDir
         throw new IOException("read past EOF");
       }
     }
+    
+    @Override
+    public short readShort() throws IOException {
+      try {
+        return buffer.getShort();
+      } catch (BufferUnderflowException e) {
+        throw new IOException("read past EOF");
+      }
+    }
 
     @Override
+    public int readInt() throws IOException {
+      try {
+        return buffer.getInt();
+      } catch (BufferUnderflowException e) {
+        throw new IOException("read past EOF");
+      }
+    }
+
+    @Override
+    public long readLong() throws IOException {
+      try {
+        return buffer.getLong();
+      } catch (BufferUnderflowException e) {
+        throw new IOException("read past EOF");
+      }
+    }
+    
+    @Override
     public long getFilePointer() {
       return buffer.position();
     }
@@ -301,7 +328,6 @@ public class MMapDirectory extends FSDir
     private final int maxBufSize;
   
     private ByteBuffer curBuf; // redundant for speed: buffers[curBufIndex]
-    private int curAvail; // redundant for speed: (bufSizes[curBufIndex] - curBuf.position())
   
     private boolean isClone = false;
     
@@ -345,38 +371,67 @@ public class MMapDirectory extends FSDir
 
     @Override
     public byte readByte() throws IOException {
-      // Performance might be improved by reading ahead into an array of
-      // e.g. 128 bytes and readByte() from there.
-      if (curAvail == 0) {
+      try {
+        return curBuf.get();
+      } catch (BufferUnderflowException e) {
         curBufIndex++;
         if (curBufIndex >= buffers.length)
           throw new IOException("read past EOF");
         curBuf = buffers[curBufIndex];
         curBuf.position(0);
-        curAvail = bufSizes[curBufIndex];
+        return curBuf.get();
       }
-      curAvail--;
-      return curBuf.get();
     }
   
     @Override
     public void readBytes(byte[] b, int offset, int len) throws IOException {
-      while (len > curAvail) {
-        curBuf.get(b, offset, curAvail);
-        len -= curAvail;
-        offset += curAvail;
-        curBufIndex++;
-        if (curBufIndex >= buffers.length)
-          throw new IOException("read past EOF");
-        curBuf = buffers[curBufIndex];
-        curBuf.position(0);
-        curAvail = bufSizes[curBufIndex];
+      try {
+        curBuf.get(b, offset, len);
+      } catch (BufferUnderflowException e) {
+        int curAvail = curBuf.remaining();
+        while (len > curAvail) {
+          curBuf.get(b, offset, curAvail);
+          len -= curAvail;
+          offset += curAvail;
+          curBufIndex++;
+          if (curBufIndex >= buffers.length)
+            throw new IOException("read past EOF");
+          curBuf = buffers[curBufIndex];
+          curBuf.position(0);
+          curAvail = curBuf.remaining();
+        }
+        curBuf.get(b, offset, len);
       }
-      curBuf.get(b, offset, len);
-      curAvail -= len;
     }
   
     @Override
+    public short readShort() throws IOException {
+      try {
+        return curBuf.getShort();
+      } catch (BufferUnderflowException e) {
+        return super.readShort();
+      }
+    }
+
+    @Override
+    public int readInt() throws IOException {
+      try {
+        return curBuf.getInt();
+      } catch (BufferUnderflowException e) {
+        return super.readInt();
+      }
+    }
+
+    @Override
+    public long readLong() throws IOException {
+      try {
+        return curBuf.getLong();
+      } catch (BufferUnderflowException e) {
+        return super.readLong();
+      }
+    }
+    
+    @Override
     public long getFilePointer() {
       return ((long) curBufIndex * maxBufSize) + curBuf.position();
     }
@@ -387,7 +442,6 @@ public class MMapDirectory extends FSDir
       curBuf = buffers[curBufIndex];
       int bufOffset = (int) (pos - ((long) curBufIndex * maxBufSize));
       curBuf.position(bufOffset);
-      curAvail = bufSizes[curBufIndex] - bufOffset;
     }
   
     @Override

Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/IOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/IOUtils.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/IOUtils.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/IOUtils.java Sun Dec 19 00:24:04 2010
@@ -64,4 +64,27 @@ public final class IOUtils {
     else if (firstIOE != null)
       throw firstIOE;
   }
+
+  /**
+   * <p>Closes all given <tt>Closeable</tt>s, suppressing all thrown exceptions. Some of the <tt>Closeable</tt>s
+   * may be null, they are ignored. After everything is closed, method either throws the first of suppressed exceptions,
+   * or completes normally.</p>
+   * @param objects         objects to call <tt>close()</tt> on
+   */
+  public static void closeSafely(Closeable... objects) throws IOException {
+    IOException firstIOE = null;
+
+    for (Closeable object : objects) {
+      try {
+        if (object != null)
+          object.close();
+      } catch (IOException ioe) {
+        if (firstIOE == null)
+          firstIOE = ioe;
+      }
+    }
+
+    if (firstIOE != null)
+      throw firstIOE;
+  }
 }

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java Sun Dec 19 00:24:04 2010
@@ -105,6 +105,5 @@ public class TestMergeSchedulerExternal 
     assertTrue(mergeCalled);
     assertTrue(excCalled);
     dir.close();
-    assertTrue(ConcurrentMergeScheduler.anyUnhandledExceptions());
   }
 }

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestSearch.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestSearch.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestSearch.java Sun Dec 19 00:24:04 2010
@@ -17,7 +17,6 @@ package org.apache.lucene;
  * limitations under the License.
  */
 
-import java.util.GregorianCalendar;
 import java.util.Random;
 import java.io.PrintWriter;
 import java.io.StringWriter;
@@ -77,7 +76,6 @@ public class TestSearch extends LuceneTe
       IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
       LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
       lmp.setUseCompoundFile(useCompoundFile);
-      lmp.setUseCompoundDocStore(useCompoundFile);
       IndexWriter writer = new IndexWriter(directory, conf);
 
       String[] docs = {

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java Sun Dec 19 00:24:04 2010
@@ -82,7 +82,6 @@ public class TestSearchForDuplicates ext
       IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
       LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
       lmp.setUseCompoundFile(useCompoundFiles);
-      lmp.setUseCompoundDocStore(useCompoundFiles);
       IndexWriter writer = new IndexWriter(directory, conf);
       if (VERBOSE) {
         System.out.println("TEST: now build index");

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java Sun Dec 19 00:24:04 2010
@@ -570,7 +570,6 @@ public class TestAddIndexes extends Luce
     Directory dir = newDirectory();
     LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy();
     lmp.setUseCompoundFile(false);
-    lmp.setUseCompoundDocStore(false);
     lmp.setMergeFactor(100);
     IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
         TEST_VERSION_CURRENT, new MockAnalyzer())
@@ -599,7 +598,6 @@ public class TestAddIndexes extends Luce
     lmp = new LogByteSizeMergePolicy();
     lmp.setMinMergeMB(0.0001);
     lmp.setUseCompoundFile(false);
-    lmp.setUseCompoundDocStore(false);
     lmp.setMergeFactor(4);
     writer = new IndexWriter(dir2, newIndexWriterConfig(TEST_VERSION_CURRENT,
         new MockAnalyzer())

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Sun Dec 19 00:24:04 2010
@@ -293,6 +293,9 @@ public class TestBackwardsCompatibility 
 
   public void testIndexOldIndex() throws IOException {
     for(int i=0;i<oldNames.length;i++) {
+      if (VERBOSE) {
+        System.out.println("TEST: oldName=" + oldNames[i]);
+      }
       unzip(getDataFile("index." + oldNames[i] + ".zip"), oldNames[i]);
       changeIndexWithAdds(random, oldNames[i]);
       rmDir(oldNames[i]);
@@ -344,6 +347,10 @@ public class TestBackwardsCompatibility 
           f = d.getField("fie\u2C77ld");
           assertEquals("field with non-ascii name", f.stringValue());
         }
+
+        TermFreqVector tfv = reader.getTermFreqVector(i, "utf8");
+        assertNotNull("docID=" + i + " index=" + dirName, tfv);
+        assertTrue(tfv instanceof TermPositionVector);
       } else
         // Only ID 7 is deleted
         assertEquals(7, i);
@@ -382,6 +389,7 @@ public class TestBackwardsCompatibility 
     Directory dir = newFSDirectory(new File(dirName));
     // open writer
     IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND));
+    writer.setInfoStream(VERBOSE ? System.out : null);
     // add 10 docs
     for(int i=0;i<10;i++) {
       addDoc(writer, 35+i);
@@ -496,7 +504,6 @@ public class TestBackwardsCompatibility 
     Directory dir = newFSDirectory(new File(dirName));
     IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(10);
     ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(doCFS);
-    ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundDocStore(doCFS);
     IndexWriter writer = new IndexWriter(dir, conf);
     
     for(int i=0;i<35;i++) {
@@ -508,7 +515,6 @@ public class TestBackwardsCompatibility 
     // open fresh writer so we get no prx file in the added segment
     conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(10);
     ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(doCFS);
-    ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundDocStore(doCFS);
     writer = new IndexWriter(dir, conf);
     addNoProxDoc(writer);
     writer.close();

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestCodecs.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestCodecs.java Sun Dec 19 00:24:04 2010
@@ -270,7 +270,7 @@ public class TestCodecs extends LuceneTe
 
     final Directory dir = newDirectory();
     this.write(fieldInfos, dir, fields);
-    final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, -1, SEGMENT, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()), fieldInfos.hasVectors());
+    final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()), fieldInfos.hasVectors());
     si.setHasProx(false);
 
     final FieldsProducer reader = si.getSegmentCodecs().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 64, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR));
@@ -318,7 +318,7 @@ public class TestCodecs extends LuceneTe
     final Directory dir = newDirectory();
 
     this.write(fieldInfos, dir, fields);
-    final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, -1, SEGMENT, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()), fieldInfos.hasVectors());
+    final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()), fieldInfos.hasVectors());
 
     final FieldsProducer terms = si.getSegmentCodecs().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 1024, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR));
 
@@ -607,7 +607,7 @@ public class TestCodecs extends LuceneTe
 
     final int termIndexInterval = this.nextInt(13, 27);
     final SegmentCodecs codecInfo = SegmentCodecs.build(fieldInfos, CodecProvider.getDefault());
-    final SegmentWriteState state = new SegmentWriteState(null, dir, SEGMENT, fieldInfos, null, 10000, 10000, termIndexInterval, codecInfo);
+    final SegmentWriteState state = new SegmentWriteState(null, dir, SEGMENT, fieldInfos, 10000, termIndexInterval, codecInfo);
 
     final FieldsConsumer consumer = state.segmentCodecs.codec().fieldsConsumer(state);
     Arrays.sort(fields);

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java Sun Dec 19 00:24:04 2010
@@ -96,7 +96,7 @@ public class TestConcurrentMergeSchedule
         writer.addDocument(doc);
         failure.setDoFail();
         try {
-          writer.flush(true, false, true);
+          writer.flush(true, true);
           if (failure.hitExc) {
             fail("failed to hit IOException");
           }

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java Sun Dec 19 00:24:04 2010
@@ -195,16 +195,13 @@ public class TestDeletionPolicy extends 
 
     final double SECONDS = 2.0;
 
-    boolean useCompoundFile = true;
-    
     Directory dir = newDirectory();
     ExpirationTimeDeletionPolicy policy = new ExpirationTimeDeletionPolicy(dir, SECONDS);
     IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
         new MockAnalyzer())
         .setIndexDeletionPolicy(policy);
     LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
-    lmp.setUseCompoundFile(useCompoundFile);
-    lmp.setUseCompoundDocStore(useCompoundFile);
+    lmp.setUseCompoundFile(true);
     IndexWriter writer = new IndexWriter(dir, conf);
     writer.close();
 
@@ -219,8 +216,7 @@ public class TestDeletionPolicy extends 
           new MockAnalyzer()).setOpenMode(
           OpenMode.APPEND).setIndexDeletionPolicy(policy);
       lmp = (LogMergePolicy) conf.getMergePolicy();
-      lmp.setUseCompoundFile(useCompoundFile);
-      lmp.setUseCompoundDocStore(useCompoundFile);
+      lmp.setUseCompoundFile(true);
       writer = new IndexWriter(dir, conf);
       for(int j=0;j<17;j++) {
         addDoc(writer);
@@ -298,7 +294,6 @@ public class TestDeletionPolicy extends 
           .setMergeScheduler(new SerialMergeScheduler());
       LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
       lmp.setUseCompoundFile(useCompoundFile);
-      lmp.setUseCompoundDocStore(useCompoundFile);
       lmp.setMergeFactor(10);
       IndexWriter writer = new IndexWriter(dir, conf);
       for(int i=0;i<107;i++) {
@@ -311,7 +306,6 @@ public class TestDeletionPolicy extends 
           OpenMode.APPEND).setIndexDeletionPolicy(policy);
       lmp = (LogMergePolicy) conf.getMergePolicy();
       lmp.setUseCompoundFile(useCompoundFile);
-      lmp.setUseCompoundDocStore(useCompoundFile);
       writer = new IndexWriter(dir, conf);
       writer.optimize();
       writer.close();
@@ -488,7 +482,6 @@ public class TestDeletionPolicy extends 
           .setMaxBufferedDocs(10);
       LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
       lmp.setUseCompoundFile(useCompoundFile);
-      lmp.setUseCompoundDocStore(useCompoundFile);
       IndexWriter writer = new IndexWriter(dir, conf);
       for(int i=0;i<107;i++) {
         addDoc(writer);
@@ -499,7 +492,6 @@ public class TestDeletionPolicy extends 
           .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
       lmp = (LogMergePolicy) conf.getMergePolicy();
       lmp.setUseCompoundFile(useCompoundFile);
-      lmp.setUseCompoundDocStore(useCompoundFile);
       writer = new IndexWriter(dir, conf);
       writer.optimize();
       writer.close();
@@ -539,7 +531,6 @@ public class TestDeletionPolicy extends 
             .setMaxBufferedDocs(10);
         LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
         lmp.setUseCompoundFile(useCompoundFile);
-        lmp.setUseCompoundDocStore(useCompoundFile);
         IndexWriter writer = new IndexWriter(dir, conf);
         for(int i=0;i<17;i++) {
           addDoc(writer);
@@ -597,7 +588,6 @@ public class TestDeletionPolicy extends 
           .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy);
       LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
       lmp.setUseCompoundFile(useCompoundFile);
-      lmp.setUseCompoundDocStore(useCompoundFile);
       IndexWriter writer = new IndexWriter(dir, conf);
       writer.close();
       Term searchTerm = new Term("content", "aaa");        
@@ -609,7 +599,6 @@ public class TestDeletionPolicy extends 
             .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
         lmp = (LogMergePolicy) conf.getMergePolicy();
         lmp.setUseCompoundFile(useCompoundFile);
-        lmp.setUseCompoundDocStore(useCompoundFile);
         writer = new IndexWriter(dir, conf);
         for(int j=0;j<17;j++) {
           addDoc(writer);
@@ -630,7 +619,6 @@ public class TestDeletionPolicy extends 
           .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
       lmp = (LogMergePolicy) conf.getMergePolicy();
       lmp.setUseCompoundFile(useCompoundFile);
-      lmp.setUseCompoundDocStore(useCompoundFile);
       writer = new IndexWriter(dir, conf);
       writer.optimize();
       // this is a commit
@@ -706,7 +694,6 @@ public class TestDeletionPolicy extends 
           .setMaxBufferedDocs(10);
       LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
       lmp.setUseCompoundFile(useCompoundFile);
-      lmp.setUseCompoundDocStore(useCompoundFile);
       IndexWriter writer = new IndexWriter(dir, conf);
       writer.close();
       Term searchTerm = new Term("content", "aaa");        
@@ -720,7 +707,6 @@ public class TestDeletionPolicy extends 
             .setMaxBufferedDocs(10);
         lmp = (LogMergePolicy) conf.getMergePolicy();
         lmp.setUseCompoundFile(useCompoundFile);
-        lmp.setUseCompoundDocStore(useCompoundFile);
         writer = new IndexWriter(dir, conf);
         for(int j=0;j<17;j++) {
           addDoc(writer);

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDoc.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDoc.java Sun Dec 19 00:24:04 2010
@@ -192,7 +192,7 @@ public class TestDoc extends LuceneTestC
       SegmentReader r1 = SegmentReader.get(true, si1, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
       SegmentReader r2 = SegmentReader.get(true, si2, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
 
-      SegmentMerger merger = new SegmentMerger(si1.dir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, merged, null, CodecProvider.getDefault(), null);
+      SegmentMerger merger = new SegmentMerger(si1.dir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, merged, null, CodecProvider.getDefault(), null, new FieldInfos());
 
       merger.add(r1);
       merger.add(r2);
@@ -201,8 +201,8 @@ public class TestDoc extends LuceneTestC
       r2.close();
       
       final SegmentInfo info = new SegmentInfo(merged, si1.docCount + si2.docCount, si1.dir,
-                                               useCompoundFile, -1, null, false, merger.hasProx(), merger.getSegmentCodecs(),
-                                               merger.hasVectors());
+                                               useCompoundFile, merger.fieldInfos().hasProx(), merger.getSegmentCodecs(),
+                                               merger.fieldInfos().hasVectors());
       
       if (useCompoundFile) {
         Collection<String> filesToDelete = merger.createCompoundFile(merged + ".cfs", info);

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java Sun Dec 19 00:24:04 2010
@@ -53,7 +53,6 @@ public class TestFieldsReader extends Lu
     dir = newDirectory();
     IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer());
     ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(false);
-    ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundDocStore(false);
     IndexWriter writer = new IndexWriter(dir, conf);
     writer.addDocument(testDoc);
     writer.close();

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java Sun Dec 19 00:24:04 2010
@@ -56,7 +56,6 @@ public class TestIndexFileDeleter extend
       addDoc(writer, i);
     }
     mergePolicy.setUseCompoundFile(false);
-    mergePolicy.setUseCompoundDocStore(false);
     for(;i<45;i++) {
       addDoc(writer, i);
     }

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java Sun Dec 19 00:24:04 2010
@@ -240,7 +240,6 @@ public class TestIndexReaderCloneNorms e
     LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy();
     lmp.setMergeFactor(3);
     lmp.setUseCompoundFile(true);
-    lmp.setUseCompoundDocStore(true);
     iw.close();
   }
 
@@ -293,7 +292,6 @@ public class TestIndexReaderCloneNorms e
     LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
     lmp.setMergeFactor(3);
     lmp.setUseCompoundFile(compound);
-    lmp.setUseCompoundDocStore(compound);
     IndexWriter iw = new IndexWriter(dir, conf);
     for (int i = 0; i < ndocs; i++) {
       iw.addDocument(newDoc());

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Sun Dec 19 00:24:04 2010
@@ -157,7 +157,7 @@ public class TestIndexWriter extends Luc
       String[] startFiles = dir.listAll();
       SegmentInfos infos = new SegmentInfos();
       infos.read(dir);
-      new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null, CodecProvider.getDefault());
+      new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, CodecProvider.getDefault());
       String[] endFiles = dir.listAll();
 
       Arrays.sort(startFiles);
@@ -866,7 +866,7 @@ public class TestIndexWriter extends Luc
       doc.add(newField("field", "aaa", Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
       for(int i=0;i<19;i++)
         writer.addDocument(doc);
-      writer.flush(false, true, true);
+      writer.flush(false, true);
       writer.close();
       SegmentInfos sis = new SegmentInfos();
       sis.read(dir);
@@ -1039,7 +1039,6 @@ public class TestIndexWriter extends Luc
       LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
       lmp.setMergeFactor(2);
       lmp.setUseCompoundFile(false);
-      lmp.setUseCompoundDocStore(false);
       Document doc = new Document();
       String contents = "aa bb cc dd ee ff gg hh ii jj kk";
 
@@ -1075,7 +1074,6 @@ public class TestIndexWriter extends Luc
         writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()));
         LogMergePolicy lmp2 = (LogMergePolicy) writer.getConfig().getMergePolicy();
         lmp2.setUseCompoundFile(false);
-        lmp2.setUseCompoundDocStore(false);
         writer.optimize();
         writer.close();
       }
@@ -2257,7 +2255,6 @@ public class TestIndexWriter extends Luc
         TEST_VERSION_CURRENT, new MockAnalyzer()));
     LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
     lmp.setUseCompoundFile(false);
-    lmp.setUseCompoundDocStore(false);
     ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
     writer.setInfoStream(new PrintStream(bos));
     writer.addDocument(new Document());

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Sun Dec 19 00:24:04 2010
@@ -689,7 +689,6 @@ public class TestIndexWriterDelete exten
 
     LogMergePolicy lmp = (LogMergePolicy) modifier.getConfig().getMergePolicy();
     lmp.setUseCompoundFile(true);
-    lmp.setUseCompoundDocStore(true);
 
     dir.failOn(failure.reset());
 

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java Sun Dec 19 00:24:04 2010
@@ -60,7 +60,7 @@ public class TestIndexWriterMerging exte
         newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).
             setMergePolicy(newLogMergePolicy(2))
     );
-
+    writer.setInfoStream(VERBOSE ? System.out : null);
     writer.addIndexes(indexA, indexB);
     writer.optimize();
     writer.close();

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java Sun Dec 19 00:24:04 2010
@@ -209,7 +209,7 @@ public class TestIndexWriterOnDiskFull e
         System.out.println("TEST: iter=" + iter);
       
       // Start with 100 bytes more than we are currently using:
-      long diskFree = diskUsage+100;
+      long diskFree = diskUsage+_TestUtil.nextInt(random, 50, 200);
       
       int method = iter;
       
@@ -226,12 +226,16 @@ public class TestIndexWriterOnDiskFull e
       }
       
       while(!done) {
+        if (VERBOSE) {
+          System.out.println("TEST: cycle...");
+        }
         
         // Make a new dir that will enforce disk usage:
         MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory(startDir));
         writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND));
         IOException err = null;
-        
+        writer.setInfoStream(VERBOSE ? System.out : null);
+
         MergeScheduler ms = writer.getConfig().getMergeScheduler();
         for(int x=0;x<2;x++) {
           if (ms instanceof ConcurrentMergeScheduler)

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java Sun Dec 19 00:24:04 2010
@@ -237,7 +237,7 @@ public class TestIndexWriterReader exten
     writer.setInfoStream(infoStream);
     // create the index
     createIndexNoClose(!optimize, "index1", writer);
-    writer.flush(false, true, true);
+    writer.flush(false, true);
 
     // create a 2nd index
     Directory dir2 = newDirectory();
@@ -317,7 +317,7 @@ public class TestIndexWriterReader exten
     writer.setInfoStream(infoStream);
     // create the index
     createIndexNoClose(!optimize, "index1", writer);
-    writer.flush(false, true, true);
+    writer.flush(false, true);
     // get a reader
     IndexReader r1 = writer.getReader();
 
@@ -524,7 +524,7 @@ public class TestIndexWriterReader exten
     IndexReader r1 = writer.getReader();
     assertEquals(0, r1.maxDoc());
     createIndexNoClose(false, "index1", writer);
-    writer.flush(!optimize, true, true);
+    writer.flush(!optimize, true);
 
     IndexReader iwr1 = writer.getReader();
     assertEquals(100, iwr1.maxDoc());
@@ -536,7 +536,7 @@ public class TestIndexWriterReader exten
       Document d = createDocument(x, "index1", 5);
       writer.addDocument(d);
     }
-    writer.flush(false, true, true);
+    writer.flush(false, true);
     // verify the reader was reopened internally
     IndexReader iwr2 = writer.getReader();
     assertTrue(iwr2 != r1);

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java Sun Dec 19 00:24:04 2010
@@ -358,47 +358,6 @@ public class TestIndexWriterWithThreads 
     _testMultipleThreadsFailure(new FailOnlyOnAbortOrFlush(true));
   }
 
-  // Throws IOException during DocumentsWriter.closeDocStore
-  private static class FailOnlyInCloseDocStore extends MockDirectoryWrapper.Failure {
-    private boolean onlyOnce;
-    public FailOnlyInCloseDocStore(boolean onlyOnce) {
-      this.onlyOnce = onlyOnce;
-    }
-    @Override
-    public void eval(MockDirectoryWrapper dir)  throws IOException {
-      if (doFail) {
-        StackTraceElement[] trace = new Exception().getStackTrace();
-        for (int i = 0; i < trace.length; i++) {
-          if ("closeDocStore".equals(trace[i].getMethodName())) {
-            if (onlyOnce)
-              doFail = false;
-            throw new IOException("now failing on purpose");
-          }
-        }
-      }
-    }
-  }
-
-  // LUCENE-1130: test IOException in closeDocStore
-  public void testIOExceptionDuringCloseDocStore() throws IOException {
-    _testSingleThreadFailure(new FailOnlyInCloseDocStore(false));
-  }
-
-  // LUCENE-1130: test IOException in closeDocStore
-  public void testIOExceptionDuringCloseDocStoreOnlyOnce() throws IOException {
-    _testSingleThreadFailure(new FailOnlyInCloseDocStore(true));
-  }
-
-  // LUCENE-1130: test IOException in closeDocStore, with threads
-  public void testIOExceptionDuringCloseDocStoreWithThreads() throws Exception {
-    _testMultipleThreadsFailure(new FailOnlyInCloseDocStore(false));
-  }
-
-  // LUCENE-1130: test IOException in closeDocStore, with threads
-  public void testIOExceptionDuringCloseDocStoreWithThreadsOnlyOnce() throws Exception {
-    _testMultipleThreadsFailure(new FailOnlyInCloseDocStore(true));
-  }
-
   // Throws IOException during DocumentsWriter.writeSegment
   private static class FailOnlyInWriteSegment extends MockDirectoryWrapper.Failure {
     private boolean onlyOnce;

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestLazyBug.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestLazyBug.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestLazyBug.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestLazyBug.java Sun Dec 19 00:24:04 2010
@@ -66,8 +66,7 @@ public class TestLazyBug extends LuceneT
           TEST_VERSION_CURRENT, new MockAnalyzer()));
       LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
       lmp.setUseCompoundFile(false);
-      lmp.setUseCompoundDocStore(false);
-      
+
       for (int d = 1; d <= NUM_DOCS; d++) {
         Document doc = new Document();
         for (int f = 1; f <= NUM_FIELDS; f++ ) {

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java Sun Dec 19 00:24:04 2010
@@ -20,6 +20,7 @@ package org.apache.lucene.index;
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -34,6 +35,7 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.Bits;
@@ -113,6 +115,9 @@ public class TestNRTThreads extends Luce
     final AtomicBoolean failed = new AtomicBoolean();
     final AtomicInteger addCount = new AtomicInteger();
     final AtomicInteger delCount = new AtomicInteger();
+
+    final List<String> delIDs = Collections.synchronizedList(new ArrayList<String>());
+
     final long stopTime = System.currentTimeMillis() + RUN_TIME_SEC*1000;
     Thread[] threads = new Thread[NUM_INDEX_THREADS];
     for(int thread=0;thread<NUM_INDEX_THREADS;thread++) {
@@ -152,7 +157,11 @@ public class TestNRTThreads extends Luce
                   for(String id : toDeleteIDs) {
                     writer.deleteDocuments(new Term("id", id));
                   }
-                  delCount.addAndGet(toDeleteIDs.size());
+                  final int count = delCount.addAndGet(toDeleteIDs.size());
+                  if (VERBOSE) {
+                    System.out.println(Thread.currentThread().getName() + ": tot " + count + " deletes");
+                  }
+                  delIDs.addAll(toDeleteIDs);
                   toDeleteIDs.clear();
                 }
                 addCount.getAndIncrement();
@@ -163,6 +172,9 @@ public class TestNRTThreads extends Luce
                 throw new RuntimeException(exc);
               }
             }
+            if (VERBOSE) {
+              System.out.println(Thread.currentThread().getName() + ": indexing done");
+            }
           }
         };
       threads[thread].setDaemon(true);
@@ -315,8 +327,20 @@ public class TestNRTThreads extends Luce
     if (VERBOSE) {
       System.out.println("TEST: done join [" + (System.currentTimeMillis()-t0) + " ms]; addCount=" + addCount + " delCount=" + delCount);
     }
+    
+    final IndexReader r2 = writer.getReader();
+    final IndexSearcher s = new IndexSearcher(r2);
+    for(String id : delIDs) {
+      final TopDocs hits = s.search(new TermQuery(new Term("id", id)), 1);
+      if (hits.totalHits != 0) {
+        fail("doc id=" + id + " is supposed to be deleted, but got docID=" + hits.scoreDocs[0].doc);
+      }
+    }
+    assertEquals("index=" + writer.segString() + " addCount=" + addCount + " delCount=" + delCount, addCount.get() - delCount.get(), r2.numDocs());
+    r2.close();
+
     writer.commit();
-    assertEquals("index=" + writer.segString(), addCount.get() - delCount.get(), writer.numDocs());
+    assertEquals("index=" + writer.segString() + " addCount=" + addCount + " delCount=" + delCount, addCount.get() - delCount.get(), writer.numDocs());
       
     writer.close(false);
     dir.close();

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNoMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNoMergePolicy.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNoMergePolicy.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNoMergePolicy.java Sun Dec 19 00:24:04 2010
@@ -35,16 +35,13 @@ public class TestNoMergePolicy extends L
     assertNull(mp.findMerges(null));
     assertNull(mp.findMergesForOptimize(null, 0, null));
     assertNull(mp.findMergesToExpungeDeletes(null));
-    assertFalse(mp.useCompoundDocStore(null));
     assertFalse(mp.useCompoundFile(null, null));
     mp.close();
   }
 
   @Test
   public void testCompoundFiles() throws Exception {
-    assertFalse(NoMergePolicy.NO_COMPOUND_FILES.useCompoundDocStore(null));
     assertFalse(NoMergePolicy.NO_COMPOUND_FILES.useCompoundFile(null, null));
-    assertTrue(NoMergePolicy.COMPOUND_FILES.useCompoundDocStore(null));
     assertTrue(NoMergePolicy.COMPOUND_FILES.useCompoundFile(null, null));
   }
 

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNorms.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNorms.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNorms.java Sun Dec 19 00:24:04 2010
@@ -154,7 +154,6 @@ public class TestNorms extends LuceneTes
     LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy();
     lmp.setMergeFactor(3);
     lmp.setUseCompoundFile(true);
-    lmp.setUseCompoundDocStore(true);
     iw.close();
   }
 
@@ -199,7 +198,6 @@ public class TestNorms extends LuceneTes
     LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy();
     lmp.setMergeFactor(3);
     lmp.setUseCompoundFile(compound);
-    lmp.setUseCompoundDocStore(compound);
     for (int i = 0; i < ndocs; i++) {
       iw.addDocument(newDoc());
     }

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestOmitTf.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestOmitTf.java Sun Dec 19 00:24:04 2010
@@ -219,7 +219,6 @@ public class TestOmitTf extends LuceneTe
     LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
     lmp.setMergeFactor(2);
     lmp.setUseCompoundFile(false);
-    lmp.setUseCompoundDocStore(false);
     Document d = new Document();
         
     Field f1 = newField("f1", "This field has term freqs", Field.Store.NO, Field.Index.ANALYZED);

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java Sun Dec 19 00:24:04 2010
@@ -137,7 +137,6 @@ public class TestPerFieldCodecSupport ex
     iwconf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer())
         .setOpenMode(OpenMode.APPEND).setCodecProvider(provider);
     ((LogMergePolicy) iwconf.getMergePolicy()).setUseCompoundFile(false);
-    ((LogMergePolicy) iwconf.getMergePolicy()).setUseCompoundDocStore(false);
     ((LogMergePolicy) iwconf.getMergePolicy()).setMergeFactor(10);
     iwconf.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
 
@@ -294,4 +293,4 @@ public class TestPerFieldCodecSupport ex
     }
     dir.close();
   }
-}
\ No newline at end of file
+}

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java Sun Dec 19 00:24:04 2010
@@ -24,7 +24,6 @@ import java.util.Random;
 import java.util.Set;
 
 import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.Document;
 import org.apache.lucene.index.TermsEnum.SeekStatus;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MockDirectoryWrapper;
@@ -73,7 +72,7 @@ public class TestPerSegmentDeletes exten
 
     // flushing without applying deletes means 
     // there will still be deletes in the segment infos
-    writer.flush(false, false, false);
+    writer.flush(false, false);
     assertTrue(writer.bufferedDeletes.any());
     
     // get reader flushes pending deletes
@@ -86,7 +85,7 @@ public class TestPerSegmentDeletes exten
     // merge segments 0 and 1
     // which should apply the delete id:2
     writer.deleteDocuments(new Term("id", "2"));
-    writer.flush(false, false, false);
+    writer.flush(false, false);
     fsmp.doMerge = true;
     fsmp.start = 0;
     fsmp.length = 2;
@@ -179,12 +178,12 @@ public class TestPerSegmentDeletes exten
       writer.addDocument(TestIndexWriterReader.createDocument(x, "5", 2));
       //System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
     }
-    writer.flush(false, true, false);
+    writer.flush(false, false);
     for (int x = 25; x < 30; x++) {
       writer.addDocument(TestIndexWriterReader.createDocument(x, "5", 2));
       //System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
     }
-    writer.flush(false, true, false);
+    writer.flush(false, false);
     
     //System.out.println("infos3:"+writer.segmentInfos);
     
@@ -287,11 +286,6 @@ public class TestPerSegmentDeletes exten
     }
     
     @Override
-    public boolean useCompoundDocStore(SegmentInfos segments) {
-      return useCompoundFile;
-    }
-    
-    @Override
     public boolean useCompoundFile(SegmentInfos segments, SegmentInfo newSegment) {
       return useCompoundFile;
     }

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java Sun Dec 19 00:24:04 2010
@@ -73,14 +73,14 @@ public class TestSegmentMerger extends L
   }
   
   public void testMerge() throws IOException {                             
-    SegmentMerger merger = new SegmentMerger(mergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, mergedSegment, null, CodecProvider.getDefault(), null);
+    SegmentMerger merger = new SegmentMerger(mergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, mergedSegment, null, CodecProvider.getDefault(), null, new FieldInfos());
     merger.add(reader1);
     merger.add(reader2);
     int docsMerged = merger.merge();
     assertTrue(docsMerged == 2);
     //Should be able to open a new SegmentReader against the new directory
-    SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, -1,
-                                                                                     null, false, merger.hasProx(), merger.getSegmentCodecs(), merger.hasVectors()),
+    SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, merger.fieldInfos().hasProx(),
+                                                                                     merger.getSegmentCodecs(), merger.fieldInfos().hasVectors()),
                                                    BufferedIndexInput.BUFFER_SIZE, true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
 
     assertTrue(mergedReader != null);

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java Sun Dec 19 00:24:04 2010
@@ -154,7 +154,6 @@ public class TestStressIndexing2 extends
     w.commit();
     LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy();
     lmp.setUseCompoundFile(false);
-    lmp.setUseCompoundDocStore(false);
     lmp.setMergeFactor(mergeFactor);
     /***
         w.setMaxMergeDocs(Integer.MAX_VALUE);
@@ -211,7 +210,6 @@ public class TestStressIndexing2 extends
       w.setInfoStream(VERBOSE ? System.out : null);
       LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy();
       lmp.setUseCompoundFile(false);
-      lmp.setUseCompoundDocStore(false);
       lmp.setMergeFactor(mergeFactor);
 
       threads = new IndexingThread[nThreads];

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java Sun Dec 19 00:24:04 2010
@@ -277,6 +277,7 @@ public class TestMultiSearcher extends L
     indexSearcher2.close();
     ramDirectory1.close();
     ramDirectory2.close();
+    searcher.close();
   }
 
   /* uncomment this when the highest score is always normalized to 1.0, even when it was < 1.0

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestParallelMultiSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestParallelMultiSearcher.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestParallelMultiSearcher.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestParallelMultiSearcher.java Sun Dec 19 00:24:04 2010
@@ -18,16 +18,34 @@ package org.apache.lucene.search;
  */
  
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.lucene.util._TestUtil;
 
 /**
  * Unit tests for the ParallelMultiSearcher 
  */
 public class TestParallelMultiSearcher extends TestMultiSearcher {
+  List<ExecutorService> pools = new ArrayList<ExecutorService>();
+
+  @Override
+  public void tearDown() throws Exception {
+    for (ExecutorService exec : pools)
+      exec.awaitTermination(1000, TimeUnit.MILLISECONDS);
+    pools.clear();
+    super.tearDown();
+  }
 
   @Override
   protected MultiSearcher getMultiSearcherInstance(Searcher[] searchers)
     throws IOException {
-    return new ParallelMultiSearcher(searchers);
+    ExecutorService exec = Executors.newFixedThreadPool(_TestUtil.nextInt(random, 2, 8));
+    pools.add(exec);
+    return new ParallelMultiSearcher(exec, searchers);
   }
 
 }

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestSort.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestSort.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestSort.java Sun Dec 19 00:24:04 2010
@@ -25,6 +25,10 @@ import java.util.BitSet;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Locale;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
@@ -51,6 +55,7 @@ import org.apache.lucene.store.LockObtai
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.DocIdBitSet;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
 
 /**
  * Unit tests for sorting code.
@@ -611,7 +616,8 @@ public class TestSort extends LuceneTest
     // Don't close the multiSearcher. it would close the full searcher too!
 
     // Do the same for a ParallelMultiSearcher
-                Searcher parallelSearcher=new ParallelMultiSearcher (full);
+    ExecutorService exec = Executors.newFixedThreadPool(_TestUtil.nextInt(random, 2, 8));
+    Searcher parallelSearcher=new ParallelMultiSearcher (exec, full);
 
     sort.setSort (new SortField ("int", SortField.INT),
                                 new SortField ("string", SortField.STRING),
@@ -622,7 +628,8 @@ public class TestSort extends LuceneTest
                                 new SortField ("string", SortField.STRING),
         new SortField ("float", SortField.FLOAT, true) );
     assertMatches (parallelSearcher, queryG, sort, "ZYXW");
-    // Don't close the parallelSearcher. it would close the full searcher too!
+    parallelSearcher.close();
+    exec.awaitTermination(1000, TimeUnit.MILLISECONDS);
   }
 
   // test sorts using a series of fields
@@ -690,8 +697,11 @@ public class TestSort extends LuceneTest
 
   // test a variety of sorts using a parallel multisearcher
   public void testParallelMultiSort() throws Exception {
-    Searcher searcher = new ParallelMultiSearcher (searchX, searchY);
+    ExecutorService exec = Executors.newFixedThreadPool(_TestUtil.nextInt(random, 2, 8));
+    Searcher searcher = new ParallelMultiSearcher (exec, searchX, searchY);
     runMultiSorts(searcher, false);
+    searcher.close();
+    exec.awaitTermination(1000, TimeUnit.MILLISECONDS);
   }
 
   // test that the relevancy scores are the same even if

Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java Sun Dec 19 00:24:04 2010
@@ -336,6 +336,11 @@ public abstract class LuceneTestCase ext
   
   @AfterClass
   public static void afterClassLuceneTestCaseJ4() {
+    int rogueThreads = threadCleanup("test class");
+    if (rogueThreads > 0) {
+      // TODO: fail here once the leaks are fixed.
+      System.err.println("RESOURCE LEAK: test class left " + rogueThreads + " thread(s) running");
+    }
     String codecDescription;
     CodecProvider cp = CodecProvider.getDefault();
 
@@ -434,13 +439,13 @@ public abstract class LuceneTestCase ext
     savedUncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler();
     Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
       public void uncaughtException(Thread t, Throwable e) {
+        testsFailed = true;
         uncaughtExceptions.add(new UncaughtExceptionEntry(t, e));
         if (savedUncaughtExceptionHandler != null)
           savedUncaughtExceptionHandler.uncaughtException(t, e);
       }
     });
     
-    ConcurrentMergeScheduler.setTestMode();
     savedBoolMaxClauseCount = BooleanQuery.getMaxClauseCount();
   }
 
@@ -469,6 +474,18 @@ public abstract class LuceneTestCase ext
     assertTrue("ensure your setUp() calls super.setUp()!!!", setup);
     setup = false;
     BooleanQuery.setMaxClauseCount(savedBoolMaxClauseCount);
+    if (!getClass().getName().startsWith("org.apache.solr")) {
+      int rogueThreads = threadCleanup("test method: '" + getName() + "'");
+      if (rogueThreads > 0) {
+        System.err.println("RESOURCE LEAK: test method: '" + getName() 
+            + "' left " + rogueThreads + " thread(s) running");
+        // TODO: fail, but print seed for now.
+        if (!testsFailed && uncaughtExceptions.isEmpty()) {
+          reportAdditionalFailureInfo();
+        }
+      }
+    }
+    Thread.setDefaultUncaughtExceptionHandler(savedUncaughtExceptionHandler);
     try {
 
       if (!uncaughtExceptions.isEmpty()) {
@@ -496,19 +513,61 @@ public abstract class LuceneTestCase ext
       // isolated in distinct test methods  
       assertSaneFieldCaches(getTestLabel());
 
-      if (ConcurrentMergeScheduler.anyUnhandledExceptions()) {
-        // Clear the failure so that we don't just keep
-        // failing subsequent test cases
-        ConcurrentMergeScheduler.clearUnhandledExceptions();
-        fail("ConcurrentMergeScheduler hit unhandled exceptions");
-      }
     } finally {
       purgeFieldCache(FieldCache.DEFAULT);
     }
-    
-    Thread.setDefaultUncaughtExceptionHandler(savedUncaughtExceptionHandler);
   }
 
+  private final static int THREAD_STOP_GRACE_MSEC = 1000;
+  // jvm-wide list of 'rogue threads' we found, so they only get reported once.
+  private final static IdentityHashMap<Thread,Boolean> rogueThreads = new IdentityHashMap<Thread,Boolean>();
+  
+  /**
+   * Looks for leftover running threads, trying to kill them off,
+   * so they don't fail future tests.
+   * returns the number of rogue threads that it found.
+   */
+  private static int threadCleanup(String context) {
+    // educated guess
+    Thread[] stillRunning = new Thread[Thread.activeCount()+1];
+    int threadCount = 0;
+    int rogueCount = 0;
+    
+    if ((threadCount = Thread.enumerate(stillRunning)) > 1) {
+      while (threadCount == stillRunning.length) {
+        // truncated response
+        stillRunning = new Thread[stillRunning.length*2];
+        threadCount = Thread.enumerate(stillRunning);
+      }
+      
+      for (int i = 0; i < threadCount; i++) {
+        Thread t = stillRunning[i];
+          
+        if (t.isAlive() && 
+            !rogueThreads.containsKey(t) && 
+            t != Thread.currentThread() &&
+            // TODO: TimeLimitingCollector starts a thread statically.... WTF?!
+            !t.getName().equals("TimeLimitedCollector timer thread")) {
+          System.err.println("WARNING: " + context  + " left thread running: " + t);
+          rogueThreads.put(t, true);
+          rogueCount++;
+          // wait on the thread to die of natural causes
+          try {
+            t.join(THREAD_STOP_GRACE_MSEC);
+          } catch (InterruptedException e) { e.printStackTrace(); }
+          // try to stop the thread:
+          t.setUncaughtExceptionHandler(null);
+          Thread.setDefaultUncaughtExceptionHandler(null);
+          t.interrupt();
+          try {
+            t.join(THREAD_STOP_GRACE_MSEC);
+          } catch (InterruptedException e) { e.printStackTrace(); }
+        }
+      }
+    }
+    return rogueCount;
+  }
+  
   /**
    * Asserts that FieldCacheSanityChecker does not detect any
    * problems with FieldCache.DEFAULT.
@@ -688,7 +747,6 @@ public abstract class LuceneTestCase ext
 
   public static LogMergePolicy newLogMergePolicy(Random r) {
     LogMergePolicy logmp = r.nextBoolean() ? new LogDocMergePolicy() : new LogByteSizeMergePolicy();
-    logmp.setUseCompoundDocStore(r.nextBoolean());
     logmp.setUseCompoundFile(r.nextBoolean());
     logmp.setCalibrateSizeByDeletes(r.nextBoolean());
     if (r.nextInt(3) == 2) {
@@ -702,14 +760,12 @@ public abstract class LuceneTestCase ext
   public static LogMergePolicy newLogMergePolicy(boolean useCFS) {
     LogMergePolicy logmp = newLogMergePolicy();
     logmp.setUseCompoundFile(useCFS);
-    logmp.setUseCompoundDocStore(useCFS);
     return logmp;
   }
 
   public static LogMergePolicy newLogMergePolicy(boolean useCFS, int mergeFactor) {
     LogMergePolicy logmp = newLogMergePolicy();
     logmp.setUseCompoundFile(useCFS);
-    logmp.setUseCompoundDocStore(useCFS);
     logmp.setMergeFactor(mergeFactor);
     return logmp;
   }
@@ -773,6 +829,11 @@ public abstract class LuceneTestCase ext
       } catch (ClassCastException e) {
         // TEST_DIRECTORY is not a sub-class of FSDirectory, so draw one at random
         fsdirClass = FS_DIRECTORIES[random.nextInt(FS_DIRECTORIES.length)];
+        
+        if (fsdirClass.indexOf(".") == -1) {// if not fully qualified, assume .store
+          fsdirClass = "org.apache.lucene.store." + fsdirClass;
+        }
+        
         clazz = Class.forName(fsdirClass).asSubclass(FSDirectory.class);
       }
       MockDirectoryWrapper dir = new MockDirectoryWrapper(random, newFSDirectoryImpl(clazz, f, lf));

Modified: lucene/dev/branches/bulkpostings/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/CHANGES.txt?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/CHANGES.txt (original)
+++ lucene/dev/branches/bulkpostings/solr/CHANGES.txt Sun Dec 19 00:24:04 2010
@@ -314,6 +314,8 @@ New Features
   Adding a parameter NOW=<time_in_ms> to the request will override the
   current time.  (Peter Sturge, yonik)
 
+* SOLR-2188: Added "maxTokenLength" argument to the factories for ClassicTokenizer,
+  StandardTokenizer, and UAX29URLEmailTokenizer. (Steven Rowe)
 
 Optimizations
 ----------------------

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/ClassicTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/ClassicTokenizerFactory.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/ClassicTokenizerFactory.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/ClassicTokenizerFactory.java Sun Dec 19 00:24:04 2010
@@ -19,6 +19,8 @@ package org.apache.solr.analysis;
 
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.standard.ClassicTokenizer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
 
 import java.io.Reader;
 import java.util.Map;
@@ -28,13 +30,20 @@ import java.util.Map;
  */
 
 public class ClassicTokenizerFactory extends BaseTokenizerFactory {
+
+  private int maxTokenLength;
+
   @Override
   public void init(Map<String,String> args) {
     super.init(args);
     assureMatchVersion();
+    maxTokenLength = getInt("maxTokenLength", 
+                            StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);
   }
 
   public Tokenizer create(Reader input) {
-    return new ClassicTokenizer(luceneMatchVersion, input);
+    ClassicTokenizer tokenizer = new ClassicTokenizer(luceneMatchVersion, input); 
+    tokenizer.setMaxTokenLength(maxTokenLength);
+    return tokenizer;
   }
 }

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/StandardTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/StandardTokenizerFactory.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/StandardTokenizerFactory.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/StandardTokenizerFactory.java Sun Dec 19 00:24:04 2010
@@ -17,6 +17,7 @@
 
 package org.apache.solr.analysis;
 
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
 
 import java.io.Reader;
@@ -27,13 +28,21 @@ import java.util.Map;
  */
 
 public class StandardTokenizerFactory extends BaseTokenizerFactory {
+  
+  private int maxTokenLength;
+  
   @Override
   public void init(Map<String,String> args) {
     super.init(args);
     assureMatchVersion();
+    maxTokenLength = getInt("maxTokenLength", 
+                            StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);
   }
 
   public StandardTokenizer create(Reader input) {
-    return new StandardTokenizer(luceneMatchVersion, input);
+    StandardTokenizer tokenizer
+      = new StandardTokenizer(luceneMatchVersion, input); 
+    tokenizer.setMaxTokenLength(maxTokenLength);
+    return tokenizer;
   }
 }

Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/UAX29URLEmailTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/UAX29URLEmailTokenizerFactory.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/UAX29URLEmailTokenizerFactory.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/UAX29URLEmailTokenizerFactory.java Sun Dec 19 00:24:04 2010
@@ -20,6 +20,7 @@ package org.apache.solr.analysis;
 
 
 
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer;
 
 import java.io.Reader;
@@ -31,13 +32,20 @@ import java.util.Map;
  */
 
 public class UAX29URLEmailTokenizerFactory extends BaseTokenizerFactory {
+
+  private int maxTokenLength;
+
   @Override
   public void init(Map<String,String> args) {
     super.init(args);
     assureMatchVersion();
+    maxTokenLength = getInt("maxTokenLength",
+                            StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);
   }
 
   public UAX29URLEmailTokenizer create(Reader input) {
-    return new UAX29URLEmailTokenizer(input);
+    UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(input); 
+    tokenizer.setMaxTokenLength(maxTokenLength);
+    return tokenizer;
   }
 }

Modified: lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/analysis/TestStandardFactories.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/analysis/TestStandardFactories.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/analysis/TestStandardFactories.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/analysis/TestStandardFactories.java Sun Dec 19 00:24:04 2010
@@ -19,6 +19,8 @@ package org.apache.solr.analysis;
 
 import java.io.Reader;
 import java.io.StringReader;
+import java.util.HashMap;
+import java.util.Map;
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
@@ -40,6 +42,24 @@ public class TestStandardFactories exten
         new String[] {"Wha\u0301t's", "this", "thing", "do" });
   }
   
+  public void testStandardTokenizerMaxTokenLength() throws Exception {
+    StringBuilder builder = new StringBuilder();
+    for (int i = 0 ; i < 100 ; ++i) {
+      builder.append("abcdefg"); // 7 * 100 = 700 char "word"
+    }
+    String longWord = builder.toString();
+    String content = "one two three " + longWord + " four five six";
+    Reader reader = new StringReader(content);
+    Map<String,String> args = new HashMap<String,String>();
+    args.put("luceneMatchVersion", DEFAULT_VERSION_PARAM.get("luceneMatchVersion"));
+    args.put("maxTokenLength", "1000");
+    StandardTokenizerFactory factory = new StandardTokenizerFactory();
+    factory.init(args);
+    Tokenizer stream = factory.create(reader);
+    assertTokenStreamContents(stream, 
+        new String[] {"one", "two", "three", longWord, "four", "five", "six" });
+  }
+  
   /**
    * Test ClassicTokenizerFactory
    */
@@ -52,6 +72,24 @@ public class TestStandardFactories exten
         new String[] {"What's", "this", "thing", "do" });
   }
   
+  public void testClassicTokenizerMaxTokenLength() throws Exception {
+    StringBuilder builder = new StringBuilder();
+    for (int i = 0 ; i < 100 ; ++i) {
+      builder.append("abcdefg"); // 7 * 100 = 700 char "word"
+    }
+    String longWord = builder.toString();
+    String content = "one two three " + longWord + " four five six";
+    Reader reader = new StringReader(content);
+    Map<String,String> args = new HashMap<String,String>();
+    args.put("luceneMatchVersion", DEFAULT_VERSION_PARAM.get("luceneMatchVersion"));
+    args.put("maxTokenLength", "1000");
+    ClassicTokenizerFactory factory = new ClassicTokenizerFactory();
+    factory.init(args);
+    Tokenizer stream = factory.create(reader);
+    assertTokenStreamContents(stream, 
+        new String[] {"one", "two", "three", longWord, "four", "five", "six" });
+  }
+  
   /**
    * Test ClassicFilterFactory
    */

Modified: lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/analysis/TestUAX29URLEmailTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/analysis/TestUAX29URLEmailTokenizerFactory.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/analysis/TestUAX29URLEmailTokenizerFactory.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/analysis/TestUAX29URLEmailTokenizerFactory.java Sun Dec 19 00:24:04 2010
@@ -19,6 +19,9 @@ package org.apache.solr.analysis;
 
 import java.io.Reader;
 import java.io.StringReader;
+import java.util.HashMap;
+import java.util.Map;
+
 import org.apache.lucene.analysis.Tokenizer;
 
 /**
@@ -152,4 +155,22 @@ public class TestUAX29URLEmailTokenizerF
         }
     );
   }
+
+  public void testMaxTokenLength() throws Exception {
+    StringBuilder builder = new StringBuilder();
+    for (int i = 0 ; i < 100 ; ++i) {
+      builder.append("abcdefg"); // 7 * 100 = 700 char "word"
+    }
+    String longWord = builder.toString();
+    String content = "one two three " + longWord + " four five six";
+    Reader reader = new StringReader(content);
+    Map<String,String> args = new HashMap<String,String>();
+    args.put("luceneMatchVersion", DEFAULT_VERSION_PARAM.get("luceneMatchVersion"));
+    args.put("maxTokenLength", "1000");
+    UAX29URLEmailTokenizerFactory factory = new UAX29URLEmailTokenizerFactory();
+    factory.init(args);
+    Tokenizer stream = factory.create(reader);
+    assertTokenStreamContents(stream, 
+        new String[] {"one", "two", "three", longWord, "four", "five", "six" });
+  }
 }



Mime
View raw message