From commits-return-1160-apmail-lucene-commits-archive=www.apache.org@lucene.apache.org Thu Jul 22 19:36:19 2010
Return-Path: <commits-return-1160-apmail-lucene-commits-archive=www.apache.org@lucene.apache.org>
Delivered-To: apmail-lucene-commits-archive@www.apache.org
Received: (qmail 34470 invoked from network); 22 Jul 2010 19:36:19 -0000
Received: from unknown (HELO mail.apache.org) (140.211.11.3)
  by 140.211.11.9 with SMTP; 22 Jul 2010 19:36:19 -0000
Received: (qmail 65148 invoked by uid 500); 22 Jul 2010 19:36:19 -0000
Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm
Precedence: bulk
List-Help: <mailto:commits-help@lucene.apache.org>
List-Unsubscribe: <mailto:commits-unsubscribe@lucene.apache.org>
List-Post: <mailto:commits@lucene.apache.org>
List-Id: <commits.lucene.apache.org>
Reply-To: dev@lucene.apache.org
Delivered-To: mailing list commits@lucene.apache.org
Received: (qmail 65135 invoked by uid 99); 22 Jul 2010 19:36:19 -0000
Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136)
    by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 22 Jul 2010 19:36:19 +0000
X-ASF-Spam-Status: No, hits=-2000.0 required=10.0
	tests=ALL_TRUSTED
X-Spam-Check-By: apache.org
Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4)
    by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 22 Jul 2010 19:36:15 +0000
Received: by eris.apache.org (Postfix, from userid 65534)
	id 986312388C18; Thu, 22 Jul 2010 19:35:00 +0000 (UTC)
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
Subject: svn commit: r966819 [10/20] - in /lucene/dev/branches/realtime_search: ./
 lucene/ lucene/backwards/ lucene/contrib/ lucene/contrib/benchmark/conf/
 lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/
 lucene/contrib/benchmark/src/...
Date: Thu, 22 Jul 2010 19:34:52 -0000
To: commits@lucene.apache.org
From: buschmi@apache.org
X-Mailer: svnmailer-1.0.8
Message-Id: <20100722193500.986312388C18@eris.apache.org>

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestPayloads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestPayloads.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestPayloads.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestPayloads.java Thu Jul 22 19:34:35 2010
@@ -41,6 +41,7 @@ import org.apache.lucene.store.Directory
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.store.MockRAMDirectory;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.UnicodeUtil;
 import org.apache.lucene.util._TestUtil;
@@ -141,8 +142,6 @@ public class TestPayloads extends Lucene
         analyzer.setPayloadData("f3", "somedata".getBytes(), 0, 3);
         writer.addDocument(d);
 
-        FlexTestUtil.verifyFlexVsPreFlex(rnd, writer);
-
         // force merge
         writer.optimize();
         // flush
@@ -154,7 +153,6 @@ public class TestPayloads extends Lucene
         assertTrue("Payload field bit should be set.", fi.fieldInfo("f2").storePayloads);
         assertTrue("Payload field bit should be set.", fi.fieldInfo("f3").storePayloads);
         reader.close();
-        FlexTestUtil.verifyFlexVsPreFlex(rnd, ram);
     }
 
     // Tests if payloads are correctly stored and loaded using both RamDirectory and FSDirectory
@@ -190,7 +188,7 @@ public class TestPayloads extends Lucene
         Term[] terms = generateTerms(fieldName, numTerms);
         StringBuilder sb = new StringBuilder();
         for (int i = 0; i < terms.length; i++) {
-            sb.append(terms[i].text);
+            sb.append(terms[i].text());
             sb.append(" ");
         }
         String content = sb.toString();
@@ -220,9 +218,7 @@ public class TestPayloads extends Lucene
             writer.addDocument(d);
         }
         
-        FlexTestUtil.verifyFlexVsPreFlex(rnd, writer);
         writer.optimize();
-        FlexTestUtil.verifyFlexVsPreFlex(rnd, writer);
         // flush
         writer.close();
         
@@ -232,83 +228,87 @@ public class TestPayloads extends Lucene
          * first we test if all payloads are stored correctly
          */        
         IndexReader reader = IndexReader.open(dir, true);
-        
+
         byte[] verifyPayloadData = new byte[payloadDataLength];
         offset = 0;
-        TermPositions[] tps = new TermPositions[numTerms];
+        DocsAndPositionsEnum[] tps = new DocsAndPositionsEnum[numTerms];
         for (int i = 0; i < numTerms; i++) {
-            tps[i] = reader.termPositions(terms[i]);
+          tps[i] = MultiFields.getTermPositionsEnum(reader,
+                                                    MultiFields.getDeletedDocs(reader),
+                                                    terms[i].field(),
+                                                    new BytesRef(terms[i].text()));
         }
         
-        while (tps[0].next()) {
+        while (tps[0].nextDoc() != DocsEnum.NO_MORE_DOCS) {
             for (int i = 1; i < numTerms; i++) {
-                tps[i].next();
+                tps[i].nextDoc();
             }
             int freq = tps[0].freq();
 
             for (int i = 0; i < freq; i++) {
                 for (int j = 0; j < numTerms; j++) {
                     tps[j].nextPosition();
-                    tps[j].getPayload(verifyPayloadData, offset);
-                    offset += tps[j].getPayloadLength();
+                    BytesRef br = tps[j].getPayload();
+                    System.arraycopy(br.bytes, br.offset, verifyPayloadData, offset, br.length);
+                    offset += br.length;
                 }
             }
         }
         
-        for (int i = 0; i < numTerms; i++) {
-            tps[i].close();
-        }
-        
         assertByteArrayEquals(payloadData, verifyPayloadData);
         
         /*
          *  test lazy skipping
          */        
-        TermPositions tp = reader.termPositions(terms[0]);
-        tp.next();
+        DocsAndPositionsEnum tp = MultiFields.getTermPositionsEnum(reader,
+                                                                   MultiFields.getDeletedDocs(reader),
+                                                                   terms[0].field(),
+                                                                   new BytesRef(terms[0].text()));
+        tp.nextDoc();
         tp.nextPosition();
         // NOTE: prior rev of this test was failing to first
         // call next here:
-        tp.next();
+        tp.nextDoc();
         // now we don't read this payload
         tp.nextPosition();
-        assertEquals("Wrong payload length.", 1, tp.getPayloadLength());
-        byte[] payload = tp.getPayload(null, 0);
-        assertEquals(payload[0], payloadData[numTerms]);
-        // NOTE: prior rev of this test was failing to first
-        // call next here:
-        tp.next();
+        BytesRef payload = tp.getPayload();
+        assertEquals("Wrong payload length.", 1, payload.length);
+        assertEquals(payload.bytes[payload.offset], payloadData[numTerms]);
+        tp.nextDoc();
         tp.nextPosition();
         
         // we don't read this payload and skip to a different document
-        tp.skipTo(5);
+        tp.advance(5);
         tp.nextPosition();
-        assertEquals("Wrong payload length.", 1, tp.getPayloadLength());
-        payload = tp.getPayload(null, 0);
-        assertEquals(payload[0], payloadData[5 * numTerms]);
+        payload = tp.getPayload();
+        assertEquals("Wrong payload length.", 1, payload.length);
+        assertEquals(payload.bytes[payload.offset], payloadData[5 * numTerms]);
                 
         
         /*
          * Test different lengths at skip points
          */
-        tp.seek(terms[1]);
-        tp.next();
+        tp = MultiFields.getTermPositionsEnum(reader,
+                                              MultiFields.getDeletedDocs(reader),
+                                              terms[1].field(),
+                                              new BytesRef(terms[1].text()));
+        tp.nextDoc();
         tp.nextPosition();
-        assertEquals("Wrong payload length.", 1, tp.getPayloadLength());
-        tp.skipTo(skipInterval - 1);
+        assertEquals("Wrong payload length.", 1, tp.getPayload().length);
+        tp.advance(skipInterval - 1);
         tp.nextPosition();
-        assertEquals("Wrong payload length.", 1, tp.getPayloadLength());
-        tp.skipTo(2 * skipInterval - 1);
+        assertEquals("Wrong payload length.", 1, tp.getPayload().length);
+        tp.advance(2 * skipInterval - 1);
         tp.nextPosition();
-        assertEquals("Wrong payload length.", 1, tp.getPayloadLength());
-        tp.skipTo(3 * skipInterval - 1);
+        assertEquals("Wrong payload length.", 1, tp.getPayload().length);
+        tp.advance(3 * skipInterval - 1);
         tp.nextPosition();
-        assertEquals("Wrong payload length.", 3 * skipInterval - 2 * numDocs - 1, tp.getPayloadLength());
+        assertEquals("Wrong payload length.", 3 * skipInterval - 2 * numDocs - 1, tp.getPayload().length);
         
         /*
          * Test multiple call of getPayload()
          */
-        tp.getPayload(null, 0);
+        assertFalse(tp.hasPayload());
         
         reader.close();
         
@@ -326,23 +326,24 @@ public class TestPayloads extends Lucene
         writer.addDocument(d);
 
         
-        FlexTestUtil.verifyFlexVsPreFlex(rnd, writer);
         writer.optimize();
-        FlexTestUtil.verifyFlexVsPreFlex(rnd, writer);
         // flush
         writer.close();
         
         reader = IndexReader.open(dir, true);
-        tp = reader.termPositions(new Term(fieldName, singleTerm));
-        tp.next();
+        tp = MultiFields.getTermPositionsEnum(reader,
+                                              MultiFields.getDeletedDocs(reader),
+                                              fieldName,
+                                              new BytesRef(singleTerm));
+        tp.nextDoc();
         tp.nextPosition();
-
-        verifyPayloadData = new byte[tp.getPayloadLength()];
-        tp.getPayload(verifyPayloadData, 0);
+        
+        BytesRef br = tp.getPayload();
+        verifyPayloadData = new byte[br.length];
         byte[] portion = new byte[1500];
         System.arraycopy(payloadData, 100, portion, 0, 1500);
         
-        assertByteArrayEquals(portion, verifyPayloadData);
+        assertByteArrayEquals(portion, br.bytes, br.offset, br.length);
         reader.close();
         
     }
@@ -389,6 +390,18 @@ public class TestPayloads extends Lucene
         }
       }    
     
+  void assertByteArrayEquals(byte[] b1, byte[] b2, int b2offset, int b2length) {
+        if (b1.length != b2length) {
+          fail("Byte arrays have different lengths: " + b1.length + ", " + b2length);
+        }
+        
+        for (int i = 0; i < b1.length; i++) {
+          if (b1[i] != b2[b2offset+i]) {
+            fail("Byte arrays different at index " + i + ": " + b1[i] + ", " + b2[b2offset+i]);
+          }
+        }
+      }    
+    
     
     /**
      * This Analyzer uses an WhitespaceTokenizer and PayloadFilter.
@@ -506,19 +519,21 @@ public class TestPayloads extends Lucene
         }
         writer.close();
         IndexReader reader = IndexReader.open(dir, true);
-        TermEnum terms = reader.terms();
-        while (terms.next()) {
-            TermPositions tp = reader.termPositions(terms.term());
-            while(tp.next()) {
-                int freq = tp.freq();
-                for (int i = 0; i < freq; i++) {
-                    tp.nextPosition();
-                    assertEquals(pool.bytesToString(tp.getPayload(new byte[5], 0)), terms.term().text);
-                }
+        TermsEnum terms = MultiFields.getFields(reader).terms(field).iterator();
+        Bits delDocs = MultiFields.getDeletedDocs(reader);
+        DocsAndPositionsEnum tp = null;
+        while (terms.next() != null) {
+          String termText = terms.term().utf8ToString();
+          tp = terms.docsAndPositions(delDocs, tp);
+          while(tp.nextDoc() != DocsEnum.NO_MORE_DOCS) {
+            int freq = tp.freq();
+            for (int i = 0; i < freq; i++) {
+              tp.nextPosition();
+              final BytesRef payload = tp.getPayload();
+              assertEquals(termText, pool.bytesToString(payload.bytes, payload.offset, payload.length));
             }
-            tp.close();
+          }
         }
-        terms.close();
         reader.close();
         
         assertEquals(pool.size(), numThreads);
@@ -537,7 +552,7 @@ public class TestPayloads extends Lucene
             this.pool = pool;
             payload = pool.get();
             generateRandomData(payload);
-            term = pool.bytesToString(payload);
+            term = pool.bytesToString(payload, 0, payload.length);
             first = true;
             payloadAtt = addAttribute(PayloadAttribute.class);
             termAtt = addAttribute(CharTermAttribute.class);
@@ -570,10 +585,9 @@ public class TestPayloads extends Lucene
             }
         }
         
-        private BytesRef utf8Result = new BytesRef(10);
-
-        synchronized String bytesToString(byte[] bytes) {
-            String s = new String(bytes);
+        static String bytesToString(byte[] bytes, int start, int length) {
+            String s = new String(bytes, start, length);
+            BytesRef utf8Result = new BytesRef(10);
             UnicodeUtil.UTF16toUTF8(s, 0, s.length(), utf8Result);
             try {
                 return new String(utf8Result.bytes, 0, utf8Result.length, "UTF-8");

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestPositionBasedTermVectorMapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestPositionBasedTermVectorMapper.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestPositionBasedTermVectorMapper.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestPositionBasedTermVectorMapper.java Thu Jul 22 19:34:35 2010
@@ -15,6 +15,7 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
 
 import java.io.IOException;
@@ -65,7 +66,7 @@ public class TestPositionBasedTermVector
     //Test single position
     for (int i = 0; i < tokens.length; i++) {
       String token = tokens[i];
-      mapper.map(token, 1, null, thePositions[i]);
+      mapper.map(new BytesRef(token), 1, null, thePositions[i]);
 
     }
     Map<String,Map<Integer,PositionBasedTermVectorMapper.TVPositionInfo>> map = mapper.getFieldToTerms();

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java Thu Jul 22 19:34:35 2010
@@ -23,6 +23,7 @@ import org.apache.lucene.store.Directory
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.util.BytesRef;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -85,9 +86,12 @@ public class TestSegmentMerger extends L
     assertTrue(newDoc2 != null);
     assertTrue(DocHelper.numFields(newDoc2) == DocHelper.numFields(doc2) - DocHelper.unstored.size());
     
-    TermDocs termDocs = mergedReader.termDocs(new Term(DocHelper.TEXT_FIELD_2_KEY, "field"));
+    DocsEnum termDocs = MultiFields.getTermDocsEnum(mergedReader,
+                                                    MultiFields.getDeletedDocs(mergedReader),
+                                                    DocHelper.TEXT_FIELD_2_KEY,
+                                                    new BytesRef("field"));
     assertTrue(termDocs != null);
-    assertTrue(termDocs.next() == true);
+    assertTrue(termDocs.nextDoc() != DocsEnum.NO_MORE_DOCS);
     
     Collection<String> stored = mergedReader.getFieldNames(IndexReader.FieldOption.INDEXED_WITH_TERMVECTOR);
     assertTrue(stored != null);
@@ -96,7 +100,7 @@ public class TestSegmentMerger extends L
     
     TermFreqVector vector = mergedReader.getTermFreqVector(0, DocHelper.TEXT_FIELD_2_KEY);
     assertTrue(vector != null);
-    String [] terms = vector.getTerms();
+    BytesRef [] terms = vector.getTerms();
     assertTrue(terms != null);
     //System.out.println("Terms size: " + terms.length);
     assertTrue(terms.length == 3);
@@ -106,7 +110,7 @@ public class TestSegmentMerger extends L
     assertTrue(vector instanceof TermPositionVector == true);
     
     for (int i = 0; i < terms.length; i++) {
-      String term = terms[i];
+      String term = terms[i].utf8ToString();
       int freq = freqs[i];
       //System.out.println("Term: " + term + " Freq: " + freq);
       assertTrue(DocHelper.FIELD_2_TEXT.indexOf(term) != -1);

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java Thu Jul 22 19:34:35 2010
@@ -23,6 +23,7 @@ import java.util.Iterator;
 import java.util.List;
 
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.BytesRef;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Fieldable;
@@ -113,33 +114,40 @@ public class TestSegmentReader extends L
   } 
   
   public void testTerms() throws IOException {
-    TermEnum terms = reader.terms();
-    assertTrue(terms != null);
-    while (terms.next() == true)
-    {
-      Term term = terms.term();
-      assertTrue(term != null);
-      //System.out.println("Term: " + term);
-      String fieldValue = (String)DocHelper.nameValues.get(term.field());
-      assertTrue(fieldValue.indexOf(term.text()) != -1);
+    FieldsEnum fields = MultiFields.getFields(reader).iterator();
+    String field;
+    while((field = fields.next()) != null) {
+      TermsEnum terms = fields.terms();
+      while(terms.next() != null) {
+        BytesRef term = terms.term();
+        assertTrue(term != null);
+        String fieldValue = (String) DocHelper.nameValues.get(field);
+        assertTrue(fieldValue.indexOf(term.utf8ToString()) != -1);
+      }
     }
     
-    TermDocs termDocs = reader.termDocs();
-    assertTrue(termDocs != null);
-    termDocs.seek(new Term(DocHelper.TEXT_FIELD_1_KEY, "field"));
-    assertTrue(termDocs.next() == true);
+    DocsEnum termDocs = MultiFields.getTermDocsEnum(reader,
+                                                    MultiFields.getDeletedDocs(reader),
+                                                    DocHelper.TEXT_FIELD_1_KEY,
+                                                    new BytesRef("field"));
+    assertTrue(termDocs.nextDoc() != DocsEnum.NO_MORE_DOCS);
+
+    termDocs = MultiFields.getTermDocsEnum(reader,
+                                           MultiFields.getDeletedDocs(reader),
+                                           DocHelper.NO_NORMS_KEY,
+                                           new BytesRef(DocHelper.NO_NORMS_TEXT));
 
-    termDocs.seek(new Term(DocHelper.NO_NORMS_KEY,  DocHelper.NO_NORMS_TEXT));
-    assertTrue(termDocs.next() == true);
+    assertTrue(termDocs.nextDoc() != DocsEnum.NO_MORE_DOCS);
 
     
-    TermPositions positions = reader.termPositions();
-    assertTrue(positions != null);
-    positions.seek(new Term(DocHelper.TEXT_FIELD_1_KEY, "field"));
+    DocsAndPositionsEnum positions = MultiFields.getTermPositionsEnum(reader,
+                                                                      MultiFields.getDeletedDocs(reader),
+                                                                      DocHelper.TEXT_FIELD_1_KEY,
+                                                                      new BytesRef("field"));
     // NOTE: prior rev of this test was failing to first
     // call next here:
-    assertTrue(positions.next());
-    assertTrue(positions.doc() == 0);
+    assertTrue(positions.nextDoc() != DocsEnum.NO_MORE_DOCS);
+    assertTrue(positions.docID() == 0);
     assertTrue(positions.nextPosition() >= 0);
   }    
   
@@ -184,11 +192,11 @@ public class TestSegmentReader extends L
   public void testTermVectors() throws IOException {
     TermFreqVector result = reader.getTermFreqVector(0, DocHelper.TEXT_FIELD_2_KEY);
     assertTrue(result != null);
-    String [] terms = result.getTerms();
+    BytesRef [] terms = result.getTerms();
     int [] freqs = result.getTermFrequencies();
     assertTrue(terms != null && terms.length == 3 && freqs != null && freqs.length == 3);
     for (int i = 0; i < terms.length; i++) {
-      String term = terms[i];
+      String term = terms[i].utf8ToString();
       int freq = freqs[i];
       assertTrue(DocHelper.FIELD_2_TEXT.indexOf(term) != -1);
       assertTrue(freq > 0);

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java Thu Jul 22 19:34:35 2010
@@ -24,6 +24,7 @@ import org.apache.lucene.store.Directory
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.util.BytesRef;
 
 import java.io.IOException;
 
@@ -56,11 +57,12 @@ public class TestSegmentTermDocs extends
     SegmentReader reader = SegmentReader.get(true, info, indexDivisor);
     assertTrue(reader != null);
     assertEquals(indexDivisor, reader.getTermInfosIndexDivisor());
-    TermDocs termDocs = reader.termDocs();
-    assertTrue(termDocs != null);
-    termDocs.seek(new Term(DocHelper.TEXT_FIELD_2_KEY, "field"));
-    if (termDocs.next() == true)    {
-      int docId = termDocs.doc();
+
+    TermsEnum terms = reader.fields().terms(DocHelper.TEXT_FIELD_2_KEY).iterator();
+    terms.seek(new BytesRef("field"));
+    DocsEnum termDocs = terms.docs(reader.getDeletedDocs(), null);
+    if (termDocs.nextDoc() != DocsEnum.NO_MORE_DOCS)    {
+      int docId = termDocs.docID();
       assertTrue(docId == 0);
       int freq = termDocs.freq();
       assertTrue(freq == 3);  
@@ -77,20 +79,21 @@ public class TestSegmentTermDocs extends
       //After adding the document, we should be able to read it back in
       SegmentReader reader = SegmentReader.get(true, info, indexDivisor);
       assertTrue(reader != null);
-      TermDocs termDocs = reader.termDocs();
-      assertTrue(termDocs != null);
-      termDocs.seek(new Term("textField2", "bad"));
-      assertTrue(termDocs.next() == false);
+      DocsEnum termDocs = reader.termDocsEnum(reader.getDeletedDocs(),
+                                              "textField2",
+                                              new BytesRef("bad"));
+
+      assertNull(termDocs);
       reader.close();
     }
     {
       //After adding the document, we should be able to read it back in
       SegmentReader reader = SegmentReader.get(true, info, indexDivisor);
       assertTrue(reader != null);
-      TermDocs termDocs = reader.termDocs();
-      assertTrue(termDocs != null);
-      termDocs.seek(new Term("junk", "bad"));
-      assertTrue(termDocs.next() == false);
+      DocsEnum termDocs = reader.termDocsEnum(reader.getDeletedDocs(),
+                                              "junk",
+                                              new BytesRef("bad"));
+      assertNull(termDocs);
       reader.close();
     }
   }
@@ -121,105 +124,125 @@ public class TestSegmentTermDocs extends
     
     IndexReader reader = IndexReader.open(dir, null, true, indexDivisor);
 
-    TermDocs tdocs = reader.termDocs();
+    DocsEnum tdocs = MultiFields.getTermDocsEnum(reader,
+                                                 MultiFields.getDeletedDocs(reader),
+                                                 ta.field(),
+                                                 new BytesRef(ta.text()));
     
     // without optimization (assumption skipInterval == 16)
     
     // with next
-    tdocs.seek(ta);
-    assertTrue(tdocs.next());
-    assertEquals(0, tdocs.doc());
-    assertEquals(4, tdocs.freq());
-    assertTrue(tdocs.next());
-    assertEquals(1, tdocs.doc());
-    assertEquals(4, tdocs.freq());
-    assertTrue(tdocs.skipTo(0));
-    assertEquals(2, tdocs.doc());
-    assertTrue(tdocs.skipTo(4));
-    assertEquals(4, tdocs.doc());
-    assertTrue(tdocs.skipTo(9));
-    assertEquals(9, tdocs.doc());
-    assertFalse(tdocs.skipTo(10));
+    assertTrue(tdocs.nextDoc() != DocsEnum.NO_MORE_DOCS);
+    assertEquals(0, tdocs.docID());
+    assertEquals(4, tdocs.freq());
+    assertTrue(tdocs.nextDoc() != DocsEnum.NO_MORE_DOCS);
+    assertEquals(1, tdocs.docID());
+    assertEquals(4, tdocs.freq());
+    assertTrue(tdocs.advance(0) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(2, tdocs.docID());
+    assertTrue(tdocs.advance(4) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(4, tdocs.docID());
+    assertTrue(tdocs.advance(9) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(9, tdocs.docID());
+    assertFalse(tdocs.advance(10) != DocsEnum.NO_MORE_DOCS);
     
     // without next
-    tdocs.seek(ta);
-    assertTrue(tdocs.skipTo(0));
-    assertEquals(0, tdocs.doc());
-    assertTrue(tdocs.skipTo(4));
-    assertEquals(4, tdocs.doc());
-    assertTrue(tdocs.skipTo(9));
-    assertEquals(9, tdocs.doc());
-    assertFalse(tdocs.skipTo(10));
+    tdocs = MultiFields.getTermDocsEnum(reader,
+                                        MultiFields.getDeletedDocs(reader),
+                                        ta.field(),
+                                        new BytesRef(ta.text()));
+    
+    assertTrue(tdocs.advance(0) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(0, tdocs.docID());
+    assertTrue(tdocs.advance(4) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(4, tdocs.docID());
+    assertTrue(tdocs.advance(9) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(9, tdocs.docID());
+    assertFalse(tdocs.advance(10) != DocsEnum.NO_MORE_DOCS);
     
     // exactly skipInterval documents and therefore with optimization
     
     // with next
-    tdocs.seek(tb);
-    assertTrue(tdocs.next());
-    assertEquals(10, tdocs.doc());
-    assertEquals(4, tdocs.freq());
-    assertTrue(tdocs.next());
-    assertEquals(11, tdocs.doc());
-    assertEquals(4, tdocs.freq());
-    assertTrue(tdocs.skipTo(5));
-    assertEquals(12, tdocs.doc());
-    assertTrue(tdocs.skipTo(15));
-    assertEquals(15, tdocs.doc());
-    assertTrue(tdocs.skipTo(24));
-    assertEquals(24, tdocs.doc());
-    assertTrue(tdocs.skipTo(25));
-    assertEquals(25, tdocs.doc());
-    assertFalse(tdocs.skipTo(26));
+    tdocs = MultiFields.getTermDocsEnum(reader,
+                                        MultiFields.getDeletedDocs(reader),
+                                        tb.field(),
+                                        new BytesRef(tb.text()));
+
+    assertTrue(tdocs.nextDoc() != DocsEnum.NO_MORE_DOCS);
+    assertEquals(10, tdocs.docID());
+    assertEquals(4, tdocs.freq());
+    assertTrue(tdocs.nextDoc() != DocsEnum.NO_MORE_DOCS);
+    assertEquals(11, tdocs.docID());
+    assertEquals(4, tdocs.freq());
+    assertTrue(tdocs.advance(5) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(12, tdocs.docID());
+    assertTrue(tdocs.advance(15) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(15, tdocs.docID());
+    assertTrue(tdocs.advance(24) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(24, tdocs.docID());
+    assertTrue(tdocs.advance(25) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(25, tdocs.docID());
+    assertFalse(tdocs.advance(26) != DocsEnum.NO_MORE_DOCS);
     
     // without next
-    tdocs.seek(tb);
-    assertTrue(tdocs.skipTo(5));
-    assertEquals(10, tdocs.doc());
-    assertTrue(tdocs.skipTo(15));
-    assertEquals(15, tdocs.doc());
-    assertTrue(tdocs.skipTo(24));
-    assertEquals(24, tdocs.doc());
-    assertTrue(tdocs.skipTo(25));
-    assertEquals(25, tdocs.doc());
-    assertFalse(tdocs.skipTo(26));
+    tdocs = MultiFields.getTermDocsEnum(reader,
+                                        MultiFields.getDeletedDocs(reader),
+                                        tb.field(),
+                                        new BytesRef(tb.text()));
+    
+    assertTrue(tdocs.advance(5) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(10, tdocs.docID());
+    assertTrue(tdocs.advance(15) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(15, tdocs.docID());
+    assertTrue(tdocs.advance(24) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(24, tdocs.docID());
+    assertTrue(tdocs.advance(25) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(25, tdocs.docID());
+    assertFalse(tdocs.advance(26) != DocsEnum.NO_MORE_DOCS);
     
     // much more than skipInterval documents and therefore with optimization
     
     // with next
-    tdocs.seek(tc);
-    assertTrue(tdocs.next());
-    assertEquals(26, tdocs.doc());
-    assertEquals(4, tdocs.freq());
-    assertTrue(tdocs.next());
-    assertEquals(27, tdocs.doc());
-    assertEquals(4, tdocs.freq());
-    assertTrue(tdocs.skipTo(5));
-    assertEquals(28, tdocs.doc());
-    assertTrue(tdocs.skipTo(40));
-    assertEquals(40, tdocs.doc());
-    assertTrue(tdocs.skipTo(57));
-    assertEquals(57, tdocs.doc());
-    assertTrue(tdocs.skipTo(74));
-    assertEquals(74, tdocs.doc());
-    assertTrue(tdocs.skipTo(75));
-    assertEquals(75, tdocs.doc());
-    assertFalse(tdocs.skipTo(76));
+    tdocs = MultiFields.getTermDocsEnum(reader,
+                                        MultiFields.getDeletedDocs(reader),
+                                        tc.field(),
+                                        new BytesRef(tc.text()));
+
+    assertTrue(tdocs.nextDoc() != DocsEnum.NO_MORE_DOCS);
+    assertEquals(26, tdocs.docID());
+    assertEquals(4, tdocs.freq());
+    assertTrue(tdocs.nextDoc() != DocsEnum.NO_MORE_DOCS);
+    assertEquals(27, tdocs.docID());
+    assertEquals(4, tdocs.freq());
+    assertTrue(tdocs.advance(5) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(28, tdocs.docID());
+    assertTrue(tdocs.advance(40) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(40, tdocs.docID());
+    assertTrue(tdocs.advance(57) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(57, tdocs.docID());
+    assertTrue(tdocs.advance(74) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(74, tdocs.docID());
+    assertTrue(tdocs.advance(75) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(75, tdocs.docID());
+    assertFalse(tdocs.advance(76) != DocsEnum.NO_MORE_DOCS);
     
     //without next
-    tdocs.seek(tc);
-    assertTrue(tdocs.skipTo(5));
-    assertEquals(26, tdocs.doc());
-    assertTrue(tdocs.skipTo(40));
-    assertEquals(40, tdocs.doc());
-    assertTrue(tdocs.skipTo(57));
-    assertEquals(57, tdocs.doc());
-    assertTrue(tdocs.skipTo(74));
-    assertEquals(74, tdocs.doc());
-    assertTrue(tdocs.skipTo(75));
-    assertEquals(75, tdocs.doc());
-    assertFalse(tdocs.skipTo(76));
+    tdocs = MultiFields.getTermDocsEnum(reader,
+                                        MultiFields.getDeletedDocs(reader),
+                                        tc.field(),
+                                        new BytesRef(tc.text()));
+    assertTrue(tdocs.advance(5) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(26, tdocs.docID());
+    assertTrue(tdocs.advance(40) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(40, tdocs.docID());
+    assertTrue(tdocs.advance(57) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(57, tdocs.docID());
+    assertTrue(tdocs.advance(74) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(74, tdocs.docID());
+    assertTrue(tdocs.advance(75) != DocsEnum.NO_MORE_DOCS);
+    assertEquals(75, tdocs.docID());
+    assertFalse(tdocs.advance(76) != DocsEnum.NO_MORE_DOCS);
     
-    tdocs.close();
     reader.close();
     dir.close();
   }

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java Thu Jul 22 19:34:35 2010
@@ -20,6 +20,7 @@ package org.apache.lucene.index;
 import java.io.IOException;
 
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
@@ -83,36 +84,32 @@ public class TestSegmentTermEnum extends
       throws IOException
   {
       IndexReader reader = IndexReader.open(dir, true);
-      TermEnum termEnum = null;
+      TermsEnum termEnum = MultiFields.getTerms(reader, "content").iterator();
 
     // create enumeration of all terms
-    termEnum = reader.terms();
     // go to the first term (aaa)
     termEnum.next();
     // assert that term is 'aaa'
-    assertEquals("aaa", termEnum.term().text());
+    assertEquals("aaa", termEnum.term().utf8ToString());
     assertEquals(200, termEnum.docFreq());
     // go to the second term (bbb)
     termEnum.next();
     // assert that term is 'bbb'
-    assertEquals("bbb", termEnum.term().text());
+    assertEquals("bbb", termEnum.term().utf8ToString());
     assertEquals(100, termEnum.docFreq());
 
-    termEnum.close();
 
-
-    // create enumeration of terms after term 'aaa', including 'aaa'
-    termEnum = reader.terms(new Term("content", "aaa"));
+    // create enumeration of terms after term 'aaa',
+    // including 'aaa'
+    termEnum.seek(new BytesRef("aaa"));
     // assert that term is 'aaa'
-    assertEquals("aaa", termEnum.term().text());
+    assertEquals("aaa", termEnum.term().utf8ToString());
     assertEquals(200, termEnum.docFreq());
     // go to term 'bbb'
     termEnum.next();
     // assert that term is 'bbb'
-    assertEquals("bbb", termEnum.term().text());
+    assertEquals("bbb", termEnum.term().utf8ToString());
     assertEquals(100, termEnum.docFreq());
-
-    termEnum.close();
   }
 
   private void addDoc(IndexWriter writer, String value) throws IOException

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestStressIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestStressIndexing.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestStressIndexing.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestStressIndexing.java Thu Jul 22 19:34:35 2010
@@ -153,8 +153,6 @@ public class TestStressIndexing extends 
 
     modifier.close();
 
-    FlexTestUtil.verifyFlexVsPreFlex(RANDOM, directory);
-
     for(int i=0;i<numThread;i++)
       assertTrue(! threads[i].failed);
 

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java Thu Jul 22 19:34:35 2010
@@ -71,8 +71,6 @@ public class TestStressIndexing2 extends
     IndexReader r = dw.writer.getReader();
     dw.writer.commit();
     verifyEquals(r, dir, "id");
-    FlexTestUtil.verifyFlexVsPreFlex(this.r, r);
-    FlexTestUtil.verifyFlexVsPreFlex(this.r, dir);
     r.close();
     dw.writer.close();
     dir.close();
@@ -94,8 +92,6 @@ public class TestStressIndexing2 extends
     // verifyEquals(dir2, dir2, "id");
 
     verifyEquals(dir1, dir2, "id");
-    FlexTestUtil.verifyFlexVsPreFlex(r, dir1);
-    FlexTestUtil.verifyFlexVsPreFlex(r, dir2);
   }
 
   public void testMultiConfig() throws Throwable {
@@ -121,9 +117,6 @@ public class TestStressIndexing2 extends
       indexSerial(docs, dir2);
       //System.out.println("TEST: verify");
       verifyEquals(dir1, dir2, "id");
-
-      FlexTestUtil.verifyFlexVsPreFlex(r, dir1);
-      FlexTestUtil.verifyFlexVsPreFlex(r, dir2);
     }
   }
 
@@ -523,8 +516,8 @@ public class TestStressIndexing2 extends
         System.out.println("v1=" + v1 + " v2=" + v2 + " i=" + i + " of " + d1.length);
       assertEquals(v1.size(), v2.size());
       int numTerms = v1.size();
-      String[] terms1 = v1.getTerms();
-      String[] terms2 = v2.getTerms();
+      BytesRef[] terms1 = v1.getTerms();
+      BytesRef[] terms2 = v2.getTerms();
       int[] freq1 = v1.getTermFrequencies();
       int[] freq2 = v2.getTermFrequencies();
       for(int j=0;j<numTerms;j++) {

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java Thu Jul 22 19:34:35 2010
@@ -32,6 +32,7 @@ import org.apache.lucene.analysis.tokena
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.store.MockRAMDirectory;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
 
 public class TestTermVectorsReader extends LuceneTestCase {
@@ -170,11 +171,11 @@ public class TestTermVectorsReader exten
     for (int j = 0; j < 5; j++) {
       TermFreqVector vector = reader.get(j, testFields[0]);
       assertTrue(vector != null);
-      String[] terms = vector.getTerms();
+      BytesRef[] terms = vector.getTerms();
       assertTrue(terms != null);
       assertTrue(terms.length == testTerms.length);
       for (int i = 0; i < terms.length; i++) {
-        String term = terms[i];
+        String term = terms[i].utf8ToString();
         //System.out.println("Term: " + term);
         assertTrue(term.equals(testTerms[i]));
       }
@@ -184,14 +185,14 @@ public class TestTermVectorsReader exten
   public void testPositionReader() throws IOException {
     TermVectorsReader reader = new TermVectorsReader(dir, seg, fieldInfos);
     TermPositionVector vector;
-    String[] terms;
+    BytesRef[] terms;
     vector = (TermPositionVector) reader.get(0, testFields[0]);
     assertTrue(vector != null);
     terms = vector.getTerms();
     assertTrue(terms != null);
     assertTrue(terms.length == testTerms.length);
     for (int i = 0; i < terms.length; i++) {
-      String term = terms[i];
+      String term = terms[i].utf8ToString();
       //System.out.println("Term: " + term);
       assertTrue(term.equals(testTerms[i]));
       int[] positions = vector.getTermPositions(i);
@@ -217,7 +218,7 @@ public class TestTermVectorsReader exten
     assertTrue(terms != null);
     assertTrue(terms.length == testTerms.length);
     for (int i = 0; i < terms.length; i++) {
-      String term = terms[i];
+      String term = terms[i].utf8ToString();
       //System.out.println("Term: " + term);
       assertTrue(term.equals(testTerms[i]));
     }
@@ -227,11 +228,11 @@ public class TestTermVectorsReader exten
     TermVectorsReader reader = new TermVectorsReader(dir, seg, fieldInfos);
     TermPositionVector vector = (TermPositionVector) reader.get(0, testFields[0]);
     assertTrue(vector != null);
-    String[] terms = vector.getTerms();
+    BytesRef[] terms = vector.getTerms();
     assertTrue(terms != null);
     assertTrue(terms.length == testTerms.length);
     for (int i = 0; i < terms.length; i++) {
-      String term = terms[i];
+      String term = terms[i].utf8ToString();
       //System.out.println("Term: " + term);
       assertTrue(term.equals(testTerms[i]));
       int[] positions = vector.getTermPositions(i);
@@ -413,7 +414,7 @@ public class TestTermVectorsReader exten
     }
 
     @Override
-    public void map(String term, int frequency, TermVectorOffsetInfo[] offsets, int[] positions) {
+    public void map(BytesRef term, int frequency, TermVectorOffsetInfo[] offsets, int[] positions) {
       if (documentNumber == -1) {
         throw new RuntimeException("Documentnumber should be set at this point!");
       }

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestTermdocPerf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestTermdocPerf.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestTermdocPerf.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestTermdocPerf.java Thu Jul 22 19:34:35 2010
@@ -30,6 +30,7 @@ import org.apache.lucene.index.IndexWrit
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.BytesRef;
 
 class RepeatingTokenStream extends TokenStream {
   public int num;
@@ -94,16 +95,18 @@ public class TestTermdocPerf extends Luc
     if (VERBOSE) System.out.println("milliseconds for creation of " + ndocs + " docs = " + (end-start));
 
     IndexReader reader = IndexReader.open(dir, true);
-    TermEnum tenum = reader.terms(new Term("foo","val"));
-    TermDocs tdocs = reader.termDocs();
+
+    TermsEnum tenum = MultiFields.getTerms(reader, "foo").iterator();
 
     start = System.currentTimeMillis();
 
     int ret=0;
+    DocsEnum tdocs = null;
     for (int i=0; i<iter; i++) {
-      tdocs.seek(tenum);
-      while (tdocs.next()) {
-        ret += tdocs.doc();
+      tenum.seek(new BytesRef("val"));
+      tdocs = tenum.docs(MultiFields.getDeletedDocs(reader), tdocs);
+      while (tdocs.nextDoc() != DocsEnum.NO_MORE_DOCS) {
+        ret += tdocs.docID();
       }
     }
 

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java Thu Jul 22 19:34:35 2010
@@ -1,5 +1,22 @@
 package org.apache.lucene.index.codecs.intblock;
 
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.store.*;
 import org.apache.lucene.index.codecs.sep.*;
@@ -34,7 +51,7 @@ public class TestIntBlockCodec extends L
     out.close();
 
     IntIndexInput in = new SimpleIntBlockIndexInput(dir, "test", 128);
-    IntIndexInput.Reader r = in.reader();
+    in.reader();
     // read no ints
     in.close();
     dir.close();

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java Thu Jul 22 19:34:35 2010
@@ -104,9 +104,9 @@ public class TestMultiAnalyzer extends B
     // direct call to (super's) getFieldQuery to demonstrate differnce
     // between phrase and multiphrase with modified default slop
     assertEquals("\"foo bar\"~99",
-                 qp.getSuperFieldQuery("","foo bar").toString());
+                 qp.getSuperFieldQuery("","foo bar", true).toString());
     assertEquals("\"(multi multi2) bar\"~99",
-                 qp.getSuperFieldQuery("","multi bar").toString());
+                 qp.getSuperFieldQuery("","multi bar", true).toString());
 
     
     // ask sublcass to parse phrase with modified default slop
@@ -243,15 +243,15 @@ public class TestMultiAnalyzer extends B
         }
 
         /** expose super's version */
-        public Query getSuperFieldQuery(String f, String t) 
+        public Query getSuperFieldQuery(String f, String t, boolean quoted) 
             throws ParseException {
-            return super.getFieldQuery(f,t);
+            return super.getFieldQuery(f,t,quoted);
         }
         /** wrap super's version */
         @Override
-        protected Query getFieldQuery(String f, String t)
+        protected Query getFieldQuery(String f, String t, boolean quoted)
             throws ParseException {
-            return new DumbQueryWrapper(getSuperFieldQuery(f,t));
+            return new DumbQueryWrapper(getSuperFieldQuery(f,t,quoted));
         }
     }
     

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java Thu Jul 22 19:34:35 2010
@@ -34,6 +34,7 @@ import org.apache.lucene.analysis.MockTo
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.DateField;
@@ -45,6 +46,7 @@ import org.apache.lucene.index.IndexWrit
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.IndexSearcher;
@@ -248,7 +250,103 @@ public class TestQueryParser extends Loc
 	 assertQueryEquals("term\u3000term\u3000term", null, "term\u0020term\u0020term");
 	 assertQueryEquals("用語\u3000用語\u3000用語", null, "用語\u0020用語\u0020用語");
   }
+
+  //individual CJK chars as terms, like StandardAnalyzer
+  private class SimpleCJKTokenizer extends Tokenizer {
+    private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+
+    public SimpleCJKTokenizer(Reader input) {
+      super(input);
+    }
+
+    @Override
+    public boolean incrementToken() throws IOException {
+      int ch = input.read();
+      if (ch < 0)
+        return false;
+      clearAttributes();
+      termAtt.setEmpty().append((char) ch);
+      return true;
+    }
+  }
+
+  private class SimpleCJKAnalyzer extends Analyzer {
+    @Override
+    public TokenStream tokenStream(String fieldName, Reader reader) {
+      return new SimpleCJKTokenizer(reader);
+    }
+  }
+
+  public void testCJKTerm() throws Exception {
+    // individual CJK chars as terms
+    SimpleCJKAnalyzer analyzer = new SimpleCJKAnalyzer(); 
+    
+    BooleanQuery expected = new BooleanQuery();
+    expected.add(new TermQuery(new Term("field", "中")), BooleanClause.Occur.SHOULD);
+    expected.add(new TermQuery(new Term("field", "国")), BooleanClause.Occur.SHOULD);
+    
+    assertEquals(expected, getQuery("中国", analyzer));
+  }
   
+  public void testCJKBoostedTerm() throws Exception {
+    // individual CJK chars as terms
+    SimpleCJKAnalyzer analyzer = new SimpleCJKAnalyzer();
+    
+    BooleanQuery expected = new BooleanQuery();
+    expected.setBoost(0.5f);
+    expected.add(new TermQuery(new Term("field", "中")), BooleanClause.Occur.SHOULD);
+    expected.add(new TermQuery(new Term("field", "国")), BooleanClause.Occur.SHOULD);
+    
+    assertEquals(expected, getQuery("中国^0.5", analyzer));
+  }
+  
+  public void testCJKPhrase() throws Exception {
+    // individual CJK chars as terms
+    SimpleCJKAnalyzer analyzer = new SimpleCJKAnalyzer();
+    
+    PhraseQuery expected = new PhraseQuery();
+    expected.add(new Term("field", "中"));
+    expected.add(new Term("field", "国"));
+    
+    assertEquals(expected, getQuery("\"中国\"", analyzer));
+  }
+  
+  public void testCJKBoostedPhrase() throws Exception {
+    // individual CJK chars as terms
+    SimpleCJKAnalyzer analyzer = new SimpleCJKAnalyzer();
+    
+    PhraseQuery expected = new PhraseQuery();
+    expected.setBoost(0.5f);
+    expected.add(new Term("field", "中"));
+    expected.add(new Term("field", "国"));
+    
+    assertEquals(expected, getQuery("\"中国\"^0.5", analyzer));
+  }
+  
+  public void testCJKSloppyPhrase() throws Exception {
+    // individual CJK chars as terms
+    SimpleCJKAnalyzer analyzer = new SimpleCJKAnalyzer();
+    
+    PhraseQuery expected = new PhraseQuery();
+    expected.setSlop(3);
+    expected.add(new Term("field", "中"));
+    expected.add(new Term("field", "国"));
+    
+    assertEquals(expected, getQuery("\"中国\"~3", analyzer));
+  }
+  
+  public void testAutoGeneratePhraseQueriesOn() throws Exception {
+    // individual CJK chars as terms
+    SimpleCJKAnalyzer analyzer = new SimpleCJKAnalyzer(); 
+  
+    PhraseQuery expected = new PhraseQuery();
+    expected.add(new Term("field", "中"));
+    expected.add(new Term("field", "国"));
+    QueryParser parser = new QueryParser(TEST_VERSION_CURRENT, "field", analyzer);
+    parser.setAutoGeneratePhraseQueries(true);
+    assertEquals(expected, parser.parse("中国"));
+  }
+
   public void testSimple() throws Exception {
     assertQueryEquals("term term term", null, "term term term");
     assertQueryEquals("türm term term", new MockAnalyzer(), "türm term term");
@@ -437,9 +535,9 @@ public class TestQueryParser extends Loc
     
     assertQueryEquals("drop AND stop AND roll", qpAnalyzer, "+drop +roll");
     assertQueryEquals("term phrase term", qpAnalyzer,
-                      "term \"phrase1 phrase2\" term");
+                      "term (phrase1 phrase2) term");
     assertQueryEquals("term AND NOT phrase term", qpAnalyzer,
-                      "+term -\"phrase1 phrase2\" term");
+                      "+term -(phrase1 phrase2) term");
     assertQueryEquals("stop^3", qpAnalyzer, "");
     assertQueryEquals("stop", qpAnalyzer, "");
     assertQueryEquals("(stop)^3", qpAnalyzer, "");
@@ -912,9 +1010,9 @@ public class TestQueryParser extends Loc
       }
 
       @Override
-      protected Query getFieldQuery(String field, String queryText) throws ParseException {
+      protected Query getFieldQuery(String field, String queryText, boolean quoted) throws ParseException {
         type[0]=3;
-        return super.getFieldQuery(field, queryText);
+        return super.getFieldQuery(field, queryText, quoted);
       }
     };
 

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/BaseTestRangeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/BaseTestRangeFilter.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/BaseTestRangeFilter.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/BaseTestRangeFilter.java Thu Jul 22 19:34:35 2010
@@ -17,131 +17,134 @@ package org.apache.lucene.search;
  * limitations under the License.
  */
 
+import java.io.IOException;
 import java.util.Random;
 
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.store.RAMDirectory;
 
 public class BaseTestRangeFilter extends LuceneTestCase {
-
-    public static final boolean F = false;
-    public static final boolean T = true;
+  
+  public static final boolean F = false;
+  public static final boolean T = true;
+  
+  protected Random rand;
+  
+  /**
+   * Collation interacts badly with hyphens -- collation produces different
+   * ordering than Unicode code-point ordering -- so two indexes are created:
+   * one which can't have negative random integers, for testing collated ranges,
+   * and the other which can have negative random integers, for all other tests.
+   */
+  class TestIndex {
+    int maxR;
+    int minR;
+    boolean allowNegativeRandomInts;
+    RAMDirectory index = new RAMDirectory();
     
-    protected Random rand;
-
-    /** 
-     * Collation interacts badly with hyphens -- collation produces different
-     * ordering than Unicode code-point ordering -- so two indexes are created:
-     * one which can't have negative random integers, for testing collated 
-     * ranges, and the other which can have negative random integers, for all
-     * other tests. 
-     */
-    class TestIndex { 
-        int maxR;
-        int minR;
-        boolean allowNegativeRandomInts;
-        RAMDirectory index = new RAMDirectory();
-
-        TestIndex(int minR, int maxR, boolean allowNegativeRandomInts) {
-            this.minR = minR;
-            this.maxR = maxR;
-            this.allowNegativeRandomInts = allowNegativeRandomInts;
-        }
+    TestIndex(int minR, int maxR, boolean allowNegativeRandomInts) {
+      this.minR = minR;
+      this.maxR = maxR;
+      this.allowNegativeRandomInts = allowNegativeRandomInts;
+    }
+  }
+  
+  IndexReader signedIndexReader;
+  IndexReader unsignedIndexReader;
+  
+  TestIndex signedIndexDir = new TestIndex(Integer.MAX_VALUE, Integer.MIN_VALUE, true);
+  TestIndex unsignedIndexDir = new TestIndex(Integer.MAX_VALUE, 0, false);
+  
+  int minId = 0;
+  int maxId = 10000;
+  
+  static final int intLength = Integer.toString(Integer.MAX_VALUE).length();
+  
+  /**
+   * a simple padding function that should work with any int
+   */
+  public static String pad(int n) {
+    StringBuilder b = new StringBuilder(40);
+    String p = "0";
+    if (n < 0) {
+      p = "-";
+      n = Integer.MAX_VALUE + n + 1;
+    }
+    b.append(p);
+    String s = Integer.toString(n);
+    for (int i = s.length(); i <= intLength; i++) {
+      b.append("0");
     }
-    TestIndex signedIndex = new TestIndex(Integer.MAX_VALUE, Integer.MIN_VALUE, true);
-    TestIndex unsignedIndex = new TestIndex(Integer.MAX_VALUE, 0, false);
+    b.append(s);
     
-    int minId = 0;
-    int maxId = 10000;
-
-    static final int intLength = Integer.toString(Integer.MAX_VALUE).length();
+    return b.toString();
+  }
+   
+  protected void setUp() throws Exception {
+    super.setUp();
+    rand = newRandom();
+    signedIndexReader = build(rand, signedIndexDir);
+    unsignedIndexReader = build(rand, unsignedIndexDir);
+  }
+  
+  protected void tearDown() throws Exception {
+    signedIndexReader.close();
+    unsignedIndexReader.close();
+    super.tearDown();
+  }
+  
+  private IndexReader build(Random random, TestIndex index) throws IOException {
+    /* build an index */
+    RandomIndexWriter writer = new RandomIndexWriter(random, index.index, 
+        new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer())
+    .setOpenMode(OpenMode.CREATE));
     
-    /**
-     * a simple padding function that should work with any int
-     */
-    public static String pad(int n) {
-        StringBuilder b = new StringBuilder(40);
-        String p = "0";
-        if (n < 0) {
-            p = "-";
-            n = Integer.MAX_VALUE + n + 1;
-        }
-        b.append(p);
-        String s = Integer.toString(n);
-        for (int i = s.length(); i <= intLength; i++) {
-            b.append("0");
-        }
-        b.append(s);
-        
-        return b.toString();
-    }
-
-    public BaseTestRangeFilter(String name) {
-	super(name);
-        rand = newRandom();
-        build(signedIndex);
-        build(unsignedIndex);
-    }
-    public BaseTestRangeFilter() {
-        rand = newRandom();
-        build(signedIndex);
-        build(unsignedIndex);
+    for (int d = minId; d <= maxId; d++) {
+      Document doc = new Document();
+      doc.add(new Field("id", pad(d), Field.Store.YES,
+          Field.Index.NOT_ANALYZED));
+      int r = index.allowNegativeRandomInts ? rand.nextInt() : rand
+          .nextInt(Integer.MAX_VALUE);
+      if (index.maxR < r) {
+        index.maxR = r;
+      }
+      if (r < index.minR) {
+        index.minR = r;
+      }
+      doc.add(new Field("rand", pad(r), Field.Store.YES,
+          Field.Index.NOT_ANALYZED));
+      doc.add(new Field("body", "body", Field.Store.YES,
+          Field.Index.NOT_ANALYZED));
+      writer.addDocument(doc);
     }
     
-    private void build(TestIndex index) {
-        try {
-            
-            /* build an index */
-          IndexWriter writer = new IndexWriter(index.index, new IndexWriterConfig(
-              TEST_VERSION_CURRENT, new MockAnalyzer())
-                  .setOpenMode(OpenMode.CREATE));
-
-          for (int d = minId; d <= maxId; d++) {
-                Document doc = new Document();
-                doc.add(new Field("id",pad(d), Field.Store.YES, Field.Index.NOT_ANALYZED));
-                int r= index.allowNegativeRandomInts 
-                       ? rand.nextInt() : rand.nextInt(Integer.MAX_VALUE);
-                if (index.maxR < r) {
-                    index.maxR = r;
-                }
-                  if (r < index.minR) {
-                    index.minR = r;
-                }
-                doc.add(new Field("rand",pad(r), Field.Store.YES, Field.Index.NOT_ANALYZED));
-                doc.add(new Field("body","body", Field.Store.YES, Field.Index.NOT_ANALYZED));
-                writer.addDocument(doc);
-            }
-            
-            writer.optimize();
-            writer.close();
-
-        } catch (Exception e) {
-            throw new RuntimeException("can't build index", e);
-        }
-
-    }
-
-    public void testPad() {
-
-        int[] tests = new int[] {
-            -9999999, -99560, -100, -3, -1, 0, 3, 9, 10, 1000, 999999999
-        };
-        for (int i = 0; i < tests.length - 1; i++) {
-            int a = tests[i];
-            int b = tests[i+1];
-            String aa = pad(a);
-            String bb = pad(b);
-            String label = a + ":" + aa + " vs " + b + ":" + bb;
-            assertEquals("length of " + label, aa.length(), bb.length());
-            assertTrue("compare less than " + label, aa.compareTo(bb) < 0);
-        }
-
+    IndexReader ir = writer.getReader();
+    writer.close();
+    return ir;
+  }
+  
+  public void testPad() {
+    
+    int[] tests = new int[] {-9999999, -99560, -100, -3, -1, 0, 3, 9, 10, 1000,
+        999999999};
+    for (int i = 0; i < tests.length - 1; i++) {
+      int a = tests[i];
+      int b = tests[i + 1];
+      String aa = pad(a);
+      String bb = pad(b);
+      String label = a + ":" + aa + " vs " + b + ":" + bb;
+      assertEquals("length of " + label, aa.length(), bb.length());
+      assertTrue("compare less than " + label, aa.compareTo(bb) < 0);
     }
-
+    
+  }
+  
 }

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java Thu Jul 22 19:34:35 2010
@@ -24,7 +24,6 @@ import org.apache.lucene.document.FieldS
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.DocsAndPositionsEnum;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.PriorityQueue;
 
@@ -298,30 +297,11 @@ final class JustCompileSearch {
     
   }
 
-  static final class JustCompileFilteredTermEnum extends FilteredTermEnum {
-
-    @Override
-    public float difference() {
-      throw new UnsupportedOperationException(UNSUPPORTED_MSG);
-    }
-
-    @Override
-    protected boolean endEnum() {
-      throw new UnsupportedOperationException(UNSUPPORTED_MSG);
-    }
-
-    @Override
-    protected boolean termCompare(Term term) {
-      throw new UnsupportedOperationException(UNSUPPORTED_MSG);
-    }
-    
-  }
-
   static final class JustCompilePhraseScorer extends PhraseScorer {
 
-    JustCompilePhraseScorer(Weight weight, DocsAndPositionsEnum[] docs, int[] offsets,
+    JustCompilePhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings,
         Similarity similarity, byte[] norms) {
-      super(weight, docs, offsets, similarity, norms);
+      super(weight, postings, similarity, norms);
     }
 
     @Override
@@ -447,6 +427,21 @@ final class JustCompileSearch {
       throw new UnsupportedOperationException(UNSUPPORTED_MSG);
     }
 
+    @Override
+    public TopDocs topDocs() {
+        throw new UnsupportedOperationException( UNSUPPORTED_MSG );
+    }
+
+    @Override
+    public TopDocs topDocs( int start ) {
+        throw new UnsupportedOperationException( UNSUPPORTED_MSG );
+    }
+
+    @Override
+    public TopDocs topDocs( int start, int end ) {
+        throw new UnsupportedOperationException( UNSUPPORTED_MSG );
+    }
+    
   }
 
   static final class JustCompileWeight extends Weight {

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/QueryUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/QueryUtils.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/QueryUtils.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/QueryUtils.java Thu Jul 22 19:34:35 2010
@@ -350,7 +350,7 @@ public class QueryUtils {
                 Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more);
               }
             }
-            this.reader = reader;
+            this.reader = lastReader[0] = reader;
             this.scorer = null;
             lastDoc[0] = -1;
           }

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestAutomatonQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestAutomatonQuery.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestAutomatonQuery.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestAutomatonQuery.java Thu Jul 22 19:34:35 2010
@@ -18,13 +18,17 @@ package org.apache.lucene.search;
  */
 
 import java.io.IOException;
+import java.util.Random;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.automaton.Automaton;
@@ -32,15 +36,18 @@ import org.apache.lucene.util.automaton.
 import org.apache.lucene.util.automaton.BasicOperations;
 
 public class TestAutomatonQuery extends LuceneTestCase {
+  private Directory directory;
+  private IndexReader reader;
   private IndexSearcher searcher;
-  
+
   private final String FN = "field";
   
   public void setUp() throws Exception {
     super.setUp();
-    RAMDirectory directory = new RAMDirectory();
-    IndexWriter writer = new IndexWriter(directory, new MockAnalyzer(), true,
-        IndexWriter.MaxFieldLength.LIMITED);
+    Random random = newRandom();
+    directory = new RAMDirectory();
+    RandomIndexWriter writer = new RandomIndexWriter(random, directory, 
+        new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
     Document doc = new Document();
     Field titleField = new Field("title", "some title", Field.Store.NO,
         Field.Index.ANALYZED);
@@ -57,13 +64,15 @@ public class TestAutomatonQuery extends 
     field.setValue("doc three has some different stuff"
         + " with numbers 1234 5678.9 and letter b");
     writer.addDocument(doc);
-    writer.optimize();
+    reader = writer.getReader();
+    searcher = new IndexSearcher(reader);
     writer.close();
-    searcher = new IndexSearcher(directory, true);
   }
   
   public void tearDown() throws Exception {
     searcher.close();
+    reader.close();
+    directory.close();
     super.tearDown();
   }
   
@@ -196,10 +205,4 @@ public class TestAutomatonQuery extends 
     assertSame(TermsEnum.EMPTY, aq.getTermsEnum(searcher.getIndexReader()));
     assertEquals(0, automatonQueryNrHits(aq));
   }
-  
-  @Deprecated
-  public void testBackwardsLayer() {
-    assertTrue(new AutomatonQuery(newTerm("bogus"), BasicAutomata
-        .makeString("piece")).hasNewAPI);
-  }
 }

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestAutomatonQueryUnicode.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestAutomatonQueryUnicode.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestAutomatonQueryUnicode.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestAutomatonQueryUnicode.java Thu Jul 22 19:34:35 2010
@@ -18,12 +18,16 @@ package org.apache.lucene.search;
  */
 
 import java.io.IOException;
+import java.util.Random;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.automaton.Automaton;
@@ -35,15 +39,18 @@ import org.apache.lucene.util.automaton.
  * and the differences between UTF-8/UTF-32 and UTF-16 binary sort order.
  */
 public class TestAutomatonQueryUnicode extends LuceneTestCase {
+  private IndexReader reader;
   private IndexSearcher searcher;
+  private Directory directory;
 
   private final String FN = "field";
 
   public void setUp() throws Exception {
     super.setUp();
-    RAMDirectory directory = new RAMDirectory();
-    IndexWriter writer = new IndexWriter(directory, new MockAnalyzer(), true,
-        IndexWriter.MaxFieldLength.LIMITED);
+    Random random = newRandom();
+    directory = new RAMDirectory();
+    RandomIndexWriter writer = new RandomIndexWriter(random, directory, 
+        new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
     Document doc = new Document();
     Field titleField = new Field("title", "some title", Field.Store.NO,
         Field.Index.ANALYZED);
@@ -79,13 +86,15 @@ public class TestAutomatonQueryUnicode e
     writer.addDocument(doc);
     field.setValue("\uFFFD\uFFFD");
     writer.addDocument(doc);
-    writer.optimize();
+    reader = writer.getReader();
+    searcher = new IndexSearcher(reader);
     writer.close();
-    searcher = new IndexSearcher(directory, true);
   }
 
   public void tearDown() throws Exception {
     searcher.close();
+    reader.close();
+    directory.close();
     super.tearDown();
   }
 

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBoolean2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBoolean2.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBoolean2.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBoolean2.java Thu Jul 22 19:34:35 2010
@@ -23,8 +23,8 @@ import java.util.Random;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.queryParser.ParseException;
@@ -42,6 +42,7 @@ public class TestBoolean2 extends Lucene
   private IndexSearcher searcher;
   private IndexSearcher bigSearcher;
   private IndexReader reader;
+  private Random rnd;
   private static int NUM_EXTRA_DOCS = 6000;
 
   public static final String field = "field";
@@ -51,8 +52,9 @@ public class TestBoolean2 extends Lucene
   @Override
   protected void setUp() throws Exception {
     super.setUp();
+    rnd = newRandom();
     RAMDirectory directory = new RAMDirectory();
-    IndexWriter writer= new IndexWriter(directory, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+    RandomIndexWriter writer= new RandomIndexWriter(rnd, directory, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
     for (int i = 0; i < docFields.length; i++) {
       Document doc = new Document();
       doc.add(new Field(field, docFields[i], Field.Store.NO, Field.Index.ANALYZED));
@@ -69,14 +71,14 @@ public class TestBoolean2 extends Lucene
     int docCount = 0;
     do {
       final Directory copy = new RAMDirectory(dir2);
-      IndexWriter w = new IndexWriter(dir2, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+      RandomIndexWriter w = new RandomIndexWriter(rnd, dir2, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
       w.addIndexes(new Directory[] {copy});
       docCount = w.maxDoc();
       w.close();
       mulFactor *= 2;
     } while(docCount < 3000);
 
-    IndexWriter w = new IndexWriter(dir2, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+    RandomIndexWriter w = new RandomIndexWriter(rnd, dir2, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
     Document doc = new Document();
     doc.add(new Field("field2", "xxx", Field.Store.NO, Field.Index.ANALYZED));
     for(int i=0;i<NUM_EXTRA_DOCS/2;i++) {
@@ -87,11 +89,9 @@ public class TestBoolean2 extends Lucene
     for(int i=0;i<NUM_EXTRA_DOCS/2;i++) {
       w.addDocument(doc);
     }
-    // optimize to 1 segment
-    w.optimize();
     reader = w.getReader();
-    w.close();
     bigSearcher = new IndexSearcher(reader);
+    w.close();
   }
 
   @Override
@@ -200,8 +200,6 @@ public class TestBoolean2 extends Lucene
   }
 
   public void testRandomQueries() throws Exception {
-    Random rnd = newRandom();
-
     String[] vals = {"w1","w2","w3","w4","w5","xx","yy","zzz"};
 
     int tot=0;

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java Thu Jul 22 19:34:35 2010
@@ -24,8 +24,8 @@ import org.apache.lucene.analysis.MockAn
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
@@ -37,16 +37,17 @@ import java.util.Random;
  */
 public class TestBooleanMinShouldMatch extends LuceneTestCase {
 
-
-    public Directory index;
-    public IndexReader r;
-    public IndexSearcher s;
+    private Random rnd;
+    private Directory index;
+    private IndexReader r;
+    private IndexSearcher s;
 
     @Override
     protected void setUp() throws Exception {
         super.setUp();
 
-
+        rnd = newRandom();
+        
         String[] data = new String [] {
             "A 1 2 3 4 5 6",
             "Z       4 5 6",
@@ -59,7 +60,7 @@ public class TestBooleanMinShouldMatch e
         };
 
         index = new RAMDirectory();
-        IndexWriter writer = new IndexWriter(index, new IndexWriterConfig(
+        RandomIndexWriter w = new RandomIndexWriter(rnd, index, new IndexWriterConfig(
         TEST_VERSION_CURRENT, new MockAnalyzer()));
 
         for (int i = 0; i < data.length; i++) {
@@ -69,17 +70,23 @@ public class TestBooleanMinShouldMatch e
             if (null != data[i]) {
                 doc.add(new Field("data", data[i], Field.Store.YES, Field.Index.ANALYZED));//Field.Text("data",data[i]));
             }
-            writer.addDocument(doc);
+            w.addDocument(doc);
         }
 
-        writer.optimize();
-        writer.close();
-
-        r = IndexReader.open(index, true);
+        r = w.getReader();
         s = new IndexSearcher(r);
-
+        w.close();
 //System.out.println("Set up " + getName());
     }
+    
+    @Override
+    protected void tearDown() throws Exception {
+      s.close();
+      r.close();
+      index.close();
+      super.tearDown();
+    }
+
 
     public void verifyNrHits(Query q, int expected) throws Exception {
         ScoreDoc[] h = s.search(q, null, 1000).scoreDocs;
@@ -295,8 +302,6 @@ public class TestBooleanMinShouldMatch e
     }
 
     public void testRandomQueries() throws Exception {
-      final Random rnd = newRandom();
-
       String field="data";
       String[] vals = {"1","2","3","4","5","6","A","Z","B","Y","Z","X","foo"};
       int maxLev=4;

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBooleanOr.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBooleanOr.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBooleanOr.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBooleanOr.java Thu Jul 22 19:34:35 2010
@@ -16,20 +16,23 @@ package org.apache.lucene.search;
  * limitations under the License.
  */
 import java.io.IOException;
+import java.util.Random;
 
 import org.apache.lucene.util.LuceneTestCase;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 
 public class TestBooleanOr extends LuceneTestCase {
@@ -43,6 +46,9 @@ public class TestBooleanOr extends Lucen
   private TermQuery c2 = new TermQuery(new Term(FIELD_C, "optimize"));
 
   private IndexSearcher searcher = null;
+  private Directory dir;
+  private IndexReader reader;
+  
 
   private int search(Query q) throws IOException {
     QueryUtils.check(q,searcher);
@@ -133,10 +139,11 @@ public class TestBooleanOr extends Lucen
     super.setUp();
 
     //
-    RAMDirectory rd = new RAMDirectory();
+    dir = new RAMDirectory();
 
+    Random random = newRandom();
     //
-    IndexWriter writer = new IndexWriter(rd, new IndexWriterConfig(
+    RandomIndexWriter writer = new RandomIndexWriter(random, dir, new IndexWriterConfig(
         TEST_VERSION_CURRENT, new MockAnalyzer()));
 
     //
@@ -154,9 +161,18 @@ public class TestBooleanOr extends Lucen
 
     //
     writer.addDocument(d);
-    writer.close();
 
+    reader = writer.getReader();
     //
-    searcher = new IndexSearcher(rd, true);
+    searcher = new IndexSearcher(reader);
+    writer.close();
+  }
+
+  @Override
+  protected void tearDown() throws Exception {
+    searcher.close();
+    reader.close();
+    dir.close();
+    super.tearDown();
   }
 }

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBooleanPrefixQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBooleanPrefixQuery.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBooleanPrefixQuery.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBooleanPrefixQuery.java Thu Jul 22 19:34:35 2010
@@ -22,8 +22,8 @@ import junit.framework.Test;
 import junit.framework.TestSuite;
 import junit.textui.TestRunner;
 import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.analysis.MockAnalyzer;
@@ -33,8 +33,6 @@ import org.apache.lucene.search.PrefixQu
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.BooleanQuery;
 
-import java.io.IOException;
-
 /**
  *
  **/
@@ -79,29 +77,27 @@ public class TestBooleanPrefixQuery exte
     Query rw1 = null;
     Query rw2 = null;
     IndexReader reader = null;
-    try {
-      IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(
+    RandomIndexWriter writer = new RandomIndexWriter(newRandom(), directory, new IndexWriterConfig(
           TEST_VERSION_CURRENT, new MockAnalyzer()));
-      for (int i = 0; i < categories.length; i++) {
-        Document doc = new Document();
-        doc.add(new Field("category", categories[i], Field.Store.YES, Field.Index.NOT_ANALYZED));
-        writer.addDocument(doc);
-      }
-      writer.close();
+    for (int i = 0; i < categories.length; i++) {
+      Document doc = new Document();
+      doc.add(new Field("category", categories[i], Field.Store.YES, Field.Index.NOT_ANALYZED));
+      writer.addDocument(doc);
+    }
+    reader = writer.getReader();
+    writer.close();
       
-      reader = IndexReader.open(directory, true);
-      PrefixQuery query = new PrefixQuery(new Term("category", "foo"));
-      rw1 = query.rewrite(reader);
+    PrefixQuery query = new PrefixQuery(new Term("category", "foo"));
+    rw1 = query.rewrite(reader);
       
-      BooleanQuery bq = new BooleanQuery();
-      bq.add(query, BooleanClause.Occur.MUST);
+    BooleanQuery bq = new BooleanQuery();
+    bq.add(query, BooleanClause.Occur.MUST);
       
-      rw2 = bq.rewrite(reader);
-    } catch (IOException e) {
-      fail(e.getMessage());
-    }
+    rw2 = bq.rewrite(reader);
 
     assertEquals("Number of Clauses Mismatch", getCount(reader, rw1), getCount(reader, rw2));
+    reader.close();
+    directory.close();
   }
 }
 

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBooleanQuery.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBooleanQuery.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBooleanQuery.java Thu Jul 22 19:34:35 2010
@@ -16,9 +16,10 @@ package org.apache.lucene.search;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import org.apache.lucene.index.IndexWriter;
+
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
@@ -61,7 +62,7 @@ public class TestBooleanQuery extends Lu
   // LUCENE-1630
   public void testNullOrSubScorer() throws Throwable {
     Directory dir = new MockRAMDirectory();
-    IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(
+    RandomIndexWriter w = new RandomIndexWriter(newRandom(), dir, new IndexWriterConfig(
         TEST_VERSION_CURRENT, new MockAnalyzer()));
     Document doc = new Document();
     doc.add(new Field("field", "a b c d", Field.Store.NO, Field.Index.ANALYZED));

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBooleanScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBooleanScorer.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBooleanScorer.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBooleanScorer.java Thu Jul 22 19:34:35 2010
@@ -23,8 +23,9 @@ import java.util.Arrays;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.store.RAMDirectory;
 
@@ -39,37 +40,34 @@ public class TestBooleanScorer extends L
 
   private static final String FIELD = "category";
   
-  public void testMethod() {
+  public void testMethod() throws Exception {
     RAMDirectory directory = new RAMDirectory();
 
     String[] values = new String[] { "1", "2", "3", "4" };
 
-    try {
-      IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
-      for (int i = 0; i < values.length; i++) {
-        Document doc = new Document();
-        doc.add(new Field(FIELD, values[i], Field.Store.YES, Field.Index.NOT_ANALYZED));
-        writer.addDocument(doc);
-      }
-      writer.close();
-
-      BooleanQuery booleanQuery1 = new BooleanQuery();
-      booleanQuery1.add(new TermQuery(new Term(FIELD, "1")), BooleanClause.Occur.SHOULD);
-      booleanQuery1.add(new TermQuery(new Term(FIELD, "2")), BooleanClause.Occur.SHOULD);
-
-      BooleanQuery query = new BooleanQuery();
-      query.add(booleanQuery1, BooleanClause.Occur.MUST);
-      query.add(new TermQuery(new Term(FIELD, "9")), BooleanClause.Occur.MUST_NOT);
-
-      IndexSearcher indexSearcher = new IndexSearcher(directory, true);
-      ScoreDoc[] hits = indexSearcher.search(query, null, 1000).scoreDocs;
-      assertEquals("Number of matched documents", 2, hits.length);
-
-    }
-    catch (IOException e) {
-      fail(e.getMessage());
+    RandomIndexWriter writer = new RandomIndexWriter(newRandom(), directory, 
+        new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+    for (int i = 0; i < values.length; i++) {
+      Document doc = new Document();
+      doc.add(new Field(FIELD, values[i], Field.Store.YES, Field.Index.NOT_ANALYZED));
+      writer.addDocument(doc);
     }
+    IndexReader ir = writer.getReader();
+    writer.close();
 
+    BooleanQuery booleanQuery1 = new BooleanQuery();
+    booleanQuery1.add(new TermQuery(new Term(FIELD, "1")), BooleanClause.Occur.SHOULD);
+    booleanQuery1.add(new TermQuery(new Term(FIELD, "2")), BooleanClause.Occur.SHOULD);
+
+    BooleanQuery query = new BooleanQuery();
+    query.add(booleanQuery1, BooleanClause.Occur.MUST);
+    query.add(new TermQuery(new Term(FIELD, "9")), BooleanClause.Occur.MUST_NOT);
+
+    IndexSearcher indexSearcher = new IndexSearcher(ir);
+    ScoreDoc[] hits = indexSearcher.search(query, null, 1000).scoreDocs;
+    assertEquals("Number of matched documents", 2, hits.length);
+    ir.close();
+    directory.close();
   }
   
   public void testEmptyBucketWithMoreDocs() throws Exception {