lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From busc...@apache.org
Subject svn commit: r966819 [14/20] - in /lucene/dev/branches/realtime_search: ./ lucene/ lucene/backwards/ lucene/contrib/ lucene/contrib/benchmark/conf/ lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ lucene/contrib/benchmark/src/...
Date Thu, 22 Jul 2010 19:34:52 GMT
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestTopScoreDocCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestTopScoreDocCollector.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestTopScoreDocCollector.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestTopScoreDocCollector.java Thu Jul 22 19:34:35 2010
@@ -17,10 +17,13 @@ package org.apache.lucene.search;
  * limitations under the License.
  */
 
+import java.util.Random;
+
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
@@ -38,12 +41,12 @@ public class TestTopScoreDocCollector ex
   public void testOutOfOrderCollection() throws Exception {
 
     Directory dir = new RAMDirectory();
-    IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+    Random random = newRandom();
+    RandomIndexWriter writer = new RandomIndexWriter(random, dir, 
+        new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
     for (int i = 0; i < 10; i++) {
       writer.addDocument(new Document());
     }
-    writer.commit();
-    writer.close();
     
     boolean[] inOrder = new boolean[] { false, true };
     String[] actualTSDCClass = new String[] {
@@ -58,7 +61,8 @@ public class TestTopScoreDocCollector ex
     // Set minNrShouldMatch to 1 so that BQ will not optimize rewrite to return
     // the clause instead of BQ.
     bq.setMinimumNumberShouldMatch(1);
-    IndexSearcher searcher = new IndexSearcher(dir, true);
+    IndexReader reader = writer.getReader();
+    IndexSearcher searcher = new IndexSearcher(reader);
     for (int i = 0; i < inOrder.length; i++) {
       TopDocsCollector<ScoreDoc> tdc = TopScoreDocCollector.create(3, inOrder[i]);
       assertEquals("org.apache.lucene.search.TopScoreDocCollector$" + actualTSDCClass[i], tdc.getClass().getName());
@@ -71,6 +75,10 @@ public class TestTopScoreDocCollector ex
         assertEquals("expected doc Id " + j + " found " + sd[j].doc, j, sd[j].doc);
       }
     }
+    writer.close();
+    searcher.close();
+    reader.close();
+    dir.close();
   }
   
 }

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestWildcard.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestWildcard.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestWildcard.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestWildcard.java Thu Jul 22 19:34:35 2010
@@ -19,25 +19,32 @@ package org.apache.lucene.search;
 
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queryParser.QueryParser;
 import org.apache.lucene.store.RAMDirectory;
 
 import java.io.IOException;
+import java.util.Random;
 
 /**
  * TestWildcard tests the '*' and '?' wildcard characters.
  */
 public class TestWildcard
     extends LuceneTestCase {
+  private Random random;
+  
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    random = newRandom();
+  }
+
   public void testEquals() {
     WildcardQuery wq1 = new WildcardQuery(new Term("field", "b*a"));
     WildcardQuery wq2 = new WildcardQuery(new Term("field", "b*a"));
@@ -195,14 +202,13 @@ public class TestWildcard
   private RAMDirectory getIndexStore(String field, String[] contents)
       throws IOException {
     RAMDirectory indexStore = new RAMDirectory();
-    IndexWriter writer = new IndexWriter(indexStore, new IndexWriterConfig(
+    RandomIndexWriter writer = new RandomIndexWriter(random, indexStore, new IndexWriterConfig(
         TEST_VERSION_CURRENT, new MockAnalyzer()));
     for (int i = 0; i < contents.length; ++i) {
       Document doc = new Document();
       doc.add(new Field(field, contents[i], Field.Store.YES, Field.Index.ANALYZED));
       writer.addDocument(doc);
     }
-    writer.optimize();
     writer.close();
 
     return indexStore;
@@ -253,7 +259,8 @@ public class TestWildcard
 
     // prepare the index
     RAMDirectory dir = new RAMDirectory();
-    IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+    RandomIndexWriter iw = new RandomIndexWriter(random, dir, 
+        new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
     for (int i = 0; i < docs.length; i++) {
       Document doc = new Document();
       doc.add(new Field(field,docs[i],Store.NO,Index.ANALYZED));
@@ -309,62 +316,4 @@ public class TestWildcard
 
     searcher.close();
   }
-  @Deprecated
-  private static final class OldWildcardQuery extends MultiTermQuery {
-    final Term term;
-  
-    OldWildcardQuery(Term term) {
-      this.term = term;
-    }
-      
-    @Override
-    protected FilteredTermEnum getEnum(IndexReader reader) throws IOException {
-      return new WildcardTermEnum(reader, term);
-    }
-    
-    @Override
-    public String toString(String field) {
-      return "OldWildcard(" + term.toString()+ ")";
-    }
-  }
-  
-  @Deprecated
-  public void testDeprecatedTermEnum() throws Exception {
-    RAMDirectory indexStore = getIndexStore("body", new String[]
-    {"metal", "metals"});
-    IndexSearcher searcher = new IndexSearcher(indexStore, true);
-    Query query1 = new TermQuery(new Term("body", "metal"));
-    Query query2 = new OldWildcardQuery(new Term("body", "metal*"));
-    Query query3 = new OldWildcardQuery(new Term("body", "m*tal"));
-    Query query4 = new OldWildcardQuery(new Term("body", "m*tal*"));
-    Query query5 = new OldWildcardQuery(new Term("body", "m*tals"));
-
-    BooleanQuery query6 = new BooleanQuery();
-    query6.add(query5, BooleanClause.Occur.SHOULD);
-
-    BooleanQuery query7 = new BooleanQuery();
-    query7.add(query3, BooleanClause.Occur.SHOULD);
-    query7.add(query5, BooleanClause.Occur.SHOULD);
-
-    // Queries do not automatically lower-case search terms:
-    Query query8 = new OldWildcardQuery(new Term("body", "M*tal*"));
-
-    assertMatches(searcher, query1, 1);
-    assertMatches(searcher, query2, 2);
-    assertMatches(searcher, query3, 1);
-    assertMatches(searcher, query4, 2);
-    assertMatches(searcher, query5, 1);
-    assertMatches(searcher, query6, 1);
-    assertMatches(searcher, query7, 2);
-    assertMatches(searcher, query8, 0);
-    assertMatches(searcher, new OldWildcardQuery(new Term("body", "*tall")), 0);
-    assertMatches(searcher, new OldWildcardQuery(new Term("body", "*tal")), 1);
-    assertMatches(searcher, new OldWildcardQuery(new Term("body", "*tal*")), 2);
-  }
-  
-  @Deprecated
-  public void testBackwardsLayer() {
-    assertTrue(new WildcardQuery(new Term("body", "metal*")).hasNewAPI);
-    assertFalse(new OldWildcardQuery(new Term("body", "metal*")).hasNewAPI);
-  }
 }

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestWildcardRandom.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestWildcardRandom.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestWildcardRandom.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestWildcardRandom.java Thu Jul 22 19:34:35 2010
@@ -26,8 +26,11 @@ import java.util.Random;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
@@ -40,13 +43,15 @@ import org.apache.lucene.util._TestUtil;
 public class TestWildcardRandom extends LuceneTestCase {
   private Searcher searcher;
   private Random random;
+  private Directory dir;
   
   @Override
   protected void setUp() throws Exception {
     super.setUp();
-    RAMDirectory dir = new RAMDirectory();
-    IndexWriter writer = new IndexWriter(dir, new MockAnalyzer(),
-        IndexWriter.MaxFieldLength.UNLIMITED);
+    random = newRandom();
+    dir = new RAMDirectory();
+    RandomIndexWriter writer = new RandomIndexWriter(random, dir, 
+        new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
     
     Document doc = new Document();
     Field field = new Field("field", "", Field.Store.NO, Field.Index.ANALYZED);
@@ -58,9 +63,9 @@ public class TestWildcardRandom extends 
       writer.addDocument(doc);
     }
     
-    writer.optimize();
+    IndexReader reader = writer.getReader();
+    searcher = new IndexSearcher(reader);
     writer.close();
-    searcher = new IndexSearcher(dir);
   }
   
   private char N() {
@@ -82,6 +87,7 @@ public class TestWildcardRandom extends 
   }
   
   private void assertPatternHits(String pattern, int numHits) throws Exception {
+    // TODO: run with different rewrites
     Query wq = new WildcardQuery(new Term("field", fillPattern(pattern)));
     TopDocs docs = searcher.search(wq, 25);
     assertEquals("Incorrect hits for pattern: " + pattern, numHits, docs.totalHits);
@@ -90,11 +96,11 @@ public class TestWildcardRandom extends 
   @Override
   protected void tearDown() throws Exception {
     searcher.close();
+    dir.close();
     super.tearDown();
   }
   
-  public void testWildcards() throws Exception {
-    random = newRandom(System.nanoTime());
+  public void testWildcards() throws Exception {;
     for (int i = 0; i < 100*_TestUtil.getRandomMultiplier(); i++) {
       assertPatternHits("NNNN", 1);
       assertPatternHits("?NNN", 10);

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/function/FunctionTestSetup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/function/FunctionTestSetup.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/function/FunctionTestSetup.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/function/FunctionTestSetup.java Thu Jul 22 19:34:35 2010
@@ -93,7 +93,6 @@ public class FunctionTestSetup extends L
   public void setUp() throws Exception {
     super.setUp();
     // prepare a small index with just a few documents.  
-    super.setUp();
     dir = new RAMDirectory();
     anlzr = new MockAnalyzer();
     IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, anlzr));

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java Thu Jul 22 19:34:35 2010
@@ -26,9 +26,10 @@ import org.apache.lucene.analysis.TokenS
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.Payload;
+import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.DefaultSimilarity;
 import org.apache.lucene.search.IndexSearcher;
@@ -39,6 +40,7 @@ import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.spans.SpanQuery;
 import org.apache.lucene.search.spans.SpanNearQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.English;
 import org.apache.lucene.util.LuceneTestCase;
@@ -47,6 +49,8 @@ import org.apache.lucene.search.Explanat
 
 public class TestPayloadNearQuery extends LuceneTestCase {
   private IndexSearcher searcher;
+  private IndexReader reader;
+  private Directory directory;
   private BoostingSimilarity similarity = new BoostingSimilarity();
   private byte[] payload2 = new byte[]{2};
   private byte[] payload4 = new byte[]{4};
@@ -101,9 +105,10 @@ public class TestPayloadNearQuery extend
   @Override
   protected void setUp() throws Exception {
     super.setUp();
-    RAMDirectory directory = new RAMDirectory();
-    IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(
-        TEST_VERSION_CURRENT, new PayloadAnalyzer()).setSimilarity(similarity));
+    directory = new RAMDirectory();
+    RandomIndexWriter writer = new RandomIndexWriter(newRandom(), directory, 
+        new IndexWriterConfig(TEST_VERSION_CURRENT, new PayloadAnalyzer())
+        .setSimilarity(similarity));
     //writer.infoStream = System.out;
     for (int i = 0; i < 1000; i++) {
       Document doc = new Document();
@@ -112,13 +117,21 @@ public class TestPayloadNearQuery extend
       doc.add(new Field("field2",  txt, Field.Store.YES, Field.Index.ANALYZED));
       writer.addDocument(doc);
     }
-    writer.optimize();
+    reader = writer.getReader();
     writer.close();
 
-    searcher = new IndexSearcher(directory, true);
+    searcher = new IndexSearcher(reader);
     searcher.setSimilarity(similarity);
   }
 
+  @Override
+  protected void tearDown() throws Exception {
+    searcher.close();
+    reader.close();
+    directory.close();
+    super.tearDown();
+  }
+
   public void test() throws IOException {
     PayloadNearQuery query;
     TopDocs hits;

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java Thu Jul 22 19:34:35 2010
@@ -34,9 +34,10 @@ import org.apache.lucene.analysis.MockTo
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.Payload;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.document.Document;
@@ -52,6 +53,7 @@ import java.io.IOException;
  **/
 public class TestPayloadTermQuery extends LuceneTestCase {
   private IndexSearcher searcher;
+  private IndexReader reader;
   private BoostingSimilarity similarity = new BoostingSimilarity();
   private byte[] payloadField = new byte[]{1};
   private byte[] payloadMultiField1 = new byte[]{2};
@@ -110,9 +112,9 @@ public class TestPayloadTermQuery extend
   protected void setUp() throws Exception {
     super.setUp();
     directory = new RAMDirectory();
-    IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(
-        TEST_VERSION_CURRENT, new PayloadAnalyzer()).setSimilarity(
-        similarity));
+    RandomIndexWriter writer = new RandomIndexWriter(newRandom(), directory, 
+        new IndexWriterConfig(TEST_VERSION_CURRENT, new PayloadAnalyzer())
+        .setSimilarity(similarity));
     //writer.infoStream = System.out;
     for (int i = 0; i < 1000; i++) {
       Document doc = new Document();
@@ -123,13 +125,21 @@ public class TestPayloadTermQuery extend
       doc.add(new Field("multiField", English.intToEnglish(i) + "  " + English.intToEnglish(i), Field.Store.YES, Field.Index.ANALYZED));
       writer.addDocument(doc);
     }
-    writer.optimize();
+    reader = writer.getReader();
     writer.close();
 
-    searcher = new IndexSearcher(directory, true);
+    searcher = new IndexSearcher(reader);
     searcher.setSimilarity(similarity);
   }
 
+  @Override
+  protected void tearDown() throws Exception {
+    searcher.close();
+    reader.close();
+    directory.close();
+    super.tearDown();
+  }
+
   public void test() throws IOException {
     PayloadTermQuery query = new PayloadTermQuery(new Term("field", "seventy"),
             new MaxPayloadFunction());

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java Thu Jul 22 19:34:35 2010
@@ -23,8 +23,9 @@ import org.apache.lucene.analysis.MockAn
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
@@ -34,6 +35,7 @@ import org.apache.lucene.search.PhraseQu
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryUtils;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.English;
 import org.apache.lucene.util.LuceneTestCase;
@@ -52,25 +54,35 @@ import org.apache.lucene.util.LuceneTest
  */
 public class TestBasics extends LuceneTestCase {
   private IndexSearcher searcher;
+  private IndexReader reader;
+  private Directory directory;
 
   @Override
   protected void setUp() throws Exception {
     super.setUp();
-    RAMDirectory directory = new RAMDirectory();
-    IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(
-        TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true)));
+    directory = new RAMDirectory();
+    RandomIndexWriter writer = new RandomIndexWriter(newRandom(), directory, 
+        new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true)));
     //writer.infoStream = System.out;
     for (int i = 0; i < 1000; i++) {
       Document doc = new Document();
       doc.add(new Field("field", English.intToEnglish(i), Field.Store.YES, Field.Index.ANALYZED));
       writer.addDocument(doc);
     }
-
+    reader = writer.getReader();
+    searcher = new IndexSearcher(reader);
     writer.close();
+  }
 
-    searcher = new IndexSearcher(directory, true);
+  @Override
+  protected void tearDown() throws Exception {
+    searcher.close();
+    reader.close();
+    directory.close();
+    super.tearDown();
   }
-  
+
+
   public void testTerm() throws Exception {
     Query query = new TermQuery(new Term("field", "seventy"));
     checkHits(query, new int[]

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java Thu Jul 22 19:34:35 2010
@@ -24,13 +24,14 @@ import org.apache.lucene.analysis.MockAn
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.CheckHits;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryUtils;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.LuceneTestCase;
 
@@ -49,12 +50,15 @@ public class TestFieldMaskingSpanQuery e
   }
 
   protected IndexSearcher searcher;
+  protected Directory directory;
+  protected IndexReader reader;
   
   @Override
   protected void setUp() throws Exception {
     super.setUp();
-    RAMDirectory directory = new RAMDirectory();
-    IndexWriter writer= new IndexWriter(directory, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+    directory = new RAMDirectory();
+    RandomIndexWriter writer= new RandomIndexWriter(newRandom(), directory, 
+        new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
     
     writer.addDocument(doc(new Field[] { field("id", "0")
                                          ,
@@ -109,14 +113,16 @@ public class TestFieldMaskingSpanQuery e
                                          field("gender", "male"),
                                          field("first",  "bubba"),
                                          field("last",   "jones")     }));
-    
+    reader = writer.getReader();
     writer.close();
-    searcher = new IndexSearcher(directory, true);
+    searcher = new IndexSearcher(reader);
   }
 
   @Override
   protected void tearDown() throws Exception {
     searcher.close();
+    reader.close();
+    directory.close();
     super.tearDown();
   }
 

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java Thu Jul 22 19:34:35 2010
@@ -20,8 +20,9 @@ package org.apache.lucene.search.spans;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queryParser.QueryParser;
 import org.apache.lucene.search.CheckHits;
@@ -29,11 +30,14 @@ import org.apache.lucene.search.Explanat
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.LuceneTestCase;
 
 public class TestNearSpansOrdered extends LuceneTestCase {
   protected IndexSearcher searcher;
+  protected Directory directory;
+  protected IndexReader reader;
 
   public static final String FIELD = "field";
   public static final QueryParser qp =
@@ -42,21 +46,25 @@ public class TestNearSpansOrdered extend
   @Override
   protected void tearDown() throws Exception {
     searcher.close();
+    reader.close();
+    directory.close();
     super.tearDown();
   }
   
   @Override
   protected void setUp() throws Exception {
     super.setUp();
-    RAMDirectory directory = new RAMDirectory();
-    IndexWriter writer= new IndexWriter(directory, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+    directory = new RAMDirectory();
+    RandomIndexWriter writer= new RandomIndexWriter(newRandom(), directory, 
+        new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
     for (int i = 0; i < docFields.length; i++) {
       Document doc = new Document();
       doc.add(new Field(FIELD, docFields[i], Field.Store.NO, Field.Index.ANALYZED));
       writer.addDocument(doc);
     }
+    reader = writer.getReader();
     writer.close();
-    searcher = new IndexSearcher(directory, true);
+    searcher = new IndexSearcher(reader);
   }
 
   protected String[] docFields = {

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java Thu Jul 22 19:34:35 2010
@@ -33,32 +33,44 @@ import org.apache.lucene.analysis.MockAn
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.util.LuceneTestCase;
 import java.io.IOException;
-import java.util.Collections;
 
 public class TestSpans extends LuceneTestCase {
   private IndexSearcher searcher;
+  private IndexReader reader;
+  private Directory directory;
 
   public static final String field = "field";
 
   @Override
   protected void setUp() throws Exception {
     super.setUp();
-    RAMDirectory directory = new RAMDirectory();
-    IndexWriter writer= new IndexWriter(directory, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+    directory = new RAMDirectory();
+    RandomIndexWriter writer= new RandomIndexWriter(newRandom(), directory, 
+        new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
     for (int i = 0; i < docFields.length; i++) {
       Document doc = new Document();
       doc.add(new Field(field, docFields[i], Field.Store.YES, Field.Index.ANALYZED));
       writer.addDocument(doc);
     }
+    reader = writer.getReader();
     writer.close();
-    searcher = new IndexSearcher(directory, true);
+    searcher = new IndexSearcher(reader);
   }
-
+  
+  @Override
+  protected void tearDown() throws Exception {
+    searcher.close();
+    reader.close();
+    directory.close();
+    super.tearDown();
+  }
+  
   private String[] docFields = {
     "w1 w2 w3 w4 w5",
     "w1 w3 w2 w3",

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java Thu Jul 22 19:34:35 2010
@@ -18,6 +18,7 @@ package org.apache.lucene.search.spans;
  */
 
 import java.io.IOException;
+import java.util.Random;
 
 import org.apache.lucene.util.LuceneTestCase;
 
@@ -26,8 +27,9 @@ import org.apache.lucene.analysis.MockTo
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.*;
 import org.apache.lucene.store.Directory;
@@ -36,136 +38,145 @@ import org.apache.lucene.store.RAMDirect
 /*******************************************************************************
  * Tests the span query bug in Lucene. It demonstrates that SpanTermQuerys don't
  * work correctly in a BooleanQuery.
- *
+ * 
  */
 public class TestSpansAdvanced extends LuceneTestCase {
-
-    // location to the index
-    protected Directory mDirectory;
-
-    protected IndexSearcher searcher;
-
-    // field names in the index
-    private final static String FIELD_ID = "ID";
-    protected final static String FIELD_TEXT = "TEXT";
-
-    /**
-     * Initializes the tests by adding 4 identical documents to the index.
-     */
-    @Override
-    protected void setUp() throws Exception {
-        super.setUp();
-
-        // create test index
-        mDirectory = new RAMDirectory();
-        final IndexWriter writer = new IndexWriter(mDirectory,
-        new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true)));
-        addDocument(writer, "1", "I think it should work.");
-        addDocument(writer, "2", "I think it should work.");
-        addDocument(writer, "3", "I think it should work.");
-        addDocument(writer, "4", "I think it should work.");
-        writer.close();
-        searcher = new IndexSearcher(mDirectory, true);
-    }
-
-    @Override
-    protected void tearDown() throws Exception {
-        searcher.close();
-        mDirectory.close();
-        mDirectory = null;
-        super.tearDown();
-    }
-
-    /**
-     * Adds the document to the index.
-     *
-     * @param writer the Lucene index writer
-     * @param id the unique id of the document
-     * @param text the text of the document
-     * @throws IOException
-     */
-    protected void addDocument(final IndexWriter writer, final String id, final String text) throws IOException {
-
-        final Document document = new Document();
-        document.add(new Field(FIELD_ID, id, Field.Store.YES, Field.Index.NOT_ANALYZED));
-        document.add(new Field(FIELD_TEXT, text, Field.Store.YES, Field.Index.ANALYZED));
-        writer.addDocument(document);
-    }
-
-    /**
-     * Tests two span queries.
-     *
-     * @throws IOException
-     */
-    public void testBooleanQueryWithSpanQueries() throws IOException {
-
-        doTestBooleanQueryWithSpanQueries(searcher,0.3884282f);
+  
+  // location to the index
+  protected Directory mDirectory;
+  protected IndexReader reader;
+  protected IndexSearcher searcher;
+  protected Random random;
+  
+  // field names in the index
+  private final static String FIELD_ID = "ID";
+  protected final static String FIELD_TEXT = "TEXT";
+  
+  /**
+   * Initializes the tests by adding 4 identical documents to the index.
+   */
+  @Override
+  protected void setUp() throws Exception {
+    super.setUp();
+    random = newRandom();
+    // create test index
+    mDirectory = new RAMDirectory();
+    final RandomIndexWriter writer = new RandomIndexWriter(random,
+        mDirectory, new IndexWriterConfig(TEST_VERSION_CURRENT,
+            new MockAnalyzer(MockTokenizer.SIMPLE, true,
+                MockTokenFilter.ENGLISH_STOPSET, true)));
+    addDocument(writer, "1", "I think it should work.");
+    addDocument(writer, "2", "I think it should work.");
+    addDocument(writer, "3", "I think it should work.");
+    addDocument(writer, "4", "I think it should work.");
+    reader = writer.getReader();
+    writer.close();
+    searcher = new IndexSearcher(reader);
+  }
+  
+  @Override
+  protected void tearDown() throws Exception {
+    searcher.close();
+    reader.close();
+    mDirectory.close();
+    mDirectory = null;
+    super.tearDown();
+  }
+  
+  /**
+   * Adds the document to the index.
+   * 
+   * @param writer the Lucene index writer
+   * @param id the unique id of the document
+   * @param text the text of the document
+   * @throws IOException
+   */
+  protected void addDocument(final RandomIndexWriter writer, final String id,
+      final String text) throws IOException {
+    
+    final Document document = new Document();
+    document.add(new Field(FIELD_ID, id, Field.Store.YES,
+        Field.Index.NOT_ANALYZED));
+    document.add(new Field(FIELD_TEXT, text, Field.Store.YES,
+        Field.Index.ANALYZED));
+    writer.addDocument(document);
+  }
+  
+  /**
+   * Tests two span queries.
+   * 
+   * @throws IOException
+   */
+  public void testBooleanQueryWithSpanQueries() throws IOException {
+    
+    doTestBooleanQueryWithSpanQueries(searcher, 0.3884282f);
+  }
+  
+  /**
+   * Tests two span queries.
+   * 
+   * @throws IOException
+   */
+  protected void doTestBooleanQueryWithSpanQueries(IndexSearcher s,
+      final float expectedScore) throws IOException {
+    
+    final Query spanQuery = new SpanTermQuery(new Term(FIELD_TEXT, "work"));
+    final BooleanQuery query = new BooleanQuery();
+    query.add(spanQuery, BooleanClause.Occur.MUST);
+    query.add(spanQuery, BooleanClause.Occur.MUST);
+    final String[] expectedIds = new String[] {"1", "2", "3", "4"};
+    final float[] expectedScores = new float[] {expectedScore, expectedScore,
+        expectedScore, expectedScore};
+    assertHits(s, query, "two span queries", expectedIds, expectedScores);
+  }
+  
+  /**
+   * Checks to see if the hits are what we expected.
+   * 
+   * @param query the query to execute
+   * @param description the description of the search
+   * @param expectedIds the expected document ids of the hits
+   * @param expectedScores the expected scores of the hits
+   * 
+   * @throws IOException
+   */
+  protected static void assertHits(Searcher s, Query query,
+      final String description, final String[] expectedIds,
+      final float[] expectedScores) throws IOException {
+    QueryUtils.check(query, s);
+    
+    final float tolerance = 1e-5f;
+    
+    // Hits hits = searcher.search(query);
+    // hits normalizes and throws things off if one score is greater than 1.0
+    TopDocs topdocs = s.search(query, null, 10000);
+    
+    /*****
+     * // display the hits System.out.println(hits.length() +
+     * " hits for search: \"" + description + '\"'); for (int i = 0; i <
+     * hits.length(); i++) { System.out.println("  " + FIELD_ID + ':' +
+     * hits.doc(i).get(FIELD_ID) + " (score:" + hits.score(i) + ')'); }
+     *****/
+    
+    // did we get the hits we expected
+    assertEquals(expectedIds.length, topdocs.totalHits);
+    for (int i = 0; i < topdocs.totalHits; i++) {
+      // System.out.println(i + " exp: " + expectedIds[i]);
+      // System.out.println(i + " field: " + hits.doc(i).get(FIELD_ID));
+      
+      int id = topdocs.scoreDocs[i].doc;
+      float score = topdocs.scoreDocs[i].score;
+      Document doc = s.doc(id);
+      assertEquals(expectedIds[i], doc.get(FIELD_ID));
+      boolean scoreEq = Math.abs(expectedScores[i] - score) < tolerance;
+      if (!scoreEq) {
+        System.out.println(i + " warning, expected score: " + expectedScores[i]
+            + ", actual " + score);
+        System.out.println(s.explain(query, id));
+      }
+      assertEquals(expectedScores[i], score, tolerance);
+      assertEquals(s.explain(query, id).getValue(), score, tolerance);
     }
-
-    /**
-     * Tests two span queries.
-     *
-     * @throws IOException
-     */
-    protected void doTestBooleanQueryWithSpanQueries(IndexSearcher s, final float expectedScore) throws IOException {
-
-        final Query spanQuery = new SpanTermQuery(new Term(FIELD_TEXT, "work"));
-        final BooleanQuery query = new BooleanQuery();
-        query.add(spanQuery, BooleanClause.Occur.MUST);
-        query.add(spanQuery, BooleanClause.Occur.MUST);
-        final String[] expectedIds = new String[] { "1", "2", "3", "4" };
-        final float[] expectedScores = new float[] { expectedScore, expectedScore, expectedScore, expectedScore };
-        assertHits(s, query, "two span queries", expectedIds, expectedScores);
-    }
-
-
-    /**
-     * Checks to see if the hits are what we expected.
-     *
-     * @param query the query to execute
-     * @param description the description of the search
-     * @param expectedIds the expected document ids of the hits
-     * @param expectedScores the expected scores of the hits
-     *
-     * @throws IOException
-     */
-    protected static void assertHits(Searcher s, Query query, final String description, final String[] expectedIds,
-            final float[] expectedScores) throws IOException {
-        QueryUtils.check(query,s);
-
-        final float tolerance = 1e-5f;
-
-        // Hits hits = searcher.search(query);
-        // hits normalizes and throws things off if one score is greater than 1.0
-        TopDocs topdocs = s.search(query,null,10000);
-
-        /*****
-        // display the hits
-        System.out.println(hits.length() + " hits for search: \"" + description + '\"');
-        for (int i = 0; i < hits.length(); i++) {
-            System.out.println("  " + FIELD_ID + ':' + hits.doc(i).get(FIELD_ID) + " (score:" + hits.score(i) + ')');
-        }
-        *****/
-
-        // did we get the hits we expected
-        assertEquals(expectedIds.length, topdocs.totalHits);
-        for (int i = 0; i < topdocs.totalHits; i++) {
-            //System.out.println(i + " exp: " + expectedIds[i]);
-            //System.out.println(i + " field: " + hits.doc(i).get(FIELD_ID));
-
-            int id = topdocs.scoreDocs[i].doc;
-            float score = topdocs.scoreDocs[i].score;
-            Document doc = s.doc(id);
-            assertEquals(expectedIds[i], doc.get(FIELD_ID));
-            boolean scoreEq = Math.abs(expectedScores[i] - score) < tolerance;
-            if (!scoreEq) {
-              System.out.println(i + " warning, expected score: " + expectedScores[i] + ", actual " + score);
-              System.out.println(s.explain(query,id));
-            }
-            assertEquals(expectedScores[i], score, tolerance);
-            assertEquals(s.explain(query,id).getValue(), score, tolerance);
-        }
-    }
-
-
+  }
+  
 }
\ No newline at end of file

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java Thu Jul 22 19:34:35 2010
@@ -23,8 +23,8 @@ import org.apache.lucene.analysis.MockAn
 import org.apache.lucene.analysis.MockTokenFilter;
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.search.*;
@@ -32,84 +32,97 @@ import org.apache.lucene.search.*;
 /*******************************************************************************
  * Some expanded tests to make sure my patch doesn't break other SpanTermQuery
  * functionality.
- *
+ * 
  */
 public class TestSpansAdvanced2 extends TestSpansAdvanced {
-    IndexSearcher searcher2;
-    /**
-     * Initializes the tests by adding documents to the index.
-     */
-    @Override
-    protected void setUp() throws Exception {
-        super.setUp();
-
-        // create test index
-        final IndexWriter writer = new IndexWriter(mDirectory,
-            new IndexWriterConfig(TEST_VERSION_CURRENT, 
-                new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true)).setOpenMode(
-                    OpenMode.APPEND));
-        addDocument(writer, "A", "Should we, could we, would we?");
-        addDocument(writer, "B", "It should.  Should it?");
-        addDocument(writer, "C", "It shouldn't.");
-        addDocument(writer, "D", "Should we, should we, should we.");
-        writer.close();
-
-        // re-open the searcher since we added more docs
-        searcher2 = new IndexSearcher(mDirectory, true);
-    }
-
-    /**
-     * Verifies that the index has the correct number of documents.
-     *
-     * @throws Exception
-     */
-    public void testVerifyIndex() throws Exception {
-        final IndexReader reader = IndexReader.open(mDirectory, true);
-        assertEquals(8, reader.numDocs());
-        reader.close();
-    }
-
-    /**
-     * Tests a single span query that matches multiple documents.
-     *
-     * @throws IOException
-     */
-    public void testSingleSpanQuery() throws IOException {
-
-        final Query spanQuery = new SpanTermQuery(new Term(FIELD_TEXT, "should"));
-        final String[] expectedIds = new String[] { "B", "D", "1", "2", "3", "4", "A" };
-        final float[] expectedScores = new float[] { 0.625f, 0.45927936f, 0.35355338f, 0.35355338f, 0.35355338f,
-                0.35355338f, 0.26516503f, };
-        assertHits(searcher2, spanQuery, "single span query", expectedIds, expectedScores);
-    }
-
-    /**
-     * Tests a single span query that matches multiple documents.
-     *
-     * @throws IOException
-     */
-    public void testMultipleDifferentSpanQueries() throws IOException {
-
-        final Query spanQuery1 = new SpanTermQuery(new Term(FIELD_TEXT, "should"));
-        final Query spanQuery2 = new SpanTermQuery(new Term(FIELD_TEXT, "we"));
-        final BooleanQuery query = new BooleanQuery();
-        query.add(spanQuery1, BooleanClause.Occur.MUST);
-        query.add(spanQuery2, BooleanClause.Occur.MUST);
-        final String[] expectedIds = new String[] { "D", "A" };
-        // these values were pre LUCENE-413
-        // final float[] expectedScores = new float[] { 0.93163157f, 0.20698164f };
-        final float[] expectedScores = new float[] { 1.0191123f, 0.93163157f };
-        assertHits(searcher2, query, "multiple different span queries", expectedIds, expectedScores);
-    }
-
-    /**
-     * Tests two span queries.
-     *
-     * @throws IOException
-     */
-    @Override
-    public void testBooleanQueryWithSpanQueries() throws IOException {
-
-        doTestBooleanQueryWithSpanQueries(searcher2, 0.73500174f);
-    }
+  IndexSearcher searcher2;
+  IndexReader reader2;
+  
+  /**
+   * Initializes the tests by adding documents to the index.
+   */
+  @Override
+  protected void setUp() throws Exception {
+    super.setUp();
+    
+    // create test index
+    final RandomIndexWriter writer = new RandomIndexWriter(random, mDirectory,
+        new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(
+            MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true))
+            .setOpenMode(OpenMode.APPEND));
+    addDocument(writer, "A", "Should we, could we, would we?");
+    addDocument(writer, "B", "It should.  Should it?");
+    addDocument(writer, "C", "It shouldn't.");
+    addDocument(writer, "D", "Should we, should we, should we.");
+    reader2 = writer.getReader();
+    writer.close();
+    
+    // re-open the searcher since we added more docs
+    searcher2 = new IndexSearcher(reader2);
+  }
+  
+  @Override
+  protected void tearDown() throws Exception {
+    searcher2.close();
+    reader2.close();
+    super.tearDown();
+  }
+  
+  /**
+   * Verifies that the index has the correct number of documents.
+   * 
+   * @throws Exception
+   */
+  public void testVerifyIndex() throws Exception {
+    final IndexReader reader = IndexReader.open(mDirectory, true);
+    assertEquals(8, reader.numDocs());
+    reader.close();
+  }
+  
+  /**
+   * Tests a single span query that matches multiple documents.
+   * 
+   * @throws IOException
+   */
+  public void testSingleSpanQuery() throws IOException {
+    
+    final Query spanQuery = new SpanTermQuery(new Term(FIELD_TEXT, "should"));
+    final String[] expectedIds = new String[] {"B", "D", "1", "2", "3", "4",
+        "A"};
+    final float[] expectedScores = new float[] {0.625f, 0.45927936f,
+        0.35355338f, 0.35355338f, 0.35355338f, 0.35355338f, 0.26516503f,};
+    assertHits(searcher2, spanQuery, "single span query", expectedIds,
+        expectedScores);
+  }
+  
+  /**
+   * Tests a single span query that matches multiple documents.
+   * 
+   * @throws IOException
+   */
+  public void testMultipleDifferentSpanQueries() throws IOException {
+    
+    final Query spanQuery1 = new SpanTermQuery(new Term(FIELD_TEXT, "should"));
+    final Query spanQuery2 = new SpanTermQuery(new Term(FIELD_TEXT, "we"));
+    final BooleanQuery query = new BooleanQuery();
+    query.add(spanQuery1, BooleanClause.Occur.MUST);
+    query.add(spanQuery2, BooleanClause.Occur.MUST);
+    final String[] expectedIds = new String[] {"D", "A"};
+    // these values were pre LUCENE-413
+    // final float[] expectedScores = new float[] { 0.93163157f, 0.20698164f };
+    final float[] expectedScores = new float[] {1.0191123f, 0.93163157f};
+    assertHits(searcher2, query, "multiple different span queries",
+        expectedIds, expectedScores);
+  }
+  
+  /**
+   * Tests two span queries.
+   * 
+   * @throws IOException
+   */
+  @Override
+  public void testBooleanQueryWithSpanQueries() throws IOException {
+    
+    doTestBooleanQueryWithSpanQueries(searcher2, 0.73500174f);
+  }
 }

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java Thu Jul 22 19:34:35 2010
@@ -72,6 +72,9 @@ public abstract class LuceneTestCase ext
   
   private volatile Thread.UncaughtExceptionHandler savedUncaughtExceptionHandler = null;
   
+  /** Used to track if setUp and tearDown are called correctly from subclasses */
+  private boolean setup;
+
   private static class UncaughtExceptionEntry {
     public final Thread thread;
     public final Throwable exception;
@@ -94,7 +97,8 @@ public abstract class LuceneTestCase ext
   @Override
   protected void setUp() throws Exception {
     super.setUp();
-    
+    assertFalse("ensure your tearDown() calls super.tearDown()!!!", setup);
+    setup = true;
     savedUncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler();
     Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
       public void uncaughtException(Thread t, Throwable e) {
@@ -128,6 +132,8 @@ public abstract class LuceneTestCase ext
 
   @Override
   protected void tearDown() throws Exception {
+    assertTrue("ensure your setUp() calls super.setUp()!!!", setup);
+    setup = false;
     BooleanQuery.setMaxClauseCount(savedBoolMaxClauseCount);
 
     try {

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/LuceneTestCaseJ4.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/LuceneTestCaseJ4.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/LuceneTestCaseJ4.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/LuceneTestCaseJ4.java Thu Jul 22 19:34:35 2010
@@ -23,6 +23,7 @@ import org.apache.lucene.search.FieldCac
 import org.apache.lucene.search.FieldCache.CacheEntry;
 import org.apache.lucene.util.FieldCacheSanityChecker.Insanity;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -33,6 +34,7 @@ import java.io.File;
 import java.io.PrintStream;
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Hashtable;
 import java.util.Iterator;
 import java.util.Random;
 import java.util.ArrayList;
@@ -106,6 +108,9 @@ public class LuceneTestCaseJ4 {
 
   private volatile Thread.UncaughtExceptionHandler savedUncaughtExceptionHandler = null;
   
+  /** Used to track if setUp and tearDown are called correctly from subclasses */
+  private boolean setup;
+
   private static class UncaughtExceptionEntry {
     public final Thread thread;
     public final Throwable exception;
@@ -155,6 +160,8 @@ public class LuceneTestCaseJ4 {
 
   @Before
   public void setUp() throws Exception {
+    Assert.assertFalse("ensure your tearDown() calls super.tearDown()!!!", setup);
+    setup = true;
     savedUncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler();
     Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
       public void uncaughtException(Thread t, Throwable e) {
@@ -191,6 +198,8 @@ public class LuceneTestCaseJ4 {
 
   @After
   public void tearDown() throws Exception {
+    Assert.assertTrue("ensure your setUp() calls super.setUp()!!!", setup);
+    setup = false;
     BooleanQuery.setMaxClauseCount(savedBoolMaxClauseCount);
     try {
 
@@ -330,6 +339,31 @@ public class LuceneTestCaseJ4 {
     return new Random(seed);
   }
 
+  private static final Map<Class<? extends LuceneTestCaseJ4>,Long> staticSeeds =
+    Collections.synchronizedMap(new WeakHashMap<Class<? extends LuceneTestCaseJ4>,Long>());
+
+  /**
+   * Returns a {@link Random} instance for generating random numbers from a beforeclass
+   * annotated method.
+   * The random seed is logged during test execution and printed to System.out on any failure
+   * for reproducing the test using {@link #newStaticRandom(Class, long)} with the recorded seed
+   * .
+   */
+  public static Random newStaticRandom(Class<? extends LuceneTestCaseJ4> clazz) {
+    return newStaticRandom(clazz, seedRnd.nextLong());
+  }
+  
+  /**
+   * Returns a {@link Random} instance for generating random numbers from a beforeclass
+   * annotated method.
+   * If an error occurs in the test that is not reproducible, you can use this method to
+   * initialize the number generator with the seed that was printed out during the failing test.
+   */
+  public static Random newStaticRandom(Class<? extends LuceneTestCaseJ4> clazz, long seed) {
+    staticSeeds.put(clazz, Long.valueOf(seed));
+    return new Random(seed);
+  }
+
   public String getName() {
     return this.name;
   }
@@ -348,6 +382,11 @@ public class LuceneTestCaseJ4 {
 
   // We get here from InterceptTestCaseEvents on the 'failed' event....
   public void reportAdditionalFailureInfo() {
+    Long staticSeed = staticSeeds.get(getClass());
+    if (staticSeed != null) {
+      System.out.println("NOTE: random static seed of testclass '" + getName() + "' was: " + staticSeed);
+    }
+    
     if (seed != null) {
       System.out.println("NOTE: random seed of testcase '" + getName() + "' was: " + seed);
     }

Propchange: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jul 22 19:34:35 2010
@@ -1,4 +1,5 @@
-/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:943137,949730
+/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:943137,949730,957490,960490,961612
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:953476-966816
 /lucene/java/branches/flex_1458/src/test/org/apache/lucene/util/TestAttributeSource.java:824912-931101
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/util/TestAttributeSource.java:909334,948516
 /lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java:924483-924731,924781,925176-925462

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestNumericUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestNumericUtils.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestNumericUtils.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestNumericUtils.java Thu Jul 22 19:34:35 2010
@@ -20,6 +20,7 @@ package org.apache.lucene.util;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Iterator;
+import java.util.Random;
 
 public class TestNumericUtils extends LuceneTestCase {
 
@@ -30,7 +31,7 @@ public class TestNumericUtils extends Lu
       NumericUtils.longToPrefixCoded(l, 0, act);
       if (last!=null) {
         // test if smaller
-        assertTrue("actual bigger than last (BytesRef)", BytesRef.getUTF8SortedAsUTF16Comparator().compare(last, act) < 0 );
+        assertTrue("actual bigger than last (BytesRef)", last.compareTo(act) < 0 );
         assertTrue("actual bigger than last (as String)", last.utf8ToString().compareTo(act.utf8ToString()) < 0 );
       }
       // test is back and forward conversion works
@@ -48,7 +49,7 @@ public class TestNumericUtils extends Lu
       NumericUtils.intToPrefixCoded(i, 0, act);
       if (last!=null) {
         // test if smaller
-        assertTrue("actual bigger than last (BytesRef)", BytesRef.getUTF8SortedAsUTF16Comparator().compare(last, act) < 0 );
+        assertTrue("actual bigger than last (BytesRef)", last.compareTo(act) < 0 );
         assertTrue("actual bigger than last (as String)", last.utf8ToString().compareTo(act.utf8ToString()) < 0 );
       }
       // test is back and forward conversion works
@@ -84,7 +85,7 @@ public class TestNumericUtils extends Lu
     
     // check sort order (prefixVals should be ascending)
     for (int i=1; i<prefixVals.length; i++) {
-      assertTrue( "check sort order", BytesRef.getUTF8SortedAsUTF16Comparator().compare(prefixVals[i-1], prefixVals[i] ) < 0 );
+      assertTrue( "check sort order", prefixVals[i-1].compareTo(prefixVals[i]) < 0 );
     }
         
     // check the prefix encoding, lower precision should have the difference to original value equal to the lower removed bits
@@ -124,7 +125,7 @@ public class TestNumericUtils extends Lu
     
     // check sort order (prefixVals should be ascending)
     for (int i=1; i<prefixVals.length; i++) {
-      assertTrue( "check sort order", BytesRef.getUTF8SortedAsUTF16Comparator().compare(prefixVals[i-1], prefixVals[i] ) < 0 );
+      assertTrue( "check sort order", prefixVals[i-1].compareTo(prefixVals[i]) < 0 );
     }
     
     // check the prefix encoding, lower precision should have the difference to original value equal to the lower removed bits
@@ -179,23 +180,30 @@ public class TestNumericUtils extends Lu
   
   // INFO: Tests for trieCodeLong()/trieCodeInt() not needed because implicitely tested by range filter tests
   
-  /** Note: The neededBounds iterator must be unsigned (easier understanding what's happening) */
-  protected void assertLongRangeSplit(final long lower, final long upper, int precisionStep,
-    final boolean useBitSet, final Iterator<Long> neededBounds
+  /** Note: The neededBounds Iterable must be unsigned (easier understanding what's happening) */
+  private void assertLongRangeSplit(final long lower, final long upper, int precisionStep,
+    final boolean useBitSet, final Iterable<Long> expectedBounds, final Iterable<Integer> expectedShifts
   ) throws Exception {
     final OpenBitSet bits=useBitSet ? new OpenBitSet(upper-lower+1) : null;
-    
+    final Iterator<Long> neededBounds = (expectedBounds == null) ? null : expectedBounds.iterator();
+    final Iterator<Integer> neededShifts = (expectedShifts == null) ? null : expectedShifts.iterator();
+
     NumericUtils.splitLongRange(new NumericUtils.LongRangeBuilder() {
       @Override
       public void addRange(long min, long max, int shift) {
         assertTrue("min, max should be inside bounds", min>=lower && min<=upper && max>=lower && max<=upper);
         if (useBitSet) for (long l=min; l<=max; l++) {
           assertFalse("ranges should not overlap", bits.getAndSet(l-lower) );
+          // extra exit condition to prevent overflow on MAX_VALUE
+          if (l == max) break;
         }
+        if (neededBounds == null || neededShifts == null)
+          return;
         // make unsigned longs for easier display and understanding
         min ^= 0x8000000000000000L;
         max ^= 0x8000000000000000L;
-        //System.out.println("Long.valueOf(0x"+Long.toHexString(min>>>shift)+"L),Long.valueOf(0x"+Long.toHexString(max>>>shift)+"L),");
+        //System.out.println("0x"+Long.toHexString(min>>>shift)+"L,0x"+Long.toHexString(max>>>shift)+"L)/*shift="+shift+"*/,");
+        assertEquals( "shift", neededShifts.next().intValue(), shift);
         assertEquals( "inner min bound", neededBounds.next().longValue(), min>>>shift);
         assertEquals( "inner max bound", neededBounds.next().longValue(), max>>>shift);
       }
@@ -208,64 +216,218 @@ public class TestNumericUtils extends Lu
     }
   }
   
+  /** LUCENE-2541: NumericRangeQuery errors with endpoints near long min and max values */
+  public void testLongExtremeValues() throws Exception {
+    // upper end extremes
+    assertLongRangeSplit(Long.MAX_VALUE, Long.MAX_VALUE, 1, true, Arrays.asList(
+      0xffffffffffffffffL,0xffffffffffffffffL
+    ), Arrays.asList(
+      0
+    ));
+    assertLongRangeSplit(Long.MAX_VALUE, Long.MAX_VALUE, 2, true, Arrays.asList(
+      0xffffffffffffffffL,0xffffffffffffffffL
+    ), Arrays.asList(
+      0
+    ));
+    assertLongRangeSplit(Long.MAX_VALUE, Long.MAX_VALUE, 4, true, Arrays.asList(
+      0xffffffffffffffffL,0xffffffffffffffffL
+    ), Arrays.asList(
+      0
+    ));
+    assertLongRangeSplit(Long.MAX_VALUE, Long.MAX_VALUE, 6, true, Arrays.asList(
+      0xffffffffffffffffL,0xffffffffffffffffL
+    ), Arrays.asList(
+      0
+    ));
+    assertLongRangeSplit(Long.MAX_VALUE, Long.MAX_VALUE, 8, true, Arrays.asList(
+      0xffffffffffffffffL,0xffffffffffffffffL
+    ), Arrays.asList(
+      0
+    ));
+    assertLongRangeSplit(Long.MAX_VALUE, Long.MAX_VALUE, 64, true, Arrays.asList(
+      0xffffffffffffffffL,0xffffffffffffffffL
+    ), Arrays.asList(
+      0
+    ));
+
+    assertLongRangeSplit(Long.MAX_VALUE-0xfL, Long.MAX_VALUE, 4, true, Arrays.asList(
+      0xfffffffffffffffL,0xfffffffffffffffL
+    ), Arrays.asList(
+      4
+    ));
+    assertLongRangeSplit(Long.MAX_VALUE-0x10L, Long.MAX_VALUE, 4, true, Arrays.asList(
+      0xffffffffffffffefL,0xffffffffffffffefL,
+      0xfffffffffffffffL,0xfffffffffffffffL
+    ), Arrays.asList(
+      0, 4
+    ));
+
+    // lower end extremes
+    assertLongRangeSplit(Long.MIN_VALUE, Long.MIN_VALUE, 1, true, Arrays.asList(
+      0x0000000000000000L,0x0000000000000000L
+    ), Arrays.asList(
+      0
+    ));
+    assertLongRangeSplit(Long.MIN_VALUE, Long.MIN_VALUE, 2, true, Arrays.asList(
+      0x0000000000000000L,0x0000000000000000L
+    ), Arrays.asList(
+      0
+    ));
+    assertLongRangeSplit(Long.MIN_VALUE, Long.MIN_VALUE, 4, true, Arrays.asList(
+      0x0000000000000000L,0x0000000000000000L
+    ), Arrays.asList(
+      0
+    ));
+    assertLongRangeSplit(Long.MIN_VALUE, Long.MIN_VALUE, 6, true, Arrays.asList(
+      0x0000000000000000L,0x0000000000000000L
+    ), Arrays.asList(
+      0
+    ));
+    assertLongRangeSplit(Long.MIN_VALUE, Long.MIN_VALUE, 8, true, Arrays.asList(
+      0x0000000000000000L,0x0000000000000000L
+    ), Arrays.asList(
+      0
+    ));
+    assertLongRangeSplit(Long.MIN_VALUE, Long.MIN_VALUE, 64, true, Arrays.asList(
+      0x0000000000000000L,0x0000000000000000L
+    ), Arrays.asList(
+      0
+    ));
+
+    assertLongRangeSplit(Long.MIN_VALUE, Long.MIN_VALUE+0xfL, 4, true, Arrays.asList(
+      0x000000000000000L,0x000000000000000L
+    ), Arrays.asList(
+      4
+    ));
+    assertLongRangeSplit(Long.MIN_VALUE, Long.MIN_VALUE+0x10L, 4, true, Arrays.asList(
+      0x0000000000000010L,0x0000000000000010L,
+      0x000000000000000L,0x000000000000000L
+    ), Arrays.asList(
+      0, 4
+    ));
+  }
+  
+  public void testRandomSplit() throws Exception {
+    final Random random = newRandom();
+    long num = 100L * _TestUtil.getRandomMultiplier();
+    for (long i=0; i < num; i++) {
+      executeOneRandomSplit(random);
+    }
+  }
+  
+  private void executeOneRandomSplit(final Random random) throws Exception {
+    long lower = randomLong(random);
+    long len = (long) random.nextInt(16384*1024); // not too large bitsets, else OOME!
+    while (lower + len < lower) { // overflow
+      lower >>= 1;
+    }
+    assertLongRangeSplit(lower, lower + len, random.nextInt(64) + 1, true, null, null);
+  }
+  
+  private long randomLong(final Random random) {
+    long val;
+    switch(random.nextInt(4)) {
+      case 0:
+        val = 1L << (random.nextInt(63)); //  patterns like 0x000000100000 (-1 yields patterns like 0x0000fff)
+        break;
+      case 1:
+        val = -1L << (random.nextInt(63)); // patterns like 0xfffff00000
+        break;
+      default:
+        val = random.nextLong();
+    }
+
+    val += random.nextInt(5)-2;
+
+    if (random.nextBoolean()) {
+      if (random.nextBoolean()) val += random.nextInt(100)-50;
+      if (random.nextBoolean()) val = ~val;
+      if (random.nextBoolean()) val = val<<1;
+      if (random.nextBoolean()) val = val>>>1;
+    }
+
+    return val;
+  }
+  
   public void testSplitLongRange() throws Exception {
     // a hard-coded "standard" range
-    assertLongRangeSplit(-5000L, 9500L, 4, true, Arrays.asList(new Long[]{
-      Long.valueOf(0x7fffffffffffec78L),Long.valueOf(0x7fffffffffffec7fL),
-      Long.valueOf(0x8000000000002510L),Long.valueOf(0x800000000000251cL),
-      Long.valueOf(0x7fffffffffffec8L), Long.valueOf(0x7fffffffffffecfL),
-      Long.valueOf(0x800000000000250L), Long.valueOf(0x800000000000250L),
-      Long.valueOf(0x7fffffffffffedL),  Long.valueOf(0x7fffffffffffefL),
-      Long.valueOf(0x80000000000020L),  Long.valueOf(0x80000000000024L),
-      Long.valueOf(0x7ffffffffffffL),   Long.valueOf(0x8000000000001L)
-    }).iterator());
+    assertLongRangeSplit(-5000L, 9500L, 4, true, Arrays.asList(
+      0x7fffffffffffec78L,0x7fffffffffffec7fL,
+      0x8000000000002510L,0x800000000000251cL,
+      0x7fffffffffffec8L, 0x7fffffffffffecfL,
+      0x800000000000250L, 0x800000000000250L,
+      0x7fffffffffffedL,  0x7fffffffffffefL,
+      0x80000000000020L,  0x80000000000024L,
+      0x7ffffffffffffL,   0x8000000000001L
+    ), Arrays.asList(
+      0, 0,
+      4, 4,
+      8, 8,
+      12
+    ));
     
     // the same with no range splitting
-    assertLongRangeSplit(-5000L, 9500L, 64, true, Arrays.asList(new Long[]{
-      Long.valueOf(0x7fffffffffffec78L),Long.valueOf(0x800000000000251cL)
-    }).iterator());
+    assertLongRangeSplit(-5000L, 9500L, 64, true, Arrays.asList(
+      0x7fffffffffffec78L,0x800000000000251cL
+    ), Arrays.asList(
+      0
+    ));
     
     // this tests optimized range splitting, if one of the inner bounds
     // is also the bound of the next lower precision, it should be used completely
-    assertLongRangeSplit(0L, 1024L+63L, 4, true, Arrays.asList(new Long[]{
-      Long.valueOf(0x800000000000040L), Long.valueOf(0x800000000000043L),
-      Long.valueOf(0x80000000000000L),  Long.valueOf(0x80000000000003L)
-    }).iterator());
+    assertLongRangeSplit(0L, 1024L+63L, 4, true, Arrays.asList(
+      0x800000000000040L, 0x800000000000043L,
+      0x80000000000000L,  0x80000000000003L
+    ), Arrays.asList(
+      4, 8
+    ));
     
     // the full long range should only consist of a lowest precision range; no bitset testing here, as too much memory needed :-)
-    assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 8, false, Arrays.asList(new Long[]{
-      Long.valueOf(0x00L),Long.valueOf(0xffL)
-    }).iterator());
+    assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 8, false, Arrays.asList(
+      0x00L,0xffL
+    ), Arrays.asList(
+      56
+    ));
 
     // the same with precisionStep=4
-    assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 4, false, Arrays.asList(new Long[]{
-      Long.valueOf(0x0L),Long.valueOf(0xfL)
-    }).iterator());
+    assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 4, false, Arrays.asList(
+      0x0L,0xfL
+    ), Arrays.asList(
+      60
+    ));
 
     // the same with precisionStep=2
-    assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 2, false, Arrays.asList(new Long[]{
-      Long.valueOf(0x0L),Long.valueOf(0x3L)
-    }).iterator());
+    assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 2, false, Arrays.asList(
+      0x0L,0x3L
+    ), Arrays.asList(
+      62
+    ));
 
     // the same with precisionStep=1
-    assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 1, false, Arrays.asList(new Long[]{
-      Long.valueOf(0x0L),Long.valueOf(0x1L)
-    }).iterator());
+    assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 1, false, Arrays.asList(
+      0x0L,0x1L
+    ), Arrays.asList(
+      63
+    ));
 
     // a inverse range should produce no sub-ranges
-    assertLongRangeSplit(9500L, -5000L, 4, false, Collections. <Long> emptyList().iterator());    
+    assertLongRangeSplit(9500L, -5000L, 4, false, Collections.<Long>emptyList(), Collections.<Integer>emptyList());    
 
     // a 0-length range should reproduce the range itsself
-    assertLongRangeSplit(9500L, 9500L, 4, false, Arrays.asList(new Long[]{
-      Long.valueOf(0x800000000000251cL),Long.valueOf(0x800000000000251cL)
-    }).iterator());
+    assertLongRangeSplit(9500L, 9500L, 4, false, Arrays.asList(
+      0x800000000000251cL,0x800000000000251cL
+    ), Arrays.asList(
+      0
+    ));
   }
 
-  /** Note: The neededBounds iterator must be unsigned (easier understanding what's happening) */
-  protected void assertIntRangeSplit(final int lower, final int upper, int precisionStep,
-    final boolean useBitSet, final Iterator<Integer> neededBounds
+  /** Note: The neededBounds Iterable must be unsigned (easier understanding what's happening) */
+  private void assertIntRangeSplit(final int lower, final int upper, int precisionStep,
+    final boolean useBitSet, final Iterable<Integer> expectedBounds, final Iterable<Integer> expectedShifts
   ) throws Exception {
     final OpenBitSet bits=useBitSet ? new OpenBitSet(upper-lower+1) : null;
+    final Iterator<Integer> neededBounds = (expectedBounds == null) ? null : expectedBounds.iterator();
+    final Iterator<Integer> neededShifts = (expectedShifts == null) ? null : expectedShifts.iterator();
     
     NumericUtils.splitIntRange(new NumericUtils.IntRangeBuilder() {
       @Override
@@ -273,11 +435,16 @@ public class TestNumericUtils extends Lu
         assertTrue("min, max should be inside bounds", min>=lower && min<=upper && max>=lower && max<=upper);
         if (useBitSet) for (int i=min; i<=max; i++) {
           assertFalse("ranges should not overlap", bits.getAndSet(i-lower) );
+          // extra exit condition to prevent overflow on MAX_VALUE
+          if (i == max) break;
         }
+        if (neededBounds == null)
+          return;
         // make unsigned ints for easier display and understanding
         min ^= 0x80000000;
         max ^= 0x80000000;
-        //System.out.println("Integer.valueOf(0x"+Integer.toHexString(min>>>shift)+"),Integer.valueOf(0x"+Integer.toHexString(max>>>shift)+"),");
+        //System.out.println("0x"+Integer.toHexString(min>>>shift)+",0x"+Integer.toHexString(max>>>shift)+")/*shift="+shift+"*/,");
+        assertEquals( "shift", neededShifts.next().intValue(), shift);
         assertEquals( "inner min bound", neededBounds.next().intValue(), min>>>shift);
         assertEquals( "inner max bound", neededBounds.next().intValue(), max>>>shift);
       }
@@ -292,55 +459,74 @@ public class TestNumericUtils extends Lu
   
   public void testSplitIntRange() throws Exception {
     // a hard-coded "standard" range
-    assertIntRangeSplit(-5000, 9500, 4, true, Arrays.asList(new Integer[]{
-      Integer.valueOf(0x7fffec78),Integer.valueOf(0x7fffec7f),
-      Integer.valueOf(0x80002510),Integer.valueOf(0x8000251c),
-      Integer.valueOf(0x7fffec8), Integer.valueOf(0x7fffecf),
-      Integer.valueOf(0x8000250), Integer.valueOf(0x8000250),
-      Integer.valueOf(0x7fffed),  Integer.valueOf(0x7fffef),
-      Integer.valueOf(0x800020),  Integer.valueOf(0x800024),
-      Integer.valueOf(0x7ffff),   Integer.valueOf(0x80001)
-    }).iterator());
+    assertIntRangeSplit(-5000, 9500, 4, true, Arrays.asList(
+      0x7fffec78,0x7fffec7f,
+      0x80002510,0x8000251c,
+      0x7fffec8, 0x7fffecf,
+      0x8000250, 0x8000250,
+      0x7fffed,  0x7fffef,
+      0x800020,  0x800024,
+      0x7ffff,   0x80001
+    ), Arrays.asList(
+      0, 0,
+      4, 4,
+      8, 8,
+      12
+    ));
     
     // the same with no range splitting
-    assertIntRangeSplit(-5000, 9500, 32, true, Arrays.asList(new Integer[]{
-      Integer.valueOf(0x7fffec78),Integer.valueOf(0x8000251c)
-    }).iterator());
+    assertIntRangeSplit(-5000, 9500, 32, true, Arrays.asList(
+      0x7fffec78,0x8000251c
+    ), Arrays.asList(
+      0
+    ));
     
     // this tests optimized range splitting, if one of the inner bounds
     // is also the bound of the next lower precision, it should be used completely
-    assertIntRangeSplit(0, 1024+63, 4, true, Arrays.asList(new Integer[]{
-      Integer.valueOf(0x8000040), Integer.valueOf(0x8000043),
-      Integer.valueOf(0x800000),  Integer.valueOf(0x800003)
-    }).iterator());
+    assertIntRangeSplit(0, 1024+63, 4, true, Arrays.asList(
+      0x8000040, 0x8000043,
+      0x800000,  0x800003
+    ), Arrays.asList(
+      4, 8
+    ));
     
     // the full int range should only consist of a lowest precision range; no bitset testing here, as too much memory needed :-)
-    assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 8, false, Arrays.asList(new Integer[]{
-      Integer.valueOf(0x00),Integer.valueOf(0xff)
-    }).iterator());
+    assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 8, false, Arrays.asList(
+      0x00,0xff
+    ), Arrays.asList(
+      24
+    ));
 
     // the same with precisionStep=4
-    assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 4, false, Arrays.asList(new Integer[]{
-      Integer.valueOf(0x0),Integer.valueOf(0xf)
-    }).iterator());
+    assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 4, false, Arrays.asList(
+      0x0,0xf
+    ), Arrays.asList(
+      28
+    ));
 
     // the same with precisionStep=2
-    assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 2, false, Arrays.asList(new Integer[]{
-      Integer.valueOf(0x0),Integer.valueOf(0x3)
-    }).iterator());
+    assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 2, false, Arrays.asList(
+      0x0,0x3
+    ), Arrays.asList(
+      30
+    ));
 
     // the same with precisionStep=1
-    assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 1, false, Arrays.asList(new Integer[]{
-      Integer.valueOf(0x0),Integer.valueOf(0x1)
-    }).iterator());
+    assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 1, false, Arrays.asList(
+      0x0,0x1
+    ), Arrays.asList(
+      31
+    ));
 
     // a inverse range should produce no sub-ranges
-    assertIntRangeSplit(9500, -5000, 4, false, Collections. <Integer> emptyList().iterator());    
+    assertIntRangeSplit(9500, -5000, 4, false, Collections.<Integer>emptyList(), Collections.<Integer>emptyList());    
 
     // a 0-length range should reproduce the range itsself
-    assertIntRangeSplit(9500, 9500, 4, false, Arrays.asList(new Integer[]{
-      Integer.valueOf(0x8000251c),Integer.valueOf(0x8000251c)
-    }).iterator());
+    assertIntRangeSplit(9500, 9500, 4, false, Arrays.asList(
+      0x8000251c,0x8000251c
+    ), Arrays.asList(
+      0
+    ));
   }
 
 }

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/AutomatonTestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/AutomatonTestUtil.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/AutomatonTestUtil.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/AutomatonTestUtil.java Thu Jul 22 19:34:35 2010
@@ -17,8 +17,17 @@ package org.apache.lucene.util.automaton
  * limitations under the License.
  */
 
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
 import java.util.Random;
+import java.util.Set;
 
+import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.UnicodeUtil;
 import org.apache.lucene.util._TestUtil;
 
@@ -65,4 +74,143 @@ public class AutomatonTestUtil {
     }
     return new String(buffer, 0, end);
   }
+  
+  // picks a random int code point that this transition
+  // accepts, avoiding the surrogates range since they are
+  // "defined" in UTF32.  Don't call this on a transition
+  // that only accepts UTF16 surrogate values!!
+  private static int getRandomCodePoint(final Random r, final Transition t) {
+    return t.min+r.nextInt(t.max-t.min+1);
+  }
+
+  public static class RandomAcceptedStrings {
+
+    private final Map<Transition,Boolean> leadsToAccept;
+    private final Automaton a;
+
+    private static class ArrivingTransition {
+      final State from;
+      final Transition t;
+      public ArrivingTransition(State from, Transition t) {
+        this.from = from;
+        this.t = t;
+      }
+    }
+
+    public RandomAcceptedStrings(Automaton a) {
+      this.a = a;
+      if (a.isSingleton()) {
+        leadsToAccept = null;
+        return;
+      }
+
+      // must use IdentityHashmap because two Transitions w/
+      // different start nodes can be considered the same
+      leadsToAccept = new IdentityHashMap<Transition,Boolean>();
+      final Map<State,List<ArrivingTransition>> allArriving = new HashMap<State,List<ArrivingTransition>>();
+
+      final LinkedList<State> q = new LinkedList<State>();
+      final Set<State> seen = new HashSet<State>();
+
+      // reverse map the transitions, so we can quickly look
+      // up all arriving transitions to a given state
+      for(State s: a.getNumberedStates()) {
+        for(int i=0;i<s.numTransitions;i++) {
+          final Transition t = s.transitionsArray[i];
+          List<ArrivingTransition> tl = allArriving.get(t.to);
+          if (tl == null) {
+            tl = new ArrayList<ArrivingTransition>();
+            allArriving.put(t.to, tl);
+          }
+          tl.add(new ArrivingTransition(s, t));
+        }
+        if (s.accept) {
+          q.add(s);
+          seen.add(s);
+        }
+      }
+
+      // Breadth-first search, from accept states,
+      // backwards:
+      while(!q.isEmpty()) {
+        final State s = q.removeFirst();
+        List<ArrivingTransition> arriving = allArriving.get(s);
+        if (arriving != null) {
+          for(ArrivingTransition at : arriving) {
+            final State from = at.from;
+            if (!seen.contains(from)) {
+              q.add(from);
+              seen.add(from);
+              leadsToAccept.put(at.t, Boolean.TRUE);
+            }
+          }
+        }
+      }
+    }
+
+    public int[] getRandomAcceptedString(Random r) {
+
+      final List<Integer> soFar = new ArrayList<Integer>();
+      if (a.isSingleton()) {
+        // accepts only one
+        final String s = a.singleton;
+      
+        int charUpto = 0;
+        while(charUpto < s.length()) {
+          final int cp = s.codePointAt(charUpto);
+          charUpto += Character.charCount(cp);
+          soFar.add(cp);
+        }
+      } else {
+
+        State s = a.initial;
+
+        while(true) {
+      
+          if (s.accept) {
+            if (s.numTransitions == 0) {
+              // stop now
+              break;
+            } else {
+              if (r.nextBoolean()) {
+                break;
+              }
+            }
+          }
+
+          if (s.numTransitions == 0) {
+            throw new RuntimeException("this automaton has dead states");
+          }
+
+          boolean cheat = r.nextBoolean();
+
+          final Transition t;
+          if (cheat) {
+            // pick a transition that we know is the fastest
+            // path to an accept state
+            List<Transition> toAccept = new ArrayList<Transition>();
+            for(int i=0;i<s.numTransitions;i++) {
+              final Transition t0 = s.transitionsArray[i];
+              if (leadsToAccept.containsKey(t0)) {
+                toAccept.add(t0);
+              }
+            }
+            if (toAccept.size() == 0) {
+              // this is OK -- it means we jumped into a cycle
+              t = s.transitionsArray[r.nextInt(s.numTransitions)];
+            } else {
+              t = toAccept.get(r.nextInt(toAccept.size()));
+            }
+          } else {
+            t = s.transitionsArray[r.nextInt(s.numTransitions)];
+          }
+          
+          soFar.add(getRandomCodePoint(r, t));
+          s = t.to;
+        }
+      }
+
+      return ArrayUtil.toIntArray(soFar);
+    }
+  }
 }

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestBasicOperations.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestBasicOperations.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestBasicOperations.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestBasicOperations.java Thu Jul 22 19:34:35 2010
@@ -95,7 +95,7 @@ public class TestBasicOperations extends
       final Automaton a = re.toAutomaton();
       assertFalse(BasicOperations.isEmpty(a));
 
-      final BasicOperations.RandomAcceptedStrings rx = new BasicOperations.RandomAcceptedStrings(a);
+      final AutomatonTestUtil.RandomAcceptedStrings rx = new AutomatonTestUtil.RandomAcceptedStrings(a);
       for(int j=0;j<ITER2;j++) {
         int[] acc = null;
         try {

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java Thu Jul 22 19:34:35 2010
@@ -35,7 +35,7 @@ public class TestDeterminizeLexicon exte
   private List<String> terms = new ArrayList<String>();
   private Random random;
   
-  public void testLexicon() {
+  public void testLexicon() throws Exception {
     random = newRandom();
     for (int i = 0; i < 3*_TestUtil.getRandomMultiplier(); i++) {
       automata.clear();
@@ -49,7 +49,7 @@ public class TestDeterminizeLexicon exte
     }
   }
   
-  public void assertLexicon() {
+  public void assertLexicon() throws Exception {
     Collections.shuffle(automata, random);
     final Automaton lex = BasicOperations.union(automata);
     lex.determinize();
@@ -59,8 +59,8 @@ public class TestDeterminizeLexicon exte
     }
     final ByteRunAutomaton lexByte = new ByteRunAutomaton(lex);
     for (String s : terms) {
-      BytesRef termByte = new BytesRef(s);
-      assertTrue(lexByte.run(termByte.bytes, 0, termByte.length));
+      byte bytes[] = s.getBytes("UTF-8");
+      assertTrue(lexByte.run(bytes, 0, bytes.length));
     }
   }
 }

Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestUTF32ToUTF8.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestUTF32ToUTF8.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestUTF32ToUTF8.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestUTF32ToUTF8.java Thu Jul 22 19:34:35 2010
@@ -173,7 +173,7 @@ public class TestUTF32ToUTF8 extends Luc
   private void assertAutomaton(Automaton automaton) throws Exception {
     CharacterRunAutomaton cra = new CharacterRunAutomaton(automaton);
     ByteRunAutomaton bra = new ByteRunAutomaton(automaton);
-    final BasicOperations.RandomAcceptedStrings ras = new BasicOperations.RandomAcceptedStrings(automaton);
+    final AutomatonTestUtil.RandomAcceptedStrings ras = new AutomatonTestUtil.RandomAcceptedStrings(automaton);
     
     for (int i = 0; i < 1000*_TestUtil.getRandomMultiplier(); i++) {
       final String string;
@@ -193,8 +193,8 @@ public class TestUTF32ToUTF8 extends Luc
           throw e;
         }
       }
-      BytesRef bytesRef = new BytesRef(string);
-      assertEquals(cra.run(string), bra.run(bytesRef.bytes, 0, bytesRef.length));
+      byte bytes[] = string.getBytes("UTF-8");
+      assertEquals(cra.run(string), bra.run(bytes, 0, bytes.length));
     }
   }
 }

Modified: lucene/dev/branches/realtime_search/modules/analysis/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/CHANGES.txt?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/CHANGES.txt (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/CHANGES.txt Thu Jul 22 19:34:35 2010
@@ -27,11 +27,14 @@ New Features
      with text contained in the required words (inverse of StopFilter).
    - o.a.l.analysis.miscellaneous.HyphenatedWordsFilter: A TokenFilter that puts 
      hyphenated words broken into two lines back together.
+   - o.a.l.analysis.miscellaneous.CapitalizationFilter: A TokenFilter that applies
+     capitalization rules to tokens.
    - o.a.l.analysis.pattern: Package for pattern-based analysis, containing a 
      CharFilter, Tokenizer, and Tokenfilter for transforming text with regexes.
    - o.a.l.analysis.synonym.SynonymFilter: A synonym filter that supports multi-word
      synonyms.
-   (... in progress)
+   - o.a.l.analysis.phonetic: Package for phonetic search, containing various
+     phonetic encoders such as Double Metaphone.
 
   * LUCENE-2413: Consolidated all Lucene analyzers into common.
     - o.a.l.analysis.KeywordAnalyzer -> o.a.l.analysis.core.KeywordAnalyzer
@@ -60,7 +63,6 @@ New Features
     - o.a.l.analysis.ReusableAnalyzerBase -> o.a.l.analysis.util.ReusableAnalyzerBase
     - o.a.l.analysis.StopwordAnalyzerBase -> o.a.l.analysis.util.StopwordAnalyzerBase
     - o.a.l.analysis.WordListLoader -> o.a.l.analysis.util.WordListLoader
-    ... (in progress)
 
 Build
 



Mime
View raw message