lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r687357 [6/6] - in /lucene/java/trunk: ./ contrib/analyzers/src/java/org/apache/lucene/analysis/br/ contrib/analyzers/src/java/org/apache/lucene/analysis/cjk/ contrib/analyzers/src/java/org/apache/lucene/analysis/cn/ contrib/analyzers/src/j...
Date Wed, 20 Aug 2008 14:38:11 GMT
Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java Wed Aug 20 07:38:07 2008
@@ -16,10 +16,10 @@
  * limitations under the License.
  */
 
-import junit.framework.TestCase;
 import org.apache.lucene.analysis.standard.StandardFilter;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
 import org.apache.lucene.util.English;
+import org.apache.lucene.util.LuceneTestCase;
 
 import java.io.IOException;
 import java.io.StringReader;
@@ -29,7 +29,7 @@
 /**
  * tests for the TeeTokenFilter and SinkTokenizer
  */
-public class TeeSinkTokenTest extends TestCase {
+public class TeeSinkTokenTest extends LuceneTestCase {
   protected StringBuffer buffer1;
   protected StringBuffer buffer2;
   protected String[] tokens1;
@@ -63,23 +63,23 @@
 
     SinkTokenizer sink1 = new SinkTokenizer(null) {
       public void add(Token t) {
-        if (t != null && t.termText().equalsIgnoreCase("The")) {
+        if (t != null && t.term().equalsIgnoreCase("The")) {
           super.add(t);
         }
       }
     };
     TokenStream source = new TeeTokenFilter(new WhitespaceTokenizer(new StringReader(buffer1.toString())), sink1);
-    Token token = null;
     int i = 0;
-    while ((token = source.next()) != null) {
-      assertTrue(token.termText() + " is not equal to " + tokens1[i], token.termText().equals(tokens1[i]) == true);
+    final Token reusableToken = new Token();
+    for (Token nextToken = source.next(reusableToken); nextToken != null; nextToken = source.next(reusableToken)) {
+      assertTrue(nextToken.term() + " is not equal to " + tokens1[i], nextToken.term().equals(tokens1[i]) == true);
       i++;
     }
     assertTrue(i + " does not equal: " + tokens1.length, i == tokens1.length);
     assertTrue("sink1 Size: " + sink1.getTokens().size() + " is not: " + 2, sink1.getTokens().size() == 2);
     i = 0;
-    while ((token = sink1.next()) != null) {
-      assertTrue(token.termText() + " is not equal to " + "The", token.termText().equalsIgnoreCase("The") == true);
+    for (Token token = sink1.next(reusableToken); token != null; token = sink1.next(reusableToken)) {
+      assertTrue(token.term() + " is not equal to " + "The", token.term().equalsIgnoreCase("The") == true);
       i++;
     }
     assertTrue(i + " does not equal: " + sink1.getTokens().size(), i == sink1.getTokens().size());
@@ -88,54 +88,54 @@
   public void testMultipleSources() throws Exception {
     SinkTokenizer theDetector = new SinkTokenizer(null) {
       public void add(Token t) {
-        if (t != null && t.termText().equalsIgnoreCase("The")) {
+        if (t != null && t.term().equalsIgnoreCase("The")) {
           super.add(t);
         }
       }
     };
     SinkTokenizer dogDetector = new SinkTokenizer(null) {
       public void add(Token t) {
-        if (t != null && t.termText().equalsIgnoreCase("Dogs")) {
+        if (t != null && t.term().equalsIgnoreCase("Dogs")) {
           super.add(t);
         }
       }
     };
     TokenStream source1 = new CachingTokenFilter(new TeeTokenFilter(new TeeTokenFilter(new WhitespaceTokenizer(new StringReader(buffer1.toString())), theDetector), dogDetector));
     TokenStream source2 = new TeeTokenFilter(new TeeTokenFilter(new WhitespaceTokenizer(new StringReader(buffer2.toString())), theDetector), dogDetector);
-    Token token = null;
     int i = 0;
-    while ((token = source1.next()) != null) {
-      assertTrue(token.termText() + " is not equal to " + tokens1[i], token.termText().equals(tokens1[i]) == true);
+    final Token reusableToken = new Token();
+    for (Token nextToken = source1.next(reusableToken); nextToken != null; nextToken = source1.next(reusableToken)) {
+      assertTrue(nextToken.term() + " is not equal to " + tokens1[i], nextToken.term().equals(tokens1[i]) == true);
       i++;
     }
     assertTrue(i + " does not equal: " + tokens1.length, i == tokens1.length);
     assertTrue("theDetector Size: " + theDetector.getTokens().size() + " is not: " + 2, theDetector.getTokens().size() == 2);
     assertTrue("dogDetector Size: " + dogDetector.getTokens().size() + " is not: " + 1, dogDetector.getTokens().size() == 1);
     i = 0;
-    while ((token = source2.next()) != null) {
-      assertTrue(token.termText() + " is not equal to " + tokens2[i], token.termText().equals(tokens2[i]) == true);
+    for (Token nextToken = source2.next(reusableToken); nextToken != null; nextToken = source2.next(reusableToken)) {
+      assertTrue(nextToken.term() + " is not equal to " + tokens2[i], nextToken.term().equals(tokens2[i]) == true);
       i++;
     }
     assertTrue(i + " does not equal: " + tokens2.length, i == tokens2.length);
     assertTrue("theDetector Size: " + theDetector.getTokens().size() + " is not: " + 4, theDetector.getTokens().size() == 4);
     assertTrue("dogDetector Size: " + dogDetector.getTokens().size() + " is not: " + 2, dogDetector.getTokens().size() == 2);
     i = 0;
-    while ((token = theDetector.next()) != null) {
-      assertTrue(token.termText() + " is not equal to " + "The", token.termText().equalsIgnoreCase("The") == true);
+    for (Token nextToken = theDetector.next(reusableToken); nextToken != null; nextToken = theDetector.next(reusableToken)) {
+      assertTrue(nextToken.term() + " is not equal to " + "The", nextToken.term().equalsIgnoreCase("The") == true);
       i++;
     }
     assertTrue(i + " does not equal: " + theDetector.getTokens().size(), i == theDetector.getTokens().size());
     i = 0;
-    while ((token = dogDetector.next()) != null) {
-      assertTrue(token.termText() + " is not equal to " + "Dogs", token.termText().equalsIgnoreCase("Dogs") == true);
+    for (Token nextToken = dogDetector.next(reusableToken); nextToken != null; nextToken = dogDetector.next(reusableToken)) {
+      assertTrue(nextToken.term() + " is not equal to " + "Dogs", nextToken.term().equalsIgnoreCase("Dogs") == true);
       i++;
     }
     assertTrue(i + " does not equal: " + dogDetector.getTokens().size(), i == dogDetector.getTokens().size());
     source1.reset();
     TokenStream lowerCasing = new LowerCaseFilter(source1);
     i = 0;
-    while ((token = lowerCasing.next()) != null) {
-      assertTrue(token.termText() + " is not equal to " + tokens1[i].toLowerCase(), token.termText().equals(tokens1[i].toLowerCase()) == true);
+    for (Token nextToken = lowerCasing.next(reusableToken); nextToken != null; nextToken = lowerCasing.next(reusableToken)) {
+      assertTrue(nextToken.term() + " is not equal to " + tokens1[i].toLowerCase(), nextToken.term().equals(tokens1[i].toLowerCase()) == true);
       i++;
     }
     assertTrue(i + " does not equal: " + tokens1.length, i == tokens1.length);
@@ -157,22 +157,21 @@
       }
       //make sure we produce the same tokens
       ModuloSinkTokenizer sink = new ModuloSinkTokenizer(tokCount[k], 100);
-      Token next = new Token();
-      TokenStream result = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), sink);
-      while ((next = result.next(next)) != null) {
+      final Token reusableToken = new Token();
+      TokenStream stream = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), sink);
+      while (stream.next(reusableToken) != null) {
       }
-      result = new ModuloTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), 100);
-      next = new Token();
+      stream = new ModuloTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), 100);
       List tmp = new ArrayList();
-      while ((next = result.next(next)) != null) {
-        tmp.add(next.clone());
+      for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+        tmp.add(nextToken.clone());
       }
       List sinkList = sink.getTokens();
       assertTrue("tmp Size: " + tmp.size() + " is not: " + sinkList.size(), tmp.size() == sinkList.size());
       for (int i = 0; i < tmp.size(); i++) {
         Token tfTok = (Token) tmp.get(i);
         Token sinkTok = (Token) sinkList.get(i);
-        assertTrue(tfTok.termText() + " is not equal to " + sinkTok.termText() + " at token: " + i, tfTok.termText().equals(sinkTok.termText()) == true);
+        assertTrue(tfTok.term() + " is not equal to " + sinkTok.term() + " at token: " + i, tfTok.term().equals(sinkTok.term()) == true);
       }
       //simulate two fields, each being analyzed once, for 20 documents
 
@@ -180,15 +179,13 @@
         int tfPos = 0;
         long start = System.currentTimeMillis();
         for (int i = 0; i < 20; i++) {
-          next = new Token();
-          result = new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString())));
-          while ((next = result.next(next)) != null) {
-            tfPos += next.getPositionIncrement();
+          stream = new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString())));
+          for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+            tfPos += nextToken.getPositionIncrement();
           }
-          next = new Token();
-          result = new ModuloTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), modCounts[j]);
-          while ((next = result.next(next)) != null) {
-            tfPos += next.getPositionIncrement();
+          stream = new ModuloTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), modCounts[j]);
+          for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+            tfPos += nextToken.getPositionIncrement();
           }
         }
         long finish = System.currentTimeMillis();
@@ -198,15 +195,14 @@
         start = System.currentTimeMillis();
         for (int i = 0; i < 20; i++) {
           sink = new ModuloSinkTokenizer(tokCount[k], modCounts[j]);
-          next = new Token();
-          result = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), sink);
-          while ((next = result.next(next)) != null) {
-            sinkPos += next.getPositionIncrement();
+          stream = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), sink);
+          for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+            sinkPos += nextToken.getPositionIncrement();
           }
           //System.out.println("Modulo--------");
-          result = sink;
-          while ((next = result.next(next)) != null) {
-            sinkPos += next.getPositionIncrement();
+          stream = sink;
+          for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+            sinkPos += nextToken.getPositionIncrement();
           }
         }
         finish = System.currentTimeMillis();
@@ -232,13 +228,15 @@
     int count = 0;
 
     //return every 100 tokens
-    public Token next(Token result) throws IOException {
-
-      while ((result = input.next(result)) != null && count % modCount != 0) {
+    public Token next(final Token reusableToken) throws IOException {
+      Token nextToken = null;
+      for (nextToken = input.next(reusableToken);
+           nextToken != null && count % modCount != 0;
+           nextToken = input.next(reusableToken)) {
         count++;
       }
       count++;
-      return result;
+      return nextToken;
     }
   }
 
@@ -254,7 +252,7 @@
 
     public void add(Token t) {
       if (t != null && count % modCount == 0) {
-        lst.add(t.clone());
+        super.add(t);
       }
       count++;
     }

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java Wed Aug 20 07:38:07 2008
@@ -17,13 +17,14 @@
  * limitations under the License.
  */
 
-import java.io.*;
-import java.util.List;
+import java.io.IOException;
+import java.io.StringReader;
 import java.util.LinkedList;
+import java.util.List;
 
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.index.Payload;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.index.Payload;
+import org.apache.lucene.util.LuceneTestCase;
 
 public class TestAnalyzers extends LuceneTestCase {
 
@@ -35,12 +36,13 @@
                                String input, 
                                String[] output) throws Exception {
     TokenStream ts = a.tokenStream("dummy", new StringReader(input));
+    final Token reusableToken  = new Token();
     for (int i=0; i<output.length; i++) {
-      Token t = ts.next();
-      assertNotNull(t);
-      assertEquals(t.termText(), output[i]);
+      Token nextToken = ts.next(reusableToken);
+      assertNotNull(nextToken);
+      assertEquals(nextToken.term(), output[i]);
     }
-    assertNull(ts.next());
+    assertNull(ts.next(reusableToken));
     ts.close();
   }
 
@@ -93,14 +95,14 @@
   }
 
   void verifyPayload(TokenStream ts) throws IOException {
-    Token t = new Token();
+    final Token reusableToken = new Token();
     for(byte b=1;;b++) {
-      t.clear();
-      t = ts.next(t);
-      if (t==null) break;
-      // System.out.println("id="+System.identityHashCode(t) + " " + t);
-      // System.out.println("payload=" + (int)t.getPayload().toByteArray()[0]);
-      assertEquals(b, t.getPayload().toByteArray()[0]);
+      reusableToken.clear();
+      Token nextToken = ts.next(reusableToken);
+      if (nextToken==null) break;
+      // System.out.println("id="+System.identityHashCode(nextToken) + " " + t);
+      // System.out.println("payload=" + (int)nextToken.getPayload().toByteArray()[0]);
+      assertEquals(b, nextToken.getPayload().toByteArray()[0]);
     }
   }
 
@@ -141,13 +143,11 @@
     super(input);
   }
 
-  public Token next() throws IOException {
+  public Token next(final Token reusableToken) throws IOException {
     if (lst == null) {
       lst = new LinkedList();
-      for(;;) {
-        Token t = input.next();
-        if (t==null) break;
-        lst.add(t);
+      for(Token nextToken = input.next(reusableToken); nextToken != null; nextToken = input.next(reusableToken)) {
+        lst.add(nextToken.clone());
       }
     }
     return lst.size()==0 ? null : (Token)lst.remove(0);
@@ -162,11 +162,12 @@
   byte[] data = new byte[1];
   Payload p = new Payload(data,0,1);
 
-  public Token next(Token target) throws IOException {
-    target = input.next(target);
-    if (target==null) return null;
-    target.setPayload(p);  // reuse the payload / byte[]
+  public Token next(final Token reusableToken) throws IOException {
+    assert reusableToken != null;
+    Token nextToken = input.next(reusableToken);
+    if (nextToken==null) return null;
+    nextToken.setPayload(p);  // reuse the payload / byte[]
     data[0]++;
-    return target;
+    return nextToken;
   }
 }
\ No newline at end of file

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java Wed Aug 20 07:38:07 2008
@@ -42,11 +42,12 @@
     TokenStream stream = new TokenStream() {
       private int index = 0;
       
-      public Token next() throws IOException {
+      public Token next(final Token reusableToken) throws IOException {
+        assert reusableToken != null;
         if (index == tokens.length) {
           return null;
         } else {
-          return new Token(tokens[index++], 0, 0);
+          return reusableToken.reinit(tokens[index++], 0, 0);
         }        
       }
       
@@ -91,10 +92,10 @@
   
   private void checkTokens(TokenStream stream) throws IOException {
     int count = 0;
-    Token token;
-    while ((token = stream.next()) != null) {
+    final Token reusableToken = new Token();
+    for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
       assertTrue(count < tokens.length);
-      assertEquals(tokens[count], token.termText());
+      assertEquals(tokens[count], nextToken.term());
       count++;
     }
     

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java Wed Aug 20 07:38:07 2008
@@ -25,81 +25,82 @@
   public void testU() throws Exception {
     TokenStream stream = new WhitespaceTokenizer(new StringReader("Des mot clés À LA CHAÎNE À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï IJ Ð Ñ Ò Ó Ô Õ Ö Ø Œ Þ Ù Ú Û Ü Ý Ÿ à á â ã ä å æ ç è é ê ë ì í î ï ij ð ñ ò ó ô õ ö ø œ ß þ ù ú û ü ý ÿ fi fl"));
     ISOLatin1AccentFilter filter = new ISOLatin1AccentFilter(stream);
-    assertEquals("Des", filter.next().termText());
-    assertEquals("mot", filter.next().termText());
-    assertEquals("cles", filter.next().termText());
-    assertEquals("A", filter.next().termText());
-    assertEquals("LA", filter.next().termText());
-    assertEquals("CHAINE", filter.next().termText());
-    assertEquals("A", filter.next().termText());
-    assertEquals("A", filter.next().termText());
-    assertEquals("A", filter.next().termText());
-    assertEquals("A", filter.next().termText());
-    assertEquals("A", filter.next().termText());
-    assertEquals("A", filter.next().termText());
-    assertEquals("AE", filter.next().termText());
-    assertEquals("C", filter.next().termText());
-    assertEquals("E", filter.next().termText());
-    assertEquals("E", filter.next().termText());
-    assertEquals("E", filter.next().termText());
-    assertEquals("E", filter.next().termText());
-    assertEquals("I", filter.next().termText());
-    assertEquals("I", filter.next().termText());
-    assertEquals("I", filter.next().termText());
-    assertEquals("I", filter.next().termText());
-    assertEquals("IJ", filter.next().termText());
-    assertEquals("D", filter.next().termText());
-    assertEquals("N", filter.next().termText());
-    assertEquals("O", filter.next().termText());
-    assertEquals("O", filter.next().termText());
-    assertEquals("O", filter.next().termText());
-    assertEquals("O", filter.next().termText());
-    assertEquals("O", filter.next().termText());
-    assertEquals("O", filter.next().termText());
-    assertEquals("OE", filter.next().termText());
-    assertEquals("TH", filter.next().termText());
-    assertEquals("U", filter.next().termText());
-    assertEquals("U", filter.next().termText());
-    assertEquals("U", filter.next().termText());
-    assertEquals("U", filter.next().termText());
-    assertEquals("Y", filter.next().termText());
-    assertEquals("Y", filter.next().termText());
-    assertEquals("a", filter.next().termText());
-    assertEquals("a", filter.next().termText());
-    assertEquals("a", filter.next().termText());
-    assertEquals("a", filter.next().termText());
-    assertEquals("a", filter.next().termText());
-    assertEquals("a", filter.next().termText());
-    assertEquals("ae", filter.next().termText());
-    assertEquals("c", filter.next().termText());
-    assertEquals("e", filter.next().termText());
-    assertEquals("e", filter.next().termText());
-    assertEquals("e", filter.next().termText());
-    assertEquals("e", filter.next().termText());
-    assertEquals("i", filter.next().termText());
-    assertEquals("i", filter.next().termText());
-    assertEquals("i", filter.next().termText());
-    assertEquals("i", filter.next().termText());
-    assertEquals("ij", filter.next().termText());
-    assertEquals("d", filter.next().termText());
-    assertEquals("n", filter.next().termText());
-    assertEquals("o", filter.next().termText());
-    assertEquals("o", filter.next().termText());
-    assertEquals("o", filter.next().termText());
-    assertEquals("o", filter.next().termText());
-    assertEquals("o", filter.next().termText());
-    assertEquals("o", filter.next().termText());
-    assertEquals("oe", filter.next().termText());
-    assertEquals("ss", filter.next().termText());
-    assertEquals("th", filter.next().termText());
-    assertEquals("u", filter.next().termText());
-    assertEquals("u", filter.next().termText());
-    assertEquals("u", filter.next().termText());
-    assertEquals("u", filter.next().termText());
-    assertEquals("y", filter.next().termText());
-    assertEquals("y", filter.next().termText());
-    assertEquals("fi", filter.next().termText());
-    assertEquals("fl", filter.next().termText());
-    assertNull(filter.next());
+    final Token reusableToken = new Token();
+    assertEquals("Des", filter.next(reusableToken).term());
+    assertEquals("mot", filter.next(reusableToken).term());
+    assertEquals("cles", filter.next(reusableToken).term());
+    assertEquals("A", filter.next(reusableToken).term());
+    assertEquals("LA", filter.next(reusableToken).term());
+    assertEquals("CHAINE", filter.next(reusableToken).term());
+    assertEquals("A", filter.next(reusableToken).term());
+    assertEquals("A", filter.next(reusableToken).term());
+    assertEquals("A", filter.next(reusableToken).term());
+    assertEquals("A", filter.next(reusableToken).term());
+    assertEquals("A", filter.next(reusableToken).term());
+    assertEquals("A", filter.next(reusableToken).term());
+    assertEquals("AE", filter.next(reusableToken).term());
+    assertEquals("C", filter.next(reusableToken).term());
+    assertEquals("E", filter.next(reusableToken).term());
+    assertEquals("E", filter.next(reusableToken).term());
+    assertEquals("E", filter.next(reusableToken).term());
+    assertEquals("E", filter.next(reusableToken).term());
+    assertEquals("I", filter.next(reusableToken).term());
+    assertEquals("I", filter.next(reusableToken).term());
+    assertEquals("I", filter.next(reusableToken).term());
+    assertEquals("I", filter.next(reusableToken).term());
+    assertEquals("IJ", filter.next(reusableToken).term());
+    assertEquals("D", filter.next(reusableToken).term());
+    assertEquals("N", filter.next(reusableToken).term());
+    assertEquals("O", filter.next(reusableToken).term());
+    assertEquals("O", filter.next(reusableToken).term());
+    assertEquals("O", filter.next(reusableToken).term());
+    assertEquals("O", filter.next(reusableToken).term());
+    assertEquals("O", filter.next(reusableToken).term());
+    assertEquals("O", filter.next(reusableToken).term());
+    assertEquals("OE", filter.next(reusableToken).term());
+    assertEquals("TH", filter.next(reusableToken).term());
+    assertEquals("U", filter.next(reusableToken).term());
+    assertEquals("U", filter.next(reusableToken).term());
+    assertEquals("U", filter.next(reusableToken).term());
+    assertEquals("U", filter.next(reusableToken).term());
+    assertEquals("Y", filter.next(reusableToken).term());
+    assertEquals("Y", filter.next(reusableToken).term());
+    assertEquals("a", filter.next(reusableToken).term());
+    assertEquals("a", filter.next(reusableToken).term());
+    assertEquals("a", filter.next(reusableToken).term());
+    assertEquals("a", filter.next(reusableToken).term());
+    assertEquals("a", filter.next(reusableToken).term());
+    assertEquals("a", filter.next(reusableToken).term());
+    assertEquals("ae", filter.next(reusableToken).term());
+    assertEquals("c", filter.next(reusableToken).term());
+    assertEquals("e", filter.next(reusableToken).term());
+    assertEquals("e", filter.next(reusableToken).term());
+    assertEquals("e", filter.next(reusableToken).term());
+    assertEquals("e", filter.next(reusableToken).term());
+    assertEquals("i", filter.next(reusableToken).term());
+    assertEquals("i", filter.next(reusableToken).term());
+    assertEquals("i", filter.next(reusableToken).term());
+    assertEquals("i", filter.next(reusableToken).term());
+    assertEquals("ij", filter.next(reusableToken).term());
+    assertEquals("d", filter.next(reusableToken).term());
+    assertEquals("n", filter.next(reusableToken).term());
+    assertEquals("o", filter.next(reusableToken).term());
+    assertEquals("o", filter.next(reusableToken).term());
+    assertEquals("o", filter.next(reusableToken).term());
+    assertEquals("o", filter.next(reusableToken).term());
+    assertEquals("o", filter.next(reusableToken).term());
+    assertEquals("o", filter.next(reusableToken).term());
+    assertEquals("oe", filter.next(reusableToken).term());
+    assertEquals("ss", filter.next(reusableToken).term());
+    assertEquals("th", filter.next(reusableToken).term());
+    assertEquals("u", filter.next(reusableToken).term());
+    assertEquals("u", filter.next(reusableToken).term());
+    assertEquals("u", filter.next(reusableToken).term());
+    assertEquals("u", filter.next(reusableToken).term());
+    assertEquals("y", filter.next(reusableToken).term());
+    assertEquals("y", filter.next(reusableToken).term());
+    assertEquals("fi", filter.next(reusableToken).term());
+    assertEquals("fl", filter.next(reusableToken).term());
+    assertNull(filter.next(reusableToken));
   }
 }

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestLengthFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestLengthFilter.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestLengthFilter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestLengthFilter.java Wed Aug 20 07:38:07 2008
@@ -27,10 +27,11 @@
     TokenStream stream = new WhitespaceTokenizer(
         new StringReader("short toolong evenmuchlongertext a ab toolong foo"));
     LengthFilter filter = new LengthFilter(stream, 2, 6);
-    assertEquals("short", filter.next().termText());
-    assertEquals("ab", filter.next().termText());
-    assertEquals("foo", filter.next().termText());
-    assertNull(filter.next());
+    final Token reusableToken = new Token();
+    assertEquals("short", filter.next(reusableToken).term());
+    assertEquals("ab", filter.next(reusableToken).term());
+    assertEquals("foo", filter.next(reusableToken).term());
+    assertNull(filter.next(reusableToken));
   }
 
 }

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java Wed Aug 20 07:38:07 2008
@@ -29,16 +29,17 @@
 
     TokenStream tokenStream = analyzer.tokenStream("field",
                                             new StringReader(text));
-    Token token = tokenStream.next();
+    final Token reusableToken = new Token();
+    Token nextToken = tokenStream.next(reusableToken);
     assertEquals("WhitespaceAnalyzer does not lowercase",
                  "Qwerty",
-                 token.termText());
+                 nextToken.term());
 
     tokenStream = analyzer.tokenStream("special",
                                             new StringReader(text));
-    token = tokenStream.next();
+    nextToken = tokenStream.next(reusableToken);
     assertEquals("SimpleAnalyzer lowercases",
                  "qwerty",
-                 token.termText());
+                 nextToken.term());
   }
 }

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java Wed Aug 20 07:38:07 2008
@@ -35,18 +35,19 @@
 
   public void assertAnalyzesTo(Analyzer a, String input, String[] expectedImages, String[] expectedTypes, int[] expectedPosIncrs) throws Exception {
     TokenStream ts = a.tokenStream("dummy", new StringReader(input));
+    final Token reusableToken = new Token();
     for (int i = 0; i < expectedImages.length; i++) {
-      Token t = ts.next();
-      assertNotNull(t);
-      assertEquals(expectedImages[i], t.termText());
+      Token nextToken = ts.next(reusableToken);
+      assertNotNull(nextToken);
+      assertEquals(expectedImages[i], nextToken.term());
       if (expectedTypes != null) {
-        assertEquals(expectedTypes[i], t.type());
+        assertEquals(expectedTypes[i], nextToken.type());
       }
       if (expectedPosIncrs != null) {
-        assertEquals(expectedPosIncrs[i], t.getPositionIncrement());
+        assertEquals(expectedPosIncrs[i], nextToken.getPositionIncrement());
       }
     }
-    assertNull(ts.next());
+    assertNull(ts.next(reusableToken));
     ts.close();
   }
 

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java Wed Aug 20 07:38:07 2008
@@ -45,9 +45,9 @@
     StringReader reader = new StringReader("This is a test of the english stop analyzer");
     TokenStream stream = stop.tokenStream("test", reader);
     assertTrue(stream != null);
-    Token token = null;
-    while ((token = stream.next()) != null) {
-      assertFalse(inValidTokens.contains(token.termText()));
+    final Token reusableToken = new Token();
+    for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+      assertFalse(inValidTokens.contains(nextToken.term()));
     }
   }
 
@@ -60,11 +60,11 @@
     StringReader reader = new StringReader("This is a good test of the english stop analyzer");
     TokenStream stream = newStop.tokenStream("test", reader);
     assertNotNull(stream);
-    Token token = null;
-    while ((token = stream.next()) != null) {
-      String text = token.termText();
+    final Token reusableToken = new Token();
+    for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+      String text = nextToken.term();
       assertFalse(stopWordsSet.contains(text));
-      assertEquals(1,token.getPositionIncrement()); // by default stop tokenizer does not apply increments.
+      assertEquals(1,nextToken.getPositionIncrement()); // by default stop tokenizer does not apply increments.
     }
   }
 
@@ -81,12 +81,12 @@
       int expectedIncr[] =                  { 1,   1, 1,          3, 1,  1,      1,            2,   1};
       TokenStream stream = newStop.tokenStream("test", reader);
       assertNotNull(stream);
-      Token token = null;
       int i = 0;
-      while ((token = stream.next()) != null) {
-        String text = token.termText();
+      final Token reusableToken = new Token();
+      for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+        String text = nextToken.term();
         assertFalse(stopWordsSet.contains(text));
-        assertEquals(expectedIncr[i++],token.getPositionIncrement());
+        assertEquals(expectedIncr[i++],nextToken.getPositionIncrement());
       }
     } finally {
       StopFilter.setEnablePositionIncrementsDefault(defaultEnable);

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopFilter.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopFilter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestStopFilter.java Wed Aug 20 07:38:07 2008
@@ -37,17 +37,19 @@
     StringReader reader = new StringReader("Now is The Time");
     String[] stopWords = new String[] { "is", "the", "Time" };
     TokenStream stream = new StopFilter(new WhitespaceTokenizer(reader), stopWords);
-    assertEquals("Now", stream.next().termText());
-    assertEquals("The", stream.next().termText());
-    assertEquals(null, stream.next());
+    final Token reusableToken = new Token();
+    assertEquals("Now", stream.next(reusableToken).term());
+    assertEquals("The", stream.next(reusableToken).term());
+    assertEquals(null, stream.next(reusableToken));
   }
 
   public void testIgnoreCase() throws IOException {
     StringReader reader = new StringReader("Now is The Time");
     String[] stopWords = new String[] { "is", "the", "Time" };
     TokenStream stream = new StopFilter(new WhitespaceTokenizer(reader), stopWords, true);
-    assertEquals("Now", stream.next().termText());
-    assertEquals(null,stream.next());
+    final Token reusableToken = new Token();
+    assertEquals("Now", stream.next(reusableToken).term());
+    assertEquals(null,stream.next(reusableToken));
   }
 
   public void testStopFilt() throws IOException {
@@ -55,9 +57,10 @@
     String[] stopWords = new String[] { "is", "the", "Time" };
     Set stopSet = StopFilter.makeStopSet(stopWords);
     TokenStream stream = new StopFilter(new WhitespaceTokenizer(reader), stopSet);
-    assertEquals("Now", stream.next().termText());
-    assertEquals("The", stream.next().termText());
-    assertEquals(null, stream.next());
+    final Token reusableToken = new Token();
+    assertEquals("Now", stream.next(reusableToken).term());
+    assertEquals("The", stream.next(reusableToken).term());
+    assertEquals(null, stream.next(reusableToken));
   }
 
   /**
@@ -109,14 +112,15 @@
   private void doTestStopPositons(StopFilter stpf, boolean enableIcrements) throws IOException {
     log("---> test with enable-increments-"+(enableIcrements?"enabled":"disabled"));
     stpf.setEnablePositionIncrements(enableIcrements);
+    final Token reusableToken = new Token();
     for (int i=0; i<20; i+=3) {
-      Token t = stpf.next();
-      log("Token "+i+": "+t);
+      Token nextToken = stpf.next(reusableToken);
+      log("Token "+i+": "+nextToken);
       String w = English.intToEnglish(i).trim();
-      assertEquals("expecting token "+i+" to be "+w,w,t.termText());
-      assertEquals("all but first token must have position increment of 3",enableIcrements?(i==0?1:3):1,t.getPositionIncrement());
+      assertEquals("expecting token "+i+" to be "+w,w,nextToken.term());
+      assertEquals("all but first token must have position increment of 3",enableIcrements?(i==0?1:3):1,nextToken.getPositionIncrement());
     }
-    assertNull(stpf.next());
+    assertNull(stpf.next(reusableToken));
   }
   
   // print debug info depending on VERBOSE

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestToken.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestToken.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestToken.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestToken.java Wed Aug 20 07:38:07 2008
@@ -17,7 +17,6 @@
  * limitations under the License.
  */
 
-import java.io.*;
 import org.apache.lucene.util.LuceneTestCase;
 
 public class TestToken extends LuceneTestCase {
@@ -26,6 +25,119 @@
     super(name);
   }
 
+  public void testCtor() throws Exception {
+    Token t = new Token();
+    char[] content = "hello".toCharArray();
+    t.setTermBuffer(content, 0, content.length);
+    char[] buf = t.termBuffer();
+    assertNotSame(t.termBuffer(), content);
+    assertEquals("hello", t.term());
+    assertEquals("word", t.type());
+    assertEquals(0, t.getFlags());
+
+    t = new Token(6, 22);
+    t.setTermBuffer(content, 0, content.length);
+    assertEquals("hello", t.term());
+    assertEquals("(hello,6,22)", t.toString());
+    assertEquals("word", t.type());
+    assertEquals(0, t.getFlags());
+
+    t = new Token(6, 22, 7);
+    t.setTermBuffer(content, 0, content.length);
+    assertEquals("hello", t.term());
+    assertEquals("(hello,6,22)", t.toString());
+    assertEquals(7, t.getFlags());
+
+    t = new Token(6, 22, "junk");
+    t.setTermBuffer(content, 0, content.length);
+    assertEquals("hello", t.term());
+    assertEquals("(hello,6,22,type=junk)", t.toString());
+    assertEquals(0, t.getFlags());
+  }
+
+  public void testResize() {
+    Token t = new Token();
+    char[] content = "hello".toCharArray();
+    t.setTermBuffer(content, 0, content.length);
+    for (int i = 0; i < 2000; i++)
+    {
+      t.resizeTermBuffer(i);
+      assertTrue(i <= t.termBuffer().length);
+      assertEquals("hello", t.term());
+    }
+  }
+
+  public void testGrow() {
+    Token t = new Token();
+    StringBuffer buf = new StringBuffer("ab");
+    for (int i = 0; i < 20; i++)
+    {
+      char[] content = buf.toString().toCharArray();
+      t.setTermBuffer(content, 0, content.length);
+      assertEquals(buf.length(), t.termLength());
+      assertEquals(buf.toString(), t.term());
+      buf.append(buf.toString());
+    }
+    assertEquals(1048576, t.termLength());
+    assertEquals(1179654, t.termBuffer().length);
+
+    // now as a string, first variant
+    t = new Token();
+    buf = new StringBuffer("ab");
+    for (int i = 0; i < 20; i++)
+    {
+      String content = buf.toString();
+      t.setTermBuffer(content, 0, content.length());
+      assertEquals(content.length(), t.termLength());
+      assertEquals(content, t.term());
+      buf.append(content);
+    }
+    assertEquals(1048576, t.termLength());
+    assertEquals(1179654, t.termBuffer().length);
+
+    // now as a string, second variant
+    t = new Token();
+    buf = new StringBuffer("ab");
+    for (int i = 0; i < 20; i++)
+    {
+      String content = buf.toString();
+      t.setTermBuffer(content);
+      assertEquals(content.length(), t.termLength());
+      assertEquals(content, t.term());
+      buf.append(content);
+    }
+    assertEquals(1048576, t.termLength());
+    assertEquals(1179654, t.termBuffer().length);
+
+    // Test for slow growth to a long term
+    t = new Token();
+    buf = new StringBuffer("a");
+    for (int i = 0; i < 20000; i++)
+    {
+      String content = buf.toString();
+      t.setTermBuffer(content);
+      assertEquals(content.length(), t.termLength());
+      assertEquals(content, t.term());
+      buf.append("a");
+    }
+    assertEquals(20000, t.termLength());
+    assertEquals(20331, t.termBuffer().length);
+
+    // Test for slow growth to a long term
+    t = new Token();
+    buf = new StringBuffer("a");
+    for (int i = 0; i < 20000; i++)
+    {
+      String content = buf.toString();
+      t.setTermBuffer(content);
+      assertEquals(content.length(), t.termLength());
+      assertEquals(content, t.term());
+      buf.append("a");
+    }
+    assertEquals(20000, t.termLength());
+    assertEquals(20331, t.termBuffer().length);
+  }
+
   public void testToString() throws Exception {
     char[] b = {'a', 'l', 'o', 'h', 'a'};
     Token t = new Token("", 0, 5);
@@ -40,10 +152,10 @@
     Token t = new Token("hello", 0, 5);
     assertEquals(t.termText(), "hello");
     assertEquals(t.termLength(), 5);
-    assertEquals(new String(t.termBuffer(), 0, 5), "hello");
+    assertEquals(t.term(), "hello");
     t.setTermText("hello2");
     assertEquals(t.termLength(), 6);
-    assertEquals(new String(t.termBuffer(), 0, 6), "hello2");
+    assertEquals(t.term(), "hello2");
     t.setTermBuffer("hello3".toCharArray(), 0, 6);
     assertEquals(t.termText(), "hello3");
 
@@ -53,4 +165,13 @@
     buffer[1] = 'o';
     assertEquals(t.termText(), "hollo3");
   }
+  
+  public void testClone() throws Exception {
+    Token t = new Token(0, 5);
+    char[] content = "hello".toCharArray();
+    t.setTermBuffer(content, 0, 5);
+    char[] buf = t.termBuffer();
+    Token copy = (Token) t.clone();
+    assertNotSame(buf, copy.termBuffer());
+  }
 }

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestDocumentWriter.java Wed Aug 20 07:38:07 2008
@@ -17,21 +17,27 @@
  * limitations under the License.
  */
 
-import org.apache.lucene.analysis.*;
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.SimpleAnalyzer;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.WhitespaceAnalyzer;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.document.Field.Index;
 import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.Field.TermVector;
-import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
 
-import java.io.IOException;
-import java.io.Reader;
-
 public class TestDocumentWriter extends LuceneTestCase {
   private RAMDirectory dir;
 
@@ -134,34 +140,30 @@
           boolean first=true;
           Token buffered;
 
-          public Token next() throws IOException {
-            return input.next();
-          }
-
-          public Token next(Token result) throws IOException {
+          public Token next(final Token reusableToken) throws IOException {
             if (buffered != null) {
-              Token t = buffered;
+              Token nextToken = buffered;
               buffered=null;
-              return t;
+              return nextToken;
             }
-            Token t = input.next(result);
-            if (t==null) return null;
-            if (Character.isDigit(t.termBuffer()[0])) {
-              t.setPositionIncrement(t.termBuffer()[0] - '0');
+            Token nextToken = input.next(reusableToken);
+            if (nextToken==null) return null;
+            if (Character.isDigit(nextToken.termBuffer()[0])) {
+              nextToken.setPositionIncrement(nextToken.termBuffer()[0] - '0');
             }
             if (first) {
               // set payload on first position only
-              t.setPayload(new Payload(new byte[]{100}));
+              nextToken.setPayload(new Payload(new byte[]{100}));
               first = false;
             }
 
             // index a "synonym" for every token
-            buffered = (Token)t.clone();
+            buffered = (Token)nextToken.clone();
             buffered.setPayload(null);
             buffered.setPositionIncrement(0);
             buffered.setTermBuffer(new char[]{'b'}, 0, 1);
 
-            return t;
+            return nextToken;
           }
         };
       }
@@ -199,11 +201,12 @@
       private String[] tokens = new String[] {"term1", "term2", "term3", "term2"};
       private int index = 0;
       
-      public Token next() throws IOException {
+      public Token next(final Token reusableToken) throws IOException {
+        assert reusableToken != null;
         if (index == tokens.length) {
           return null;
         } else {
-          return new Token(tokens[index++], 0, 0);
+          return reusableToken.reinit(tokens[index++], 0, 0);
         }        
       }
       

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java Wed Aug 20 07:38:07 2008
@@ -1786,11 +1786,11 @@
         return new TokenFilter(new StandardTokenizer(reader)) {
           private int count = 0;
 
-          public Token next() throws IOException {
+          public Token next(final Token reusableToken) throws IOException {
             if (count++ == 5) {
               throw new IOException();
             }
-            return input.next();
+            return input.next(reusableToken);
           }
         };
       }
@@ -1909,10 +1909,10 @@
       this.fieldName = fieldName;
     }
 
-    public Token next(Token result) throws IOException {
+    public Token next(final Token reusableToken) throws IOException {
       if (this.fieldName.equals("crash") && count++ >= 4)
         throw new IOException("I'm experiencing problems");
-      return input.next(result);
+      return input.next(reusableToken);
     }
 
     public void reset() throws IOException {
@@ -3574,13 +3574,13 @@
   public void testNegativePositions() throws Throwable {
     SinkTokenizer tokens = new SinkTokenizer();
     Token t = new Token();
-    t.setTermText("a");
+    t.setTermBuffer("a");
     t.setPositionIncrement(0);
     tokens.add(t);
-    t.setTermText("b");
+    t.setTermBuffer("b");
     t.setPositionIncrement(1);
     tokens.add(t);
-    t.setTermText("c");
+    t.setTermBuffer("c");
     tokens.add(t);
 
     MockRAMDirectory dir = new MockRAMDirectory();

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java Wed Aug 20 07:38:07 2008
@@ -103,12 +103,13 @@
       super(input);
     }
 
-    public Token next() throws IOException {
-      Token t = input.next();
-      if (t != null) {
-        t.setPayload(new Payload(new byte[] { (byte) count++ }));
+    public Token next(final Token reusableToken) throws IOException {
+      assert reusableToken != null;
+      Token nextToken = input.next(reusableToken);
+      if (nextToken != null) {
+        nextToken.setPayload(new Payload(new byte[] { (byte) count++ }));
       }
-      return t;
+      return nextToken;
     }
 
   }

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestPayloads.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestPayloads.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestPayloads.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestPayloads.java Wed Aug 20 07:38:07 2008
@@ -450,23 +450,24 @@
             this.offset = offset;
         }
         
-        public Token next(Token token) throws IOException {
-            token = input.next(token);
-            if (token != null) {
+        public Token next(final Token reusableToken) throws IOException {
+            assert reusableToken != null;
+            Token nextToken = input.next(reusableToken);
+            if (nextToken != null) {
                 if (offset + length <= data.length) {
                     Payload p = null;
                     if (p == null) {
                         p = new Payload();
-                        token.setPayload(p);
+                        nextToken.setPayload(p);
                     }
                     p.setData(data, offset, length);
                     offset += length;                
                 } else {
-                    token.setPayload(null);
+                    nextToken.setPayload(null);
                 }
             }
             
-            return token;
+            return nextToken;
         }
     }
     
@@ -536,11 +537,11 @@
             first = true;
         }
         
-        public Token next() throws IOException {
-            if (!first) return null;            
-            Token t = new Token(term, 0, 0);
-            t.setPayload(new Payload(payload));
-            return t;        
+        public Token next(final Token reusableToken) throws IOException {
+            if (!first) return null;
+            reusableToken.reinit(term, 0, 0);
+            reusableToken.setPayload(new Payload(payload));
+            return reusableToken;
         }
         
         public void close() throws IOException {

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestTermVectorsReader.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestTermVectorsReader.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestTermVectorsReader.java Wed Aug 20 07:38:07 2008
@@ -118,20 +118,17 @@
 
   private class MyTokenStream extends TokenStream {
     int tokenUpto;
-    public Token next() {
+    public Token next(final Token reusableToken) {
       if (tokenUpto >= tokens.length)
         return null;
       else {
-        final Token t = new Token();
         final TestToken testToken = tokens[tokenUpto++];
-        t.setTermText(testToken.text);
+        reusableToken.reinit(testToken.text, testToken.startOffset, testToken.endOffset);
         if (tokenUpto > 1)
-          t.setPositionIncrement(testToken.pos - tokens[tokenUpto-2].pos);
+          reusableToken.setPositionIncrement(testToken.pos - tokens[tokenUpto-2].pos);
         else
-          t.setPositionIncrement(testToken.pos+1);
-        t.setStartOffset(testToken.startOffset);
-        t.setEndOffset(testToken.endOffset);
-        return t;
+          reusableToken.setPositionIncrement(testToken.pos+1);
+        return reusableToken;
       }
     }
   }

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestTermdocPerf.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestTermdocPerf.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestTermdocPerf.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestTermdocPerf.java Wed Aug 20 07:38:07 2008
@@ -40,11 +40,12 @@
   Token t;
 
    public RepeatingTokenStream(String val) {
-     t = new Token(val,0,val.length());
+     t = new Token(0,val.length());
+     t.setTermBuffer(val);
    }
 
-   public Token next() throws IOException {
-     return --num<0 ? null : t;
+   public Token next(final Token reusableToken) throws IOException {
+     return --num<0 ? null : (Token) t.clone();
    }
 }
 

Modified: lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java Wed Aug 20 07:38:07 2008
@@ -141,34 +141,34 @@
 
   private final class TestFilter extends TokenFilter {
     
-    private org.apache.lucene.analysis.Token prevToken;
+    private Token prevToken;
     
     public TestFilter(TokenStream in) {
       super(in);
     }
 
-    public final org.apache.lucene.analysis.Token next() throws java.io.IOException {
+    public final Token next(final Token reusableToken) throws java.io.IOException {
       if (multiToken > 0) {
-        org.apache.lucene.analysis.Token token = 
-          new org.apache.lucene.analysis.Token("multi"+(multiToken+1), prevToken.startOffset(),
-          prevToken.endOffset(), prevToken.type());
-        token.setPositionIncrement(0);
+        reusableToken.reinit("multi"+(multiToken+1), prevToken.startOffset(), prevToken.endOffset(), prevToken.type());
+        reusableToken.setPositionIncrement(0);
         multiToken--;
-        return token;
+        return reusableToken;
       } else {
-        org.apache.lucene.analysis.Token t = input.next();
-        prevToken = t;
-        if (t == null)
+        Token nextToken = input.next(reusableToken);
+        if (nextToken == null) {
+          prevToken = null;
           return null;
-        String text = t.termText();
+        }
+        prevToken = (Token) nextToken.clone();
+        String text = nextToken.term();
         if (text.equals("triplemulti")) {
           multiToken = 2;
-          return t;
+          return nextToken;
         } else if (text.equals("multi")) {
           multiToken = 1;
-          return t;
+          return nextToken;
         } else {
-          return t;
+          return nextToken;
         }
       }
     }
@@ -197,22 +197,16 @@
       super(in);
     }
 
-    public final org.apache.lucene.analysis.Token next() throws java.io.IOException {
-      for (Token t = input.next(); t != null; t = input.next()) {
-        if (t.termText().equals("the")) {
+    public final Token next(final Token reusableToken) throws java.io.IOException {
+      for (Token nextToken = input.next(reusableToken); nextToken != null; nextToken = input.next(reusableToken)) {
+        if (nextToken.term().equals("the")) {
           // stopword, do nothing
-        } else if (t.termText().equals("quick")) {
-          org.apache.lucene.analysis.Token token = 
-            new org.apache.lucene.analysis.Token(t.termText(), t.startOffset(),
-                t.endOffset(), t.type());
-          token.setPositionIncrement(2);
-          return token;
+        } else if (nextToken.term().equals("quick")) {
+          nextToken.setPositionIncrement(2);
+          return nextToken;
         } else {
-          org.apache.lucene.analysis.Token token = 
-            new org.apache.lucene.analysis.Token(t.termText(), t.startOffset(),
-                t.endOffset(), t.type());
-          token.setPositionIncrement(1);
-          return token;
+          nextToken.setPositionIncrement(1);
+          return nextToken;
         }
       }
       return null;

Modified: lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiFieldQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiFieldQueryParser.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiFieldQueryParser.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestMultiFieldQueryParser.java Wed Aug 20 07:38:07 2008
@@ -319,7 +319,7 @@
     }
 
     private static class EmptyTokenStream extends TokenStream {
-      public Token next() {
+      public Token next(final Token reusableToken) {
         return null;
       }
     }

Modified: lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestQueryParser.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestQueryParser.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/queryParser/TestQueryParser.java Wed Aug 20 07:38:07 2008
@@ -75,19 +75,20 @@
     boolean inPhrase = false;
     int savedStart = 0, savedEnd = 0;
 
-    public Token next() throws IOException {
+    public Token next(final Token reusableToken) throws IOException {
+      assert reusableToken != null;
       if (inPhrase) {
         inPhrase = false;
-        return new Token("phrase2", savedStart, savedEnd);
+        return reusableToken.reinit("phrase2", savedStart, savedEnd);
       } else
-        for (Token token = input.next(); token != null; token = input.next()) {
-          if (token.termText().equals("phrase")) {
+        for (Token nextToken = input.next(reusableToken); nextToken != null; nextToken = input.next(reusableToken)) {
+          if (nextToken.term().equals("phrase")) {
             inPhrase = true;
-            savedStart = token.startOffset();
-            savedEnd = token.endOffset();
-            return new Token("phrase1", savedStart, savedEnd);
-          } else if (!token.termText().equals("stop"))
-            return token;
+            savedStart = nextToken.startOffset();
+            savedEnd = nextToken.endOffset();
+            return nextToken.reinit("phrase1", savedStart, savedEnd);
+          } else if (!nextToken.term().equals("stop"))
+            return nextToken;
         }
       return null;
     }

Modified: lucene/java/trunk/src/test/org/apache/lucene/search/TestPositionIncrement.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/search/TestPositionIncrement.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/search/TestPositionIncrement.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/search/TestPositionIncrement.java Wed Aug 20 07:38:07 2008
@@ -49,13 +49,14 @@
           private final int[] INCREMENTS = {1, 2, 1, 0, 1};
           private int i = 0;
 
-          public Token next() {
+          public Token next(final Token reusableToken) {
+            assert reusableToken != null;
             if (i == TOKENS.length)
               return null;
-            Token t = new Token(TOKENS[i], i, i);
-            t.setPositionIncrement(INCREMENTS[i]);
+            reusableToken.reinit(TOKENS[i], i, i);
+            reusableToken.setPositionIncrement(INCREMENTS[i]);
             i++;
-            return t;
+            return reusableToken;
           }
         };
       }
@@ -204,11 +205,9 @@
     Analyzer analyzer = new WhitespaceAnalyzer();
     TokenStream ts = analyzer.tokenStream("field",
                                 new StringReader("one two three four five"));
-
-    while (true) {
-      Token token = ts.next();
-      if (token == null) break;
-      assertEquals(token.termText(), 1, token.getPositionIncrement());
+    final Token reusableToken = new Token();
+    for (Token nextToken = ts.next(reusableToken); nextToken != null; nextToken = ts.next(reusableToken)) {
+      assertEquals(nextToken.term(), 1, nextToken.getPositionIncrement());
     }
   }
 }

Modified: lucene/java/trunk/src/test/org/apache/lucene/search/payloads/TestBoostingTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/search/payloads/TestBoostingTermQuery.java?rev=687357&r1=687356&r2=687357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/search/payloads/TestBoostingTermQuery.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/search/payloads/TestBoostingTermQuery.java Wed Aug 20 07:38:07 2008
@@ -16,21 +16,31 @@
  * limitations under the License.
  */
 
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.analysis.*;
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.LowerCaseTokenizer;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.Payload;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.search.*;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.CheckHits;
+import org.apache.lucene.search.DefaultSimilarity;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.spans.Spans;
 import org.apache.lucene.search.spans.TermSpans;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.English;
-
-import java.io.IOException;
-import java.io.Reader;
+import org.apache.lucene.util.LuceneTestCase;
 
 public class TestBoostingTermQuery extends LuceneTestCase {
   private IndexSearcher searcher;
@@ -62,22 +72,23 @@
       this.fieldName = fieldName;
     }
 
-    public Token next() throws IOException {
-      Token result = input.next();
-      if (result != null) {
+    public Token next(final Token reusableToken) throws IOException {
+      assert reusableToken != null;
+      Token nextToken = input.next(reusableToken);
+      if (nextToken != null) {
         if (fieldName.equals("field")) {
-          result.setPayload(new Payload(payloadField));
+          nextToken.setPayload(new Payload(payloadField));
         } else if (fieldName.equals("multiField")) {
           if (numSeen % 2 == 0) {
-            result.setPayload(new Payload(payloadMultiField1));
+            nextToken.setPayload(new Payload(payloadMultiField1));
           } else {
-            result.setPayload(new Payload(payloadMultiField2));
+            nextToken.setPayload(new Payload(payloadMultiField2));
           }
           numSeen++;
         }
 
       }
-      return result;
+      return nextToken;
     }
   }
 



Mime
View raw message