lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rm...@apache.org
Subject svn commit: r1592339 [2/2] - in /lucene/dev/trunk/lucene: analysis/common/src/test/org/apache/lucene/analysis/ar/ analysis/common/src/test/org/apache/lucene/analysis/ckb/ analysis/common/src/test/org/apache/lucene/analysis/compound/ analysis/common/src...
Date Sun, 04 May 2014 12:19:36 GMT
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java Sun May  4 12:19:35 2014
@@ -20,6 +20,7 @@ package org.apache.lucene.analysis.wikip
 
 import java.io.StringReader;
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Random;
 import java.util.Set;
 import java.util.HashSet;
@@ -39,7 +40,7 @@ public class WikipediaTokenizerTest exte
 
   public void testSimple() throws Exception {
     String text = "This is a [[Category:foo]]";
-    WikipediaTokenizer tf = new WikipediaTokenizer();
+    WikipediaTokenizer tf = new WikipediaTokenizer(newAttributeFactory(), WikipediaTokenizer.TOKENS_ONLY, Collections.<String>emptySet());
     tf.setReader(new StringReader(text));
     assertTokenStreamContents(tf,
         new String[] { "This", "is", "a", "foo" },
@@ -62,7 +63,7 @@ public class WikipediaTokenizerTest exte
         + " [http://foo.boo.com/test/test/ Test Test] [http://foo.boo.com/test/test/test.html Test Test]"
         + " [http://foo.boo.com/test/test/test.html?g=b&c=d Test Test] <ref>Citation</ref> <sup>martian</sup> <span class=\"glue\">code</span>";
     
-    WikipediaTokenizer tf = new WikipediaTokenizer();
+    WikipediaTokenizer tf = new WikipediaTokenizer(newAttributeFactory(), WikipediaTokenizer.TOKENS_ONLY, Collections.<String>emptySet());
     tf.setReader(new StringReader(test));
     assertTokenStreamContents(tf, 
       new String[] {"link", "This", "is", "a",
@@ -104,7 +105,7 @@ public class WikipediaTokenizerTest exte
   }
 
   public void testLinkPhrases() throws Exception {
-    WikipediaTokenizer tf = new WikipediaTokenizer();
+    WikipediaTokenizer tf = new WikipediaTokenizer(newAttributeFactory(), WikipediaTokenizer.TOKENS_ONLY, Collections.<String>emptySet());
     tf.setReader(new StringReader(LINK_PHRASES));
     checkLinkPhrases(tf);
   }
@@ -118,7 +119,7 @@ public class WikipediaTokenizerTest exte
 
   public void testLinks() throws Exception {
     String test = "[http://lucene.apache.org/java/docs/index.html#news here] [http://lucene.apache.org/java/docs/index.html?b=c here] [https://lucene.apache.org/java/docs/index.html?b=c here]";
-    WikipediaTokenizer tf = new WikipediaTokenizer();
+    WikipediaTokenizer tf = new WikipediaTokenizer(newAttributeFactory(), WikipediaTokenizer.TOKENS_ONLY, Collections.<String>emptySet());
     tf.setReader(new StringReader(test));
     assertTokenStreamContents(tf,
         new String[] { "http://lucene.apache.org/java/docs/index.html#news", "here",
@@ -134,7 +135,7 @@ public class WikipediaTokenizerTest exte
     untoks.add(WikipediaTokenizer.CATEGORY);
     untoks.add(WikipediaTokenizer.ITALICS);
     //should be exactly the same, regardless of untoks
-    WikipediaTokenizer tf = new WikipediaTokenizer(WikipediaTokenizer.TOKENS_ONLY, untoks);
+    WikipediaTokenizer tf = new WikipediaTokenizer(newAttributeFactory(), WikipediaTokenizer.TOKENS_ONLY, untoks);
     tf.setReader(new StringReader(LINK_PHRASES));
     checkLinkPhrases(tf);
     String test = "[[Category:a b c d]] [[Category:e f g]] [[link here]] [[link there]] ''italics here'' something ''more italics'' [[Category:h   i   j]]";
@@ -155,7 +156,7 @@ public class WikipediaTokenizerTest exte
     untoks.add(WikipediaTokenizer.ITALICS);
     String test = "[[Category:a b c d]] [[Category:e f g]] [[link here]] [[link there]] ''italics here'' something ''more italics'' [[Category:h   i   j]]";
     //should output all the indivual tokens plus the untokenized tokens as well.  Untokenized tokens
-    WikipediaTokenizer tf = new WikipediaTokenizer(WikipediaTokenizer.BOTH, untoks);
+    WikipediaTokenizer tf = new WikipediaTokenizer(newAttributeFactory(), WikipediaTokenizer.BOTH, untoks);
     tf.setReader(new StringReader(test));
     assertTokenStreamContents(tf,
         new String[] { "a b c d", "a", "b", "c", "d", "e f g", "e", "f", "g",
@@ -167,7 +168,7 @@ public class WikipediaTokenizerTest exte
        );
     
     // now check the flags, TODO: add way to check flags from BaseTokenStreamTestCase?
-    tf = new WikipediaTokenizer(WikipediaTokenizer.BOTH, untoks);
+    tf = new WikipediaTokenizer(newAttributeFactory(), WikipediaTokenizer.BOTH, untoks);
     tf.setReader(new StringReader(test));
     int expectedFlags[] = new int[] { UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, 0, 
         0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0 };
@@ -187,11 +188,12 @@ public class WikipediaTokenizerTest exte
 
       @Override
       protected TokenStreamComponents createComponents(String fieldName) {
-        Tokenizer tokenizer = new WikipediaTokenizer();
+        Tokenizer tokenizer = new WikipediaTokenizer(newAttributeFactory(), WikipediaTokenizer.TOKENS_ONLY, Collections.<String>emptySet());
         return new TokenStreamComponents(tokenizer, tokenizer);
       } 
     };
-    checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
+    // TODO: properly support positionLengthAttribute
+    checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER, 20, false, false);
   }
   
   /** blast some random large strings through the analyzer */
@@ -201,10 +203,11 @@ public class WikipediaTokenizerTest exte
 
       @Override
       protected TokenStreamComponents createComponents(String fieldName) {
-        Tokenizer tokenizer = new WikipediaTokenizer();
+        Tokenizer tokenizer = new WikipediaTokenizer(newAttributeFactory(), WikipediaTokenizer.TOKENS_ONLY, Collections.<String>emptySet());
         return new TokenStreamComponents(tokenizer, tokenizer);
       } 
     };
-    checkRandomData(random, a, 100*RANDOM_MULTIPLIER, 8192);
+    // TODO: properly support positionLengthAttribute
+    checkRandomData(random, a, 100*RANDOM_MULTIPLIER, 8192, false, false);
   }
 }

Modified: lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java Sun May  4 12:19:35 2014
@@ -77,7 +77,7 @@ public class TestICUNormalizer2CharFilte
     CharFilter reader = new ICUNormalizer2CharFilter(new StringReader(input),
       Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE));
 
-    Tokenizer tokenStream = new NGramTokenizer(TEST_VERSION_CURRENT, 1, 1);
+    Tokenizer tokenStream = new NGramTokenizer(TEST_VERSION_CURRENT, newAttributeFactory(), 1, 1);
     tokenStream.setReader(reader);
 
     assertTokenStreamContents(tokenStream,

Modified: lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java Sun May  4 12:19:35 2014
@@ -42,7 +42,7 @@ public class TestICUTokenizer extends Ba
     sb.append(whitespace);
     sb.append("testing 1234");
     String input = sb.toString();
-    ICUTokenizer tokenizer = new ICUTokenizer(new DefaultICUTokenizerConfig(false));
+    ICUTokenizer tokenizer = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false));
     tokenizer.setReader(new StringReader(input));
     assertTokenStreamContents(tokenizer, new String[] { "testing", "1234" });
   }
@@ -53,7 +53,7 @@ public class TestICUTokenizer extends Ba
       sb.append('a');
     }
     String input = sb.toString();
-    ICUTokenizer tokenizer = new ICUTokenizer(new DefaultICUTokenizerConfig(false));
+    ICUTokenizer tokenizer = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false));
     tokenizer.setReader(new StringReader(input));
     char token[] = new char[4096];
     Arrays.fill(token, 'a');
@@ -70,7 +70,7 @@ public class TestICUTokenizer extends Ba
   private Analyzer a = new Analyzer() {
     @Override
     protected TokenStreamComponents createComponents(String fieldName) {
-      Tokenizer tokenizer = new ICUTokenizer(new DefaultICUTokenizerConfig(false));
+      Tokenizer tokenizer = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false));
       TokenFilter filter = new ICUNormalizer2Filter(tokenizer);
       return new TokenStreamComponents(tokenizer, filter);
     }

Modified: lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java Sun May  4 12:19:35 2014
@@ -30,7 +30,7 @@ public class TestICUTokenizerCJK extends
   Analyzer a = new Analyzer() {
     @Override
     protected TokenStreamComponents createComponents(String fieldName) {
-      return new TokenStreamComponents(new ICUTokenizer());
+      return new TokenStreamComponents(new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(true)));
     }
   };
   

Modified: lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java Sun May  4 12:19:35 2014
@@ -33,7 +33,7 @@ public class TestICUTokenizerFactory ext
     Reader reader = new StringReader("การที่ได้ต้องแสดงว่างานดี  This is a test ກວ່າດອກ");
     ICUTokenizerFactory factory = new ICUTokenizerFactory(new HashMap<String,String>());
     factory.inform(new ClasspathResourceLoader(getClass()));
-    Tokenizer stream = factory.create();
+    Tokenizer stream = factory.create(newAttributeFactory());
     stream.setReader(reader);
     assertTokenStreamContents(stream,
         new String[] { "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี",
@@ -48,7 +48,7 @@ public class TestICUTokenizerFactory ext
     args.put(ICUTokenizerFactory.RULEFILES, "Latn:Latin-break-only-on-whitespace.rbbi");
     ICUTokenizerFactory factory = new ICUTokenizerFactory(args);
     factory.inform(new ClasspathResourceLoader(this.getClass()));
-    Tokenizer stream = factory.create();
+    Tokenizer stream = factory.create(newAttributeFactory());
     stream.setReader(reader);
     assertTokenStreamContents(stream,
         new String[] { "Don't,break.at?/(punct)!", "\u201Cnice\u201D", "85_At:all;", "`really\"",  "+2=3$5,&813", "!@#%$^)(*@#$" },
@@ -62,7 +62,7 @@ public class TestICUTokenizerFactory ext
     args.put(ICUTokenizerFactory.RULEFILES, "Latn:Latin-dont-break-on-hyphens.rbbi");
     ICUTokenizerFactory factory = new ICUTokenizerFactory(args);
     factory.inform(new ClasspathResourceLoader(getClass()));
-    Tokenizer stream = factory.create();
+    Tokenizer stream = factory.create(newAttributeFactory());
     stream.setReader(reader);
     assertTokenStreamContents(stream,
         new String[] { "One-two", "punch",
@@ -82,7 +82,7 @@ public class TestICUTokenizerFactory ext
     args.put(ICUTokenizerFactory.RULEFILES, "Cyrl:KeywordTokenizer.rbbi,Thai:KeywordTokenizer.rbbi");
     ICUTokenizerFactory factory = new ICUTokenizerFactory(args);
     factory.inform(new ClasspathResourceLoader(getClass()));
-    Tokenizer stream = factory.create();
+    Tokenizer stream = factory.create(newAttributeFactory());
     stream.setReader(reader);
     assertTokenStreamContents(stream, new String[] { "Some", "English",
         "Немного русский.  ",

Modified: lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java Sun May  4 12:19:35 2014
@@ -41,7 +41,7 @@ public class TestWithCJKBigramFilter ext
   private Analyzer analyzer = new Analyzer() {
     @Override
     protected TokenStreamComponents createComponents(String fieldName) {
-      Tokenizer source = new ICUTokenizer(new DefaultICUTokenizerConfig(false));
+      Tokenizer source = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false));
       TokenStream result = new CJKBigramFilter(source);
       return new TokenStreamComponents(source, new StopFilter(TEST_VERSION_CURRENT, result, CharArraySet.EMPTY_SET));
     }
@@ -56,7 +56,7 @@ public class TestWithCJKBigramFilter ext
   private Analyzer analyzer2 = new Analyzer() {
     @Override
     protected TokenStreamComponents createComponents(String fieldName) {
-      Tokenizer source = new ICUTokenizer(new DefaultICUTokenizerConfig(false));
+      Tokenizer source = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false));
       // we put this before the CJKBigramFilter, because the normalization might combine
       // some halfwidth katakana forms, which will affect the bigramming.
       TokenStream result = new ICUNormalizer2Filter(source);

Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java Sun May  4 12:19:35 2014
@@ -36,7 +36,7 @@ public class TestExtendedMode extends Ba
     
     @Override
     protected TokenStreamComponents createComponents(String fieldName) {
-      Tokenizer tokenizer = new JapaneseTokenizer(null, true, Mode.EXTENDED);
+      Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, Mode.EXTENDED);
       return new TokenStreamComponents(tokenizer, tokenizer);
     }
   };

Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java Sun May  4 12:19:35 2014
@@ -32,7 +32,7 @@ public class TestJapaneseBaseFormFilter 
   private Analyzer analyzer = new Analyzer() {
     @Override
     protected TokenStreamComponents createComponents(String fieldName) {
-      Tokenizer tokenizer = new JapaneseTokenizer(null, true, JapaneseTokenizer.DEFAULT_MODE);
+      Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.DEFAULT_MODE);
       return new TokenStreamComponents(tokenizer, new JapaneseBaseFormFilter(tokenizer));
     }
   };
@@ -48,7 +48,7 @@ public class TestJapaneseBaseFormFilter 
     Analyzer a = new Analyzer() {
       @Override
       protected TokenStreamComponents createComponents(String fieldName) {
-        Tokenizer source = new JapaneseTokenizer(null, true, JapaneseTokenizer.DEFAULT_MODE);
+        Tokenizer source = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.DEFAULT_MODE);
         TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
         return new TokenStreamComponents(source, new JapaneseBaseFormFilter(sink));
       }

Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilterFactory.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilterFactory.java Sun May  4 12:19:35 2014
@@ -32,7 +32,7 @@ public class TestJapaneseBaseFormFilterF
   public void testBasics() throws IOException {
     JapaneseTokenizerFactory tokenizerFactory = new JapaneseTokenizerFactory(new HashMap<String,String>());
     tokenizerFactory.inform(new StringMockResourceLoader(""));
-    TokenStream ts = tokenizerFactory.create();
+    TokenStream ts = tokenizerFactory.create(newAttributeFactory());
     ((Tokenizer)ts).setReader(new StringReader("それはまだ実験段階にあります"));
     JapaneseBaseFormFilterFactory factory = new JapaneseBaseFormFilterFactory(new HashMap<String,String>());
     ts = factory.create(ts);

Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java Sun May  4 12:19:35 2014
@@ -45,7 +45,7 @@ public class TestJapaneseIterationMarkCh
   private Analyzer japaneseAnalyzer = new Analyzer() {
     @Override
     protected TokenStreamComponents createComponents(String fieldName) {
-      Tokenizer tokenizer = new JapaneseTokenizer(null, false, JapaneseTokenizer.Mode.SEARCH);
+      Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, false, JapaneseTokenizer.Mode.SEARCH);
       return new TokenStreamComponents(tokenizer, tokenizer);
     }
 

Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilterFactory.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilterFactory.java Sun May  4 12:19:35 2014
@@ -50,7 +50,7 @@ public class TestJapaneseIterationMarkCh
     CharFilter filter = filterFactory.create(
         new StringReader("時々馬鹿々々しいところゞゝゝミスヾ")
     );
-    TokenStream tokenStream = tokenizerFactory.create();
+    TokenStream tokenStream = tokenizerFactory.create(newAttributeFactory());
     ((Tokenizer)tokenStream).setReader(filter);
     assertTokenStreamContents(tokenStream, new String[]{"時時", "馬鹿馬鹿しい", "ところどころ", "ミ", "スズ"});
   }
@@ -67,7 +67,7 @@ public class TestJapaneseIterationMarkCh
     CharFilter filter = filterFactory.create(
         new StringReader("時々馬鹿々々しいところゞゝゝミスヾ")
     );
-    TokenStream tokenStream = tokenizerFactory.create();
+    TokenStream tokenStream = tokenizerFactory.create(newAttributeFactory());
     ((Tokenizer)tokenStream).setReader(filter);
     assertTokenStreamContents(tokenStream, new String[]{"時時", "馬鹿馬鹿しい", "ところ", "ゞ", "ゝ", "ゝ", "ミス", "ヾ"});
   }
@@ -84,7 +84,7 @@ public class TestJapaneseIterationMarkCh
     CharFilter filter = filterFactory.create(
         new StringReader("時々馬鹿々々しいところゞゝゝミスヾ")
     );
-    TokenStream tokenStream = tokenizerFactory.create();
+    TokenStream tokenStream = tokenizerFactory.create(newAttributeFactory());
     ((Tokenizer)tokenStream).setReader(filter);
     assertTokenStreamContents(tokenStream, new String[]{"時々", "馬鹿", "々", "々", "しい", "ところどころ", "ミ", "スズ"});
   }

Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilterFactory.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilterFactory.java Sun May  4 12:19:35 2014
@@ -32,7 +32,7 @@ public class TestJapaneseKatakanaStemFil
   public void testKatakanaStemming() throws IOException {
     JapaneseTokenizerFactory tokenizerFactory = new JapaneseTokenizerFactory(new HashMap<String,String>());
     tokenizerFactory.inform(new StringMockResourceLoader(""));
-    TokenStream tokenStream = tokenizerFactory.create();
+    TokenStream tokenStream = tokenizerFactory.create(newAttributeFactory());
     ((Tokenizer)tokenStream).setReader(new StringReader("明後日パーティーに行く予定がある。図書館で資料をコピーしました。"));
     JapaneseKatakanaStemFilterFactory filterFactory = new JapaneseKatakanaStemFilterFactory(new HashMap<String,String>());;
     assertTokenStreamContents(filterFactory.create(tokenStream),

Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilter.java Sun May  4 12:19:35 2014
@@ -35,7 +35,7 @@ public class TestJapaneseReadingFormFilt
   private Analyzer katakanaAnalyzer = new Analyzer() {
     @Override
     protected TokenStreamComponents createComponents(String fieldName) {
-      Tokenizer tokenizer = new JapaneseTokenizer(null, true, JapaneseTokenizer.Mode.SEARCH);
+      Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.Mode.SEARCH);
       return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(tokenizer, false));
     }
   };
@@ -43,7 +43,7 @@ public class TestJapaneseReadingFormFilt
   private Analyzer romajiAnalyzer = new Analyzer() {
     @Override
     protected TokenStreamComponents createComponents(String fieldName) {
-      Tokenizer tokenizer = new JapaneseTokenizer(null, true, JapaneseTokenizer.Mode.SEARCH);
+      Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.Mode.SEARCH);
       return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(tokenizer, true));
     }
   };
@@ -59,7 +59,7 @@ public class TestJapaneseReadingFormFilt
     Analyzer a = new Analyzer() {
       @Override
       protected TokenStreamComponents createComponents(String fieldName) {
-        Tokenizer tokenizer = new JapaneseTokenizer(null, true, JapaneseTokenizer.Mode.SEARCH);
+        Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.Mode.SEARCH);
         TokenStream stream = new CJKWidthFilter(tokenizer);
         return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, false));
       }
@@ -79,7 +79,7 @@ public class TestJapaneseReadingFormFilt
     Analyzer a = new Analyzer() {
       @Override
       protected TokenStreamComponents createComponents(String fieldName) {
-        Tokenizer tokenizer = new JapaneseTokenizer(null, true, JapaneseTokenizer.Mode.SEARCH);
+        Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.Mode.SEARCH);
         TokenStream stream = new CJKWidthFilter(tokenizer);
         return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, true));
       }

Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java Sun May  4 12:19:35 2014
@@ -62,7 +62,7 @@ public class TestJapaneseTokenizer exten
   private Analyzer analyzer = new Analyzer() {
     @Override
     protected TokenStreamComponents createComponents(String fieldName) {
-      Tokenizer tokenizer = new JapaneseTokenizer(readDict(), false, Mode.SEARCH);
+      Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, Mode.SEARCH);
       return new TokenStreamComponents(tokenizer, tokenizer);
     }
   };
@@ -70,7 +70,7 @@ public class TestJapaneseTokenizer exten
   private Analyzer analyzerNormal = new Analyzer() {
     @Override
     protected TokenStreamComponents createComponents(String fieldName) {
-      Tokenizer tokenizer = new JapaneseTokenizer(readDict(), false, Mode.NORMAL);
+      Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, Mode.NORMAL);
       return new TokenStreamComponents(tokenizer, tokenizer);
     }
   };
@@ -78,7 +78,7 @@ public class TestJapaneseTokenizer exten
   private Analyzer analyzerNoPunct = new Analyzer() {
     @Override
     protected TokenStreamComponents createComponents(String fieldName) {
-      Tokenizer tokenizer = new JapaneseTokenizer(readDict(), true, Mode.SEARCH);
+      Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), true, Mode.SEARCH);
       return new TokenStreamComponents(tokenizer, tokenizer);
     }
   };
@@ -86,7 +86,7 @@ public class TestJapaneseTokenizer exten
   private Analyzer extendedModeAnalyzerNoPunct = new Analyzer() {
     @Override
     protected TokenStreamComponents createComponents(String fieldName) {
-      Tokenizer tokenizer = new JapaneseTokenizer(readDict(), true, Mode.EXTENDED);
+      Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), true, Mode.EXTENDED);
       return new TokenStreamComponents(tokenizer, tokenizer);
     }
   };
@@ -202,7 +202,7 @@ public class TestJapaneseTokenizer exten
                     new Analyzer() {
                       @Override
                       protected TokenStreamComponents createComponents(String fieldName) {
-                        Tokenizer tokenizer = new JapaneseTokenizer(readDict(), false, Mode.SEARCH);
+                        Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, Mode.SEARCH);
                         TokenStream graph = new MockGraphTokenFilter(random(), tokenizer);
                         return new TokenStreamComponents(tokenizer, graph);
                       }
@@ -352,7 +352,7 @@ public class TestJapaneseTokenizer exten
     final Analyzer analyzer = new Analyzer() {
       @Override
       protected TokenStreamComponents createComponents(String fieldName) {
-        JapaneseTokenizer tokenizer = new JapaneseTokenizer(readDict(), false, Mode.SEARCH);
+        JapaneseTokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, Mode.SEARCH);
         tokenizer.setGraphvizFormatter(gv2);
         return new TokenStreamComponents(tokenizer, tokenizer);
       }

Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizerFactory.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizerFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizerFactory.java Sun May  4 12:19:35 2014
@@ -34,7 +34,7 @@ public class TestJapaneseTokenizerFactor
   public void testSimple() throws IOException {
     JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(new HashMap<String,String>());
     factory.inform(new StringMockResourceLoader(""));
-    TokenStream ts = factory.create();
+    TokenStream ts = factory.create(newAttributeFactory());
     ((Tokenizer)ts).setReader(new StringReader("これは本ではない"));
     assertTokenStreamContents(ts,
         new String[] { "これ", "は", "本", "で", "は", "ない" },
@@ -49,7 +49,7 @@ public class TestJapaneseTokenizerFactor
   public void testDefaults() throws IOException {
     JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(new HashMap<String,String>());
     factory.inform(new StringMockResourceLoader(""));
-    TokenStream ts = factory.create();
+    TokenStream ts = factory.create(newAttributeFactory());
     ((Tokenizer)ts).setReader(new StringReader("シニアソフトウェアエンジニア"));
     assertTokenStreamContents(ts,
                               new String[] { "シニア", "シニアソフトウェアエンジニア", "ソフトウェア", "エンジニア" }
@@ -64,7 +64,7 @@ public class TestJapaneseTokenizerFactor
     args.put("mode", "normal");
     JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(args);
     factory.inform(new StringMockResourceLoader(""));
-    TokenStream ts = factory.create();
+    TokenStream ts = factory.create(newAttributeFactory());
     ((Tokenizer)ts).setReader(new StringReader("シニアソフトウェアエンジニア"));
     assertTokenStreamContents(ts,
         new String[] { "シニアソフトウェアエンジニア" }
@@ -85,7 +85,7 @@ public class TestJapaneseTokenizerFactor
     args.put("userDictionary", "userdict.txt");
     JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(args);
     factory.inform(new StringMockResourceLoader(userDict));
-    TokenStream ts = factory.create();
+    TokenStream ts = factory.create(newAttributeFactory());
     ((Tokenizer)ts).setReader(new StringReader("関西国際空港に行った"));
     assertTokenStreamContents(ts,
         new String[] { "関西", "国際", "空港", "に",  "行っ",  "た" }
@@ -100,7 +100,7 @@ public class TestJapaneseTokenizerFactor
     args.put("discardPunctuation", "false");
     JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(args);
     factory.inform(new StringMockResourceLoader(""));
-    TokenStream ts = factory.create();
+    TokenStream ts = factory.create(newAttributeFactory());
     ((Tokenizer)ts).setReader(new StringReader("今ノルウェーにいますが、来週の頭日本に戻ります。楽しみにしています!お寿司が食べたいな。。。"));
     assertTokenStreamContents(ts,
         new String[] { "今", "ノルウェー", "に", "い", "ます", "が", "、",

Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestSearchMode.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestSearchMode.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestSearchMode.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestSearchMode.java Sun May  4 12:19:35 2014
@@ -34,7 +34,7 @@ public class TestSearchMode extends Base
   private final Analyzer analyzer = new Analyzer() {
     @Override
     protected TokenStreamComponents createComponents(String fieldName) {
-      Tokenizer tokenizer = new JapaneseTokenizer(null, true, Mode.SEARCH);
+      Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, Mode.SEARCH);
       return new TokenStreamComponents(tokenizer, tokenizer);
     }
   };

Modified: lucene/dev/trunk/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/DoubleMetaphoneFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/DoubleMetaphoneFilterTest.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/DoubleMetaphoneFilterTest.java (original)
+++ lucene/dev/trunk/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/DoubleMetaphoneFilterTest.java Sun May  4 12:19:35 2014
@@ -17,7 +17,6 @@
 package org.apache.lucene.analysis.phonetic;
 
 import java.io.IOException;
-import java.io.StringReader;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.BaseTokenStreamTestCase;
@@ -25,54 +24,47 @@ import org.apache.lucene.analysis.MockTo
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
 import org.apache.lucene.util.TestUtil;
 
 public class DoubleMetaphoneFilterTest extends BaseTokenStreamTestCase {
-  
-  private TokenStream whitespaceTokenizer(String data) throws IOException {
-    WhitespaceTokenizer whitespaceTokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
-    whitespaceTokenizer.setReader(new StringReader(data));
-    return whitespaceTokenizer;
-  }
 
   public void testSize4FalseInject() throws Exception {
-    TokenStream stream = whitespaceTokenizer("international");
+    TokenStream stream = whitespaceMockTokenizer("international");
     TokenStream filter = new DoubleMetaphoneFilter(stream, 4, false);
     assertTokenStreamContents(filter, new String[] { "ANTR" });
   }
 
   public void testSize4TrueInject() throws Exception {
-    TokenStream stream = whitespaceTokenizer("international");
+    TokenStream stream = whitespaceMockTokenizer("international");
     TokenStream filter = new DoubleMetaphoneFilter(stream, 4, true);
     assertTokenStreamContents(filter, new String[] { "international", "ANTR" });
   }
 
   public void testAlternateInjectFalse() throws Exception {
-    TokenStream stream = whitespaceTokenizer("Kuczewski");
+    TokenStream stream = whitespaceMockTokenizer("Kuczewski");
     TokenStream filter = new DoubleMetaphoneFilter(stream, 4, false);
     assertTokenStreamContents(filter, new String[] { "KSSK", "KXFS" });
   }
 
   public void testSize8FalseInject() throws Exception {
-    TokenStream stream = whitespaceTokenizer("international");
+    TokenStream stream = whitespaceMockTokenizer("international");
     TokenStream filter = new DoubleMetaphoneFilter(stream, 8, false);
     assertTokenStreamContents(filter, new String[] { "ANTRNXNL" });
   }
 
   public void testNonConvertableStringsWithInject() throws Exception {
-    TokenStream stream = whitespaceTokenizer("12345 #$%@#^%&");
+    TokenStream stream = whitespaceMockTokenizer("12345 #$%@#^%&");
     TokenStream filter = new DoubleMetaphoneFilter(stream, 8, true);
     assertTokenStreamContents(filter, new String[] { "12345", "#$%@#^%&" });
   }
 
   public void testNonConvertableStringsWithoutInject() throws Exception {
-    TokenStream stream = whitespaceTokenizer("12345 #$%@#^%&");
+    TokenStream stream = whitespaceMockTokenizer("12345 #$%@#^%&");
     TokenStream filter = new DoubleMetaphoneFilter(stream, 8, false);
     assertTokenStreamContents(filter, new String[] { "12345", "#$%@#^%&" });
     
     // should have something after the stream
-    stream = whitespaceTokenizer("12345 #$%@#^%& hello");
+    stream = whitespaceMockTokenizer("12345 #$%@#^%& hello");
     filter = new DoubleMetaphoneFilter(stream, 8, false);
     assertTokenStreamContents(filter, new String[] { "12345", "#$%@#^%&", "HL" });
   }

Modified: lucene/dev/trunk/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestBeiderMorseFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestBeiderMorseFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestBeiderMorseFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestBeiderMorseFilter.java Sun May  4 12:19:35 2014
@@ -110,7 +110,7 @@ public class TestBeiderMorseFilter exten
   }
   
   public void testCustomAttribute() throws IOException {
-    TokenStream stream = new KeywordTokenizer();
+    TokenStream stream = new MockTokenizer(MockTokenizer.KEYWORD, false);
     ((Tokenizer)stream).setReader(new StringReader("D'Angelo"));
     stream = new PatternKeywordMarkerFilter(stream, Pattern.compile(".*"));
     stream = new BeiderMorseFilter(stream, new PhoneticEngine(NameType.GENERIC, RuleType.EXACT, true));

Modified: lucene/dev/trunk/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestHMMChineseTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestHMMChineseTokenizerFactory.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestHMMChineseTokenizerFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestHMMChineseTokenizerFactory.java Sun May  4 12:19:35 2014
@@ -34,7 +34,7 @@ public class TestHMMChineseTokenizerFact
   public void testSimple() throws Exception {
     Reader reader = new StringReader("我购买了道具和服装。");
     TokenizerFactory factory = new HMMChineseTokenizerFactory(new HashMap<String,String>());
-    Tokenizer tokenizer = factory.create();
+    Tokenizer tokenizer = factory.create(newAttributeFactory());
     tokenizer.setReader(reader);
     // TODO: fix smart chinese to not emit punctuation tokens
     // at the moment: you have to clean up with WDF, or use the stoplist, etc

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestToken.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestToken.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestToken.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestToken.java Sun May  4 12:19:35 2014
@@ -40,16 +40,9 @@ public class TestToken extends LuceneTes
     assertEquals("word", t.type());
     assertEquals(0, t.getFlags());
 
-    t = new Token(6, 22);
-    t.copyBuffer(content, 0, content.length);
-    assertEquals("hello", t.toString());
-    assertEquals("hello", t.toString());
-    assertEquals(6, t.startOffset());
-    assertEquals(22, t.endOffset());
-    assertEquals("word", t.type());
-    assertEquals(0, t.getFlags());
-
-    t = new Token(6, 22, 7);
+    t = new Token();
+    t.setOffset(6, 22);
+    t.setFlags(7);
     t.copyBuffer(content, 0, content.length);
     assertEquals("hello", t.toString());
     assertEquals("hello", t.toString());
@@ -58,7 +51,9 @@ public class TestToken extends LuceneTes
     assertEquals("word", t.type());
     assertEquals(7, t.getFlags());
 
-    t = new Token(6, 22, "junk");
+    t = new Token();
+    t.setOffset(6, 22);
+    t.setType("junk");
     t.copyBuffer(content, 0, content.length);
     assertEquals("hello", t.toString());
     assertEquals("hello", t.toString());
@@ -174,7 +169,8 @@ public class TestToken extends LuceneTes
   }
   
   public void testClone() throws Exception {
-    Token t = new Token(0, 5);
+    Token t = new Token();
+    t.setOffset(0, 5);
     char[] content = "hello".toCharArray();
     t.copyBuffer(content, 0, 5);
     char[] buf = t.buffer();
@@ -195,7 +191,8 @@ public class TestToken extends LuceneTes
     assertEquals("", t.toString());
     assertEquals("", copy.toString());
 
-    t = new Token(0, 5);
+    t = new Token();
+    t.setOffset(0, 5);
     char[] content = "hello".toCharArray();
     t.copyBuffer(content, 0, 5);
     char[] buf = t.buffer();
@@ -245,7 +242,8 @@ public class TestToken extends LuceneTes
   }
 
   public void testAttributeReflection() throws Exception {
-    Token t = new Token("foobar", 6, 22, 8);
+    Token t = new Token("foobar", 6, 22);
+    t.setFlags(8);
     TestUtil.assertAttributeReflection(t,
         new HashMap<String, Object>() {{
           put(CharTermAttribute.class.getName() + "#term", "foobar");

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java Sun May  4 12:19:35 2014
@@ -591,7 +591,6 @@ public class TestPayloads extends Lucene
     Field field = new TextField("field", "", Field.Store.NO);
     TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
     ((Tokenizer)ts).setReader(new StringReader("here we go"));
-    assertFalse(ts.hasAttribute(PayloadAttribute.class));
     field.setTokenStream(ts);
     doc.add(field);
     writer.addDocument(doc);
@@ -603,7 +602,6 @@ public class TestPayloads extends Lucene
     writer.addDocument(doc);
     ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
     ((Tokenizer)ts).setReader(new StringReader("another"));
-    assertFalse(ts.hasAttribute(PayloadAttribute.class));
     field.setTokenStream(ts);
     writer.addDocument(doc);
     DirectoryReader reader = writer.getReader();
@@ -625,7 +623,6 @@ public class TestPayloads extends Lucene
     Field field = new TextField("field", "", Field.Store.NO);
     TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
     ((Tokenizer)ts).setReader(new StringReader("here we go"));
-    assertFalse(ts.hasAttribute(PayloadAttribute.class));
     field.setTokenStream(ts);
     doc.add(field);
     Field field2 = new TextField("field", "", Field.Store.NO);
@@ -638,8 +635,6 @@ public class TestPayloads extends Lucene
     Field field3 = new TextField("field", "", Field.Store.NO);
     ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
     ((Tokenizer)ts).setReader(new StringReader("nopayload"));
-
-    assertFalse(ts.hasAttribute(PayloadAttribute.class));
     field3.setTokenStream(ts);
     doc.add(field3);
     writer.addDocument(doc);

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java Sun May  4 12:19:35 2014
@@ -51,7 +51,6 @@ public class TestPayloadsOnVectors exten
     Field field = new Field("field", "", customType);
     TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
     ((Tokenizer)ts).setReader(new StringReader("here we go"));
-    assertFalse(ts.hasAttribute(PayloadAttribute.class));
     field.setTokenStream(ts);
     doc.add(field);
     writer.addDocument(doc);
@@ -65,7 +64,6 @@ public class TestPayloadsOnVectors exten
     
     ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
     ((Tokenizer)ts).setReader(new StringReader("another"));
-    assertFalse(ts.hasAttribute(PayloadAttribute.class));
     field.setTokenStream(ts);
     writer.addDocument(doc);
     
@@ -96,7 +94,6 @@ public class TestPayloadsOnVectors exten
     Field field = new Field("field", "", customType);
     TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
     ((Tokenizer)ts).setReader(new StringReader("here we go"));
-    assertFalse(ts.hasAttribute(PayloadAttribute.class));
     field.setTokenStream(ts);
     doc.add(field);
     Field field2 = new Field("field", "", customType);
@@ -109,7 +106,6 @@ public class TestPayloadsOnVectors exten
     Field field3 = new Field("field", "", customType);
     ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
     ((Tokenizer)ts).setReader(new StringReader("nopayload"));
-    assertFalse(ts.hasAttribute(PayloadAttribute.class));
     field3.setTokenStream(ts);
     doc.add(field3);
     writer.addDocument(doc);

Modified: lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java (original)
+++ lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java Sun May  4 12:19:35 2014
@@ -67,7 +67,8 @@ public class TokenGroup {
           tot += score;
         }
       }
-      Token token = new Token(termStartOffset, termEndOffset);
+      Token token = new Token();
+      token.setOffset(termStartOffset, termEndOffset);
       token.setEmpty().append(termAtt);
       tokens[numTokens] = token;
       scores[numTokens] = score;

Modified: lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java (original)
+++ lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java Sun May  4 12:19:35 2014
@@ -314,10 +314,10 @@ public class HighlighterPhraseTest exten
     public void reset() {
       this.i = -1;
       this.tokens = new Token[] {
-          new Token(new char[] { 't', 'h', 'e' }, 0, 3, 0, 3),
-          new Token(new char[] { 'f', 'o', 'x' }, 0, 3, 4, 7),
-          new Token(new char[] { 'd', 'i', 'd' }, 0, 3, 8, 11),
-          new Token(new char[] { 'j', 'u', 'm', 'p' }, 0, 4, 16, 20) };
+          new Token("the", 0, 3),
+          new Token("fox", 4, 7),
+          new Token("did", 8, 11),
+          new Token("jump", 16, 20) };
       this.tokens[3].setPositionIncrement(2);
     }
   }
@@ -354,10 +354,10 @@ public class HighlighterPhraseTest exten
     public void reset() {
       this.i = -1;
       this.tokens = new Token[] {
-          new Token(new char[] { 't', 'h', 'e' }, 0, 3, 0, 3),
-          new Token(new char[] { 'f', 'o', 'x' }, 0, 3, 4, 7),
-          new Token(new char[] { 'j', 'u', 'm', 'p' }, 0, 4, 8, 14),
-          new Token(new char[] { 'j', 'u', 'm', 'p', 'e', 'd' }, 0, 6, 8, 14) };
+          new Token("the", 0, 3),
+          new Token("fox", 4, 7),
+          new Token("jump", 8, 14),
+          new Token("jumped", 8, 14) };
       this.tokens[3].setPositionIncrement(0);
     }
   }

Modified: lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original)
+++ lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Sun May  4 12:19:35 2014
@@ -2013,7 +2013,8 @@ final class SynonymTokenizer extends Tok
       }
       st = new StringTokenizer(expansions, ",");
       if (st.hasMoreTokens()) {
-        currentRealToken = new Token(realOffsetAtt.startOffset(), realOffsetAtt.endOffset());
+        currentRealToken = new Token();
+        currentRealToken.setOffset(realOffsetAtt.startOffset(), realOffsetAtt.endOffset());
         currentRealToken.copyBuffer(realTermAtt.buffer(), 0, realTermAtt.length());
       }
       

Modified: lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java (original)
+++ lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java Sun May  4 12:19:35 2014
@@ -78,12 +78,12 @@ public class TokenSourcesTest extends Lu
     public void reset() {
       this.i = -1;
       this.tokens = new Token[] {
-          new Token(new char[] {'t', 'h', 'e'}, 0, 3, 0, 3),
-          new Token(new char[] {'{', 'f', 'o', 'x', '}'}, 0, 5, 0, 7),
-          new Token(new char[] {'f', 'o', 'x'}, 0, 3, 4, 7),
-          new Token(new char[] {'d', 'i', 'd'}, 0, 3, 8, 11),
-          new Token(new char[] {'n', 'o', 't'}, 0, 3, 12, 15),
-          new Token(new char[] {'j', 'u', 'm', 'p'}, 0, 4, 16, 20)};
+          new Token("the", 0, 3),
+          new Token("{fox}", 0, 7),
+          new Token("fox", 4, 7),
+          new Token("did", 8, 11),
+          new Token("not", 12, 15),
+          new Token("jump", 16, 20)};
       this.tokens[1].setPositionIncrement(0);
     }
   }

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java Sun May  4 12:19:35 2014
@@ -39,6 +39,7 @@ import org.apache.lucene.index.RandomInd
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Attribute;
 import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LineFileDocs;
 import org.apache.lucene.util.LuceneTestCase;
@@ -933,5 +934,18 @@ public abstract class BaseTokenStreamTes
     mockTokenizer.setReader(new StringReader(input));
     return mockTokenizer;
   }
-
+  
+  /** Returns a new AttributeFactory impl */
+  public static AttributeFactory newAttributeFactory(Random random) {
+    if (random.nextBoolean()) {
+      return Token.TOKEN_ATTRIBUTE_FACTORY;
+    } else {
+      return AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY;
+    }
+  }
+  
+  /** Returns a new AttributeFactory impl */
+  public static AttributeFactory newAttributeFactory() {
+    return newAttributeFactory(random());
+  }
 }

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java Sun May  4 12:19:35 2014
@@ -98,7 +98,7 @@ public class MockTokenizer extends Token
   }
 
   public MockTokenizer(CharacterRunAutomaton runAutomaton, boolean lowerCase, int maxTokenLength) {
-    this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, runAutomaton, lowerCase, maxTokenLength);
+    this(BaseTokenStreamTestCase.newAttributeFactory(), runAutomaton, lowerCase, maxTokenLength);
   }
 
   public MockTokenizer(CharacterRunAutomaton runAutomaton, boolean lowerCase) {



Mime
View raw message