lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rm...@apache.org
Subject svn commit: r950008 [1/4] - in /lucene/dev/trunk: lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/ lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/ lucene/contrib/highlighter/src/test/org/apache/lucene/search...
Date Tue, 01 Jun 2010 10:35:16 GMT
Author: rmuir
Date: Tue Jun  1 10:35:13 2010
New Revision: 950008

URL: http://svn.apache.org/viewvc?rev=950008&view=rev
Log:
LUCENE-2372: switch over remaining uses of TermAttribute

Modified:
    lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
    lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java
    lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java
    lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java
    lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java
    lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java
    lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java
    lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java
    lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java
    lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
    lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
    lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java
    lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
    lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
    lucene/dev/trunk/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java
    lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
    lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
    lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
    lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java
    lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java
    lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java
    lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj
    lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java
    lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java
    lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java
    lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java
    lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java
    lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java
    lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java
    lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java
    lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynonymTokenFilter.java
    lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java
    lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java
    lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java
    lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicStemFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianStemFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cn/ChineseFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cz/CzechStemFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/de/GermanStemFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianNormalizationFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiNormalizationFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiStemFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/in/IndicNormalizationFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PatternAnalyzer.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLowerCaseFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishLowerCaseFilter.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java
    lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex
    lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java
    lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestElision.java
    lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAndSuffixAwareTokenFilter.java
    lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAwareTokenFilter.java
    lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestRemoveDuplicatesTokenFilter.java
    lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestSingleTokenTokenFilter.java
    lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java
    lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilterTest.java
    lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java
    lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java
    lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/position/PositionFilterTest.java
    lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java
    lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java
    lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java
    lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java
    lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java
    lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java
    lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java
    lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymFilter.java
    lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java
    lucene/dev/trunk/modules/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilter.java
    lucene/dev/trunk/modules/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java
    lucene/dev/trunk/modules/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java
    lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/BufferedTokenStream.java
    lucene/dev/trunk/solr/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
    lucene/dev/trunk/solr/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java
    lucene/dev/trunk/solr/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java
    lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/SpellCheckComponent.java
    lucene/dev/trunk/solr/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java
    lucene/dev/trunk/solr/src/java/org/apache/solr/spelling/SpellingQueryConverter.java
    lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestBufferedTokenStream.java
    lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestRemoveDuplicatesTokenFilterFactory.java
    lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestSynonymMap.java
    lucene/dev/trunk/solr/src/test/org/apache/solr/spelling/SimpleQueryConverter.java
    lucene/dev/trunk/solr/src/test/org/apache/solr/spelling/SpellingQueryConverterTest.java
    lucene/dev/trunk/solr/src/webapp/web/admin/analysis.jsp

Modified: lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (original)
+++ lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java Tue Jun  1 10:35:13 2010
@@ -26,9 +26,10 @@ import java.util.List;
 import java.util.Locale;
 
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.benchmark.BenchmarkTestCase;
 import org.apache.lucene.benchmark.byTask.feeds.DocMaker;
 import org.apache.lucene.benchmark.byTask.feeds.ReutersQueryMaker;
@@ -918,11 +919,11 @@ public class TestPerfTasksLogic extends 
     TokenStream ts2 = a2.tokenStream("bogus", new StringReader(text));
     ts1.reset();
     ts2.reset();
-    TermAttribute termAtt1 = ts1.addAttribute(TermAttribute.class);
-    TermAttribute termAtt2 = ts2.addAttribute(TermAttribute.class);
+    CharTermAttribute termAtt1 = ts1.addAttribute(CharTermAttribute.class);
+    CharTermAttribute termAtt2 = ts2.addAttribute(CharTermAttribute.class);
     assertTrue(ts1.incrementToken());
     assertTrue(ts2.incrementToken());
-    assertEquals(termAtt1.term(), termAtt2.term());
+    assertEquals(termAtt1.toString(), termAtt2.toString());
     assertFalse(ts1.incrementToken());
     assertFalse(ts2.incrementToken());
     ts1.close();
@@ -994,21 +995,7 @@ public class TestPerfTasksLogic extends 
   
   private void assertEqualShingle
     (Analyzer analyzer, String text, String[] expected) throws Exception {
-    TokenStream stream = analyzer.tokenStream("bogus", new StringReader(text));
-    stream.reset();
-    TermAttribute termAtt = stream.addAttribute(TermAttribute.class);
-    int termNum = 0;
-    while (stream.incrementToken()) {
-      assertTrue("Extra output term(s), starting with '"
-                 + new String(termAtt.termBuffer(), 0, termAtt.termLength()) + "'",
-                 termNum < expected.length);
-      assertEquals("Mismatch in output term # " + termNum + " - ", 
-                   expected[termNum],
-                   new String(termAtt.termBuffer(), 0, termAtt.termLength()));
-      ++termNum;
-    }
-    assertEquals("Too few output terms", expected.length, termNum);
-    stream.close();
+    BaseTokenStreamTestCase.assertAnalyzesTo(analyzer, text, expected);
   }
   
   private String[] getShingleConfig(String params) { 

Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java Tue Jun  1 10:35:13 2010
@@ -23,9 +23,9 @@ import java.util.Iterator;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.util.PriorityQueue;
 
 /**
@@ -191,7 +191,7 @@ public class Highlighter
 		ArrayList<TextFragment> docFrags = new ArrayList<TextFragment>();
 		StringBuilder newText=new StringBuilder();
 		
-	    TermAttribute termAtt = tokenStream.addAttribute(TermAttribute.class);
+	    CharTermAttribute termAtt = tokenStream.addAttribute(CharTermAttribute.class);
 	    OffsetAttribute offsetAtt = tokenStream.addAttribute(OffsetAttribute.class);
 	    tokenStream.addAttribute(PositionIncrementAttribute.class);
 	    tokenStream.reset();
@@ -225,7 +225,7 @@ public class Highlighter
 					(offsetAtt.startOffset()>text.length())
 					)						
 				{
-					throw new InvalidTokenOffsetsException("Token "+ termAtt.term()
+					throw new InvalidTokenOffsetsException("Token "+ termAtt.toString()
 							+" exceeds length of provided text sized "+text.length());
 				}
 				if((tokenGroup.numTokens>0)&&(tokenGroup.isDistinct()))

Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java Tue Jun  1 10:35:13 2010
@@ -25,8 +25,8 @@ import java.util.Set;
 
 import org.apache.lucene.analysis.CachingTokenFilter;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.memory.MemoryIndex;
 import org.apache.lucene.search.Query;
@@ -46,7 +46,7 @@ public class QueryScorer implements Scor
   private float maxTermWeight;
   private int position = -1;
   private String defaultField;
-  private TermAttribute termAtt;
+  private CharTermAttribute termAtt;
   private PositionIncrementAttribute posIncAtt;
   private boolean expandMultiTermQuery = true;
   private Query query;
@@ -145,7 +145,7 @@ public class QueryScorer implements Scor
    */
   public float getTokenScore() {
     position += posIncAtt.getPositionIncrement();
-    String termText = termAtt.term();
+    String termText = termAtt.toString();
 
     WeightedSpanTerm weightedSpanTerm;
 
@@ -175,7 +175,7 @@ public class QueryScorer implements Scor
    */
   public TokenStream init(TokenStream tokenStream) throws IOException {
     position = -1;
-    termAtt = tokenStream.addAttribute(TermAttribute.class);
+    termAtt = tokenStream.addAttribute(CharTermAttribute.class);
     posIncAtt = tokenStream.addAttribute(PositionIncrementAttribute.class);
     if(!skipInitExtractor) {
       if(fieldWeightedSpanTerms != null) {

Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java Tue Jun  1 10:35:13 2010
@@ -21,7 +21,7 @@ import java.util.HashMap;
 import java.util.HashSet;
 
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.Query;
 
@@ -41,7 +41,7 @@ public class QueryTermScorer implements 
   float maxTermWeight = 0;
   private HashMap<String,WeightedTerm> termsToFind;
 
-  private TermAttribute termAtt;
+  private CharTermAttribute termAtt;
 
   /**
    * 
@@ -95,7 +95,7 @@ public class QueryTermScorer implements 
    * @see org.apache.lucene.search.highlight.Scorer#init(org.apache.lucene.analysis.TokenStream)
    */
   public TokenStream init(TokenStream tokenStream) {
-    termAtt = tokenStream.addAttribute(TermAttribute.class);
+    termAtt = tokenStream.addAttribute(CharTermAttribute.class);
     return null;
   }
 
@@ -118,7 +118,7 @@ public class QueryTermScorer implements 
    * @see org.apache.lucene.search.highlight.Scorer#getTokenScore()
    */
   public float getTokenScore() {
-    String termText = termAtt.term();
+    String termText = termAtt.toString();
 
     WeightedTerm queryTerm = termsToFind.get(termText);
     if (queryTerm == null) {

Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java Tue Jun  1 10:35:13 2010
@@ -20,9 +20,9 @@ package org.apache.lucene.search.highlig
 import java.util.List;
 
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.search.spans.Spans;
 
 
@@ -38,7 +38,7 @@ public class SimpleSpanFragmenter implem
   private QueryScorer queryScorer;
   private int waitForPos = -1;
   private int textSize;
-  private TermAttribute termAtt;
+  private CharTermAttribute termAtt;
   private PositionIncrementAttribute posIncAtt;
   private OffsetAttribute offsetAtt;
 
@@ -70,7 +70,7 @@ public class SimpleSpanFragmenter implem
       return false;
     }
 
-    WeightedSpanTerm wSpanTerm = queryScorer.getWeightedSpanTerm(termAtt.term());
+    WeightedSpanTerm wSpanTerm = queryScorer.getWeightedSpanTerm(termAtt.toString());
 
     if (wSpanTerm != null) {
       List<PositionSpan> positionSpans = wSpanTerm.getPositionSpans();
@@ -101,7 +101,7 @@ public class SimpleSpanFragmenter implem
     position = -1;
     currentNumFrags = 1;
     textSize = originalText.length();
-    termAtt = tokenStream.addAttribute(TermAttribute.class);
+    termAtt = tokenStream.addAttribute(CharTermAttribute.class);
     posIncAtt = tokenStream.addAttribute(PositionIncrementAttribute.class);
     offsetAtt = tokenStream.addAttribute(OffsetAttribute.class);
   }

Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java Tue Jun  1 10:35:13 2010
@@ -19,8 +19,8 @@ package org.apache.lucene.search.highlig
 
 import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 
 /**
  * One, or several overlapping tokens, along with the score(s) and the scope of
@@ -38,11 +38,11 @@ public class TokenGroup {
   int matchStartOffset, matchEndOffset;
 
   private OffsetAttribute offsetAtt;
-  private TermAttribute termAtt;
+  private CharTermAttribute termAtt;
 
   public TokenGroup(TokenStream tokenStream) {
     offsetAtt = tokenStream.addAttribute(OffsetAttribute.class);
-    termAtt = tokenStream.addAttribute(TermAttribute.class);
+    termAtt = tokenStream.addAttribute(CharTermAttribute.class);
   }
 
   void addToken(float score) {
@@ -68,7 +68,7 @@ public class TokenGroup {
         }
       }
       Token token = new Token(termStartOffset, termEndOffset);
-      token.setTermBuffer(termAtt.term());
+      token.setEmpty().append(termAtt);
       tokens[numTokens] = token;
       scores[numTokens] = score;
       numTokens++;

Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java Tue Jun  1 10:35:13 2010
@@ -29,8 +29,8 @@ import java.util.Comparator;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.TermFreqVector;
@@ -153,13 +153,13 @@ public class TokenSources {
 
       int currentToken = 0;
 
-      TermAttribute termAtt;
+      CharTermAttribute termAtt;
 
       OffsetAttribute offsetAtt;
 
       StoredTokenStream(Token tokens[]) {
         this.tokens = tokens;
-        termAtt = addAttribute(TermAttribute.class);
+        termAtt = addAttribute(CharTermAttribute.class);
         offsetAtt = addAttribute(OffsetAttribute.class);
       }
 
@@ -170,7 +170,7 @@ public class TokenSources {
         }
         Token token = tokens[currentToken++];
         clearAttributes();
-        termAtt.setTermBuffer(token.term());
+        termAtt.setEmpty().append(token);
         offsetAtt.setOffset(token.startOffset(), token.endOffset());
         return true;
       }
@@ -204,9 +204,8 @@ public class TokenSources {
           unsortedTokens = new ArrayList<Token>();
         }
         for (int tp = 0; tp < offsets.length; tp++) {
-          Token token = new Token(offsets[tp].getStartOffset(), offsets[tp]
+          Token token = new Token(terms[t], offsets[tp].getStartOffset(), offsets[tp]
               .getEndOffset());
-          token.setTermBuffer(terms[t]);
           unsortedTokens.add(token);
         }
       } else {

Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java Tue Jun  1 10:35:13 2010
@@ -25,9 +25,9 @@ import java.util.List;
 
 import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.index.TermPositionVector;
 import org.apache.lucene.index.TermVectorOffsetInfo;
 
@@ -37,7 +37,7 @@ public final class TokenStreamFromTermPo
 
   private Iterator<Token> tokensAtCurrentPosition;
 
-  private TermAttribute termAttribute;
+  private CharTermAttribute termAttribute;
 
   private PositionIncrementAttribute positionIncrementAttribute;
 
@@ -51,7 +51,7 @@ public final class TokenStreamFromTermPo
    */
   public TokenStreamFromTermPositionVector(
       final TermPositionVector termPositionVector) {
-    termAttribute = addAttribute(TermAttribute.class);
+    termAttribute = addAttribute(CharTermAttribute.class);
     positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
     offsetAttribute = addAttribute(OffsetAttribute.class);
     final String[] terms = termPositionVector.getTerms();
@@ -65,7 +65,7 @@ public final class TokenStreamFromTermPo
               offsets[j].getStartOffset(), offsets[j].getEndOffset());
         } else {
           token = new Token();
-          token.setTermBuffer(terms[i]);
+          token.setEmpty().append(terms[i]);
         }
         // Yes - this is the position, not the increment! This is for
         // sorting. This value
@@ -100,7 +100,7 @@ public final class TokenStreamFromTermPo
     if (this.tokensAtCurrentPosition.hasNext()) {
       final Token next = this.tokensAtCurrentPosition.next();
       clearAttributes();
-      termAttribute.setTermBuffer(next.term());
+      termAttribute.setEmpty().append(next);
       positionIncrementAttribute.setPositionIncrement(next
           .getPositionIncrement());
       offsetAttribute.setOffset(next.startOffset(), next.endOffset());

Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java Tue Jun  1 10:35:13 2010
@@ -25,7 +25,7 @@ import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Field.Index;
@@ -296,16 +296,11 @@ public class HighlighterPhraseTest exten
 
     private int i = -1;
 
-    private TermAttribute termAttribute;
-
-    private OffsetAttribute offsetAttribute;
-
-    private PositionIncrementAttribute positionIncrementAttribute;
+    private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
+    private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);
+    private final PositionIncrementAttribute positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
 
     public TokenStreamSparse() {
-      termAttribute = addAttribute(TermAttribute.class);
-      offsetAttribute = addAttribute(OffsetAttribute.class);
-      positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
       reset();
     }
 
@@ -316,8 +311,7 @@ public class HighlighterPhraseTest exten
         return false;
       }
       clearAttributes();
-      termAttribute.setTermBuffer(this.tokens[i].term(), 0, this.tokens[i]
-          .term().length());
+      termAttribute.setEmpty().append(this.tokens[i]);
       offsetAttribute.setOffset(this.tokens[i].startOffset(), this.tokens[i]
           .endOffset());
       positionIncrementAttribute.setPositionIncrement(this.tokens[i]
@@ -342,16 +336,11 @@ public class HighlighterPhraseTest exten
 
     private int i = -1;
 
-    private TermAttribute termAttribute;
-
-    private OffsetAttribute offsetAttribute;
-
-    private PositionIncrementAttribute positionIncrementAttribute;
+    private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
+    private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);
+    private final PositionIncrementAttribute positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
 
     public TokenStreamConcurrent() {
-      termAttribute = addAttribute(TermAttribute.class);
-      offsetAttribute = addAttribute(OffsetAttribute.class);
-      positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
       reset();
     }
 
@@ -362,8 +351,7 @@ public class HighlighterPhraseTest exten
         return false;
       }
       clearAttributes();
-      termAttribute.setTermBuffer(this.tokens[i].term(), 0, this.tokens[i]
-          .term().length());
+      termAttribute.setEmpty().append(this.tokens[i]);
       offsetAttribute.setOffset(this.tokens[i].startOffset(), this.tokens[i]
           .endOffset());
       positionIncrementAttribute.setPositionIncrement(this.tokens[i]

Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Tue Jun  1 10:35:13 2010
@@ -41,7 +41,7 @@ import org.apache.lucene.analysis.TokenS
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.NumericField;
@@ -1424,13 +1424,10 @@ public class HighlighterTest extends Bas
     return new TokenStream() {
       Iterator<Token> iter;
       List<Token> lst;
-      private TermAttribute termAtt;
-      private PositionIncrementAttribute posIncrAtt;
-      private OffsetAttribute offsetAtt;
+      private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+      private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+      private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
       {
-        termAtt = addAttribute(TermAttribute.class);
-        posIncrAtt = addAttribute(PositionIncrementAttribute.class);
-        offsetAtt = addAttribute(OffsetAttribute.class);
         lst = new ArrayList<Token>();
         Token t;
         t = createToken("hi", 0, 2);
@@ -1456,7 +1453,7 @@ public class HighlighterTest extends Bas
         if(iter.hasNext()) {
           Token token =  iter.next();
           clearAttributes();
-          termAtt.setTermBuffer(token.term());
+          termAtt.setEmpty().append(token);
           posIncrAtt.setPositionIncrement(token.getPositionIncrement());
           offsetAtt.setOffset(token.startOffset(), token.endOffset());
           return true;
@@ -1473,13 +1470,10 @@ public class HighlighterTest extends Bas
     return new TokenStream() {
       Iterator<Token> iter;
       List<Token> lst;
-      private TermAttribute termAtt;
-      private PositionIncrementAttribute posIncrAtt;
-      private OffsetAttribute offsetAtt;
+      private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+      private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+      private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
       {
-        termAtt = addAttribute(TermAttribute.class);
-        posIncrAtt = addAttribute(PositionIncrementAttribute.class);
-        offsetAtt = addAttribute(OffsetAttribute.class);
         lst = new ArrayList<Token>();
         Token t;
         t = createToken("hispeed", 0, 8);
@@ -1505,7 +1499,7 @@ public class HighlighterTest extends Bas
         if(iter.hasNext()) {
           Token token = iter.next();
           clearAttributes();
-          termAtt.setTermBuffer(token.term());
+          termAtt.setEmpty().append(token);
           posIncrAtt.setPositionIncrement(token.getPositionIncrement());
           offsetAtt.setOffset(token.startOffset(), token.endOffset());
           return true;
@@ -1762,9 +1756,7 @@ public class HighlighterTest extends Bas
 
   private static Token createToken(String term, int start, int offset)
   {
-    Token token = new Token(start, offset);
-    token.setTermBuffer(term);
-    return token;
+    return new Token(term, start, offset);
   }
 
 }
@@ -1795,7 +1787,7 @@ final class SynonymAnalyzer extends Anal
   @Override
   public TokenStream tokenStream(String arg0, Reader arg1) {
     Tokenizer stream = new MockTokenizer(arg1, MockTokenizer.SIMPLE, true);
-    stream.addAttribute(TermAttribute.class);
+    stream.addAttribute(CharTermAttribute.class);
     stream.addAttribute(PositionIncrementAttribute.class);
     stream.addAttribute(OffsetAttribute.class);
     return new SynonymTokenizer(stream, synonyms);
@@ -1811,21 +1803,21 @@ final class SynonymTokenizer extends Tok
   private Token currentRealToken = null;
   private Map<String,String> synonyms;
   StringTokenizer st = null;
-  private TermAttribute realTermAtt;
+  private CharTermAttribute realTermAtt;
   private PositionIncrementAttribute realPosIncrAtt;
   private OffsetAttribute realOffsetAtt;
-  private TermAttribute termAtt;
+  private CharTermAttribute termAtt;
   private PositionIncrementAttribute posIncrAtt;
   private OffsetAttribute offsetAtt;
 
   public SynonymTokenizer(TokenStream realStream, Map<String,String> synonyms) {
     this.realStream = realStream;
     this.synonyms = synonyms;
-    realTermAtt = realStream.addAttribute(TermAttribute.class);
+    realTermAtt = realStream.addAttribute(CharTermAttribute.class);
     realPosIncrAtt = realStream.addAttribute(PositionIncrementAttribute.class);
     realOffsetAtt = realStream.addAttribute(OffsetAttribute.class);
 
-    termAtt = addAttribute(TermAttribute.class);
+    termAtt = addAttribute(CharTermAttribute.class);
     posIncrAtt = addAttribute(PositionIncrementAttribute.class);
     offsetAtt = addAttribute(OffsetAttribute.class);
   }
@@ -1840,25 +1832,25 @@ final class SynonymTokenizer extends Tok
       }
       //Token nextRealToken = new Token(, offsetAtt.startOffset(), offsetAtt.endOffset());
       clearAttributes();
-      termAtt.setTermBuffer(realTermAtt.term());
+      termAtt.copyBuffer(realTermAtt.buffer(), 0, realTermAtt.length());
       offsetAtt.setOffset(realOffsetAtt.startOffset(), realOffsetAtt.endOffset());
       posIncrAtt.setPositionIncrement(realPosIncrAtt.getPositionIncrement());
 
-      String expansions =  synonyms.get(realTermAtt.term());
+      String expansions =  synonyms.get(realTermAtt.toString());
       if (expansions == null) {
         return true;
       }
       st = new StringTokenizer(expansions, ",");
       if (st.hasMoreTokens()) {
         currentRealToken = new Token(realOffsetAtt.startOffset(), realOffsetAtt.endOffset());
-        currentRealToken.setTermBuffer(realTermAtt.term());
+        currentRealToken.copyBuffer(realTermAtt.buffer(), 0, realTermAtt.length());
       }
       
       return true;
     } else {
       String tok = st.nextToken();
       clearAttributes();
-      termAtt.setTermBuffer(tok);
+      termAtt.setEmpty().append(tok);
       offsetAtt.setOffset(currentRealToken.startOffset(), currentRealToken.endOffset());
       posIncrAtt.setPositionIncrement(0);
       if (!st.hasMoreTokens()) {

Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java Tue Jun  1 10:35:13 2010
@@ -26,8 +26,8 @@ import org.apache.lucene.analysis.MockAn
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Field.Index;
@@ -221,14 +221,14 @@ public abstract class AbstractTestCase e
       ch = 0;
     }
 
-    TermAttribute termAtt = addAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
     OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
     @Override
     public boolean incrementToken() throws IOException {
       if( !getNextPartialSnippet() )
         return false;
       clearAttributes();
-      termAtt.setTermBuffer(snippet, startTerm, lenTerm);
+      termAtt.setEmpty().append(snippet, startTerm, startTerm + lenTerm);
       offsetAtt.setOffset(correctOffset(startOffset), correctOffset(startOffset + lenTerm));
       return true;
     }

Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java Tue Jun  1 10:35:13 2010
@@ -25,7 +25,7 @@ import java.util.Set;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.util.AttributeImpl;
@@ -301,7 +301,7 @@ public class IndexTimeSynonymTest extend
     @Override
     public TokenStream tokenStream(String fieldName, Reader reader) {      
       TokenStream ts = new TokenStream(Token.TOKEN_ATTRIBUTE_FACTORY) {
-        final AttributeImpl reusableToken = (AttributeImpl) addAttribute(TermAttribute.class);
+        final AttributeImpl reusableToken = (AttributeImpl) addAttribute(CharTermAttribute.class);
         int p = 0;
         
         @Override

Modified: lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java Tue Jun  1 10:35:13 2010
@@ -561,7 +561,7 @@ public class InstantiatedIndexWriter imp
           // untokenized
           String fieldVal = field.stringValue();
           Token token = new Token(0, fieldVal.length(), "untokenized");
-          token.setTermBuffer(fieldVal);
+          token.setEmpty().append(fieldVal);
           tokens.add(token);
           fieldSetting.fieldLength++;
         }
@@ -596,10 +596,10 @@ public class InstantiatedIndexWriter imp
 
       for (Token token : eField_Tokens.getValue()) {
 
-        TermDocumentInformationFactory termDocumentInformationFactory = termDocumentInformationFactoryByTermText.get(token.term());
+        TermDocumentInformationFactory termDocumentInformationFactory = termDocumentInformationFactoryByTermText.get(token.toString());
         if (termDocumentInformationFactory == null) {
           termDocumentInformationFactory = new TermDocumentInformationFactory();
-          termDocumentInformationFactoryByTermText.put(token.term(), termDocumentInformationFactory);
+          termDocumentInformationFactoryByTermText.put(token.toString(), termDocumentInformationFactory);
         }
         //termDocumentInformationFactory.termFrequency++;
 

Modified: lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (original)
+++ lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java Tue Jun  1 10:35:13 2010
@@ -25,7 +25,7 @@ import java.util.List;
 import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexReader;
@@ -278,7 +278,7 @@ public class TestIndicesEquals extends L
               tokens.add(t);
               tokens.add(createToken("fin", 7, 9));
               TokenStream ts = new TokenStream(Token.TOKEN_ATTRIBUTE_FACTORY) {
-                final AttributeImpl reusableToken = (AttributeImpl) addAttribute(TermAttribute.class);
+                final AttributeImpl reusableToken = (AttributeImpl) addAttribute(CharTermAttribute.class);
                 Iterator<Token> it = tokens.iterator();
                 
                 @Override
@@ -601,16 +601,12 @@ public class TestIndicesEquals extends L
 
   private static Token createToken(String term, int start, int offset)
   {
-    Token token = new Token(start, offset);
-    token.setTermBuffer(term);
-    return token;
+    return new Token(term, start, offset);
   }
 
   private static Token createToken(String term, int start, int offset, String type)
   {
-    Token token = new Token(start, offset, type);
-    token.setTermBuffer(term);
-    return token;
+    return new Token(term, start, offset, type);
   }
 
 

Modified: lucene/dev/trunk/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java (original)
+++ lucene/dev/trunk/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java Tue Jun  1 10:35:13 2010
@@ -36,8 +36,8 @@ import jline.ConsoleReader;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Fieldable;
 import org.apache.lucene.index.IndexReader;
@@ -303,14 +303,14 @@ class LuceneMethods {
           int position = 0;
           // Tokenize field and add to postingTable
           TokenStream stream = analyzer.tokenStream(fieldName, reader);
-          TermAttribute termAtt = stream.addAttribute(TermAttribute.class);
+          CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
           PositionIncrementAttribute posIncrAtt = stream.addAttribute(PositionIncrementAttribute.class);
           
           try {
             while (stream.incrementToken()) {
               position += (posIncrAtt.getPositionIncrement() - 1);
               position++;
-              String name = termAtt.term();
+              String name = termAtt.toString();
               Integer Count = tokenMap.get(name);
               if (Count == null) { // not in there yet
                 tokenMap.put(name, Integer.valueOf(1)); //first one

Modified: lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Tue Jun  1 10:35:13 2010
@@ -30,9 +30,10 @@ import java.util.Map;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.FieldSelector;
 import org.apache.lucene.index.IndexReader;
@@ -51,6 +52,7 @@ import org.apache.lucene.search.Searcher
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.Similarity;
 import org.apache.lucene.store.RAMDirectory; // for javadocs
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Constants; // for javadocs
 
 /**
@@ -276,8 +278,8 @@ public class MemoryIndex implements Seri
     return new TokenStream() {
       private Iterator<T> iter = keywords.iterator();
       private int start = 0;
-      private TermAttribute termAtt = addAttribute(TermAttribute.class);
-      private OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+      private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+      private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
       
       @Override
       public boolean incrementToken() {
@@ -289,8 +291,8 @@ public class MemoryIndex implements Seri
         
         String term = obj.toString();
         clearAttributes();
-        termAtt.setTermBuffer(term);
-        offsetAtt.setOffset(start, start+termAtt.termLength());
+        termAtt.setEmpty().append(term);
+        offsetAtt.setOffset(start, start+termAtt.length());
         start += term.length() + 1; // separate words by 1 (blank) character
         return true;
       }
@@ -340,13 +342,15 @@ public class MemoryIndex implements Seri
       int numOverlapTokens = 0;
       int pos = -1;
       
-      TermAttribute termAtt = stream.addAttribute(TermAttribute.class);
+      TermToBytesRefAttribute termAtt = stream.addAttribute(TermToBytesRefAttribute.class);
       PositionIncrementAttribute posIncrAttribute = stream.addAttribute(PositionIncrementAttribute.class);
       OffsetAttribute offsetAtt = stream.addAttribute(OffsetAttribute.class);
-      
+      BytesRef ref = new BytesRef(10);
       stream.reset();
       while (stream.incrementToken()) {
-        String term = termAtt.term();
+        termAtt.toBytesRef(ref);
+        // TODO: support non-UTF8 strings (like numerics) here
+        String term = ref.utf8ToString();
         if (term.length() == 0) continue; // nothing to do
 //        if (DEBUG) System.err.println("token='" + term + "'");
         numTokens++;

Modified: lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (original)
+++ lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java Tue Jun  1 10:35:13 2010
@@ -26,7 +26,7 @@ import java.util.Iterator;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.util.BytesRef;
@@ -185,14 +185,14 @@ public class FuzzyLikeThisQuery extends 
     {
         if(f.queryString==null) return;
         TokenStream ts=analyzer.tokenStream(f.fieldName,new StringReader(f.queryString));
-        TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
+        CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
         
         int corpusNumDocs=reader.numDocs();
         Term internSavingTemplateTerm =new Term(f.fieldName); //optimization to avoid constructing new Term() objects
         HashSet<String> processedTerms=new HashSet<String>();
         while (ts.incrementToken()) 
         {
-                String term = termAtt.term();
+                String term = termAtt.toString();
         	if(!processedTerms.contains(term))
         	{
                   processedTerms.add(term);

Modified: lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java (original)
+++ lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java Tue Jun  1 10:35:13 2010
@@ -32,7 +32,7 @@ import java.util.Set;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
@@ -884,10 +884,10 @@ public final class MoreLikeThis {
 		   TokenStream ts = analyzer.tokenStream(fieldName, r);
 			int tokenCount=0;
 			// for every token
-			TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
+			CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
 			
 			while (ts.incrementToken()) {
-				String word = termAtt.term();
+				String word = termAtt.toString();
 				tokenCount++;
 				if(tokenCount>maxNumTokensParsed)
 				{

Modified: lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java (original)
+++ lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java Tue Jun  1 10:35:13 2010
@@ -22,7 +22,7 @@ import java.util.Set;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
@@ -86,12 +86,12 @@ public final class SimilarityQueries
 										  throws IOException
 	{	
 		TokenStream ts = a.tokenStream( field, new StringReader( body));
-		TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
+		CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
 		
 		BooleanQuery tmp = new BooleanQuery();
 		Set<String> already = new HashSet<String>(); // ignore dups
 		while (ts.incrementToken()) {
-		  String word = termAtt.term();
+		  String word = termAtt.toString();
 			// ignore opt stop words
 			if ( stop != null &&
 				 stop.contains( word)) continue;

Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java Tue Jun  1 10:35:13 2010
@@ -24,7 +24,7 @@ import java.util.List;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.queryParser.ParseException;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.Version;
@@ -107,7 +107,7 @@ public class AnalyzingQueryParser extend
 
     // get Analyzer from superclass and tokenize the term
     TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr));
-    TermAttribute termAtt = source.addAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
     
     int countTokens = 0;
     while (true) {
@@ -116,7 +116,7 @@ public class AnalyzingQueryParser extend
       } catch (IOException e) {
         break;
       }
-      String term = termAtt.term();
+      String term = termAtt.toString();
       if (!"".equals(term)) {
         try {
           tlist.set(countTokens++, term);
@@ -190,7 +190,7 @@ public class AnalyzingQueryParser extend
     // get Analyzer from superclass and tokenize the term
     TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr));
     List<String> tlist = new ArrayList<String>();
-    TermAttribute termAtt = source.addAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
     
     while (true) {
       try {
@@ -198,7 +198,7 @@ public class AnalyzingQueryParser extend
       } catch (IOException e) {
         break;
       }
-      tlist.add(termAtt.term());
+      tlist.add(termAtt.toString());
     }
 
     try {
@@ -237,13 +237,13 @@ public class AnalyzingQueryParser extend
       throws ParseException {
     // get Analyzer from superclass and tokenize the term
     TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr));
-    TermAttribute termAtt = source.addAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
     String nextToken = null;
     boolean multipleTokens = false;
     
     try {
       if (source.incrementToken()) {
-        nextToken = termAtt.term();
+        nextToken = termAtt.toString();
       }
       multipleTokens = source.incrementToken();
     } catch (IOException e) {
@@ -273,13 +273,13 @@ public class AnalyzingQueryParser extend
       throws ParseException {
     // get Analyzer from superclass and tokenize the terms
     TokenStream source = getAnalyzer().tokenStream(field, new StringReader(part1));
-    TermAttribute termAtt = source.addAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
     boolean multipleTokens = false;
 
     // part1
     try {
       if (source.incrementToken()) {
-        part1 = termAtt.term();
+        part1 = termAtt.toString();
       }
       multipleTokens = source.incrementToken();
     } catch (IOException e) {
@@ -297,11 +297,11 @@ public class AnalyzingQueryParser extend
 
     // part2
     source = getAnalyzer().tokenStream(field, new StringReader(part2));
-    termAtt = source.addAttribute(TermAttribute.class);
+    termAtt = source.addAttribute(CharTermAttribute.class);
     
     try {
       if (source.incrementToken()) {
-        part2 = termAtt.term();
+        part2 = termAtt.toString();
       }
       multipleTokens = source.incrementToken();
     } catch (IOException e) {

Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java Tue Jun  1 10:35:13 2010
@@ -307,7 +307,7 @@ public class PrecedenceQueryParser imple
     List<AttributeSource.State> list = new ArrayList<AttributeSource.State>();
     int positionCount = 0;
     boolean severalTokensAtSamePosition = false;
-    TermAttribute termAtt = source.addAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
     PositionIncrementAttribute posincrAtt = source.addAttribute(PositionIncrementAttribute.class);
 
     try {
@@ -328,7 +328,7 @@ public class PrecedenceQueryParser imple
       return null;
     else if (list.size() == 1) {
       source.restoreState(list.get(0));
-      return new TermQuery(new Term(field, termAtt.term()));
+      return new TermQuery(new Term(field, termAtt.toString()));
     } else {
       if (severalTokensAtSamePosition) {
         if (positionCount == 1) {
@@ -337,7 +337,7 @@ public class PrecedenceQueryParser imple
           for (int i = 0; i < list.size(); i++) {
             source.restoreState(list.get(i));
             TermQuery currentQuery = new TermQuery(
-                new Term(field, termAtt.term()));
+                new Term(field, termAtt.toString()));
             q.add(currentQuery, BooleanClause.Occur.SHOULD);
           }
           return q;
@@ -352,7 +352,7 @@ public class PrecedenceQueryParser imple
               mpq.add(multiTerms.toArray(new Term[0]));
               multiTerms.clear();
             }
-            multiTerms.add(new Term(field, termAtt.term()));
+            multiTerms.add(new Term(field, termAtt.toString()));
           }
           mpq.add(multiTerms.toArray(new Term[0]));
           return mpq;
@@ -363,7 +363,7 @@ public class PrecedenceQueryParser imple
         q.setSlop(phraseSlop);
         for (int i = 0; i < list.size(); i++) {
           source.restoreState(list.get(i));
-          q.add(new Term(field, termAtt.term()));
+          q.add(new Term(field, termAtt.toString()));
         }
         return q;
       }

Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj Tue Jun  1 10:35:13 2010
@@ -331,7 +331,7 @@ public class PrecedenceQueryParser {
     List<AttributeSource.State> list = new ArrayList<AttributeSource.State>();
     int positionCount = 0;
     boolean severalTokensAtSamePosition = false;
-    TermAttribute termAtt = source.addAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
     PositionIncrementAttribute posincrAtt = source.addAttribute(PositionIncrementAttribute.class);
 
     try {
@@ -352,7 +352,7 @@ public class PrecedenceQueryParser {
       return null;
     else if (list.size() == 1) {
       source.restoreState(list.get(0));
-      return new TermQuery(new Term(field, termAtt.term()));
+      return new TermQuery(new Term(field, termAtt.toString()));
     } else {
       if (severalTokensAtSamePosition) {
         if (positionCount == 1) {
@@ -361,7 +361,7 @@ public class PrecedenceQueryParser {
           for (int i = 0; i < list.size(); i++) {
             source.restoreState(list.get(i));
             TermQuery currentQuery = new TermQuery(
-                new Term(field, termAtt.term()));
+                new Term(field, termAtt.toString()));
             q.add(currentQuery, BooleanClause.Occur.SHOULD);
           }
           return q;
@@ -376,7 +376,7 @@ public class PrecedenceQueryParser {
               mpq.add(multiTerms.toArray(new Term[0]));
               multiTerms.clear();
             }
-            multiTerms.add(new Term(field, termAtt.term()));
+            multiTerms.add(new Term(field, termAtt.toString()));
           }
           mpq.add(multiTerms.toArray(new Term[0]));
           return mpq;
@@ -387,7 +387,7 @@ public class PrecedenceQueryParser {
         q.setSlop(phraseSlop);
         for (int i = 0; i < list.size(); i++) {
           source.restoreState(list.get(i));
-          q.add(new Term(field, termAtt.term()));
+          q.add(new Term(field, termAtt.toString()));
         }
         return q;
       }

Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java Tue Jun  1 10:35:13 2010
@@ -26,8 +26,8 @@ import java.util.List;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.CachingTokenFilter;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.queryParser.core.QueryNodeException;
 import org.apache.lucene.queryParser.core.config.QueryConfigHandler;
 import org.apache.lucene.queryParser.core.nodes.FieldQueryNode;
@@ -162,11 +162,11 @@ public class AnalyzerQueryNodeProcessor 
         // ignore
       }
 
-      if (!buffer.hasAttribute(TermAttribute.class)) {
+      if (!buffer.hasAttribute(CharTermAttribute.class)) {
         return new NoTokenFoundQueryNode();
       }
 
-      TermAttribute termAtt = buffer.getAttribute(TermAttribute.class);
+      CharTermAttribute termAtt = buffer.getAttribute(CharTermAttribute.class);
 
       if (numTokens == 0) {
         return new NoTokenFoundQueryNode();
@@ -177,7 +177,7 @@ public class AnalyzerQueryNodeProcessor 
           boolean hasNext;
           hasNext = buffer.incrementToken();
           assert hasNext == true;
-          term = termAtt.term();
+          term = termAtt.toString();
 
         } catch (IOException e) {
           // safe to ignore, because we know the number of tokens
@@ -197,7 +197,7 @@ public class AnalyzerQueryNodeProcessor 
             try {
               boolean hasNext = buffer.incrementToken();
               assert hasNext == true;
-              term = termAtt.term();
+              term = termAtt.toString();
 
             } catch (IOException e) {
               // safe to ignore, because we know the number of tokens
@@ -224,7 +224,7 @@ public class AnalyzerQueryNodeProcessor 
             try {
               boolean hasNext = buffer.incrementToken();
               assert hasNext == true;
-              term = termAtt.term();
+              term = termAtt.toString();
               if (posIncrAtt != null) {
                 positionIncrement = posIncrAtt.getPositionIncrement();
               }
@@ -290,7 +290,7 @@ public class AnalyzerQueryNodeProcessor 
           try {
             boolean hasNext = buffer.incrementToken();
             assert hasNext == true;
-            term = termAtt.term();
+            term = termAtt.toString();
 
             if (posIncrAtt != null) {
               positionIncrement = posIncrAtt.getPositionIncrement();

Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java Tue Jun  1 10:35:13 2010
@@ -23,8 +23,8 @@ import org.apache.lucene.analysis.MockTo
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.document.DateTools;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.FuzzyQuery;
@@ -68,7 +68,7 @@ public class TestPrecedenceQueryParser e
     boolean inPhrase = false;
     int savedStart = 0, savedEnd = 0;
 
-    TermAttribute termAtt = addAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
     OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
     
     @Override
@@ -76,19 +76,19 @@ public class TestPrecedenceQueryParser e
       clearAttributes();
       if (inPhrase) {
         inPhrase = false;
-        termAtt.setTermBuffer("phrase2");
+        termAtt.setEmpty().append("phrase2");
         offsetAtt.setOffset(savedStart, savedEnd);
         return true;
       } else
         while(input.incrementToken())
-          if (termAtt.term().equals("phrase")) {
+          if (termAtt.toString().equals("phrase")) {
             inPhrase = true;
             savedStart = offsetAtt.startOffset();
             savedEnd = offsetAtt.endOffset();
-            termAtt.setTermBuffer("phrase1");
+            termAtt.setEmpty().append("phrase1");
             offsetAtt.setOffset(savedStart, savedEnd);
             return true;
-          } else if (!termAtt.term().equals("stop"))
+          } else if (!termAtt.toString().equals("stop"))
             return true;
       return false;
     }

Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java Tue Jun  1 10:35:13 2010
@@ -23,9 +23,9 @@ import org.apache.lucene.analysis.Analyz
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.queryParser.core.QueryNodeException;
 import org.apache.lucene.queryParser.standard.config.DefaultOperatorAttribute.Operator;
@@ -163,24 +163,19 @@ public class TestMultiAnalyzerQPHelper e
     private int prevStartOffset;
     private int prevEndOffset;
 
-    TermAttribute termAtt;
-    PositionIncrementAttribute posIncrAtt;
-    OffsetAttribute offsetAtt;
-    TypeAttribute typeAtt;
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+    private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+    private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
 
     public TestFilter(TokenStream in) {
       super(in);
-      termAtt = addAttribute(TermAttribute.class);
-      posIncrAtt = addAttribute(PositionIncrementAttribute.class);
-      offsetAtt = addAttribute(OffsetAttribute.class);
-      typeAtt = addAttribute(TypeAttribute.class);
-
     }
 
     @Override
     public final boolean incrementToken() throws java.io.IOException {
       if (multiToken > 0) {
-        termAtt.setTermBuffer("multi" + (multiToken + 1));
+        termAtt.setEmpty().append("multi" + (multiToken + 1));
         offsetAtt.setOffset(prevStartOffset, prevEndOffset);
         typeAtt.setType(prevType);
         posIncrAtt.setPositionIncrement(0);
@@ -194,7 +189,7 @@ public class TestMultiAnalyzerQPHelper e
         prevType = typeAtt.type();
         prevStartOffset = offsetAtt.startOffset();
         prevEndOffset = offsetAtt.endOffset();
-        String text = termAtt.term();
+        String text = termAtt.toString();
         if (text.equals("triplemulti")) {
           multiToken = 2;
           return true;
@@ -228,21 +223,19 @@ public class TestMultiAnalyzerQPHelper e
 
   private class TestPosIncrementFilter extends TokenFilter {
 
-    TermAttribute termAtt;
-    PositionIncrementAttribute posIncrAtt;
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
 
     public TestPosIncrementFilter(TokenStream in) {
       super(in);
-      termAtt = addAttribute(TermAttribute.class);
-      posIncrAtt = addAttribute(PositionIncrementAttribute.class);
     }
 
     @Override
     public final boolean incrementToken() throws java.io.IOException {
       while (input.incrementToken()) {
-        if (termAtt.term().equals("the")) {
+        if (termAtt.toString().equals("the")) {
           // stopword, do nothing
-        } else if (termAtt.term().equals("quick")) {
+        } else if (termAtt.toString().equals("quick")) {
           posIncrAtt.setPositionIncrement(2);
           return true;
         } else {

Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java Tue Jun  1 10:35:13 2010
@@ -23,9 +23,9 @@ import org.apache.lucene.analysis.Analyz
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.queryParser.ParseException;
 import org.apache.lucene.util.LuceneTestCase;
@@ -157,24 +157,19 @@ public class TestMultiAnalyzerWrapper ex
     private int prevStartOffset;
     private int prevEndOffset;
 
-    TermAttribute termAtt;
-    PositionIncrementAttribute posIncrAtt;
-    OffsetAttribute offsetAtt;
-    TypeAttribute typeAtt;
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+    private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+    private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
 
     public TestFilter(TokenStream in) {
       super(in);
-      termAtt = addAttribute(TermAttribute.class);
-      posIncrAtt = addAttribute(PositionIncrementAttribute.class);
-      offsetAtt = addAttribute(OffsetAttribute.class);
-      typeAtt = addAttribute(TypeAttribute.class);
-
     }
 
     @Override
     public final boolean incrementToken() throws java.io.IOException {
       if (multiToken > 0) {
-        termAtt.setTermBuffer("multi" + (multiToken + 1));
+        termAtt.setEmpty().append("multi" + (multiToken + 1));
         offsetAtt.setOffset(prevStartOffset, prevEndOffset);
         typeAtt.setType(prevType);
         posIncrAtt.setPositionIncrement(0);
@@ -188,7 +183,7 @@ public class TestMultiAnalyzerWrapper ex
         prevType = typeAtt.type();
         prevStartOffset = offsetAtt.startOffset();
         prevEndOffset = offsetAtt.endOffset();
-        String text = termAtt.term();
+        String text = termAtt.toString();
         if (text.equals("triplemulti")) {
           multiToken = 2;
           return true;
@@ -222,21 +217,19 @@ public class TestMultiAnalyzerWrapper ex
 
   private class TestPosIncrementFilter extends TokenFilter {
 
-    TermAttribute termAtt;
-    PositionIncrementAttribute posIncrAtt;
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
 
     public TestPosIncrementFilter(TokenStream in) {
       super(in);
-      termAtt = addAttribute(TermAttribute.class);
-      posIncrAtt = addAttribute(PositionIncrementAttribute.class);
     }
 
     @Override
     public final boolean incrementToken() throws java.io.IOException {
       while (input.incrementToken()) {
-        if (termAtt.term().equals("the")) {
+        if (termAtt.toString().equals("the")) {
           // stopword, do nothing
-        } else if (termAtt.term().equals("quick")) {
+        } else if (termAtt.toString().equals("quick")) {
           posIncrAtt.setPositionIncrement(2);
           return true;
         } else {

Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java Tue Jun  1 10:35:13 2010
@@ -37,8 +37,8 @@ import org.apache.lucene.analysis.MockTo
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.document.DateField;
 import org.apache.lucene.document.DateTools;
@@ -96,8 +96,8 @@ public class TestQPHelper extends Locali
   public static Analyzer qpAnalyzer = new QPTestAnalyzer();
 
   public static final class QPTestFilter extends TokenFilter {
-    TermAttribute termAtt;
-    OffsetAttribute offsetAtt;
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
 
     /**
      * Filter which discards the token 'stop' and which expands the token
@@ -105,8 +105,6 @@ public class TestQPHelper extends Locali
      */
     public QPTestFilter(TokenStream in) {
       super(in);
-      termAtt = addAttribute(TermAttribute.class);
-      offsetAtt = addAttribute(OffsetAttribute.class);
     }
 
     boolean inPhrase = false;
@@ -117,19 +115,19 @@ public class TestQPHelper extends Locali
       if (inPhrase) {
         inPhrase = false;
         clearAttributes();
-        termAtt.setTermBuffer("phrase2");
+        termAtt.setEmpty().append("phrase2");
         offsetAtt.setOffset(savedStart, savedEnd);
         return true;
       } else
         while (input.incrementToken()) {
-          if (termAtt.term().equals("phrase")) {
+          if (termAtt.toString().equals("phrase")) {
             inPhrase = true;
             savedStart = offsetAtt.startOffset();
             savedEnd = offsetAtt.endOffset();
-            termAtt.setTermBuffer("phrase1");
+            termAtt.setEmpty().append("phrase1");
             offsetAtt.setOffset(savedStart, savedEnd);
             return true;
-          } else if (!termAtt.term().equals("stop"))
+          } else if (!termAtt.toString().equals("stop"))
             return true;
         }
       return false;
@@ -1158,7 +1156,7 @@ public class TestQPHelper extends Locali
   private class CannedTokenStream extends TokenStream {
     private int upto = 0;
     final PositionIncrementAttribute posIncr = addAttribute(PositionIncrementAttribute.class);
-    final TermAttribute term = addAttribute(TermAttribute.class);
+    final CharTermAttribute term = addAttribute(CharTermAttribute.class);
     @Override
     public boolean incrementToken() {
       clearAttributes();
@@ -1167,16 +1165,16 @@ public class TestQPHelper extends Locali
       }
       if (upto == 0) {
         posIncr.setPositionIncrement(1);
-        term.setTermBuffer("a");
+        term.setEmpty().append("a");
       } else if (upto == 1) {
         posIncr.setPositionIncrement(1);
-        term.setTermBuffer("b");
+        term.setEmpty().append("b");
       } else if (upto == 2) {
         posIncr.setPositionIncrement(0);
-        term.setTermBuffer("c");
+        term.setEmpty().append("c");
       } else {
         posIncr.setPositionIncrement(0);
-        term.setTermBuffer("d");
+        term.setEmpty().append("d");
       }
       upto++;
       return true;



Mime
View raw message