Return-Path: Delivered-To: apmail-lucene-commits-archive@www.apache.org Received: (qmail 75592 invoked from network); 1 Jun 2010 10:35:47 -0000 Received: from unknown (HELO mail.apache.org) (140.211.11.3) by 140.211.11.9 with SMTP; 1 Jun 2010 10:35:47 -0000 Received: (qmail 14555 invoked by uid 500); 1 Jun 2010 10:35:46 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 14548 invoked by uid 99); 1 Jun 2010 10:35:46 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 01 Jun 2010 10:35:46 +0000 X-ASF-Spam-Status: No, hits=-1736.4 required=10.0 tests=ALL_TRUSTED,AWL X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 01 Jun 2010 10:35:41 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id A7294238890A; Tue, 1 Jun 2010 10:35:18 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r950008 [1/4] - in /lucene/dev/trunk: lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/ lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/ lucene/contrib/highlighter/src/test/org/apache/lucene/search... Date: Tue, 01 Jun 2010 10:35:16 -0000 To: commits@lucene.apache.org From: rmuir@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20100601103518.A7294238890A@eris.apache.org> Author: rmuir Date: Tue Jun 1 10:35:13 2010 New Revision: 950008 URL: http://svn.apache.org/viewvc?rev=950008&view=rev Log: LUCENE-2372: switch over remaining uses of TermAttribute Modified: lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java lucene/dev/trunk/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynonymTokenFilter.java lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicStemFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianStemFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cn/ChineseFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cz/CzechStemFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/de/GermanStemFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianNormalizationFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiNormalizationFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiStemFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/in/IndicNormalizationFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PatternAnalyzer.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLowerCaseFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishLowerCaseFilter.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestElision.java lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAndSuffixAwareTokenFilter.java lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAwareTokenFilter.java lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestRemoveDuplicatesTokenFilter.java lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestSingleTokenTokenFilter.java lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilterTest.java lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/position/PositionFilterTest.java lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymFilter.java lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java lucene/dev/trunk/modules/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilter.java lucene/dev/trunk/modules/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java lucene/dev/trunk/modules/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/BufferedTokenStream.java lucene/dev/trunk/solr/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java lucene/dev/trunk/solr/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java lucene/dev/trunk/solr/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/SpellCheckComponent.java lucene/dev/trunk/solr/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java lucene/dev/trunk/solr/src/java/org/apache/solr/spelling/SpellingQueryConverter.java lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestBufferedTokenStream.java lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestRemoveDuplicatesTokenFilterFactory.java lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestSynonymMap.java lucene/dev/trunk/solr/src/test/org/apache/solr/spelling/SimpleQueryConverter.java lucene/dev/trunk/solr/src/test/org/apache/solr/spelling/SpellingQueryConverterTest.java lucene/dev/trunk/solr/src/webapp/web/admin/analysis.jsp Modified: lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (original) +++ lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java Tue Jun 1 10:35:13 2010 @@ -26,9 +26,10 @@ import java.util.List; import java.util.Locale; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.benchmark.BenchmarkTestCase; import org.apache.lucene.benchmark.byTask.feeds.DocMaker; import org.apache.lucene.benchmark.byTask.feeds.ReutersQueryMaker; @@ -918,11 +919,11 @@ public class TestPerfTasksLogic extends TokenStream ts2 = a2.tokenStream("bogus", new StringReader(text)); ts1.reset(); ts2.reset(); - TermAttribute termAtt1 = ts1.addAttribute(TermAttribute.class); - TermAttribute termAtt2 = ts2.addAttribute(TermAttribute.class); + CharTermAttribute termAtt1 = ts1.addAttribute(CharTermAttribute.class); + CharTermAttribute termAtt2 = ts2.addAttribute(CharTermAttribute.class); assertTrue(ts1.incrementToken()); assertTrue(ts2.incrementToken()); - assertEquals(termAtt1.term(), termAtt2.term()); + assertEquals(termAtt1.toString(), termAtt2.toString()); assertFalse(ts1.incrementToken()); assertFalse(ts2.incrementToken()); ts1.close(); @@ -994,21 +995,7 @@ public class TestPerfTasksLogic extends private void assertEqualShingle (Analyzer analyzer, String text, String[] expected) throws Exception { - TokenStream stream = analyzer.tokenStream("bogus", new StringReader(text)); - stream.reset(); - TermAttribute termAtt = stream.addAttribute(TermAttribute.class); - int termNum = 0; - while (stream.incrementToken()) { - assertTrue("Extra output term(s), starting with '" - + new String(termAtt.termBuffer(), 0, termAtt.termLength()) + "'", - termNum < expected.length); - assertEquals("Mismatch in output term # " + termNum + " - ", - expected[termNum], - new String(termAtt.termBuffer(), 0, termAtt.termLength())); - ++termNum; - } - assertEquals("Too few output terms", expected.length, termNum); - stream.close(); + BaseTokenStreamTestCase.assertAnalyzesTo(analyzer, text, expected); } private String[] getShingleConfig(String params) { Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java (original) +++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java Tue Jun 1 10:35:13 2010 @@ -23,9 +23,9 @@ import java.util.Iterator; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; import org.apache.lucene.util.PriorityQueue; /** @@ -191,7 +191,7 @@ public class Highlighter ArrayList docFrags = new ArrayList(); StringBuilder newText=new StringBuilder(); - TermAttribute termAtt = tokenStream.addAttribute(TermAttribute.class); + CharTermAttribute termAtt = tokenStream.addAttribute(CharTermAttribute.class); OffsetAttribute offsetAtt = tokenStream.addAttribute(OffsetAttribute.class); tokenStream.addAttribute(PositionIncrementAttribute.class); tokenStream.reset(); @@ -225,7 +225,7 @@ public class Highlighter (offsetAtt.startOffset()>text.length()) ) { - throw new InvalidTokenOffsetsException("Token "+ termAtt.term() + throw new InvalidTokenOffsetsException("Token "+ termAtt.toString() +" exceeds length of provided text sized "+text.length()); } if((tokenGroup.numTokens>0)&&(tokenGroup.isDistinct())) Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java (original) +++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java Tue Jun 1 10:35:13 2010 @@ -25,8 +25,8 @@ import java.util.Set; import org.apache.lucene.analysis.CachingTokenFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.memory.MemoryIndex; import org.apache.lucene.search.Query; @@ -46,7 +46,7 @@ public class QueryScorer implements Scor private float maxTermWeight; private int position = -1; private String defaultField; - private TermAttribute termAtt; + private CharTermAttribute termAtt; private PositionIncrementAttribute posIncAtt; private boolean expandMultiTermQuery = true; private Query query; @@ -145,7 +145,7 @@ public class QueryScorer implements Scor */ public float getTokenScore() { position += posIncAtt.getPositionIncrement(); - String termText = termAtt.term(); + String termText = termAtt.toString(); WeightedSpanTerm weightedSpanTerm; @@ -175,7 +175,7 @@ public class QueryScorer implements Scor */ public TokenStream init(TokenStream tokenStream) throws IOException { position = -1; - termAtt = tokenStream.addAttribute(TermAttribute.class); + termAtt = tokenStream.addAttribute(CharTermAttribute.class); posIncAtt = tokenStream.addAttribute(PositionIncrementAttribute.class); if(!skipInitExtractor) { if(fieldWeightedSpanTerms != null) { Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java (original) +++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java Tue Jun 1 10:35:13 2010 @@ -21,7 +21,7 @@ import java.util.HashMap; import java.util.HashSet; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.Query; @@ -41,7 +41,7 @@ public class QueryTermScorer implements float maxTermWeight = 0; private HashMap termsToFind; - private TermAttribute termAtt; + private CharTermAttribute termAtt; /** * @@ -95,7 +95,7 @@ public class QueryTermScorer implements * @see org.apache.lucene.search.highlight.Scorer#init(org.apache.lucene.analysis.TokenStream) */ public TokenStream init(TokenStream tokenStream) { - termAtt = tokenStream.addAttribute(TermAttribute.class); + termAtt = tokenStream.addAttribute(CharTermAttribute.class); return null; } @@ -118,7 +118,7 @@ public class QueryTermScorer implements * @see org.apache.lucene.search.highlight.Scorer#getTokenScore() */ public float getTokenScore() { - String termText = termAtt.term(); + String termText = termAtt.toString(); WeightedTerm queryTerm = termsToFind.get(termText); if (queryTerm == null) { Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java (original) +++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java Tue Jun 1 10:35:13 2010 @@ -20,9 +20,9 @@ package org.apache.lucene.search.highlig import java.util.List; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; import org.apache.lucene.search.spans.Spans; @@ -38,7 +38,7 @@ public class SimpleSpanFragmenter implem private QueryScorer queryScorer; private int waitForPos = -1; private int textSize; - private TermAttribute termAtt; + private CharTermAttribute termAtt; private PositionIncrementAttribute posIncAtt; private OffsetAttribute offsetAtt; @@ -70,7 +70,7 @@ public class SimpleSpanFragmenter implem return false; } - WeightedSpanTerm wSpanTerm = queryScorer.getWeightedSpanTerm(termAtt.term()); + WeightedSpanTerm wSpanTerm = queryScorer.getWeightedSpanTerm(termAtt.toString()); if (wSpanTerm != null) { List positionSpans = wSpanTerm.getPositionSpans(); @@ -101,7 +101,7 @@ public class SimpleSpanFragmenter implem position = -1; currentNumFrags = 1; textSize = originalText.length(); - termAtt = tokenStream.addAttribute(TermAttribute.class); + termAtt = tokenStream.addAttribute(CharTermAttribute.class); posIncAtt = tokenStream.addAttribute(PositionIncrementAttribute.class); offsetAtt = tokenStream.addAttribute(OffsetAttribute.class); } Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java (original) +++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java Tue Jun 1 10:35:13 2010 @@ -19,8 +19,8 @@ package org.apache.lucene.search.highlig import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; /** * One, or several overlapping tokens, along with the score(s) and the scope of @@ -38,11 +38,11 @@ public class TokenGroup { int matchStartOffset, matchEndOffset; private OffsetAttribute offsetAtt; - private TermAttribute termAtt; + private CharTermAttribute termAtt; public TokenGroup(TokenStream tokenStream) { offsetAtt = tokenStream.addAttribute(OffsetAttribute.class); - termAtt = tokenStream.addAttribute(TermAttribute.class); + termAtt = tokenStream.addAttribute(CharTermAttribute.class); } void addToken(float score) { @@ -68,7 +68,7 @@ public class TokenGroup { } } Token token = new Token(termStartOffset, termEndOffset); - token.setTermBuffer(termAtt.term()); + token.setEmpty().append(termAtt); tokens[numTokens] = token; scores[numTokens] = score; numTokens++; Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java (original) +++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java Tue Jun 1 10:35:13 2010 @@ -29,8 +29,8 @@ import java.util.Comparator; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.TermFreqVector; @@ -153,13 +153,13 @@ public class TokenSources { int currentToken = 0; - TermAttribute termAtt; + CharTermAttribute termAtt; OffsetAttribute offsetAtt; StoredTokenStream(Token tokens[]) { this.tokens = tokens; - termAtt = addAttribute(TermAttribute.class); + termAtt = addAttribute(CharTermAttribute.class); offsetAtt = addAttribute(OffsetAttribute.class); } @@ -170,7 +170,7 @@ public class TokenSources { } Token token = tokens[currentToken++]; clearAttributes(); - termAtt.setTermBuffer(token.term()); + termAtt.setEmpty().append(token); offsetAtt.setOffset(token.startOffset(), token.endOffset()); return true; } @@ -204,9 +204,8 @@ public class TokenSources { unsortedTokens = new ArrayList(); } for (int tp = 0; tp < offsets.length; tp++) { - Token token = new Token(offsets[tp].getStartOffset(), offsets[tp] + Token token = new Token(terms[t], offsets[tp].getStartOffset(), offsets[tp] .getEndOffset()); - token.setTermBuffer(terms[t]); unsortedTokens.add(token); } } else { Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java (original) +++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java Tue Jun 1 10:35:13 2010 @@ -25,9 +25,9 @@ import java.util.List; import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; import org.apache.lucene.index.TermPositionVector; import org.apache.lucene.index.TermVectorOffsetInfo; @@ -37,7 +37,7 @@ public final class TokenStreamFromTermPo private Iterator tokensAtCurrentPosition; - private TermAttribute termAttribute; + private CharTermAttribute termAttribute; private PositionIncrementAttribute positionIncrementAttribute; @@ -51,7 +51,7 @@ public final class TokenStreamFromTermPo */ public TokenStreamFromTermPositionVector( final TermPositionVector termPositionVector) { - termAttribute = addAttribute(TermAttribute.class); + termAttribute = addAttribute(CharTermAttribute.class); positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class); offsetAttribute = addAttribute(OffsetAttribute.class); final String[] terms = termPositionVector.getTerms(); @@ -65,7 +65,7 @@ public final class TokenStreamFromTermPo offsets[j].getStartOffset(), offsets[j].getEndOffset()); } else { token = new Token(); - token.setTermBuffer(terms[i]); + token.setEmpty().append(terms[i]); } // Yes - this is the position, not the increment! This is for // sorting. This value @@ -100,7 +100,7 @@ public final class TokenStreamFromTermPo if (this.tokensAtCurrentPosition.hasNext()) { final Token next = this.tokensAtCurrentPosition.next(); clearAttributes(); - termAttribute.setTermBuffer(next.term()); + termAttribute.setEmpty().append(next); positionIncrementAttribute.setPositionIncrement(next .getPositionIncrement()); offsetAttribute.setOffset(next.startOffset(), next.endOffset()); Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java (original) +++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java Tue Jun 1 10:35:13 2010 @@ -25,7 +25,7 @@ import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Index; @@ -296,16 +296,11 @@ public class HighlighterPhraseTest exten private int i = -1; - private TermAttribute termAttribute; - - private OffsetAttribute offsetAttribute; - - private PositionIncrementAttribute positionIncrementAttribute; + private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class); + private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class); + private final PositionIncrementAttribute positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class); public TokenStreamSparse() { - termAttribute = addAttribute(TermAttribute.class); - offsetAttribute = addAttribute(OffsetAttribute.class); - positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class); reset(); } @@ -316,8 +311,7 @@ public class HighlighterPhraseTest exten return false; } clearAttributes(); - termAttribute.setTermBuffer(this.tokens[i].term(), 0, this.tokens[i] - .term().length()); + termAttribute.setEmpty().append(this.tokens[i]); offsetAttribute.setOffset(this.tokens[i].startOffset(), this.tokens[i] .endOffset()); positionIncrementAttribute.setPositionIncrement(this.tokens[i] @@ -342,16 +336,11 @@ public class HighlighterPhraseTest exten private int i = -1; - private TermAttribute termAttribute; - - private OffsetAttribute offsetAttribute; - - private PositionIncrementAttribute positionIncrementAttribute; + private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class); + private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class); + private final PositionIncrementAttribute positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class); public TokenStreamConcurrent() { - termAttribute = addAttribute(TermAttribute.class); - offsetAttribute = addAttribute(OffsetAttribute.class); - positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class); reset(); } @@ -362,8 +351,7 @@ public class HighlighterPhraseTest exten return false; } clearAttributes(); - termAttribute.setTermBuffer(this.tokens[i].term(), 0, this.tokens[i] - .term().length()); + termAttribute.setEmpty().append(this.tokens[i]); offsetAttribute.setOffset(this.tokens[i].startOffset(), this.tokens[i] .endOffset()); positionIncrementAttribute.setPositionIncrement(this.tokens[i] Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original) +++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Tue Jun 1 10:35:13 2010 @@ -41,7 +41,7 @@ import org.apache.lucene.analysis.TokenS import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericField; @@ -1424,13 +1424,10 @@ public class HighlighterTest extends Bas return new TokenStream() { Iterator iter; List lst; - private TermAttribute termAtt; - private PositionIncrementAttribute posIncrAtt; - private OffsetAttribute offsetAtt; + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class); + private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); { - termAtt = addAttribute(TermAttribute.class); - posIncrAtt = addAttribute(PositionIncrementAttribute.class); - offsetAtt = addAttribute(OffsetAttribute.class); lst = new ArrayList(); Token t; t = createToken("hi", 0, 2); @@ -1456,7 +1453,7 @@ public class HighlighterTest extends Bas if(iter.hasNext()) { Token token = iter.next(); clearAttributes(); - termAtt.setTermBuffer(token.term()); + termAtt.setEmpty().append(token); posIncrAtt.setPositionIncrement(token.getPositionIncrement()); offsetAtt.setOffset(token.startOffset(), token.endOffset()); return true; @@ -1473,13 +1470,10 @@ public class HighlighterTest extends Bas return new TokenStream() { Iterator iter; List lst; - private TermAttribute termAtt; - private PositionIncrementAttribute posIncrAtt; - private OffsetAttribute offsetAtt; + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class); + private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); { - termAtt = addAttribute(TermAttribute.class); - posIncrAtt = addAttribute(PositionIncrementAttribute.class); - offsetAtt = addAttribute(OffsetAttribute.class); lst = new ArrayList(); Token t; t = createToken("hispeed", 0, 8); @@ -1505,7 +1499,7 @@ public class HighlighterTest extends Bas if(iter.hasNext()) { Token token = iter.next(); clearAttributes(); - termAtt.setTermBuffer(token.term()); + termAtt.setEmpty().append(token); posIncrAtt.setPositionIncrement(token.getPositionIncrement()); offsetAtt.setOffset(token.startOffset(), token.endOffset()); return true; @@ -1762,9 +1756,7 @@ public class HighlighterTest extends Bas private static Token createToken(String term, int start, int offset) { - Token token = new Token(start, offset); - token.setTermBuffer(term); - return token; + return new Token(term, start, offset); } } @@ -1795,7 +1787,7 @@ final class SynonymAnalyzer extends Anal @Override public TokenStream tokenStream(String arg0, Reader arg1) { Tokenizer stream = new MockTokenizer(arg1, MockTokenizer.SIMPLE, true); - stream.addAttribute(TermAttribute.class); + stream.addAttribute(CharTermAttribute.class); stream.addAttribute(PositionIncrementAttribute.class); stream.addAttribute(OffsetAttribute.class); return new SynonymTokenizer(stream, synonyms); @@ -1811,21 +1803,21 @@ final class SynonymTokenizer extends Tok private Token currentRealToken = null; private Map synonyms; StringTokenizer st = null; - private TermAttribute realTermAtt; + private CharTermAttribute realTermAtt; private PositionIncrementAttribute realPosIncrAtt; private OffsetAttribute realOffsetAtt; - private TermAttribute termAtt; + private CharTermAttribute termAtt; private PositionIncrementAttribute posIncrAtt; private OffsetAttribute offsetAtt; public SynonymTokenizer(TokenStream realStream, Map synonyms) { this.realStream = realStream; this.synonyms = synonyms; - realTermAtt = realStream.addAttribute(TermAttribute.class); + realTermAtt = realStream.addAttribute(CharTermAttribute.class); realPosIncrAtt = realStream.addAttribute(PositionIncrementAttribute.class); realOffsetAtt = realStream.addAttribute(OffsetAttribute.class); - termAtt = addAttribute(TermAttribute.class); + termAtt = addAttribute(CharTermAttribute.class); posIncrAtt = addAttribute(PositionIncrementAttribute.class); offsetAtt = addAttribute(OffsetAttribute.class); } @@ -1840,25 +1832,25 @@ final class SynonymTokenizer extends Tok } //Token nextRealToken = new Token(, offsetAtt.startOffset(), offsetAtt.endOffset()); clearAttributes(); - termAtt.setTermBuffer(realTermAtt.term()); + termAtt.copyBuffer(realTermAtt.buffer(), 0, realTermAtt.length()); offsetAtt.setOffset(realOffsetAtt.startOffset(), realOffsetAtt.endOffset()); posIncrAtt.setPositionIncrement(realPosIncrAtt.getPositionIncrement()); - String expansions = synonyms.get(realTermAtt.term()); + String expansions = synonyms.get(realTermAtt.toString()); if (expansions == null) { return true; } st = new StringTokenizer(expansions, ","); if (st.hasMoreTokens()) { currentRealToken = new Token(realOffsetAtt.startOffset(), realOffsetAtt.endOffset()); - currentRealToken.setTermBuffer(realTermAtt.term()); + currentRealToken.copyBuffer(realTermAtt.buffer(), 0, realTermAtt.length()); } return true; } else { String tok = st.nextToken(); clearAttributes(); - termAtt.setTermBuffer(tok); + termAtt.setEmpty().append(tok); offsetAtt.setOffset(currentRealToken.startOffset(), currentRealToken.endOffset()); posIncrAtt.setPositionIncrement(0); if (!st.hasMoreTokens()) { Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java (original) +++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java Tue Jun 1 10:35:13 2010 @@ -26,8 +26,8 @@ import org.apache.lucene.analysis.MockAn import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Index; @@ -221,14 +221,14 @@ public abstract class AbstractTestCase e ch = 0; } - TermAttribute termAtt = addAttribute(TermAttribute.class); + CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); @Override public boolean incrementToken() throws IOException { if( !getNextPartialSnippet() ) return false; clearAttributes(); - termAtt.setTermBuffer(snippet, startTerm, lenTerm); + termAtt.setEmpty().append(snippet, startTerm, startTerm + lenTerm); offsetAtt.setOffset(correctOffset(startOffset), correctOffset(startOffset + lenTerm)); return true; } Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java (original) +++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java Tue Jun 1 10:35:13 2010 @@ -25,7 +25,7 @@ import java.util.Set; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.util.AttributeImpl; @@ -301,7 +301,7 @@ public class IndexTimeSynonymTest extend @Override public TokenStream tokenStream(String fieldName, Reader reader) { TokenStream ts = new TokenStream(Token.TOKEN_ATTRIBUTE_FACTORY) { - final AttributeImpl reusableToken = (AttributeImpl) addAttribute(TermAttribute.class); + final AttributeImpl reusableToken = (AttributeImpl) addAttribute(CharTermAttribute.class); int p = 0; @Override Modified: lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (original) +++ lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java Tue Jun 1 10:35:13 2010 @@ -561,7 +561,7 @@ public class InstantiatedIndexWriter imp // untokenized String fieldVal = field.stringValue(); Token token = new Token(0, fieldVal.length(), "untokenized"); - token.setTermBuffer(fieldVal); + token.setEmpty().append(fieldVal); tokens.add(token); fieldSetting.fieldLength++; } @@ -596,10 +596,10 @@ public class InstantiatedIndexWriter imp for (Token token : eField_Tokens.getValue()) { - TermDocumentInformationFactory termDocumentInformationFactory = termDocumentInformationFactoryByTermText.get(token.term()); + TermDocumentInformationFactory termDocumentInformationFactory = termDocumentInformationFactoryByTermText.get(token.toString()); if (termDocumentInformationFactory == null) { termDocumentInformationFactory = new TermDocumentInformationFactory(); - termDocumentInformationFactoryByTermText.put(token.term(), termDocumentInformationFactory); + termDocumentInformationFactoryByTermText.put(token.toString(), termDocumentInformationFactory); } //termDocumentInformationFactory.termFrequency++; Modified: lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (original) +++ lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java Tue Jun 1 10:35:13 2010 @@ -25,7 +25,7 @@ import java.util.List; import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; @@ -278,7 +278,7 @@ public class TestIndicesEquals extends L tokens.add(t); tokens.add(createToken("fin", 7, 9)); TokenStream ts = new TokenStream(Token.TOKEN_ATTRIBUTE_FACTORY) { - final AttributeImpl reusableToken = (AttributeImpl) addAttribute(TermAttribute.class); + final AttributeImpl reusableToken = (AttributeImpl) addAttribute(CharTermAttribute.class); Iterator it = tokens.iterator(); @Override @@ -601,16 +601,12 @@ public class TestIndicesEquals extends L private static Token createToken(String term, int start, int offset) { - Token token = new Token(start, offset); - token.setTermBuffer(term); - return token; + return new Token(term, start, offset); } private static Token createToken(String term, int start, int offset, String type) { - Token token = new Token(start, offset, type); - token.setTermBuffer(term); - return token; + return new Token(term, start, offset, type); } Modified: lucene/dev/trunk/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java (original) +++ lucene/dev/trunk/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java Tue Jun 1 10:35:13 2010 @@ -36,8 +36,8 @@ import jline.ConsoleReader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.Fieldable; import org.apache.lucene.index.IndexReader; @@ -303,14 +303,14 @@ class LuceneMethods { int position = 0; // Tokenize field and add to postingTable TokenStream stream = analyzer.tokenStream(fieldName, reader); - TermAttribute termAtt = stream.addAttribute(TermAttribute.class); + CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class); PositionIncrementAttribute posIncrAtt = stream.addAttribute(PositionIncrementAttribute.class); try { while (stream.incrementToken()) { position += (posIncrAtt.getPositionIncrement() - 1); position++; - String name = termAtt.term(); + String name = termAtt.toString(); Integer Count = tokenMap.get(name); if (Count == null) { // not in there yet tokenMap.put(name, Integer.valueOf(1)); //first one Modified: lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original) +++ lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Tue Jun 1 10:35:13 2010 @@ -30,9 +30,10 @@ import java.util.Map; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; +import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; import org.apache.lucene.index.IndexReader; @@ -51,6 +52,7 @@ import org.apache.lucene.search.Searcher import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Similarity; import org.apache.lucene.store.RAMDirectory; // for javadocs +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Constants; // for javadocs /** @@ -276,8 +278,8 @@ public class MemoryIndex implements Seri return new TokenStream() { private Iterator iter = keywords.iterator(); private int start = 0; - private TermAttribute termAtt = addAttribute(TermAttribute.class); - private OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); @Override public boolean incrementToken() { @@ -289,8 +291,8 @@ public class MemoryIndex implements Seri String term = obj.toString(); clearAttributes(); - termAtt.setTermBuffer(term); - offsetAtt.setOffset(start, start+termAtt.termLength()); + termAtt.setEmpty().append(term); + offsetAtt.setOffset(start, start+termAtt.length()); start += term.length() + 1; // separate words by 1 (blank) character return true; } @@ -340,13 +342,15 @@ public class MemoryIndex implements Seri int numOverlapTokens = 0; int pos = -1; - TermAttribute termAtt = stream.addAttribute(TermAttribute.class); + TermToBytesRefAttribute termAtt = stream.addAttribute(TermToBytesRefAttribute.class); PositionIncrementAttribute posIncrAttribute = stream.addAttribute(PositionIncrementAttribute.class); OffsetAttribute offsetAtt = stream.addAttribute(OffsetAttribute.class); - + BytesRef ref = new BytesRef(10); stream.reset(); while (stream.incrementToken()) { - String term = termAtt.term(); + termAtt.toBytesRef(ref); + // TODO: support non-UTF8 strings (like numerics) here + String term = ref.utf8ToString(); if (term.length() == 0) continue; // nothing to do // if (DEBUG) System.err.println("token='" + term + "'"); numTokens++; Modified: lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (original) +++ lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java Tue Jun 1 10:35:13 2010 @@ -26,7 +26,7 @@ import java.util.Iterator; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.util.BytesRef; @@ -185,14 +185,14 @@ public class FuzzyLikeThisQuery extends { if(f.queryString==null) return; TokenStream ts=analyzer.tokenStream(f.fieldName,new StringReader(f.queryString)); - TermAttribute termAtt = ts.addAttribute(TermAttribute.class); + CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); int corpusNumDocs=reader.numDocs(); Term internSavingTemplateTerm =new Term(f.fieldName); //optimization to avoid constructing new Term() objects HashSet processedTerms=new HashSet(); while (ts.incrementToken()) { - String term = termAtt.term(); + String term = termAtt.toString(); if(!processedTerms.contains(term)) { processedTerms.add(term); Modified: lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java (original) +++ lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java Tue Jun 1 10:35:13 2010 @@ -32,7 +32,7 @@ import java.util.Set; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; @@ -884,10 +884,10 @@ public final class MoreLikeThis { TokenStream ts = analyzer.tokenStream(fieldName, r); int tokenCount=0; // for every token - TermAttribute termAtt = ts.addAttribute(TermAttribute.class); + CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); while (ts.incrementToken()) { - String word = termAtt.term(); + String word = termAtt.toString(); tokenCount++; if(tokenCount>maxNumTokensParsed) { Modified: lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java (original) +++ lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java Tue Jun 1 10:35:13 2010 @@ -22,7 +22,7 @@ import java.util.Set; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; @@ -86,12 +86,12 @@ public final class SimilarityQueries throws IOException { TokenStream ts = a.tokenStream( field, new StringReader( body)); - TermAttribute termAtt = ts.addAttribute(TermAttribute.class); + CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); BooleanQuery tmp = new BooleanQuery(); Set already = new HashSet(); // ignore dups while (ts.incrementToken()) { - String word = termAtt.term(); + String word = termAtt.toString(); // ignore opt stop words if ( stop != null && stop.contains( word)) continue; Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java (original) +++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java Tue Jun 1 10:35:13 2010 @@ -24,7 +24,7 @@ import java.util.List; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.queryParser.ParseException; import org.apache.lucene.search.Query; import org.apache.lucene.util.Version; @@ -107,7 +107,7 @@ public class AnalyzingQueryParser extend // get Analyzer from superclass and tokenize the term TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr)); - TermAttribute termAtt = source.addAttribute(TermAttribute.class); + CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); int countTokens = 0; while (true) { @@ -116,7 +116,7 @@ public class AnalyzingQueryParser extend } catch (IOException e) { break; } - String term = termAtt.term(); + String term = termAtt.toString(); if (!"".equals(term)) { try { tlist.set(countTokens++, term); @@ -190,7 +190,7 @@ public class AnalyzingQueryParser extend // get Analyzer from superclass and tokenize the term TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr)); List tlist = new ArrayList(); - TermAttribute termAtt = source.addAttribute(TermAttribute.class); + CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); while (true) { try { @@ -198,7 +198,7 @@ public class AnalyzingQueryParser extend } catch (IOException e) { break; } - tlist.add(termAtt.term()); + tlist.add(termAtt.toString()); } try { @@ -237,13 +237,13 @@ public class AnalyzingQueryParser extend throws ParseException { // get Analyzer from superclass and tokenize the term TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr)); - TermAttribute termAtt = source.addAttribute(TermAttribute.class); + CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); String nextToken = null; boolean multipleTokens = false; try { if (source.incrementToken()) { - nextToken = termAtt.term(); + nextToken = termAtt.toString(); } multipleTokens = source.incrementToken(); } catch (IOException e) { @@ -273,13 +273,13 @@ public class AnalyzingQueryParser extend throws ParseException { // get Analyzer from superclass and tokenize the terms TokenStream source = getAnalyzer().tokenStream(field, new StringReader(part1)); - TermAttribute termAtt = source.addAttribute(TermAttribute.class); + CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); boolean multipleTokens = false; // part1 try { if (source.incrementToken()) { - part1 = termAtt.term(); + part1 = termAtt.toString(); } multipleTokens = source.incrementToken(); } catch (IOException e) { @@ -297,11 +297,11 @@ public class AnalyzingQueryParser extend // part2 source = getAnalyzer().tokenStream(field, new StringReader(part2)); - termAtt = source.addAttribute(TermAttribute.class); + termAtt = source.addAttribute(CharTermAttribute.class); try { if (source.incrementToken()) { - part2 = termAtt.term(); + part2 = termAtt.toString(); } multipleTokens = source.incrementToken(); } catch (IOException e) { Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java (original) +++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java Tue Jun 1 10:35:13 2010 @@ -307,7 +307,7 @@ public class PrecedenceQueryParser imple List list = new ArrayList(); int positionCount = 0; boolean severalTokensAtSamePosition = false; - TermAttribute termAtt = source.addAttribute(TermAttribute.class); + CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); PositionIncrementAttribute posincrAtt = source.addAttribute(PositionIncrementAttribute.class); try { @@ -328,7 +328,7 @@ public class PrecedenceQueryParser imple return null; else if (list.size() == 1) { source.restoreState(list.get(0)); - return new TermQuery(new Term(field, termAtt.term())); + return new TermQuery(new Term(field, termAtt.toString())); } else { if (severalTokensAtSamePosition) { if (positionCount == 1) { @@ -337,7 +337,7 @@ public class PrecedenceQueryParser imple for (int i = 0; i < list.size(); i++) { source.restoreState(list.get(i)); TermQuery currentQuery = new TermQuery( - new Term(field, termAtt.term())); + new Term(field, termAtt.toString())); q.add(currentQuery, BooleanClause.Occur.SHOULD); } return q; @@ -352,7 +352,7 @@ public class PrecedenceQueryParser imple mpq.add(multiTerms.toArray(new Term[0])); multiTerms.clear(); } - multiTerms.add(new Term(field, termAtt.term())); + multiTerms.add(new Term(field, termAtt.toString())); } mpq.add(multiTerms.toArray(new Term[0])); return mpq; @@ -363,7 +363,7 @@ public class PrecedenceQueryParser imple q.setSlop(phraseSlop); for (int i = 0; i < list.size(); i++) { source.restoreState(list.get(i)); - q.add(new Term(field, termAtt.term())); + q.add(new Term(field, termAtt.toString())); } return q; } Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj (original) +++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj Tue Jun 1 10:35:13 2010 @@ -331,7 +331,7 @@ public class PrecedenceQueryParser { List list = new ArrayList(); int positionCount = 0; boolean severalTokensAtSamePosition = false; - TermAttribute termAtt = source.addAttribute(TermAttribute.class); + CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); PositionIncrementAttribute posincrAtt = source.addAttribute(PositionIncrementAttribute.class); try { @@ -352,7 +352,7 @@ public class PrecedenceQueryParser { return null; else if (list.size() == 1) { source.restoreState(list.get(0)); - return new TermQuery(new Term(field, termAtt.term())); + return new TermQuery(new Term(field, termAtt.toString())); } else { if (severalTokensAtSamePosition) { if (positionCount == 1) { @@ -361,7 +361,7 @@ public class PrecedenceQueryParser { for (int i = 0; i < list.size(); i++) { source.restoreState(list.get(i)); TermQuery currentQuery = new TermQuery( - new Term(field, termAtt.term())); + new Term(field, termAtt.toString())); q.add(currentQuery, BooleanClause.Occur.SHOULD); } return q; @@ -376,7 +376,7 @@ public class PrecedenceQueryParser { mpq.add(multiTerms.toArray(new Term[0])); multiTerms.clear(); } - multiTerms.add(new Term(field, termAtt.term())); + multiTerms.add(new Term(field, termAtt.toString())); } mpq.add(multiTerms.toArray(new Term[0])); return mpq; @@ -387,7 +387,7 @@ public class PrecedenceQueryParser { q.setSlop(phraseSlop); for (int i = 0; i < list.size(); i++) { source.restoreState(list.get(i)); - q.add(new Term(field, termAtt.term())); + q.add(new Term(field, termAtt.toString())); } return q; } Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java (original) +++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java Tue Jun 1 10:35:13 2010 @@ -26,8 +26,8 @@ import java.util.List; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.CachingTokenFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; import org.apache.lucene.queryParser.core.QueryNodeException; import org.apache.lucene.queryParser.core.config.QueryConfigHandler; import org.apache.lucene.queryParser.core.nodes.FieldQueryNode; @@ -162,11 +162,11 @@ public class AnalyzerQueryNodeProcessor // ignore } - if (!buffer.hasAttribute(TermAttribute.class)) { + if (!buffer.hasAttribute(CharTermAttribute.class)) { return new NoTokenFoundQueryNode(); } - TermAttribute termAtt = buffer.getAttribute(TermAttribute.class); + CharTermAttribute termAtt = buffer.getAttribute(CharTermAttribute.class); if (numTokens == 0) { return new NoTokenFoundQueryNode(); @@ -177,7 +177,7 @@ public class AnalyzerQueryNodeProcessor boolean hasNext; hasNext = buffer.incrementToken(); assert hasNext == true; - term = termAtt.term(); + term = termAtt.toString(); } catch (IOException e) { // safe to ignore, because we know the number of tokens @@ -197,7 +197,7 @@ public class AnalyzerQueryNodeProcessor try { boolean hasNext = buffer.incrementToken(); assert hasNext == true; - term = termAtt.term(); + term = termAtt.toString(); } catch (IOException e) { // safe to ignore, because we know the number of tokens @@ -224,7 +224,7 @@ public class AnalyzerQueryNodeProcessor try { boolean hasNext = buffer.incrementToken(); assert hasNext == true; - term = termAtt.term(); + term = termAtt.toString(); if (posIncrAtt != null) { positionIncrement = posIncrAtt.getPositionIncrement(); } @@ -290,7 +290,7 @@ public class AnalyzerQueryNodeProcessor try { boolean hasNext = buffer.incrementToken(); assert hasNext == true; - term = termAtt.term(); + term = termAtt.toString(); if (posIncrAtt != null) { positionIncrement = posIncrAtt.getPositionIncrement(); Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java (original) +++ lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java Tue Jun 1 10:35:13 2010 @@ -23,8 +23,8 @@ import org.apache.lucene.analysis.MockTo import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; import org.apache.lucene.document.DateTools; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.FuzzyQuery; @@ -68,7 +68,7 @@ public class TestPrecedenceQueryParser e boolean inPhrase = false; int savedStart = 0, savedEnd = 0; - TermAttribute termAtt = addAttribute(TermAttribute.class); + CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); @Override @@ -76,19 +76,19 @@ public class TestPrecedenceQueryParser e clearAttributes(); if (inPhrase) { inPhrase = false; - termAtt.setTermBuffer("phrase2"); + termAtt.setEmpty().append("phrase2"); offsetAtt.setOffset(savedStart, savedEnd); return true; } else while(input.incrementToken()) - if (termAtt.term().equals("phrase")) { + if (termAtt.toString().equals("phrase")) { inPhrase = true; savedStart = offsetAtt.startOffset(); savedEnd = offsetAtt.endOffset(); - termAtt.setTermBuffer("phrase1"); + termAtt.setEmpty().append("phrase1"); offsetAtt.setOffset(savedStart, savedEnd); return true; - } else if (!termAtt.term().equals("stop")) + } else if (!termAtt.toString().equals("stop")) return true; return false; } Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java (original) +++ lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java Tue Jun 1 10:35:13 2010 @@ -23,9 +23,9 @@ import org.apache.lucene.analysis.Analyz import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; import org.apache.lucene.analysis.tokenattributes.TypeAttribute; import org.apache.lucene.queryParser.core.QueryNodeException; import org.apache.lucene.queryParser.standard.config.DefaultOperatorAttribute.Operator; @@ -163,24 +163,19 @@ public class TestMultiAnalyzerQPHelper e private int prevStartOffset; private int prevEndOffset; - TermAttribute termAtt; - PositionIncrementAttribute posIncrAtt; - OffsetAttribute offsetAtt; - TypeAttribute typeAtt; + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class); + private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); + private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class); public TestFilter(TokenStream in) { super(in); - termAtt = addAttribute(TermAttribute.class); - posIncrAtt = addAttribute(PositionIncrementAttribute.class); - offsetAtt = addAttribute(OffsetAttribute.class); - typeAtt = addAttribute(TypeAttribute.class); - } @Override public final boolean incrementToken() throws java.io.IOException { if (multiToken > 0) { - termAtt.setTermBuffer("multi" + (multiToken + 1)); + termAtt.setEmpty().append("multi" + (multiToken + 1)); offsetAtt.setOffset(prevStartOffset, prevEndOffset); typeAtt.setType(prevType); posIncrAtt.setPositionIncrement(0); @@ -194,7 +189,7 @@ public class TestMultiAnalyzerQPHelper e prevType = typeAtt.type(); prevStartOffset = offsetAtt.startOffset(); prevEndOffset = offsetAtt.endOffset(); - String text = termAtt.term(); + String text = termAtt.toString(); if (text.equals("triplemulti")) { multiToken = 2; return true; @@ -228,21 +223,19 @@ public class TestMultiAnalyzerQPHelper e private class TestPosIncrementFilter extends TokenFilter { - TermAttribute termAtt; - PositionIncrementAttribute posIncrAtt; + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class); public TestPosIncrementFilter(TokenStream in) { super(in); - termAtt = addAttribute(TermAttribute.class); - posIncrAtt = addAttribute(PositionIncrementAttribute.class); } @Override public final boolean incrementToken() throws java.io.IOException { while (input.incrementToken()) { - if (termAtt.term().equals("the")) { + if (termAtt.toString().equals("the")) { // stopword, do nothing - } else if (termAtt.term().equals("quick")) { + } else if (termAtt.toString().equals("quick")) { posIncrAtt.setPositionIncrement(2); return true; } else { Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java (original) +++ lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java Tue Jun 1 10:35:13 2010 @@ -23,9 +23,9 @@ import org.apache.lucene.analysis.Analyz import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; import org.apache.lucene.analysis.tokenattributes.TypeAttribute; import org.apache.lucene.queryParser.ParseException; import org.apache.lucene.util.LuceneTestCase; @@ -157,24 +157,19 @@ public class TestMultiAnalyzerWrapper ex private int prevStartOffset; private int prevEndOffset; - TermAttribute termAtt; - PositionIncrementAttribute posIncrAtt; - OffsetAttribute offsetAtt; - TypeAttribute typeAtt; + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class); + private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); + private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class); public TestFilter(TokenStream in) { super(in); - termAtt = addAttribute(TermAttribute.class); - posIncrAtt = addAttribute(PositionIncrementAttribute.class); - offsetAtt = addAttribute(OffsetAttribute.class); - typeAtt = addAttribute(TypeAttribute.class); - } @Override public final boolean incrementToken() throws java.io.IOException { if (multiToken > 0) { - termAtt.setTermBuffer("multi" + (multiToken + 1)); + termAtt.setEmpty().append("multi" + (multiToken + 1)); offsetAtt.setOffset(prevStartOffset, prevEndOffset); typeAtt.setType(prevType); posIncrAtt.setPositionIncrement(0); @@ -188,7 +183,7 @@ public class TestMultiAnalyzerWrapper ex prevType = typeAtt.type(); prevStartOffset = offsetAtt.startOffset(); prevEndOffset = offsetAtt.endOffset(); - String text = termAtt.term(); + String text = termAtt.toString(); if (text.equals("triplemulti")) { multiToken = 2; return true; @@ -222,21 +217,19 @@ public class TestMultiAnalyzerWrapper ex private class TestPosIncrementFilter extends TokenFilter { - TermAttribute termAtt; - PositionIncrementAttribute posIncrAtt; + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class); public TestPosIncrementFilter(TokenStream in) { super(in); - termAtt = addAttribute(TermAttribute.class); - posIncrAtt = addAttribute(PositionIncrementAttribute.class); } @Override public final boolean incrementToken() throws java.io.IOException { while (input.incrementToken()) { - if (termAtt.term().equals("the")) { + if (termAtt.toString().equals("the")) { // stopword, do nothing - } else if (termAtt.term().equals("quick")) { + } else if (termAtt.toString().equals("quick")) { posIncrAtt.setPositionIncrement(2); return true; } else { Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java?rev=950008&r1=950007&r2=950008&view=diff ============================================================================== --- lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java (original) +++ lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java Tue Jun 1 10:35:13 2010 @@ -37,8 +37,8 @@ import org.apache.lucene.analysis.MockTo import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.document.DateField; import org.apache.lucene.document.DateTools; @@ -96,8 +96,8 @@ public class TestQPHelper extends Locali public static Analyzer qpAnalyzer = new QPTestAnalyzer(); public static final class QPTestFilter extends TokenFilter { - TermAttribute termAtt; - OffsetAttribute offsetAtt; + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); /** * Filter which discards the token 'stop' and which expands the token @@ -105,8 +105,6 @@ public class TestQPHelper extends Locali */ public QPTestFilter(TokenStream in) { super(in); - termAtt = addAttribute(TermAttribute.class); - offsetAtt = addAttribute(OffsetAttribute.class); } boolean inPhrase = false; @@ -117,19 +115,19 @@ public class TestQPHelper extends Locali if (inPhrase) { inPhrase = false; clearAttributes(); - termAtt.setTermBuffer("phrase2"); + termAtt.setEmpty().append("phrase2"); offsetAtt.setOffset(savedStart, savedEnd); return true; } else while (input.incrementToken()) { - if (termAtt.term().equals("phrase")) { + if (termAtt.toString().equals("phrase")) { inPhrase = true; savedStart = offsetAtt.startOffset(); savedEnd = offsetAtt.endOffset(); - termAtt.setTermBuffer("phrase1"); + termAtt.setEmpty().append("phrase1"); offsetAtt.setOffset(savedStart, savedEnd); return true; - } else if (!termAtt.term().equals("stop")) + } else if (!termAtt.toString().equals("stop")) return true; } return false; @@ -1158,7 +1156,7 @@ public class TestQPHelper extends Locali private class CannedTokenStream extends TokenStream { private int upto = 0; final PositionIncrementAttribute posIncr = addAttribute(PositionIncrementAttribute.class); - final TermAttribute term = addAttribute(TermAttribute.class); + final CharTermAttribute term = addAttribute(CharTermAttribute.class); @Override public boolean incrementToken() { clearAttributes(); @@ -1167,16 +1165,16 @@ public class TestQPHelper extends Locali } if (upto == 0) { posIncr.setPositionIncrement(1); - term.setTermBuffer("a"); + term.setEmpty().append("a"); } else if (upto == 1) { posIncr.setPositionIncrement(1); - term.setTermBuffer("b"); + term.setEmpty().append("b"); } else if (upto == 2) { posIncr.setPositionIncrement(0); - term.setTermBuffer("c"); + term.setEmpty().append("c"); } else { posIncr.setPositionIncrement(0); - term.setTermBuffer("d"); + term.setEmpty().append("d"); } upto++; return true;