lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rm...@apache.org
Subject svn commit: r950008 [2/4] - in /lucene/dev/trunk: lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/ lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/ lucene/contrib/highlighter/src/test/org/apache/lucene/search...
Date Tue, 01 Jun 2010 10:35:16 GMT
Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java Tue Jun  1 10:35:13 2010
@@ -36,7 +36,7 @@ import org.apache.lucene.analysis.MockTo
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.DateField;
 import org.apache.lucene.document.DateTools;
 import org.apache.lucene.document.Document;
@@ -93,8 +93,8 @@ public class TestQueryParserWrapper exte
   public static Analyzer qpAnalyzer = new QPTestAnalyzer();
 
   public static final class QPTestFilter extends TokenFilter {
-    TermAttribute termAtt;
-    OffsetAttribute offsetAtt;
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
 
     /**
      * Filter which discards the token 'stop' and which expands the token
@@ -102,8 +102,6 @@ public class TestQueryParserWrapper exte
      */
     public QPTestFilter(TokenStream in) {
       super(in);
-      termAtt = addAttribute(TermAttribute.class);
-      offsetAtt = addAttribute(OffsetAttribute.class);
     }
 
     boolean inPhrase = false;
@@ -114,19 +112,19 @@ public class TestQueryParserWrapper exte
       if (inPhrase) {
         inPhrase = false;
         clearAttributes();
-        termAtt.setTermBuffer("phrase2");
+        termAtt.setEmpty().append("phrase2");
         offsetAtt.setOffset(savedStart, savedEnd);
         return true;
       } else
         while (input.incrementToken()) {
-          if (termAtt.term().equals("phrase")) {
+          if (termAtt.toString().equals("phrase")) {
             inPhrase = true;
             savedStart = offsetAtt.startOffset();
             savedEnd = offsetAtt.endOffset();
-            termAtt.setTermBuffer("phrase1");
+            termAtt.setEmpty().append("phrase1");
             offsetAtt.setOffset(savedStart, savedEnd);
             return true;
-          } else if (!termAtt.term().equals("stop"))
+          } else if (!termAtt.toString().equals("stop"))
             return true;
         }
       return false;

Modified: lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java (original)
+++ lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java Tue Jun  1 10:35:13 2010
@@ -29,7 +29,7 @@ import java.util.Set;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
@@ -117,10 +117,10 @@ public final class SynExpand {
 
 		// [1] Parse query into separate words so that when we expand we can avoid dups
 		TokenStream ts = a.tokenStream( field, new StringReader( query));
-		TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
+		CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
 		
 		while (ts.incrementToken()) {
-		  String word = termAtt.term();
+		  String word = termAtt.toString();
 			if ( already.add( word))
 				top.add( word);
 		}

Modified: lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java (original)
+++ lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java Tue Jun  1 10:35:13 2010
@@ -28,7 +28,7 @@ import java.util.Set;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
@@ -125,10 +125,10 @@ public class SynLookup {
 
 		// [1] Parse query into separate words so that when we expand we can avoid dups
 		TokenStream ts = a.tokenStream( field, new StringReader( query));
-    TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
     
 		while (ts.incrementToken()) {
-			String word = termAtt.term();
+			String word = termAtt.toString();
 			if ( already.add( word))
 				top.add( word);
 		}

Modified: lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynonymTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynonymTokenFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynonymTokenFilter.java (original)
+++ lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynonymTokenFilter.java Tue Jun  1 10:35:13 2010
@@ -21,8 +21,8 @@ import java.io.IOException;
 
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.util.AttributeSource;
 
@@ -45,9 +45,9 @@ public class SynonymTokenFilter extends 
   private AttributeSource.State current = null;
   private int todo = 0;
   
-  private TermAttribute termAtt;
-  private TypeAttribute typeAtt;
-  private PositionIncrementAttribute posIncrAtt;
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
+  private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
   
   /**
    * Creates an instance for the given underlying stream and synonym table.
@@ -71,10 +71,6 @@ public class SynonymTokenFilter extends 
     
     this.synonyms = synonyms;
     this.maxSynonyms = maxSynonyms;
-    
-    this.termAtt = addAttribute(TermAttribute.class);
-    this.typeAtt = addAttribute(TypeAttribute.class);
-    this.posIncrAtt = addAttribute(PositionIncrementAttribute.class);
   }
   
   /** Returns the next token in the stream, or null at EOS. */
@@ -89,7 +85,7 @@ public class SynonymTokenFilter extends 
     
     if (!input.incrementToken()) return false; // EOS; iterator exhausted 
     
-    stack = synonyms.getSynonyms(termAtt.term()); // push onto stack
+    stack = synonyms.getSynonyms(termAtt.toString()); // push onto stack
     if (stack.length > maxSynonyms) randomize(stack);
     index = 0;
     current = captureState();
@@ -110,7 +106,7 @@ public class SynonymTokenFilter extends 
    */
   protected boolean createToken(String synonym, AttributeSource.State current) {
     restoreState(current);
-    termAtt.setTermBuffer(synonym);
+    termAtt.setEmpty().append(synonym);
     typeAtt.setType(SYNONYM_TOKEN_TYPE);
     posIncrAtt.setPositionIncrement(0);
     return true;

Modified: lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java (original)
+++ lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java Tue Jun  1 10:35:13 2010
@@ -10,7 +10,7 @@ import java.util.Set;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.search.similar.MoreLikeThisQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.xmlparser.DOMUtils;
@@ -77,11 +77,11 @@ public class LikeThisQueryBuilder implem
 		    for (int i = 0; i < fields.length; i++)
             {
                 TokenStream ts = analyzer.tokenStream(fields[i],new StringReader(stopWords));
-                TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
+                CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
                 try
                 {
 	                while(ts.incrementToken()) {
-	                    stopWordsSet.add(termAtt.term());
+	                    stopWordsSet.add(termAtt.toString());
 	                }
                 }
                 catch(IOException ioe)

Modified: lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java (original)
+++ lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java Tue Jun  1 10:35:13 2010
@@ -6,7 +6,7 @@ import java.util.ArrayList;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.spans.SpanOrQuery;
 import org.apache.lucene.search.spans.SpanQuery;
@@ -56,10 +56,10 @@ public class SpanOrTermsBuilder extends 
 		{
 			ArrayList<SpanQuery> clausesList=new ArrayList<SpanQuery>();
 			TokenStream ts=analyzer.tokenStream(fieldName,new StringReader(value));
-			TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
+			CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
 			
 	    while (ts.incrementToken()) {
-			    SpanTermQuery stq=new SpanTermQuery(new Term(fieldName, termAtt.term()));
+			    SpanTermQuery stq=new SpanTermQuery(new Term(fieldName, termAtt.toString()));
 			    clausesList.add(stq);
 			}
 			SpanOrQuery soq=new SpanOrQuery(clausesList.toArray(new SpanQuery[clausesList.size()]));

Modified: lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java (original)
+++ lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java Tue Jun  1 10:35:13 2010
@@ -5,7 +5,7 @@ import java.io.StringReader;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.search.TermsFilter;
@@ -57,7 +57,7 @@ public class TermsFilterBuilder implemen
 		String text = DOMUtils.getNonBlankTextOrFail(e);
 		String fieldName = DOMUtils.getAttributeWithInheritanceOrFail(e, "fieldName");
 		TokenStream ts = analyzer.tokenStream(fieldName, new StringReader(text));
-    TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
     
 		try
 		{
@@ -65,11 +65,11 @@ public class TermsFilterBuilder implemen
 	      while (ts.incrementToken()) {
 				if (term == null)
 				{
-					term = new Term(fieldName, termAtt.term());
+					term = new Term(fieldName, termAtt.toString());
 				} else
 				{
 //					 create from previous to save fieldName.intern overhead
-					term = term.createTerm(termAtt.term()); 
+					term = term.createTerm(termAtt.toString()); 
 				}
 				tf.addTerm(term);
 			}

Modified: lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java (original)
+++ lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java Tue Jun  1 10:35:13 2010
@@ -5,7 +5,7 @@ import java.io.StringReader;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
@@ -57,16 +57,16 @@ public class TermsQueryBuilder implement
 		TokenStream ts = analyzer.tokenStream(fieldName, new StringReader(text));
 		try
 		{
-		  TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
+		  CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
 			Term term = null;
 			while (ts.incrementToken()) {
 				if (term == null)
 				{
-					term = new Term(fieldName, termAtt.term());
+					term = new Term(fieldName, termAtt.toString());
 				} else
 				{
 //					 create from previous to save fieldName.intern overhead
-					term = term.createTerm(termAtt.term()); 
+					term = term.createTerm(termAtt.toString()); 
 				}
 				bq.add(new BooleanClause(new TermQuery(term),BooleanClause.Occur.SHOULD));
 			}

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilter.java Tue Jun  1 10:35:13 2010
@@ -21,7 +21,7 @@ import java.io.IOException;
 
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 /**
  * A {@link TokenFilter} that applies {@link ArabicNormalizer} to normalize the orthography.
@@ -29,21 +29,18 @@ import org.apache.lucene.analysis.tokena
  */
 
 public final class ArabicNormalizationFilter extends TokenFilter {
-
-  private final ArabicNormalizer normalizer;
-  private final TermAttribute termAtt;
+  private final ArabicNormalizer normalizer = new ArabicNormalizer();
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
   
   public ArabicNormalizationFilter(TokenStream input) {
     super(input);
-    normalizer = new ArabicNormalizer();
-    termAtt = addAttribute(TermAttribute.class);
   }
 
   @Override
   public boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
-      int newlen = normalizer.normalize(termAtt.termBuffer(), termAtt.termLength());
-      termAtt.setTermLength(newlen);
+      int newlen = normalizer.normalize(termAtt.buffer(), termAtt.length());
+      termAtt.setLength(newlen);
       return true;
     }
     return false;

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicStemFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicStemFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicStemFilter.java Tue Jun  1 10:35:13 2010
@@ -23,7 +23,7 @@ import org.apache.lucene.analysis.miscel
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 /**
  * A {@link TokenFilter} that applies {@link ArabicStemmer} to stem Arabic words..
@@ -35,24 +35,20 @@ import org.apache.lucene.analysis.tokena
  * @see KeywordMarkerFilter */
 
 public final class ArabicStemFilter extends TokenFilter {
-
-  private final ArabicStemmer stemmer;
-  private final TermAttribute termAtt;
-  private final KeywordAttribute keywordAttr;
+  private final ArabicStemmer stemmer = new ArabicStemmer();
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
   
   public ArabicStemFilter(TokenStream input) {
     super(input);
-    stemmer = new ArabicStemmer();
-    termAtt = addAttribute(TermAttribute.class);
-    keywordAttr = addAttribute(KeywordAttribute.class);
   }
 
   @Override
   public boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
       if(!keywordAttr.isKeyword()) {
-        final int newlen = stemmer.stem(termAtt.termBuffer(), termAtt.termLength());
-        termAtt.setTermLength(newlen);
+        final int newlen = stemmer.stem(termAtt.buffer(), termAtt.length());
+        termAtt.setLength(newlen);
       }
       return true;
     } else {

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianStemFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianStemFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianStemFilter.java Tue Jun  1 10:35:13 2010
@@ -23,7 +23,7 @@ import org.apache.lucene.analysis.miscel
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 /**
  * A {@link TokenFilter} that applies {@link BulgarianStemmer} to stem Bulgarian
@@ -35,23 +35,20 @@ import org.apache.lucene.analysis.tokena
  * </p>
  */
 public final class BulgarianStemFilter extends TokenFilter {
-  private final BulgarianStemmer stemmer;
-  private final TermAttribute termAtt;
-  private final KeywordAttribute keywordAttr;
+  private final BulgarianStemmer stemmer = new BulgarianStemmer();
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
   
   public BulgarianStemFilter(final TokenStream input) {
     super(input);
-    stemmer = new BulgarianStemmer();
-    termAtt = addAttribute(TermAttribute.class);
-    keywordAttr = addAttribute(KeywordAttribute.class);
   }
   
   @Override
   public boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
       if(!keywordAttr.isKeyword()) {
-        final int newlen = stemmer.stem(termAtt.termBuffer(), termAtt.termLength());
-        termAtt.setTermLength(newlen);
+        final int newlen = stemmer.stem(termAtt.buffer(), termAtt.length());
+        termAtt.setLength(newlen);
       }
       return true;
     } else {

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemFilter.java Tue Jun  1 10:35:13 2010
@@ -24,7 +24,7 @@ import org.apache.lucene.analysis.miscel
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 /**
  * A {@link TokenFilter} that applies {@link BrazilianStemmer}.
@@ -41,10 +41,10 @@ public final class BrazilianStemFilter e
   /**
    * {@link BrazilianStemmer} in use by this filter.
    */
-  private BrazilianStemmer stemmer = null;
+  private BrazilianStemmer stemmer = new BrazilianStemmer();
   private Set<?> exclusions = null;
-  private final TermAttribute termAtt;
-  private final KeywordAttribute keywordAttr;
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
 
   /**
    * Creates a new BrazilianStemFilter 
@@ -53,9 +53,6 @@ public final class BrazilianStemFilter e
    */
   public BrazilianStemFilter(TokenStream in) {
     super(in);
-    stemmer = new BrazilianStemmer();
-    termAtt = addAttribute(TermAttribute.class);
-    keywordAttr = addAttribute(KeywordAttribute.class);
   }
   
   /**
@@ -74,13 +71,13 @@ public final class BrazilianStemFilter e
   @Override
   public boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
-      final String term = termAtt.term();
+      final String term = termAtt.toString();
       // Check the exclusion table.
       if (!keywordAttr.isKeyword() && (exclusions == null || !exclusions.contains(term))) {
         final String s = stemmer.stem(term);
         // If not stemmed, don't waste the time adjusting the token.
         if ((s != null) && !s.equals(term))
-          termAtt.setTermBuffer(s);
+          termAtt.setEmpty().append(s);
       }
       return true;
     } else {

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java Tue Jun  1 10:35:13 2010
@@ -22,7 +22,7 @@ import java.io.Reader;
 
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.util.AttributeSource;
 
@@ -98,9 +98,9 @@ public final class CJKTokenizer extends 
      */
     private boolean preIsTokened = false;
 
-    private TermAttribute termAtt;
-    private OffsetAttribute offsetAtt;
-    private TypeAttribute typeAtt;
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+    private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
     
     //~ Constructors -----------------------------------------------------------
 
@@ -111,23 +111,14 @@ public final class CJKTokenizer extends 
      */
     public CJKTokenizer(Reader in) {
       super(in);
-      init();
     }
 
     public CJKTokenizer(AttributeSource source, Reader in) {
       super(source, in);
-      init();
     }
 
     public CJKTokenizer(AttributeFactory factory, Reader in) {
       super(factory, in);
-      init();
-    }
-    
-    private void init() {
-      termAtt = addAttribute(TermAttribute.class);
-      offsetAtt = addAttribute(OffsetAttribute.class);
-      typeAtt = addAttribute(TypeAttribute.class);
     }
     
     //~ Methods ----------------------------------------------------------------
@@ -287,7 +278,7 @@ public final class CJKTokenizer extends 
         }
       
         if (length > 0) {
-          termAtt.setTermBuffer(buffer, 0, length);
+          termAtt.copyBuffer(buffer, 0, length);
           offsetAtt.setOffset(correctOffset(start), correctOffset(start+length));
           typeAtt.setType(TOKEN_TYPE_NAMES[tokenType]);
           return true;

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cn/ChineseFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cn/ChineseFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cn/ChineseFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cn/ChineseFilter.java Tue Jun  1 10:35:13 2010
@@ -23,7 +23,7 @@ import java.util.Arrays;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.core.StopFilter;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.util.CharArraySet;
 import org.apache.lucene.util.Version;
 
@@ -61,21 +61,20 @@ public final class ChineseFilter extends
 
     private CharArraySet stopTable;
 
-    private TermAttribute termAtt;
+    private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
     
     public ChineseFilter(TokenStream in) {
         super(in);
 
         stopTable = new CharArraySet(Version.LUCENE_CURRENT, Arrays.asList(STOP_WORDS), false);
-        termAtt = addAttribute(TermAttribute.class);
     }
 
     @Override
     public boolean incrementToken() throws IOException {
 
         while (input.incrementToken()) {
-            char text[] = termAtt.termBuffer();
-            int termLength = termAtt.termLength();
+            char text[] = termAtt.buffer();
+            int termLength = termAtt.length();
 
           // why not key off token type here assuming ChineseTokenizer comes first?
             if (!stopTable.contains(text, 0, termLength)) {

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java Tue Jun  1 10:35:13 2010
@@ -23,8 +23,8 @@ import java.io.Reader;
 
 import org.apache.lucene.analysis.standard.StandardTokenizer;
 import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.util.AttributeSource;
 
 
@@ -62,24 +62,16 @@ public final class ChineseTokenizer exte
 
     public ChineseTokenizer(Reader in) {
       super(in);
-      init();
     }
 
     public ChineseTokenizer(AttributeSource source, Reader in) {
       super(source, in);
-      init();
     }
 
     public ChineseTokenizer(AttributeFactory factory, Reader in) {
       super(factory, in);
-      init();
     }
-    
-    private void init() {
-      termAtt = addAttribute(TermAttribute.class);
-      offsetAtt = addAttribute(OffsetAttribute.class);
-    }
-    
+       
     private int offset = 0, bufferIndex=0, dataLen=0;
     private final static int MAX_WORD_LEN = 255;
     private final static int IO_BUFFER_SIZE = 1024;
@@ -90,8 +82,8 @@ public final class ChineseTokenizer exte
     private int length;
     private int start;
 
-    private TermAttribute termAtt;
-    private OffsetAttribute offsetAtt;
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
     
     private final void push(char c) {
 
@@ -105,7 +97,7 @@ public final class ChineseTokenizer exte
         if (length>0) {
             //System.out.println(new String(buffer, 0,
             //length));
-          termAtt.setTermBuffer(buffer, 0, length);
+          termAtt.copyBuffer(buffer, 0, length);
           offsetAtt.setOffset(correctOffset(start), correctOffset(start+length));
           return true;
         }

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java Tue Jun  1 10:35:13 2010
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.tokena
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.analysis.util.CharArraySet;
 import org.apache.lucene.util.Version;
@@ -69,12 +69,12 @@ public abstract class CompoundWordTokenF
   protected final int maxSubwordSize;
   protected final boolean onlyLongestMatch;
   
-  private TermAttribute termAtt;
-  private OffsetAttribute offsetAtt;
-  private FlagsAttribute flagsAtt;
-  private PositionIncrementAttribute posIncAtt;
-  private TypeAttribute typeAtt;
-  private PayloadAttribute payloadAtt;
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+  private final FlagsAttribute flagsAtt = addAttribute(FlagsAttribute.class);
+  private final PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
+  private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
+  private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class);
   
   private final Token wrapper = new Token();
   /**
@@ -160,13 +160,6 @@ public abstract class CompoundWordTokenF
       this.dictionary = new CharArraySet(matchVersion, dictionary.size(), false);
       addAllLowerCase(this.dictionary, dictionary);
     }
-    
-    termAtt = addAttribute(TermAttribute.class);
-    offsetAtt = addAttribute(OffsetAttribute.class);
-    flagsAtt = addAttribute(FlagsAttribute.class);
-    posIncAtt = addAttribute(PositionIncrementAttribute.class);
-    typeAtt = addAttribute(TypeAttribute.class);
-    payloadAtt = addAttribute(PayloadAttribute.class);
   }
 
   /**
@@ -192,7 +185,7 @@ public abstract class CompoundWordTokenF
   
   private final void setToken(final Token token) throws IOException {
     clearAttributes();
-    termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength());
+    termAtt.copyBuffer(token.buffer(), 0, token.length());
     flagsAtt.setFlags(token.getFlags());
     typeAtt.setType(token.type());
     offsetAtt.setOffset(token.startOffset(), token.endOffset());
@@ -210,7 +203,7 @@ public abstract class CompoundWordTokenF
     if (!input.incrementToken())
       return false;
     
-    wrapper.setTermBuffer(termAtt.termBuffer(), 0, termAtt.termLength());
+    wrapper.copyBuffer(termAtt.buffer(), 0, termAtt.length());
     wrapper.setStartOffset(offsetAtt.startOffset());
     wrapper.setEndOffset(offsetAtt.endOffset());
     wrapper.setFlags(flagsAtt.getFlags());
@@ -248,7 +241,7 @@ public abstract class CompoundWordTokenF
   protected final Token createToken(final int offset, final int length,
       final Token prototype) {
     int newStart = prototype.startOffset() + offset;
-    Token t = prototype.clone(prototype.termBuffer(), offset, length, newStart, newStart+length);
+    Token t = prototype.clone(prototype.buffer(), offset, length, newStart, newStart+length);
     t.setPositionIncrement(0);
     return t;
   }
@@ -258,7 +251,7 @@ public abstract class CompoundWordTokenF
     tokens.add((Token) token.clone());
 
     // Only words longer than minWordSize get processed
-    if (token.termLength() < this.minWordSize) {
+    if (token.length() < this.minWordSize) {
       return;
     }
     

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java Tue Jun  1 10:35:13 2010
@@ -191,22 +191,22 @@ public class DictionaryCompoundWordToken
   @Override
   protected void decomposeInternal(final Token token) {
     // Only words longer than minWordSize get processed
-    if (token.termLength() < this.minWordSize) {
+    if (token.length() < this.minWordSize) {
       return;
     }
     
-    char[] lowerCaseTermBuffer=makeLowerCaseCopy(token.termBuffer());
+    char[] lowerCaseTermBuffer=makeLowerCaseCopy(token.buffer());
     
-    for (int i=0;i<token.termLength()-this.minSubwordSize;++i) {
+    for (int i=0;i<token.length()-this.minSubwordSize;++i) {
         Token longestMatchToken=null;
         for (int j=this.minSubwordSize-1;j<this.maxSubwordSize;++j) {
-            if(i+j>token.termLength()) {
+            if(i+j>token.length()) {
                 break;
             }
             if(dictionary.contains(lowerCaseTermBuffer, i, j)) {
                 if (this.onlyLongestMatch) {
                    if (longestMatchToken!=null) {
-                     if (longestMatchToken.termLength()<j) {
+                     if (longestMatchToken.length()<j) {
                        longestMatchToken=createToken(i,j,token);
                      }
                    } else {

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java Tue Jun  1 10:35:13 2010
@@ -302,15 +302,15 @@ public class HyphenationCompoundWordToke
   @Override
   protected void decomposeInternal(final Token token) {
     // get the hyphenation points
-    Hyphenation hyphens = hyphenator.hyphenate(token.termBuffer(), 0, token
-        .termLength(), 1, 1);
+    Hyphenation hyphens = hyphenator.hyphenate(token.buffer(), 0, token
+        .length(), 1, 1);
     // No hyphen points found -> exit
     if (hyphens == null) {
       return;
     }
 
     final int[] hyp = hyphens.getHyphenationPoints();
-    char[] lowerCaseTermBuffer=makeLowerCaseCopy(token.termBuffer());
+    char[] lowerCaseTermBuffer=makeLowerCaseCopy(token.buffer());
 
     for (int i = 0; i < hyp.length; ++i) {
       int remaining = hyp.length - i;
@@ -335,7 +335,7 @@ public class HyphenationCompoundWordToke
         if (dictionary == null || dictionary.contains(lowerCaseTermBuffer, start, partLength)) {
           if (this.onlyLongestMatch) {
             if (longestMatchToken != null) {
-              if (longestMatchToken.termLength() < partLength) {
+              if (longestMatchToken.length() < partLength) {
                 longestMatchToken = createToken(start, partLength, token);
               }
             } else {
@@ -352,7 +352,7 @@ public class HyphenationCompoundWordToke
           // characters
           if (this.onlyLongestMatch) {
             if (longestMatchToken != null) {
-              if (longestMatchToken.termLength() < partLength - 1) {
+              if (longestMatchToken.length() < partLength - 1) {
                 longestMatchToken = createToken(start, partLength - 1, token);
               }
             } else {

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cz/CzechStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cz/CzechStemFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cz/CzechStemFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cz/CzechStemFilter.java Tue Jun  1 10:35:13 2010
@@ -6,7 +6,7 @@ import org.apache.lucene.analysis.miscel
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -37,23 +37,20 @@ import org.apache.lucene.analysis.tokena
  * @see KeywordMarkerFilter
  */
 public final class CzechStemFilter extends TokenFilter {
-  private final CzechStemmer stemmer;
-  private final TermAttribute termAtt;
-  private final KeywordAttribute keywordAttr;
+  private final CzechStemmer stemmer = new CzechStemmer();
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
   
   public CzechStemFilter(TokenStream input) {
     super(input);
-    stemmer = new CzechStemmer();
-    termAtt = addAttribute(TermAttribute.class);
-    keywordAttr = addAttribute(KeywordAttribute.class);
   }
 
   @Override
   public boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
       if(!keywordAttr.isKeyword()) {
-        final int newlen = stemmer.stem(termAtt.termBuffer(), termAtt.termLength());
-        termAtt.setTermLength(newlen);
+        final int newlen = stemmer.stem(termAtt.buffer(), termAtt.length());
+        termAtt.setLength(newlen);
       }
       return true;
     } else {

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/de/GermanStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/de/GermanStemFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/de/GermanStemFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/de/GermanStemFilter.java Tue Jun  1 10:35:13 2010
@@ -24,7 +24,7 @@ import org.apache.lucene.analysis.miscel
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 /**
  * A {@link TokenFilter} that stems German words. 
@@ -45,11 +45,11 @@ public final class GermanStemFilter exte
     /**
      * The actual token in the input stream.
      */
-    private GermanStemmer stemmer = null;
+    private GermanStemmer stemmer = new GermanStemmer();
     private Set<?> exclusionSet = null;
 
-    private final TermAttribute termAtt;
-    private final KeywordAttribute keywordAttr;
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
 
     /**
      * Creates a {@link GermanStemFilter} instance
@@ -58,9 +58,6 @@ public final class GermanStemFilter exte
     public GermanStemFilter( TokenStream in )
     {
       super(in);
-      stemmer = new GermanStemmer();
-      termAtt = addAttribute(TermAttribute.class);
-      keywordAttr = addAttribute(KeywordAttribute.class);
     }
 
     /**
@@ -80,13 +77,13 @@ public final class GermanStemFilter exte
     @Override
     public boolean incrementToken() throws IOException {
       if (input.incrementToken()) {
-        String term = termAtt.term();
+        String term = termAtt.toString();
         // Check the exclusion table.
         if (!keywordAttr.isKeyword() && (exclusionSet == null || !exclusionSet.contains(term))) {
           String s = stemmer.stem(term);
           // If not stemmed, don't waste the time adjusting the token.
           if ((s != null) && !s.equals(term))
-            termAtt.setTermBuffer(s);
+            termAtt.setEmpty().append(s);
         }
         return true;
       } else {

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianNormalizationFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianNormalizationFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianNormalizationFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianNormalizationFilter.java Tue Jun  1 10:35:13 2010
@@ -21,7 +21,7 @@ import java.io.IOException;
 
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 /**
  * A {@link TokenFilter} that applies {@link PersianNormalizer} to normalize the
@@ -30,22 +30,19 @@ import org.apache.lucene.analysis.tokena
  */
 
 public final class PersianNormalizationFilter extends TokenFilter {
-
-  private final PersianNormalizer normalizer;
-  private final TermAttribute termAtt;
+  private final PersianNormalizer normalizer = new PersianNormalizer();
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
 
   public PersianNormalizationFilter(TokenStream input) {
     super(input);
-    normalizer = new PersianNormalizer();
-    termAtt = addAttribute(TermAttribute.class);
   }
 
   @Override
   public boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
-      final int newlen = normalizer.normalize(termAtt.termBuffer(), termAtt
-          .termLength());
-      termAtt.setTermLength(newlen);
+      final int newlen = normalizer.normalize(termAtt.buffer(), 
+          termAtt.length());
+      termAtt.setLength(newlen);
       return true;
     } 
     return false;

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java Tue Jun  1 10:35:13 2010
@@ -23,7 +23,7 @@ import java.util.Arrays;
 import org.apache.lucene.analysis.standard.StandardTokenizer; // for javadocs
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.TokenFilter;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.util.CharArraySet;
 import org.apache.lucene.util.Version;
 
@@ -37,7 +37,7 @@ import org.apache.lucene.util.Version;
  */
 public final class ElisionFilter extends TokenFilter {
   private CharArraySet articles = CharArraySet.EMPTY_SET;
-  private final TermAttribute termAtt;
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
   private static final CharArraySet DEFAULT_ARTICLES = CharArraySet.unmodifiableSet(
       new CharArraySet(Version.LUCENE_CURRENT, Arrays.asList(
           "l", "m", "t", "qu", "n", "s", "j"), true));
@@ -100,7 +100,6 @@ public final class ElisionFilter extends
     super(input);
     this.articles = CharArraySet.unmodifiableSet(
         new CharArraySet(matchVersion, articles, true));
-    termAtt = addAttribute(TermAttribute.class);
   }
 
   /**
@@ -115,13 +114,13 @@ public final class ElisionFilter extends
   }
 
   /**
-   * Increments the {@link TokenStream} with a {@link TermAttribute} without elisioned start
+   * Increments the {@link TokenStream} with a {@link CharTermAttribute} without elisioned start
    */
   @Override
   public final boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
-      char[] termBuffer = termAtt.termBuffer();
-      int termLength = termAtt.termLength();
+      char[] termBuffer = termAtt.buffer();
+      int termLength = termAtt.length();
 
       int minPoz = Integer.MAX_VALUE;
       for (int i = 0; i < apostrophes.length; i++) {
@@ -137,8 +136,8 @@ public final class ElisionFilter extends
 
       // An apostrophe has been found. If the prefix is an article strip it off.
       if (minPoz != Integer.MAX_VALUE
-          && articles.contains(termAtt.termBuffer(), 0, minPoz)) {
-        termAtt.setTermBuffer(termAtt.termBuffer(), minPoz + 1, termAtt.termLength() - (minPoz + 1));
+          && articles.contains(termAtt.buffer(), 0, minPoz)) {
+        termAtt.copyBuffer(termAtt.buffer(), minPoz + 1, termAtt.length() - (minPoz + 1));
       }
 
       return true;

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java Tue Jun  1 10:35:13 2010
@@ -22,7 +22,7 @@ import org.apache.lucene.analysis.TokenF
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.snowball.SnowballFilter;
 import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 import java.io.IOException;
 import java.util.HashSet;
@@ -51,17 +51,14 @@ public final class FrenchStemFilter exte
 	/**
 	 * The actual token in the input stream.
 	 */
-	private FrenchStemmer stemmer = null;
+	private FrenchStemmer stemmer = new FrenchStemmer();
 	private Set<?> exclusions = null;
 	
-	private final TermAttribute termAtt;
-  private final KeywordAttribute keywordAttr;
+	private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
 
 	public FrenchStemFilter( TokenStream in ) {
-          super(in);
-		stemmer = new FrenchStemmer();
-		termAtt = addAttribute(TermAttribute.class);
-    keywordAttr = addAttribute(KeywordAttribute.class);
+    super(in);
 	}
 
   /**
@@ -82,14 +79,14 @@ public final class FrenchStemFilter exte
 	@Override
 	public boolean incrementToken() throws IOException {
 	  if (input.incrementToken()) {
-	    String term = termAtt.term();
+	    String term = termAtt.toString();
 
 	    // Check the exclusion table
 	    if ( !keywordAttr.isKeyword() && (exclusions == null || !exclusions.contains( term )) ) {
 	      String s = stemmer.stem( term );
 	      // If not stemmed, don't waste the time  adjusting the token.
 	      if ((s != null) && !s.equals( term ) )
-	        termAtt.setTermBuffer(s);
+	        termAtt.setEmpty().append(s);
 	    }
 	    return true;
 	  } else {

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiNormalizationFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiNormalizationFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiNormalizationFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiNormalizationFilter.java Tue Jun  1 10:35:13 2010
@@ -23,7 +23,7 @@ import org.apache.lucene.analysis.miscel
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 /**
  * A {@link TokenFilter} that applies {@link HindiNormalizer} to normalize the
@@ -39,7 +39,7 @@ import org.apache.lucene.analysis.tokena
 public final class HindiNormalizationFilter extends TokenFilter {
 
   private final HindiNormalizer normalizer = new HindiNormalizer();
-  private final TermAttribute termAtt = addAttribute(TermAttribute.class);
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
   private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);
   
   public HindiNormalizationFilter(TokenStream input) {
@@ -50,8 +50,8 @@ public final class HindiNormalizationFil
   public boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
       if (!keywordAtt.isKeyword())
-        termAtt.setTermLength(normalizer.normalize(termAtt.termBuffer(), 
-            termAtt.termLength()));
+        termAtt.setLength(normalizer.normalize(termAtt.buffer(), 
+            termAtt.length()));
       return true;
     } 
     return false;

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiStemFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiStemFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiStemFilter.java Tue Jun  1 10:35:13 2010
@@ -22,13 +22,13 @@ import java.io.IOException;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 /**
  * A {@link TokenFilter} that applies {@link HindiStemmer} to stem Hindi words.
  */
 public final class HindiStemFilter extends TokenFilter {
-  private final TermAttribute termAtt = addAttribute(TermAttribute.class);
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
   private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);
   private final HindiStemmer stemmer = new HindiStemmer();
   
@@ -40,7 +40,7 @@ public final class HindiStemFilter exten
   public boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
       if (!keywordAtt.isKeyword())
-        termAtt.setTermLength(stemmer.stem(termAtt.termBuffer(), termAtt.termLength()));
+        termAtt.setLength(stemmer.stem(termAtt.buffer(), termAtt.length()));
       return true;
     } else {
       return false;

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/in/IndicNormalizationFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/in/IndicNormalizationFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/in/IndicNormalizationFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/in/IndicNormalizationFilter.java Tue Jun  1 10:35:13 2010
@@ -21,14 +21,14 @@ import java.io.IOException;
 
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 /**
  * A {@link TokenFilter} that applies {@link IndicNormalizer} to normalize text
  * in Indian Languages.
  */
 public final class IndicNormalizationFilter extends TokenFilter {
-  private final TermAttribute termAtt = addAttribute(TermAttribute.class);
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
   private final IndicNormalizer normalizer = new IndicNormalizer();
   
   public IndicNormalizationFilter(TokenStream input) {
@@ -38,7 +38,7 @@ public final class IndicNormalizationFil
   @Override
   public boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
-      termAtt.setTermLength(normalizer.normalize(termAtt.termBuffer(), termAtt.termLength()));
+      termAtt.setLength(normalizer.normalize(termAtt.buffer(), termAtt.length()));
       return true;
     } else {
       return false;

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PatternAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PatternAnalyzer.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PatternAnalyzer.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PatternAnalyzer.java Tue Jun  1 10:35:13 2010
@@ -30,8 +30,8 @@ import org.apache.lucene.analysis.Analyz
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.core.StopAnalyzer;
 import org.apache.lucene.analysis.core.StopFilter;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.analysis.util.CharArraySet;
 import org.apache.lucene.util.Version;
 
@@ -332,8 +332,8 @@ public final class PatternAnalyzer exten
     private Matcher matcher;
     private int pos = 0;
     private static final Locale locale = Locale.getDefault();
-    private TermAttribute termAtt = addAttribute(TermAttribute.class);
-    private OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
     
     public PatternTokenizer(String str, Pattern pattern, boolean toLowerCase) {
       this.str = str;
@@ -360,7 +360,7 @@ public final class PatternAnalyzer exten
         if (start != end) { // non-empty match (header/trailer)
           String text = str.substring(start, end);
           if (toLowerCase) text = text.toLowerCase(locale);
-          termAtt.setTermBuffer(text);
+          termAtt.setEmpty().append(text);
           offsetAtt.setOffset(start, end);
           return true;
         }
@@ -392,8 +392,8 @@ public final class PatternAnalyzer exten
     private final boolean toLowerCase;
     private final Set<?> stopWords;
     private static final Locale locale = Locale.getDefault();
-    private TermAttribute termAtt = addAttribute(TermAttribute.class);
-    private OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
     
     public FastStringTokenizer(String str, boolean isLetter, boolean toLowerCase, Set<?> stopWords) {
       this.str = str;
@@ -446,7 +446,7 @@ public final class PatternAnalyzer exten
       {
         return false;
       }
-      termAtt.setTermBuffer(text);
+      termAtt.setEmpty().append(text);
       offsetAtt.setOffset(start, i);
       return true;
     }

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java Tue Jun  1 10:35:13 2010
@@ -23,7 +23,7 @@ import org.apache.lucene.analysis.tokena
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.index.Payload;
 
@@ -44,14 +44,14 @@ public class PrefixAwareTokenFilter exte
   private TokenStream prefix;
   private TokenStream suffix;
   
-  private TermAttribute termAtt;
+  private CharTermAttribute termAtt;
   private PositionIncrementAttribute posIncrAtt;
   private PayloadAttribute payloadAtt;
   private OffsetAttribute offsetAtt;
   private TypeAttribute typeAtt;
   private FlagsAttribute flagsAtt;
 
-  private TermAttribute p_termAtt;
+  private CharTermAttribute p_termAtt;
   private PositionIncrementAttribute p_posIncrAtt;
   private PayloadAttribute p_payloadAtt;
   private OffsetAttribute p_offsetAtt;
@@ -64,14 +64,14 @@ public class PrefixAwareTokenFilter exte
     this.prefix = prefix;
     prefixExhausted = false;
     
-    termAtt = addAttribute(TermAttribute.class);
+    termAtt = addAttribute(CharTermAttribute.class);
     posIncrAtt = addAttribute(PositionIncrementAttribute.class);
     payloadAtt = addAttribute(PayloadAttribute.class);
     offsetAtt = addAttribute(OffsetAttribute.class);
     typeAtt = addAttribute(TypeAttribute.class);
     flagsAtt = addAttribute(FlagsAttribute.class);
 
-    p_termAtt = prefix.addAttribute(TermAttribute.class);
+    p_termAtt = prefix.addAttribute(CharTermAttribute.class);
     p_posIncrAtt = prefix.addAttribute(PositionIncrementAttribute.class);
     p_payloadAtt = prefix.addAttribute(PayloadAttribute.class);
     p_offsetAtt = prefix.addAttribute(OffsetAttribute.class);
@@ -115,7 +115,7 @@ public class PrefixAwareTokenFilter exte
   private void setCurrentToken(Token token) {
     if (token == null) return;
     clearAttributes();
-    termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength());
+    termAtt.copyBuffer(token.buffer(), 0, token.length());
     posIncrAtt.setPositionIncrement(token.getPositionIncrement());
     flagsAtt.setFlags(token.getFlags());
     offsetAtt.setOffset(token.startOffset(), token.endOffset());
@@ -125,7 +125,7 @@ public class PrefixAwareTokenFilter exte
   
   private Token getNextPrefixInputToken(Token token) throws IOException {
     if (!prefix.incrementToken()) return null;
-    token.setTermBuffer(p_termAtt.termBuffer(), 0, p_termAtt.termLength());
+    token.copyBuffer(p_termAtt.buffer(), 0, p_termAtt.length());
     token.setPositionIncrement(p_posIncrAtt.getPositionIncrement());
     token.setFlags(p_flagsAtt.getFlags());
     token.setOffset(p_offsetAtt.startOffset(), p_offsetAtt.endOffset());
@@ -136,7 +136,7 @@ public class PrefixAwareTokenFilter exte
 
   private Token getNextSuffixInputToken(Token token) throws IOException {
     if (!suffix.incrementToken()) return null;
-    token.setTermBuffer(termAtt.termBuffer(), 0, termAtt.termLength());
+    token.copyBuffer(termAtt.buffer(), 0, termAtt.length());
     token.setPositionIncrement(posIncrAtt.getPositionIncrement());
     token.setFlags(flagsAtt.getFlags());
     token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java Tue Jun  1 10:35:13 2010
@@ -22,7 +22,7 @@ import java.io.IOException;
 import org.apache.lucene.util.AttributeImpl;
 import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 /**
  * A {@link TokenStream} containing a single token.
@@ -41,7 +41,7 @@ public final class SingleTokenTokenStrea
     assert token != null;
     this.singleToken = (Token) token.clone();
     
-    tokenAtt = (AttributeImpl) addAttribute(TermAttribute.class);
+    tokenAtt = (AttributeImpl) addAttribute(CharTermAttribute.class);
     assert (tokenAtt instanceof Token);
   }
 

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java Tue Jun  1 10:35:13 2010
@@ -23,7 +23,7 @@ import java.util.Map;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.util.CharArrayMap;
 import org.apache.lucene.util.Version;
 
@@ -34,7 +34,7 @@ import org.apache.lucene.util.Version;
 public final class StemmerOverrideFilter extends TokenFilter {
   private final CharArrayMap<String> dictionary;
   
-  private final TermAttribute termAtt = addAttribute(TermAttribute.class);
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
   private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);
   
   /**
@@ -56,9 +56,9 @@ public final class StemmerOverrideFilter
   public boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
       if (!keywordAtt.isKeyword()) { // don't muck with already-keyworded terms
-        String stem = dictionary.get(termAtt.termBuffer(), 0, termAtt.termLength());
+        String stem = dictionary.get(termAtt.buffer(), 0, termAtt.length());
         if (stem != null) {
-          termAtt.setTermBuffer(stem);
+          termAtt.setEmpty().append(stem);
           keywordAtt.setKeyword(true);
         }
       }

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java Tue Jun  1 10:35:13 2010
@@ -20,7 +20,7 @@ package org.apache.lucene.analysis.ngram
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 import java.io.IOException;
 
@@ -72,8 +72,8 @@ public final class EdgeNGramTokenFilter 
   private int curGramSize;
   private int tokStart;
   
-  private final TermAttribute termAtt;
-  private final OffsetAttribute offsetAtt;
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
 
   /**
    * Creates EdgeNGramTokenFilter that can generate n-grams in the sizes of the given range
@@ -101,8 +101,6 @@ public final class EdgeNGramTokenFilter 
     this.minGram = minGram;
     this.maxGram = maxGram;
     this.side = side;
-    this.termAtt = addAttribute(TermAttribute.class);
-    this.offsetAtt = addAttribute(OffsetAttribute.class);
   }
 
   /**
@@ -124,8 +122,8 @@ public final class EdgeNGramTokenFilter 
         if (!input.incrementToken()) {
           return false;
         } else {
-          curTermBuffer = termAtt.termBuffer().clone();
-          curTermLength = termAtt.termLength();
+          curTermBuffer = termAtt.buffer().clone();
+          curTermLength = termAtt.length();
           curGramSize = minGram;
           tokStart = offsetAtt.startOffset();
         }
@@ -138,7 +136,7 @@ public final class EdgeNGramTokenFilter 
           int end = start + curGramSize;
           clearAttributes();
           offsetAtt.setOffset(tokStart + start, tokStart + end);
-          termAtt.setTermBuffer(curTermBuffer, start, curGramSize);
+          termAtt.copyBuffer(curTermBuffer, start, curGramSize);
           curGramSize++;
           return true;
         }

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java Tue Jun  1 10:35:13 2010
@@ -18,8 +18,8 @@ package org.apache.lucene.analysis.ngram
  */
 
 import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.util.AttributeSource;
 
 import java.io.IOException;
@@ -37,8 +37,8 @@ public final class EdgeNGramTokenizer ex
   public static final int DEFAULT_MAX_GRAM_SIZE = 1;
   public static final int DEFAULT_MIN_GRAM_SIZE = 1;
   
-  private TermAttribute termAtt;
-  private OffsetAttribute offsetAtt;
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
 
   /** Specifies which side of the input the n-gram should be generated from */
   public static enum Side {
@@ -173,10 +173,6 @@ public final class EdgeNGramTokenizer ex
     this.minGram = minGram;
     this.maxGram = maxGram;
     this.side = side;
-    
-    this.termAtt = addAttribute(TermAttribute.class);
-    this.offsetAtt = addAttribute(OffsetAttribute.class);
-
   }
 
   /** Returns the next token in the stream, or null at EOS. */
@@ -206,7 +202,7 @@ public final class EdgeNGramTokenizer ex
     // grab gramSize chars from front or back
     int start = side == Side.FRONT ? 0 : inLen - gramSize;
     int end = start + gramSize;
-    termAtt.setTermBuffer(inStr, start, gramSize);
+    termAtt.setEmpty().append(inStr, start, end);
     offsetAtt.setOffset(correctOffset(start), correctOffset(end));
     gramSize++;
     return true;

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java Tue Jun  1 10:35:13 2010
@@ -22,7 +22,7 @@ import java.io.IOException;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 /**
  * Tokenizes the input into n-grams of the given size(s).
@@ -39,8 +39,8 @@ public final class NGramTokenFilter exte
   private int curPos;
   private int tokStart;
   
-  private TermAttribute termAtt;
-  private OffsetAttribute offsetAtt;
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
 
   /**
    * Creates NGramTokenFilter with given min and max n-grams.
@@ -58,9 +58,6 @@ public final class NGramTokenFilter exte
     }
     this.minGram = minGram;
     this.maxGram = maxGram;
-    
-    this.termAtt = addAttribute(TermAttribute.class);
-    this.offsetAtt = addAttribute(OffsetAttribute.class);
   }
 
   /**
@@ -79,8 +76,8 @@ public final class NGramTokenFilter exte
         if (!input.incrementToken()) {
           return false;
         } else {
-          curTermBuffer = termAtt.termBuffer().clone();
-          curTermLength = termAtt.termLength();
+          curTermBuffer = termAtt.buffer().clone();
+          curTermLength = termAtt.length();
           curGramSize = minGram;
           curPos = 0;
           tokStart = offsetAtt.startOffset();
@@ -89,7 +86,7 @@ public final class NGramTokenFilter exte
       while (curGramSize <= maxGram) {
         while (curPos+curGramSize <= curTermLength) {     // while there is input
           clearAttributes();
-          termAtt.setTermBuffer(curTermBuffer, curPos, curGramSize);
+          termAtt.copyBuffer(curTermBuffer, curPos, curGramSize);
           offsetAtt.setOffset(tokStart + curPos, tokStart + curPos + curGramSize);
           curPos++;
           return true;

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java Tue Jun  1 10:35:13 2010
@@ -18,8 +18,8 @@ package org.apache.lucene.analysis.ngram
  */
 
 import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.util.AttributeSource;
 
 import java.io.IOException;
@@ -39,8 +39,8 @@ public final class NGramTokenizer extend
   private String inStr;
   private boolean started = false;
   
-  private TermAttribute termAtt;
-  private OffsetAttribute offsetAtt;
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
 
   /**
    * Creates NGramTokenizer with given min and max n-grams.
@@ -94,9 +94,6 @@ public final class NGramTokenizer extend
     }
     this.minGram = minGram;
     this.maxGram = maxGram;
-    
-    this.termAtt = addAttribute(TermAttribute.class);
-    this.offsetAtt = addAttribute(OffsetAttribute.class);    
   }
 
   /** Returns the next token in the stream, or null at EOS. */
@@ -123,7 +120,7 @@ public final class NGramTokenizer extend
 
     int oldPos = pos;
     pos++;
-    termAtt.setTermBuffer(inStr, oldPos, gramSize);
+    termAtt.setEmpty().append(inStr, oldPos, oldPos+gramSize);
     offsetAtt.setOffset(correctOffset(oldPos), correctOffset(oldPos+gramSize));
     return true;
   }

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java Tue Jun  1 10:35:13 2010
@@ -28,7 +28,7 @@ import org.apache.lucene.analysis.TokenF
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.snowball.SnowballFilter;
 import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 /**
  * A {@link TokenFilter} that stems Dutch words. 
@@ -52,17 +52,14 @@ public final class DutchStemFilter exten
   /**
    * The actual token in the input stream.
    */
-  private DutchStemmer stemmer = null;
+  private DutchStemmer stemmer = new DutchStemmer();
   private Set<?> exclusions = null;
   
-  private final TermAttribute termAtt;
-  private final KeywordAttribute keywordAttr;
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
 
   public DutchStemFilter(TokenStream _in) {
     super(_in);
-    stemmer = new DutchStemmer();
-    termAtt = addAttribute(TermAttribute.class);
-    keywordAttr = addAttribute(KeywordAttribute.class);
   }
 
   /**
@@ -99,14 +96,14 @@ public final class DutchStemFilter exten
   @Override
   public boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
-      final String term = termAtt.term();
+      final String term = termAtt.toString();
 
       // Check the exclusion table.
       if (!keywordAttr.isKeyword() && (exclusions == null || !exclusions.contains(term))) {
         final String s = stemmer.stem(term);
         // If not stemmed, don't waste the time adjusting the token.
         if ((s != null) && !s.equals(term))
-          termAtt.setTermBuffer(s);
+          termAtt.setEmpty().append(s);
       }
       return true;
     } else {

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilter.java Tue Jun  1 10:35:13 2010
@@ -21,7 +21,7 @@ import java.io.IOException;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 
 /**
@@ -39,15 +39,13 @@ import org.apache.lucene.analysis.tokena
 public final class DelimitedPayloadTokenFilter extends TokenFilter {
   public static final char DEFAULT_DELIMITER = '|';
   private final char delimiter;
-  private final TermAttribute termAtt;
-  private final PayloadAttribute payAtt;
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final PayloadAttribute payAtt = addAttribute(PayloadAttribute.class);
   private final PayloadEncoder encoder;
 
 
   public DelimitedPayloadTokenFilter(TokenStream input, char delimiter, PayloadEncoder encoder) {
     super(input);
-    termAtt = addAttribute(TermAttribute.class);
-    payAtt = addAttribute(PayloadAttribute.class);
     this.delimiter = delimiter;
     this.encoder = encoder;
   }
@@ -55,12 +53,12 @@ public final class DelimitedPayloadToken
   @Override
   public boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
-      final char[] buffer = termAtt.termBuffer();
-      final int length = termAtt.termLength();
+      final char[] buffer = termAtt.buffer();
+      final int length = termAtt.length();
       for (int i = 0; i < length; i++) {
         if (buffer[i] == delimiter) {
           payAtt.setPayload(encoder.encode(buffer, i + 1, (length - (i + 1))));
-          termAtt.setTermLength(i); // simply set a new length
+          termAtt.setLength(i); // simply set a new length
           return true;
         }
       }

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java Tue Jun  1 10:35:13 2010
@@ -35,16 +35,14 @@ public class NumericPayloadTokenFilter e
   private String typeMatch;
   private Payload thePayload;
 
-  private PayloadAttribute payloadAtt;
-  private TypeAttribute typeAtt;
+  private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class);
+  private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
 
   public NumericPayloadTokenFilter(TokenStream input, float payload, String typeMatch) {
     super(input);
     //Need to encode the payload
     thePayload = new Payload(PayloadHelper.encodeFloat(payload));
     this.typeMatch = typeMatch;
-    payloadAtt = addAttribute(PayloadAttribute.class);
-    typeAtt = addAttribute(TypeAttribute.class);
   }
 
   @Override

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java Tue Jun  1 10:35:13 2010
@@ -33,13 +33,11 @@ import org.apache.lucene.index.Payload;
  *
  **/
 public class TokenOffsetPayloadTokenFilter extends TokenFilter {
-  protected OffsetAttribute offsetAtt;
-  protected PayloadAttribute payAtt;
+  private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+  private final PayloadAttribute payAtt = addAttribute(PayloadAttribute.class);
 
   public TokenOffsetPayloadTokenFilter(TokenStream input) {
     super(input);
-    offsetAtt = addAttribute(OffsetAttribute.class);
-    payAtt = addAttribute(PayloadAttribute.class);
   }
 
   @Override

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java Tue Jun  1 10:35:13 2010
@@ -33,13 +33,11 @@ import java.io.IOException;
  *
  **/
 public class TypeAsPayloadTokenFilter extends TokenFilter {
-  private PayloadAttribute payloadAtt;
-  private TypeAttribute typeAtt;
+  private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class);
+  private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
 
   public TypeAsPayloadTokenFilter(TokenStream input) {
     super(input);
-    payloadAtt = addAttribute(PayloadAttribute.class);
-    typeAtt = addAttribute(TypeAttribute.class);
   }
 
 

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java Tue Jun  1 10:35:13 2010
@@ -35,7 +35,7 @@ public final class PositionFilter extend
   /** The first token must have non-zero positionIncrement **/
   private boolean firstTokenPositioned = false;
   
-  private PositionIncrementAttribute posIncrAtt;
+  private PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
 
   /**
    * Constructs a PositionFilter that assigns a position increment of zero to
@@ -45,7 +45,6 @@ public final class PositionFilter extend
    */
   public PositionFilter(final TokenStream input) {
     super(input);
-    posIncrAtt = addAttribute(PositionIncrementAttribute.class);
   }
 
   /**



Mime
View raw message