lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject svn commit: r833968 [2/10] - in /lucene/java/branches/flex_1458: ./ contrib/ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/ contrib/analyzers/common/src/java/org/apache/...
Date Mon, 09 Nov 2009 04:02:02 GMT
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java Mon Nov  9 04:01:50 2009
@@ -149,6 +149,7 @@
     return tree;
   }
 
+  @Override
   protected void decomposeInternal(final Token token) {
     // get the hyphenation points
     Hyphenation hyphens = hyphenator.hyphenate(token.termBuffer(), 0, token

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/CharVector.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/CharVector.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/CharVector.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/CharVector.java Mon Nov  9 04:01:50 2009
@@ -81,6 +81,7 @@
     n = 0;
   }
 
+  @Override
   public Object clone() {
     CharVector cv = new CharVector((char[]) array.clone(), blockSize);
     cv.n = this.n;

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/Hyphen.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/Hyphen.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/Hyphen.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/Hyphen.java Mon Nov  9 04:01:50 2009
@@ -51,6 +51,7 @@
     postBreak = null;
   }
 
+  @Override
   public String toString() {
     if (noBreak == null && postBreak == null && preBreak != null
         && preBreak.equals("-")) {

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java Mon Nov  9 04:01:50 2009
@@ -466,6 +466,7 @@
     insert(pattern, (char) k);
   }
 
+  @Override
   public void printStats() {
     System.out.println("Value space size = "
         + Integer.toString(vspace.length()));

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternParser.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternParser.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternParser.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternParser.java Mon Nov  9 04:01:50 2009
@@ -266,6 +266,7 @@
   //
   // EntityResolver methods
   //
+  @Override
   public InputSource resolveEntity(String publicId, String systemId) {
     return HyphenationDTDGenerator.generateDTD();
   }
@@ -278,6 +279,7 @@
    * @see org.xml.sax.ContentHandler#startElement(java.lang.String,
    *      java.lang.String, java.lang.String, org.xml.sax.Attributes)
    */
+  @Override
   public void startElement(String uri, String local, String raw,
       Attributes attrs) {
     if (local.equals("hyphen-char")) {
@@ -307,6 +309,7 @@
    * @see org.xml.sax.ContentHandler#endElement(java.lang.String,
    *      java.lang.String, java.lang.String)
    */
+  @Override
   public void endElement(String uri, String local, String raw) {
 
     if (token.length() > 0) {
@@ -343,6 +346,7 @@
   /**
    * @see org.xml.sax.ContentHandler#characters(char[], int, int)
    */
+  @Override
   public void characters(char ch[], int start, int length) {
     StringBuffer chars = new StringBuffer(length);
     chars.append(ch, start, length);
@@ -376,6 +380,7 @@
   /**
    * @see org.xml.sax.ErrorHandler#warning(org.xml.sax.SAXParseException)
    */
+  @Override
   public void warning(SAXParseException ex) {
     errMsg = "[Warning] " + getLocationString(ex) + ": " + ex.getMessage();
   }
@@ -383,6 +388,7 @@
   /**
    * @see org.xml.sax.ErrorHandler#error(org.xml.sax.SAXParseException)
    */
+  @Override
   public void error(SAXParseException ex) {
     errMsg = "[Error] " + getLocationString(ex) + ": " + ex.getMessage();
   }
@@ -390,6 +396,7 @@
   /**
    * @see org.xml.sax.ErrorHandler#fatalError(org.xml.sax.SAXParseException)
    */
+  @Override
   public void fatalError(SAXParseException ex) throws SAXException {
     errMsg = "[Fatal Error] " + getLocationString(ex) + ": " + ex.getMessage();
     throw ex;

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java Mon Nov  9 04:01:50 2009
@@ -348,6 +348,7 @@
     return length;
   }
 
+  @Override
   public Object clone() {
     TernaryTree t = new TernaryTree();
     t.lo = (char[]) this.lo.clone();
@@ -483,6 +484,7 @@
         child = c;
       }
 
+      @Override
       public Object clone() {
         return new Item(parent, child);
       }

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java Mon Nov  9 04:01:50 2009
@@ -140,6 +140,7 @@
 	 * @return  A {@link TokenStream} built from a {@link StandardTokenizer} filtered with
 	 * 			{@link StandardFilter}, {@link LowerCaseFilter}, and {@link StopFilter}
 	 */
+	@Override
 	public final TokenStream tokenStream( String fieldName, Reader reader ) {
                 TokenStream result = new StandardTokenizer( matchVersion, reader );
 		result = new StandardFilter( result );
@@ -161,6 +162,7 @@
      * @return  A {@link TokenStream} built from a {@link StandardTokenizer} filtered with
      *          {@link StandardFilter}, {@link LowerCaseFilter}, and {@link StopFilter}
      */
+	@Override
 	public TokenStream reusableTokenStream(String fieldName, Reader reader)
       throws IOException {
       SavedStreams streams = (SavedStreams) getPreviousTokenStream();

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java Mon Nov  9 04:01:50 2009
@@ -149,6 +149,7 @@
    *         {@link StandardFilter}, {@link LowerCaseFilter}, {@link StopFilter}, and
    *         {@link GermanStemFilter}
    */
+  @Override
   public TokenStream tokenStream(String fieldName, Reader reader) {
     TokenStream result = new StandardTokenizer(matchVersion, reader);
     result = new StandardFilter(result);
@@ -172,6 +173,7 @@
    *         {@link StandardFilter}, {@link LowerCaseFilter}, {@link StopFilter}, and
    *         {@link GermanStemFilter}
    */
+  @Override
   public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
     if (overridesTokenStreamMethod) {
       // LUCENE-1678: force fallback to tokenStream() if we

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanStemFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanStemFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanStemFilter.java Mon Nov  9 04:01:50 2009
@@ -61,6 +61,7 @@
     /**
      * @return  Returns true for next token in the stream, or false at EOS
      */
+    @Override
     public boolean incrementToken() throws IOException {
       if (input.incrementToken()) {
         String term = termAtt.term();

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java Mon Nov  9 04:01:50 2009
@@ -99,6 +99,7 @@
      * @return  A {@link TokenStream} built from a {@link StandardTokenizer} filtered with
      *                  {@link GreekLowerCaseFilter} and {@link StopFilter}
      */
+    @Override
     public TokenStream tokenStream(String fieldName, Reader reader)
     {
         TokenStream result = new StandardTokenizer(matchVersion, reader);
@@ -120,6 +121,7 @@
      * @return  A {@link TokenStream} built from a {@link StandardTokenizer} filtered with
      *                  {@link GreekLowerCaseFilter} and {@link StopFilter}
      */
+    @Override
     public TokenStream reusableTokenStream(String fieldName, Reader reader) 
       throws IOException {
       SavedStreams streams = (SavedStreams) getPreviousTokenStream();

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekLowerCaseFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekLowerCaseFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekLowerCaseFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekLowerCaseFilter.java Mon Nov  9 04:01:50 2009
@@ -37,6 +37,7 @@
     	termAtt = addAttribute(TermAttribute.class);
     }
 
+    @Override
     public boolean incrementToken() throws IOException {
       if (input.incrementToken()) {
         char[] chArray = termAtt.termBuffer();

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java Mon Nov  9 04:01:50 2009
@@ -152,6 +152,7 @@
    *         {@link ArabicNormalizationFilter},
    *         {@link PersianNormalizationFilter} and Persian Stop words
    */
+  @Override
   public TokenStream tokenStream(String fieldName, Reader reader) {
     TokenStream result = new ArabicLetterTokenizer(reader);
     result = new LowerCaseFilter(result);
@@ -181,6 +182,7 @@
    *         {@link ArabicNormalizationFilter},
    *         {@link PersianNormalizationFilter} and Persian Stop words
    */
+  @Override
   public TokenStream reusableTokenStream(String fieldName, Reader reader)
       throws IOException {
     SavedStreams streams = (SavedStreams) getPreviousTokenStream();

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianNormalizationFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianNormalizationFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianNormalizationFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianNormalizationFilter.java Mon Nov  9 04:01:50 2009
@@ -40,6 +40,7 @@
     termAtt = addAttribute(TermAttribute.class);
   }
 
+  @Override
   public boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
       final int newlen = normalizer.normalize(termAtt.termBuffer(), termAtt

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java Mon Nov  9 04:01:50 2009
@@ -19,11 +19,9 @@
 
 import java.io.IOException;
 import java.util.Set;
-import java.util.HashSet;
 import java.util.Arrays;
-import java.util.Iterator;
 import org.apache.lucene.analysis.standard.StandardTokenizer; // for javadocs
-import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.CharArraySet;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
@@ -37,17 +35,16 @@
  * @see <a href="http://fr.wikipedia.org/wiki/%C3%89lision">Elision in Wikipedia</a>
  */
 public final class ElisionFilter extends TokenFilter {
-  private Set articles = null;
+  private CharArraySet articles = null;
   private TermAttribute termAtt;
   
   private static char[] apostrophes = {'\'', '’'};
 
-  public void setArticles(Set articles) {
-    this.articles = new HashSet();
-    Iterator iter = articles.iterator();
-    while (iter.hasNext()) {
-      this.articles.add(((String) iter.next()).toLowerCase());
-    }
+  public void setArticles(Set<?> articles) {
+    if (articles instanceof CharArraySet)
+      this.articles = (CharArraySet) articles;
+    else
+      this.articles = new CharArraySet(articles, true);
   }
 
   /**
@@ -55,15 +52,15 @@
    */
   protected ElisionFilter(TokenStream input) {
     super(input);
-    this.articles = new HashSet(Arrays.asList(new String[] { "l", "m", "t",
-        "qu", "n", "s", "j" }));
+    this.articles = new CharArraySet(Arrays.asList(
+        "l", "m", "t", "qu", "n", "s", "j"), true);
     termAtt = addAttribute(TermAttribute.class);
   }
 
   /**
    * Constructs an elision filter with a Set of stop words
    */
-  public ElisionFilter(TokenStream input, Set articles) {
+  public ElisionFilter(TokenStream input, Set<?> articles) {
     super(input);
     setArticles(articles);
     termAtt = addAttribute(TermAttribute.class);
@@ -74,13 +71,14 @@
    */
   public ElisionFilter(TokenStream input, String[] articles) {
     super(input);
-    setArticles(new HashSet(Arrays.asList(articles)));
+    this.articles = new CharArraySet(Arrays.asList(articles), true);
     termAtt = addAttribute(TermAttribute.class);
   }
 
   /**
    * Increments the {@link TokenStream} with a {@link TermAttribute} without elisioned start
    */
+  @Override
   public final boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
       char[] termBuffer = termAtt.termBuffer();
@@ -100,7 +98,7 @@
 
       // An apostrophe has been found. If the prefix is an article strip it off.
       if (minPoz != Integer.MAX_VALUE
-          && articles.contains(new String(termAtt.termBuffer(), 0, minPoz).toLowerCase())) {
+          && articles.contains(termAtt.termBuffer(), 0, minPoz)) {
         termAtt.setTermBuffer(termAtt.termBuffer(), minPoz + 1, termAtt.termLength() - (minPoz + 1));
       }
 

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java Mon Nov  9 04:01:50 2009
@@ -155,6 +155,7 @@
    *         filtered with {@link StandardFilter}, {@link StopFilter}, 
    *         {@link FrenchStemFilter} and {@link LowerCaseFilter}
    */
+  @Override
   public final TokenStream tokenStream(String fieldName, Reader reader) {
     TokenStream result = new StandardTokenizer(matchVersion, reader);
     result = new StandardFilter(result);
@@ -179,6 +180,7 @@
    *         filtered with {@link StandardFilter}, {@link StopFilter}, 
    *         {@link FrenchStemFilter} and {@link LowerCaseFilter}
    */
+  @Override
   public TokenStream reusableTokenStream(String fieldName, Reader reader)
       throws IOException {
     SavedStreams streams = (SavedStreams) getPreviousTokenStream();

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java Mon Nov  9 04:01:50 2009
@@ -59,6 +59,7 @@
 	/**
 	 * @return  Returns true for the next token in the stream, or false at EOS
 	 */
+	@Override
 	public boolean incrementToken() throws IOException {
 	  if (input.incrementToken()) {
 	    String term = termAtt.term();

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/EmptyTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/EmptyTokenStream.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/EmptyTokenStream.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/EmptyTokenStream.java Mon Nov  9 04:01:50 2009
@@ -27,6 +27,7 @@
  */
 public final class EmptyTokenStream extends TokenStream {
 
+  @Override
   public final boolean incrementToken() throws IOException {
     return false;
   }

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAndSuffixAwareTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAndSuffixAwareTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAndSuffixAwareTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAndSuffixAwareTokenFilter.java Mon Nov  9 04:01:50 2009
@@ -35,11 +35,13 @@
   public PrefixAndSuffixAwareTokenFilter(TokenStream prefix, TokenStream input, TokenStream suffix) {
     super(suffix);
     prefix = new PrefixAwareTokenFilter(prefix, input) {
+      @Override
       public Token updateSuffixToken(Token suffixToken, Token lastInputToken) {
         return PrefixAndSuffixAwareTokenFilter.this.updateInputToken(suffixToken, lastInputToken);
       }
     };
     this.suffix = new PrefixAwareTokenFilter(prefix, suffix) {
+      @Override
       public Token updateSuffixToken(Token suffixToken, Token lastInputToken) {
         return PrefixAndSuffixAwareTokenFilter.this.updateSuffixToken(suffixToken, lastInputToken);
       }
@@ -59,15 +61,18 @@
   }
 
 
+  @Override
   public final boolean incrementToken() throws IOException {
     return suffix.incrementToken();
   }
 
+  @Override
   public void reset() throws IOException {
     suffix.reset();
   }
 
 
+  @Override
   public void close() throws IOException {
     suffix.close();
   }

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java Mon Nov  9 04:01:50 2009
@@ -84,6 +84,7 @@
 
   private boolean prefixExhausted;
 
+  @Override
   public final boolean incrementToken() throws IOException {
     if (!prefixExhausted) {
       Token nextToken = getNextPrefixInputToken(reusableToken);
@@ -156,11 +157,13 @@
     return suffixToken;
   }
 
+  @Override
   public void close() throws IOException {
     prefix.close();
     suffix.close();
   }
 
+  @Override
   public void reset() throws IOException {
     super.reset();
     if (prefix != null) {

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java Mon Nov  9 04:01:50 2009
@@ -45,6 +45,7 @@
     assert (tokenAtt instanceof Token);
   }
 
+  @Override
   public final boolean incrementToken() throws IOException {
     if (exhausted) {
       return false;
@@ -56,6 +57,7 @@
     }
   }
 
+  @Override
   public void reset() throws IOException {
     exhausted = false;
   }

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java Mon Nov  9 04:01:50 2009
@@ -39,10 +39,16 @@
   public static enum Side {
 
     /** Get the n-gram from the front of the input */
-    FRONT { public String getLabel() { return "front"; } },
+    FRONT {
+      @Override
+      public String getLabel() { return "front"; }
+    },
 
     /** Get the n-gram from the end of the input */
-    BACK  { public String getLabel() { return "back"; } };
+    BACK  {
+      @Override
+      public String getLabel() { return "back"; }
+    };
 
     public abstract String getLabel();
 
@@ -117,6 +123,7 @@
     this(input, Side.getSide(sideLabel), minGram, maxGram);
   }
 
+  @Override
   public final boolean incrementToken() throws IOException {
     while (true) {
       if (curTermBuffer == null) {
@@ -144,6 +151,7 @@
     }
   }
 
+  @Override
   public void reset() throws IOException {
     super.reset();
     curTermBuffer = null;

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java Mon Nov  9 04:01:50 2009
@@ -44,10 +44,16 @@
   public static enum Side {
 
     /** Get the n-gram from the front of the input */
-    FRONT { public String getLabel() { return "front"; } },
+    FRONT {
+      @Override
+      public String getLabel() { return "front"; }
+    },
 
     /** Get the n-gram from the end of the input */
-    BACK  { public String getLabel() { return "back"; } };
+    BACK  {
+      @Override
+      public String getLabel() { return "back"; }
+    };
 
     public abstract String getLabel();
 
@@ -174,14 +180,15 @@
   }
 
   /** Returns the next token in the stream, or null at EOS. */
+  @Override
   public final boolean incrementToken() throws IOException {
     clearAttributes();
     // if we are just starting, read the whole input
     if (!started) {
       started = true;
       char[] chars = new char[1024];
-      input.read(chars);
-      inStr = new String(chars).trim();  // remove any leading or trailing spaces
+      int charsRead = input.read(chars);
+      inStr = new String(chars, 0, charsRead).trim();  // remove any leading or trailing spaces
       inLen = inStr.length();
       gramSize = minGram;
     }
@@ -205,17 +212,20 @@
     return true;
   }
   
+  @Override
   public final void end() {
     // set final offset
     final int finalOffset = inLen;
     this.offsetAtt.setOffset(finalOffset, finalOffset);
   }    
 
+  @Override
   public void reset(Reader input) throws IOException {
     super.reset(input);
     reset();
   }
 
+  @Override
   public void reset() throws IOException {
     super.reset();
     started = false;

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java Mon Nov  9 04:01:50 2009
@@ -72,6 +72,7 @@
   }
 
   /** Returns the next token in the stream, or null at EOS. */
+  @Override
   public final boolean incrementToken() throws IOException {
     while (true) {
       if (curTermBuffer == null) {
@@ -98,6 +99,7 @@
     }
   }
 
+  @Override
   public void reset() throws IOException {
     super.reset();
     curTermBuffer = null;

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java Mon Nov  9 04:01:50 2009
@@ -101,6 +101,7 @@
   }
 
   /** Returns the next token in the stream, or null at EOS. */
+  @Override
   public final boolean incrementToken() throws IOException {
     clearAttributes();
     if (!started) {
@@ -128,17 +129,20 @@
     return true;
   }
   
+  @Override
   public final void end() {
     // set final offset
     final int finalOffset = inLen;
     this.offsetAtt.setOffset(finalOffset, finalOffset);
   }    
   
+  @Override
   public void reset(Reader input) throws IOException {
     super.reset(input);
     reset();
   }
 
+  @Override
   public void reset() throws IOException {
     super.reset();
     started = false;

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java Mon Nov  9 04:01:50 2009
@@ -188,6 +188,7 @@
    *   filtered with {@link StandardFilter}, {@link StopFilter}, 
    *   and {@link DutchStemFilter}
    */
+  @Override
   public TokenStream tokenStream(String fieldName, Reader reader) {
     TokenStream result = new StandardTokenizer(matchVersion, reader);
     result = new StandardFilter(result);
@@ -210,6 +211,7 @@
    *   filtered with {@link StandardFilter}, {@link StopFilter}, 
    *   and {@link DutchStemFilter}
    */
+  @Override
   public TokenStream reusableTokenStream(String fieldName, Reader reader)
       throws IOException {
     if (overridesTokenStreamMethod) {

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java Mon Nov  9 04:01:50 2009
@@ -69,6 +69,7 @@
   /**
    * Returns the next token in the stream, or null at EOS
    */
+  @Override
   public boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
       String term = termAtt.term();

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilter.java Mon Nov  9 04:01:50 2009
@@ -59,6 +59,7 @@
     this.encoder = encoder;
   }
 
+  @Override
   public boolean incrementToken() throws IOException {
     boolean result = false;
     if (input.incrementToken()) {

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java Mon Nov  9 04:01:50 2009
@@ -48,6 +48,7 @@
     typeAtt = addAttribute(TypeAttribute.class);
   }
 
+  @Override
   public final boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
       if (typeAtt.type().equals(typeMatch))

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java Mon Nov  9 04:01:50 2009
@@ -43,6 +43,7 @@
     payAtt = addAttribute(PayloadAttribute.class);
   }
 
+  @Override
   public final boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
       byte[] data = new byte[8];

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java Mon Nov  9 04:01:50 2009
@@ -44,6 +44,7 @@
   }
 
 
+  @Override
   public final boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
       String type = typeAtt.type();

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java Mon Nov  9 04:01:50 2009
@@ -62,6 +62,7 @@
     this.positionIncrement = positionIncrement;
   }
 
+  @Override
   public final boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
       if (firstTokenPositioned) {
@@ -75,6 +76,7 @@
     }
   }
 
+  @Override
   public void reset() throws IOException {
     super.reset();
     firstTokenPositioned = false;

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java Mon Nov  9 04:01:50 2009
@@ -172,6 +172,7 @@
     return stopWords.size();
   }
 
+  @Override
   public TokenStream tokenStream(String fieldName, Reader reader) {
     TokenStream result;
     try {
@@ -198,6 +199,7 @@
     TokenStream withStopFilter;
   };
   
+  @Override
   public TokenStream reusableTokenStream(String fieldName, Reader reader)
       throws IOException {
     if (overridesTokenStreamMethod) {

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilter.java Mon Nov  9 04:01:50 2009
@@ -88,6 +88,7 @@
     termAtt = addAttribute(TermAttribute.class);
   }
 
+  @Override
   public boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
       int len = termAtt.termLength();

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java Mon Nov  9 04:01:50 2009
@@ -97,6 +97,7 @@
      *   {@link RussianLowerCaseFilter}, {@link StopFilter}, 
      *   and {@link RussianStemFilter}
      */
+    @Override
     public TokenStream tokenStream(String fieldName, Reader reader)
     {
         TokenStream result = new RussianLetterTokenizer(reader);
@@ -121,6 +122,7 @@
      *   {@link RussianLowerCaseFilter}, {@link StopFilter}, 
      *   and {@link RussianStemFilter}
      */
+    @Override
     public TokenStream reusableTokenStream(String fieldName, Reader reader) 
       throws IOException {
     SavedStreams streams = (SavedStreams) getPreviousTokenStream();

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianLetterTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianLetterTokenizer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianLetterTokenizer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianLetterTokenizer.java Mon Nov  9 04:01:50 2009
@@ -49,6 +49,7 @@
      * Collects only characters which satisfy
      * {@link Character#isLetter(char)}.
      */
+    @Override
     protected boolean isTokenChar(char c)
     {
         if (Character.isLetter(c) || (c >= '0' && c <= '9'))

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java Mon Nov  9 04:01:50 2009
@@ -51,6 +51,7 @@
     /**
      * Returns the next token in the stream, or null at EOS
      */
+    @Override
     public final boolean incrementToken() throws IOException
     {
       if (input.incrementToken()) {

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapper.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapper.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapper.java Mon Nov  9 04:01:50 2009
@@ -98,6 +98,7 @@
     this.outputUnigrams = outputUnigrams;
   }
 
+  @Override
   public TokenStream tokenStream(String fieldName, Reader reader) {
     TokenStream wrapped;
     try {
@@ -116,6 +117,7 @@
     ShingleFilter shingle;
   };
   
+  @Override
   public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
     if (overridesTokenStreamMethod) {
       // LUCENE-1678: force fallback to tokenStream() if we

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java Mon Nov  9 04:01:50 2009
@@ -163,6 +163,7 @@
   /* (non-Javadoc)
    * @see org.apache.lucene.analysis.TokenStream#next()
    */
+  @Override
   public final boolean incrementToken() throws IOException {
     while (true) {
       if (nextToken == null) {
@@ -328,6 +329,7 @@
     return true;
   }
 
+  @Override
   public void reset() throws IOException {
     super.reset();
     nextToken = null;

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java Mon Nov  9 04:01:50 2009
@@ -116,7 +116,7 @@
  */
 public final class ShingleMatrixFilter extends TokenStream {
 
-  public static Character defaultSpacerCharacter = new Character('_');
+  public static Character defaultSpacerCharacter = Character.valueOf('_');
   public static TokenSettingsCodec defaultSettingsCodec = new OneDimensionalNonWeightedTokenSettingsCodec();
   public static boolean ignoringSinglePrefixOrSuffixShingleByDefault = false;
 
@@ -345,6 +345,7 @@
   private Set<List<Token>> shinglesSeen = new HashSet<List<Token>>();
 
 
+  @Override
   public void reset() throws IOException {
     permutations = null;
     shinglesSeen.clear();
@@ -355,6 +356,7 @@
 
   private Token reusableToken = new Token();
 
+  @Override
   public final boolean incrementToken() throws IOException {
     if (matrix == null) {
       matrix = new Matrix();
@@ -743,6 +745,7 @@
         return Matrix.this.columns.indexOf(this);
       }
 
+      @Override
       public String toString() {
         return "Column{" +
             "first=" + first +
@@ -807,6 +810,7 @@
 //          return ret;
 //        }
 
+        @Override
         public String toString() {
           return "Row{" +
               "index=" + getIndex() +
@@ -863,6 +867,7 @@
       };
     }
 
+    @Override
     public String toString() {
       return "Matrix{" +
           "columns=" + columns +
@@ -920,17 +925,21 @@
    */
   public static class OneDimensionalNonWeightedTokenSettingsCodec extends TokenSettingsCodec {
 
+    @Override
     public TokenPositioner getTokenPositioner(Token token) throws IOException {
       return TokenPositioner.newColumn;
     }
 
+    @Override
     public void setTokenPositioner(Token token, TokenPositioner tokenPositioner) {
     }
 
+    @Override
     public float getWeight(Token token) {
       return 1f;
     }
 
+    @Override
     public void setWeight(Token token, float weight) {
     }
 
@@ -944,6 +953,7 @@
    */
   public static class TwoDimensionalNonWeightedSynonymTokenSettingsCodec extends TokenSettingsCodec {
 
+    @Override
     public TokenPositioner getTokenPositioner(Token token) throws IOException {
       if (token.getPositionIncrement() == 0) {
         return TokenPositioner.newRow;
@@ -952,14 +962,17 @@
       }
     }
 
+    @Override
     public void setTokenPositioner(Token token, TokenPositioner tokenPositioner) {
       throw new UnsupportedOperationException();
     }
 
+    @Override
     public float getWeight(Token token) {
       return 1f;
     }
 
+    @Override
     public void setWeight(Token token, float weight) {
     }
 
@@ -981,6 +994,7 @@
      * @return the token flags int value as TokenPosition
      * @throws IOException
      */
+    @Override
     public TokenPositioner getTokenPositioner(Token token) throws IOException {
       switch (token.getFlags()) {
         case 0:
@@ -999,6 +1013,7 @@
      * @param token
      * @param tokenPositioner
      */
+    @Override
     public void setTokenPositioner(Token token, TokenPositioner tokenPositioner) {
       token.setFlags(tokenPositioner.getIndex());
     }
@@ -1009,6 +1024,7 @@
      * @param token
      * @return 32 bit float
      */
+    @Override
     public float getWeight(Token token) {
       if (token.getPayload() == null || token.getPayload().getData() == null) {
         return 1f;
@@ -1022,6 +1038,7 @@
      * @param token
      * @param weight
      */
+    @Override
     public void setWeight(Token token, float weight) {
       if (weight == 1f) {
         token.setPayload(null);

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkFilter.java Mon Nov  9 04:01:50 2009
@@ -49,6 +49,7 @@
     this.dateFormat = dateFormat; 
   }
 
+  @Override
   public boolean accept(AttributeSource source) {
     if (termAtt == null) {
       termAtt = source.addAttribute(TermAttribute.class);

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/TokenRangeSinkFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/TokenRangeSinkFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/TokenRangeSinkFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/TokenRangeSinkFilter.java Mon Nov  9 04:01:50 2009
@@ -37,6 +37,7 @@
   }
 
   
+  @Override
   public boolean accept(AttributeSource source) {
     try {
       if (count >= lower && count < upper){
@@ -48,6 +49,7 @@
     }
   }
 
+  @Override
   public void reset() throws IOException {
     count = 0;
   }

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/TokenTypeSinkFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/TokenTypeSinkFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/TokenTypeSinkFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/TokenTypeSinkFilter.java Mon Nov  9 04:01:50 2009
@@ -29,6 +29,7 @@
     this.typeToMatch = typeToMatch;
   }
 
+  @Override
   public boolean accept(AttributeSource source) {
     if (typeAtt == null) {
       typeAtt = source.addAttribute(TypeAttribute.class);

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java Mon Nov  9 04:01:50 2009
@@ -43,6 +43,7 @@
     this.matchVersion = matchVersion;
   }
   
+  @Override
   public TokenStream tokenStream(String fieldName, Reader reader) {
     TokenStream ts = new StandardTokenizer(matchVersion, reader);
     ts = new StandardFilter(ts);
@@ -57,6 +58,7 @@
     TokenStream result;
   };
   
+  @Override
   public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
     if (overridesTokenStreamMethod) {
       // LUCENE-1678: force fallback to tokenStream() if we

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java Mon Nov  9 04:01:50 2009
@@ -48,6 +48,7 @@
     offsetAtt = addAttribute(OffsetAttribute.class);
   }
   
+  @Override
   public final boolean incrementToken() throws IOException {
     if (thaiState != null) {
       int start = breaker.current();
@@ -82,6 +83,7 @@
     return false;
   }
   
+  @Override
   public void reset() throws IOException {
     super.reset();
     thaiState = null;

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java Mon Nov  9 04:01:50 2009
@@ -65,6 +65,7 @@
      * convenience to show the behavior of the tokenizer
      */
     private class JustChineseTokenizerAnalyzer extends Analyzer {
+      @Override
       public TokenStream tokenStream(String fieldName, Reader reader) {
         return new ChineseTokenizer(reader);
       }   
@@ -75,6 +76,7 @@
      * convenience to show the behavior of the filter.
      */
     private class JustChineseFilterAnalyzer extends Analyzer {
+      @Override
       public TokenStream tokenStream(String fieldName, Reader reader) {
         return new ChineseFilter(new WhitespaceTokenizer(reader));
       }

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java Mon Nov  9 04:01:50 2009
@@ -51,6 +51,7 @@
 
   private static byte[] patternsFileContent;
 
+  @Override
   protected void setUp() throws Exception {
     super.setUp();
     getHyphenationPatternFileContents();

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java Mon Nov  9 04:01:50 2009
@@ -51,6 +51,7 @@
    * An input stream that always throws IOException for testing.
    */
   private class UnreliableInputStream extends InputStream {
+    @Override
     public int read() throws IOException {
       throw new IOException();
     }

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java Mon Nov  9 04:01:50 2009
@@ -76,6 +76,7 @@
       super(matchVersion);
     }
 
+    @Override
     public TokenStream tokenStream(String fieldName, Reader reader) {
       return new WhitespaceTokenizer(reader);
     }

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java Mon Nov  9 04:01:50 2009
@@ -29,6 +29,7 @@
 public class EdgeNGramTokenFilterTest extends BaseTokenStreamTestCase {
   private TokenStream input;
 
+  @Override
   public void setUp() throws Exception {
     super.setUp();
     input = new WhitespaceTokenizer(new StringReader("abcde"));

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java Mon Nov  9 04:01:50 2009
@@ -28,6 +28,7 @@
 public class EdgeNGramTokenizerTest extends BaseTokenStreamTestCase {
   private StringReader input;
 
+  @Override
   public void setUp() throws Exception {
     super.setUp();
     input = new StringReader("abcde");

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java Mon Nov  9 04:01:50 2009
@@ -30,6 +30,7 @@
 public class NGramTokenFilterTest extends BaseTokenStreamTestCase {
     private TokenStream input;
     
+    @Override
     public void setUp() throws Exception {
         super.setUp();
         input = new WhitespaceTokenizer(new StringReader("abcde"));

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java Mon Nov  9 04:01:50 2009
@@ -30,6 +30,7 @@
 public class NGramTokenizerTest extends BaseTokenStreamTestCase {
     private StringReader input;
     
+    @Override
     public void setUp() throws Exception {
         super.setUp();
         input = new StringReader("abcde");

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/nl/TestDutchStemmer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/nl/TestDutchStemmer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/nl/TestDutchStemmer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/nl/TestDutchStemmer.java Mon Nov  9 04:01:50 2009
@@ -134,6 +134,7 @@
     public DutchSubclassAnalyzer(Version matchVersion) {
       super(matchVersion);
     }
+    @Override
     public TokenStream tokenStream(String fieldName, Reader reader) {
       return new WhitespaceTokenizer(reader);
     }

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java Mon Nov  9 04:01:50 2009
@@ -69,6 +69,7 @@
       typeAtt = addAttribute(TypeAttribute.class);
     }
     
+    @Override
     public boolean incrementToken() throws IOException {
       if (input.incrementToken()) {
         if (termAtt.term().equals("dogs"))

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java Mon Nov  9 04:01:50 2009
@@ -65,6 +65,7 @@
       typeAtt = addAttribute(TypeAttribute.class);
     }
 
+    @Override
     public boolean incrementToken() throws IOException {
       if (input.incrementToken()) {
         typeAtt.setType(String.valueOf(Character.toUpperCase(termAtt.termBuffer()[0])));

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/position/PositionFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/position/PositionFilterTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/position/PositionFilterTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/position/PositionFilterTest.java Mon Nov  9 04:01:50 2009
@@ -38,6 +38,7 @@
       termAtt = addAttribute(TermAttribute.class);
     }
 
+    @Override
     public final boolean incrementToken() throws IOException {
       clearAttributes();
       if (index < testToken.length) {
@@ -47,6 +48,7 @@
         return false;
       }
     }
+    @Override
     public void reset() {
       index = 0;
     }

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java Mon Nov  9 04:01:50 2009
@@ -47,6 +47,7 @@
   IndexReader reader;
   QueryAutoStopWordAnalyzer protectedAnalyzer;
 
+  @Override
   protected void setUp() throws Exception {
     super.setUp();
     dir = new RAMDirectory();
@@ -66,6 +67,7 @@
     protectedAnalyzer = new QueryAutoStopWordAnalyzer(Version.LUCENE_CURRENT, appAnalyzer);
   }
 
+  @Override
   protected void tearDown() throws Exception {
     reader.close();
     super.tearDown();
@@ -154,6 +156,7 @@
       super(matchVersion, new WhitespaceAnalyzer());
     }
     
+    @Override
     public TokenStream tokenStream(String fieldName, Reader reader) {
       return new WhitespaceTokenizer(reader);
     }    
@@ -172,6 +175,7 @@
    */
   private class NonreusableAnalyzer extends Analyzer {
     int invocationCount = 0;
+    @Override
     public TokenStream tokenStream(String fieldName, Reader reader) {
       if (++invocationCount % 2 == 0)
         return new WhitespaceTokenizer(reader);

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java Mon Nov  9 04:01:50 2009
@@ -42,6 +42,7 @@
 
     private File dataDir;
 
+    @Override
     protected void setUp() throws Exception
     {
       super.setUp();

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianStem.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianStem.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianStem.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianStem.java Mon Nov  9 04:01:50 2009
@@ -37,6 +37,7 @@
     /**
      * @see TestCase#setUp()
      */
+    @Override
     protected void setUp() throws Exception
     {
         super.setUp();
@@ -73,6 +74,7 @@
     /**
      * @see TestCase#tearDown()
      */
+    @Override
     protected void tearDown() throws Exception
     {
         super.tearDown();

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java Mon Nov  9 04:01:50 2009
@@ -221,6 +221,7 @@
       super(org.apache.lucene.util.Version.LUCENE_CURRENT);
     }
   
+    @Override
     public TokenStream tokenStream(String fieldName, Reader reader) {
       return new WhitespaceTokenizer(reader);
     }  
@@ -240,6 +241,7 @@
    */
   private class NonreusableAnalyzer extends Analyzer {
     int invocationCount = 0;
+    @Override
     public TokenStream tokenStream(String fieldName, Reader reader) {
       if (++invocationCount % 2 == 0)
         return new WhitespaceTokenizer(reader);

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java Mon Nov  9 04:01:50 2009
@@ -48,6 +48,7 @@
       this.typeAtt = addAttribute(TypeAttribute.class);
     }
 
+    @Override
     public final boolean incrementToken() throws IOException {
       clearAttributes();
       if (index < testToken.length) {
@@ -201,6 +202,7 @@
   };
 
 
+  @Override
   protected void setUp() throws Exception {
     super.setUp();
     testTokenWithHoles = new Token[] {

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java Mon Nov  9 04:01:50 2009
@@ -537,6 +537,7 @@
 
     private Iterator iterator;
 
+    @Override
     public boolean incrementToken() throws IOException {
       if (iterator == null) {
         iterator = tokens.iterator();
@@ -556,6 +557,7 @@
     }
 
 
+    @Override
     public void reset() throws IOException {
       iterator = null;
     }

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java Mon Nov  9 04:01:50 2009
@@ -76,6 +76,7 @@
       typeAtt = addAttribute(TypeAttribute.class);
     }
 
+    @Override
     public final boolean incrementToken() throws IOException {
       if (!input.incrementToken()) return false;
       

Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java Mon Nov  9 04:01:50 2009
@@ -90,6 +90,26 @@
 			new String[] { "ประโยค", "ว่า", "quick", "brown", "fox", "jumped", "over", "lazy", "dogs" });
 	}
 	
+	/*
+	 * Test that position increments are adjusted correctly for stopwords.
+	 */
+	public void testPositionIncrements() throws Exception {
+	  ThaiAnalyzer analyzer = new ThaiAnalyzer(Version.LUCENE_CURRENT);
+
+	  assertAnalyzesTo(analyzer, "ประโยคว่า the ประโยคว่า",
+	          new String[] { "ประโยค", "ว่า", "ประโยค", "ว่า" },
+	          new int[] { 0, 6, 14, 20 },
+	          new int[] { 6, 9, 20, 23 },
+	          new int[] { 1, 1, 2, 1 });
+	 
+	  // case that a stopword is adjacent to thai text, with no whitespace
+	  assertAnalyzesTo(analyzer, "ประโยคว่าtheประโยคว่า",
+	      new String[] { "ประโยค", "ว่า", "ประโยค", "ว่า" },
+	      new int[] { 0, 6, 12, 18 },
+	      new int[] { 6, 9, 18, 21 },
+	      new int[] { 1, 1, 2, 1 });
+	}
+	
 	public void testReusableTokenStream() throws Exception {
 	  ThaiAnalyzer analyzer = new ThaiAnalyzer(Version.LUCENE_CURRENT);
 	  assertAnalyzesToReuse(analyzer, "", new String[] {});
@@ -112,6 +132,7 @@
           public ThaiSubclassAnalyzer(Version matchVersion) {
             super(matchVersion);
           }
+	  @Override
 	  public TokenStream tokenStream(String fieldName, Reader reader) {
 	    return new WhitespaceTokenizer(reader);
 	  }

Modified: lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java Mon Nov  9 04:01:50 2009
@@ -73,6 +73,7 @@
     typeAtt = addAttribute(TypeAttribute.class);    
   }
   
+  @Override
   public boolean incrementToken() throws IOException {
     clearAttributes();
     buffer.setLength(0);
@@ -122,11 +123,13 @@
     }
   }
 
+  @Override
   public void reset() throws IOException {
     super.reset();
     tokenStart = tokenEnd = 0;
   }
 
+  @Override
   public void reset(Reader input) throws IOException {
     super.reset(input);
     reset();

Modified: lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java Mon Nov  9 04:01:50 2009
@@ -60,7 +60,7 @@
  */
 public class SmartChineseAnalyzer extends Analyzer {
 
-  private final Set stopWords;
+  private final Set<?> stopWords;
   
   private static final String DEFAULT_STOPWORD_FILE = "stopwords.txt";
   
@@ -144,6 +144,7 @@
     this.matchVersion = matchVersion;
   }
 
+  @Override
   public TokenStream tokenStream(String fieldName, Reader reader) {
     TokenStream result = new SentenceTokenizer(reader);
     result = new WordTokenFilter(result);
@@ -163,6 +164,7 @@
     TokenStream filteredTokenStream;
   }
   
+  @Override
   public TokenStream reusableTokenStream(String fieldName, Reader reader)
       throws IOException {
     SavedStreams streams = (SavedStreams) getPreviousTokenStream();

Modified: lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordSegmenter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordSegmenter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordSegmenter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordSegmenter.java Mon Nov  9 04:01:50 2009
@@ -17,7 +17,7 @@
 
 package org.apache.lucene.analysis.cn.smart;
 
-import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 import org.apache.lucene.analysis.cn.smart.hhmm.HHMMSegmenter;
@@ -45,18 +45,19 @@
    * @param startOffset start offset of sentence
    * @return {@link List} of {@link SegToken}
    */
-  public List segmentSentence(String sentence, int startOffset) {
-
-    List segTokenList = hhmmSegmenter.process(sentence);
-
-    List result = new ArrayList();
+  public List<SegToken> segmentSentence(String sentence, int startOffset) {
 
+    List<SegToken> segTokenList = hhmmSegmenter.process(sentence);
     // tokens from sentence, excluding WordType.SENTENCE_BEGIN and WordType.SENTENCE_END
-    for (int i = 1; i < segTokenList.size() - 1; i++) {
-      result.add(convertSegToken((SegToken) segTokenList.get(i), sentence, startOffset));
-    }
+    List<SegToken> result = Collections.emptyList();
+    
+    if (segTokenList.size() > 2) // if its not an empty sentence
+      result = segTokenList.subList(1, segTokenList.size() - 1);
+    
+    for (SegToken st : result)
+      convertSegToken(st, sentence, startOffset);
+    
     return result;
-
   }
 
   /**

Modified: lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java Mon Nov  9 04:01:50 2009
@@ -40,9 +40,9 @@
 
   private WordSegmenter wordSegmenter;
 
-  private Iterator tokenIter;
+  private Iterator<SegToken> tokenIter;
 
-  private List tokenBuffer;
+  private List<SegToken> tokenBuffer;
   
   private TermAttribute termAtt;
   private OffsetAttribute offsetAtt;
@@ -61,6 +61,7 @@
     typeAtt = addAttribute(TypeAttribute.class);
   }
   
+  @Override
   public boolean incrementToken() throws IOException {   
     if (tokenIter == null || !tokenIter.hasNext()) {
       // there are no remaining tokens from the current sentence... are there more sentences?
@@ -78,15 +79,17 @@
         return false; // no more sentences, end of stream!
       }
     } 
-    
+    // WordTokenFilter must clear attributes, as it is creating new tokens.
+    clearAttributes();
     // There are remaining tokens from the current sentence, return the next one. 
-    SegToken nextWord = (SegToken) tokenIter.next();
+    SegToken nextWord = tokenIter.next();
     termAtt.setTermBuffer(nextWord.charArray, 0, nextWord.charArray.length);
     offsetAtt.setOffset(nextWord.startOffset, nextWord.endOffset);
     typeAtt.setType("word");
     return true;
   }
 
+  @Override
   public void reset() throws IOException {
     super.reset();
     tokenIter = null;



Mime
View raw message