lucene-java-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Erik Hatcher <e...@ehatchersolutions.com>
Subject Re: implementing a TokenFilter for aliases
Date Thu, 04 Dec 2003 23:22:24 GMT
On Thursday, December 4, 2003, at 05:00  PM, Allen Atamer wrote:
> This is the code that I have so far for the next Method within 
> AliasFilter.
> After reading some posts, I also got the idea to call
> setPositionIncrement(). Neither way works, because when I search for 
> the
> alias, no search results come back.

Rather than trying to analyze your code, rather I'm providing you with 
a utility class that you can use to see what the analysis process is 
doing.

Use the displayTokensWithPositions method using your analyzer on text 
you're having issues with and let us know the results.

	Erik


package lia.analysis;

import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.Analyzer;
import java.io.StringReader;
import java.io.IOException;
import java.util.ArrayList;

public class AnalyzerUtils {
   public static Token[] tokensFromAnalysis
                (Analyzer analyzer, String text) throws IOException {
     TokenStream stream =
            analyzer.tokenStream("contents", new StringReader(text));
     ArrayList tokenList = new ArrayList();
     while (true) {
       Token token = stream.next();
       if (token == null) break;

       tokenList.add(token);
     }

     return (Token[]) tokenList.toArray(new Token[0]);
   }

   public static void displayTokens
                (Analyzer analyzer, String text) throws IOException {
     Token[] tokens = tokensFromAnalysis(analyzer, text);

     for (int i = 0; i < tokens.length; i++) {
       Token token = tokens[i];

       System.out.print("[" + token.termText() + "] ");
     }
   }

   public static void displayTokensWithPositions
                (Analyzer analyzer, String text) throws IOException {
     Token[] tokens = tokensFromAnalysis(analyzer, text);

     int position = 0;

     for (int i = 0; i < tokens.length; i++) {
       Token token = tokens[i];

       int increment = token.getPositionIncrement();

       if (increment > 0) {
         position = position + increment;
         System.out.println();
         System.out.print(position + ": ");
       }

       System.out.print("[" + token.termText() + "] ");
     }
   }

}


---------------------------------------------------------------------
To unsubscribe, e-mail: lucene-user-unsubscribe@jakarta.apache.org
For additional commands, e-mail: lucene-user-help@jakarta.apache.org


Mime
View raw message