lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From o...@apache.org
Subject cvs commit: jakarta-lucene/src/test/org/apache/lucene/analysis TestStopAnalyzer.java
Date Thu, 05 Feb 2004 13:45:57 GMT
otis        2004/02/05 05:45:57

  Modified:    src/test/org/apache/lucene/analysis TestStopAnalyzer.java
  Log:
  - Applied a patch from Grant Ingersoll
  
  Revision  Changes    Path
  1.3       +62 -46    jakarta-lucene/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java
  
  Index: TestStopAnalyzer.java
  ===================================================================
  RCS file: /home/cvs/jakarta-lucene/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java,v
  retrieving revision 1.2
  retrieving revision 1.3
  diff -u -r1.2 -r1.3
  --- TestStopAnalyzer.java	8 Dec 2003 16:16:32 -0000	1.2
  +++ TestStopAnalyzer.java	5 Feb 2004 13:45:55 -0000	1.3
  @@ -1,60 +1,76 @@
   package org.apache.lucene.analysis;
   
  +/**
  + * Copyright 2004 The Apache Software Foundation
  + *
  + * Licensed under the Apache License, Version 2.0 (the "License");
  + * you may not use this file except in compliance with the License.
  + * You may obtain a copy of the License at
  + *
  + *     http://www.apache.org/licenses/LICENSE-2.0
  + *
  + * Unless required by applicable law or agreed to in writing, software
  + * distributed under the License is distributed on an "AS IS" BASIS,
  + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  + * See the License for the specific language governing permissions and
  + * limitations under the License.
  + */
  +
   import junit.framework.TestCase;
  +
   import java.io.StringReader;
  -import java.util.ArrayList;
  -import org.apache.lucene.index.IndexWriter;
  -import org.apache.lucene.index.Term;
  -import org.apache.lucene.store.RAMDirectory;
  -import org.apache.lucene.document.Document;
  -import org.apache.lucene.document.Field;
  -import org.apache.lucene.search.IndexSearcher;
  -import org.apache.lucene.search.PhraseQuery;
  -import org.apache.lucene.search.Hits;
  +import java.io.IOException;
  +import java.util.Set;
  +import java.util.HashSet;
   
   public class TestStopAnalyzer extends TestCase {
  -  private StopAnalyzer stopAnalyzer = new StopAnalyzer();
  +  private StopAnalyzer stop = new StopAnalyzer();
   
  -  public Token[] tokensFromAnalyzer(Analyzer analyzer, String text)
  -                                                  throws Exception {
  -    TokenStream stream =
  -      analyzer.tokenStream("contents", new StringReader(text));
  -    ArrayList tokenList = new ArrayList();
  -    while (true) {
  -      Token token = stream.next();
  -      if (token == null) break;
  +  private Set inValidTokens = new HashSet();
  +  public TestStopAnalyzer(String s) {
  +    super(s);
  +  }
   
  -      tokenList.add(token);
  +  protected void setUp() {
  +    for (int i = 0; i < StopAnalyzer.ENGLISH_STOP_WORDS.length; i++) {
  +      inValidTokens.add(StopAnalyzer.ENGLISH_STOP_WORDS[i]);
       }
  -
  -    return (Token[]) tokenList.toArray(new Token[0]);
     }
   
  +  public void testDefaults() {
  +    assertTrue(stop != null);
  +    StringReader reader = new StringReader("This is a test of the english stop analyzer");
  +    TokenStream stream = stop.tokenStream("test", reader);
  +    assertTrue(stream != null);
  +    Token token = null;
  +    try {
  +      while ((token = stream.next()) != null)
  +      {
  +        assertTrue(inValidTokens.contains(token.termText()) == false);
  +      }
  +    } catch (IOException e) {
  +      assertTrue(false);
  +    }
  +  }
   
  -  public void testPhraseQuery() throws Exception {
  -    RAMDirectory directory = new RAMDirectory();
  -    IndexWriter writer = new IndexWriter(directory, stopAnalyzer, true);
  -    Document doc = new Document();
  -    doc.add(Field.Text("field", "the stop words are here"));
  -    writer.addDocument(doc);
  -    writer.close();
  -
  -    IndexSearcher searcher = new IndexSearcher(directory);
  -
  -    // valid exact phrase query
  -    PhraseQuery query = new PhraseQuery();
  -    query.add(new Term("field","stop"));
  -    query.add(new Term("field","words"));
  -    Hits hits = searcher.search(query);
  -    assertEquals(1, hits.length());
  -
  -    // currently StopAnalyzer does not leave "holes", so this matches.
  -    query = new PhraseQuery();
  -    query.add(new Term("field", "words"));
  -    query.add(new Term("field", "here"));
  -    hits = searcher.search(query);
  -    assertEquals(1, hits.length());
  -
  -    searcher.close();
  +  public void testStopList() {
  +    Set stopWordsSet = new HashSet();
  +    stopWordsSet.add("good");
  +    stopWordsSet.add("test");
  +    stopWordsSet.add("analyzer");
  +    StopAnalyzer newStop = new StopAnalyzer((String[])stopWordsSet.toArray(new String[3]));
  +    StringReader reader = new StringReader("This is a good test of the english stop analyzer");
  +    TokenStream stream = newStop.tokenStream("test", reader);
  +    assertTrue(stream != null);
  +    Token token = null;
  +    try {
  +      while ((token = stream.next()) != null)
  +      {
  +        String text = token.termText();
  +        assertTrue(stopWordsSet.contains(text) == false);
  +      }
  +    } catch (IOException e) {
  +      assertTrue(false);
  +    }
     }
   }
  
  
  

---------------------------------------------------------------------
To unsubscribe, e-mail: lucene-dev-unsubscribe@jakarta.apache.org
For additional commands, e-mail: lucene-dev-help@jakarta.apache.org


Mime
View raw message