lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rm...@apache.org
Subject svn commit: r1102290 - in /lucene/dev/trunk/lucene: contrib/highlighter/src/java/org/apache/lucene/search/highlight/ contrib/highlighter/src/test/org/apache/lucene/search/highlight/ contrib/queries/src/java/org/apache/lucene/search/ contrib/queries/src...
Date Thu, 12 May 2011 13:32:33 GMT
Author: rmuir
Date: Thu May 12 13:32:32 2011
New Revision: 1102290

URL: http://svn.apache.org/viewvc?rev=1102290&view=rev
Log:
LUCENE-3064: add checks to MockTokenizer to enforce proper consumption

Modified:
    lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java
    lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
    lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java
    lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
    lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
    lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java
    lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java
    lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java
    lucene/dev/trunk/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java
    lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java
    lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java
    lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java
    lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java
    lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
    lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java
    lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java
    lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLongPostings.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java

Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java
(original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java
Thu May 12 13:32:32 2011
@@ -355,6 +355,7 @@ public class Highlighter
 			{
 				try
 				{
+				  tokenStream.end();
 					tokenStream.close();
 				}
 				catch (Exception e)

Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
(original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
Thu May 12 13:32:32 2011
@@ -1093,6 +1093,10 @@ public class HighlighterTest extends Bas
   }
 
   public void testMaxSizeHighlight() throws Exception {
+    final MockAnalyzer analyzer = new MockAnalyzer(random, MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET,
true);
+    // we disable MockTokenizer checks because we will forcefully limit the 
+    // tokenstream and call end() before incrementToken() returns false.
+    analyzer.setEnableChecks(false);
     TestHighlightRunner helper = new TestHighlightRunner() {
 
       @Override
@@ -1122,7 +1126,10 @@ public class HighlighterTest extends Bas
       public void run() throws Exception {
         String goodWord = "goodtoken";
         CharacterRunAutomaton stopWords = new CharacterRunAutomaton(BasicAutomata.makeString("stoppedtoken"));
-
+        // we disable MockTokenizer checks because we will forcefully limit the 
+        // tokenstream and call end() before incrementToken() returns false.
+        final MockAnalyzer analyzer = new MockAnalyzer(random, MockTokenizer.SIMPLE, true,
stopWords, true);
+        analyzer.setEnableChecks(false);
         TermQuery query = new TermQuery(new Term("data", goodWord));
 
         String match;
@@ -1134,13 +1141,13 @@ public class HighlighterTest extends Bas
           sb.append("stoppedtoken");
         }
         SimpleHTMLFormatter fm = new SimpleHTMLFormatter();
-        Highlighter hg = getHighlighter(query, "data", new MockAnalyzer(random, MockTokenizer.SIMPLE,
true, stopWords, true).tokenStream(
+        Highlighter hg = getHighlighter(query, "data", analyzer.tokenStream(
             "data", new StringReader(sb.toString())), fm);// new Highlighter(fm,
         // new
         // QueryTermScorer(query));
         hg.setTextFragmenter(new NullFragmenter());
         hg.setMaxDocCharsToAnalyze(100);
-        match = hg.getBestFragment(new MockAnalyzer(random, MockTokenizer.SIMPLE, true, stopWords,
true), "data", sb.toString());
+        match = hg.getBestFragment(analyzer, "data", sb.toString());
         assertTrue("Matched text should be no more than 100 chars in length ", match.length()
< hg
             .getMaxDocCharsToAnalyze());
 
@@ -1151,7 +1158,7 @@ public class HighlighterTest extends Bas
         // + whitespace)
         sb.append(" ");
         sb.append(goodWord);
-        match = hg.getBestFragment(new MockAnalyzer(random, MockTokenizer.SIMPLE, true, stopWords,
true), "data", sb.toString());
+        match = hg.getBestFragment(analyzer, "data", sb.toString());
         assertTrue("Matched text should be no more than 100 chars in length ", match.length()
< hg
             .getMaxDocCharsToAnalyze());
       }
@@ -1726,6 +1733,11 @@ final class SynonymAnalyzer extends Anal
     stream.addAttribute(CharTermAttribute.class);
     stream.addAttribute(PositionIncrementAttribute.class);
     stream.addAttribute(OffsetAttribute.class);
+    try {
+      stream.reset();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
     return new SynonymTokenizer(stream, synonyms);
   }
 }

Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java
(original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java
Thu May 12 13:32:32 2011
@@ -28,32 +28,38 @@ import org.apache.lucene.analysis.TokenS
 public class OffsetLimitTokenFilterTest extends BaseTokenStreamTestCase {
   
   public void testFilter() throws Exception {
-    TokenStream stream = new MockTokenizer(new StringReader(
+    // we disable MockTokenizer checks because we will forcefully limit the 
+    // tokenstream and call end() before incrementToken() returns false.
+    MockTokenizer stream = new MockTokenizer(new StringReader(
         "short toolong evenmuchlongertext a ab toolong foo"),
         MockTokenizer.WHITESPACE, false);
+    stream.setEnableChecks(false);
     OffsetLimitTokenFilter filter = new OffsetLimitTokenFilter(stream, 10);
     assertTokenStreamContents(filter, new String[] {"short", "toolong"});
     
     stream = new MockTokenizer(new StringReader(
     "short toolong evenmuchlongertext a ab toolong foo"),
     MockTokenizer.WHITESPACE, false);
+    stream.setEnableChecks(false);
     filter = new OffsetLimitTokenFilter(stream, 12);
     assertTokenStreamContents(filter, new String[] {"short", "toolong"});
     
     stream = new MockTokenizer(new StringReader(
         "short toolong evenmuchlongertext a ab toolong foo"),
         MockTokenizer.WHITESPACE, false);
+    stream.setEnableChecks(false);
     filter = new OffsetLimitTokenFilter(stream, 30);
     assertTokenStreamContents(filter, new String[] {"short", "toolong",
         "evenmuchlongertext"});
     
-    
+    // TODO: This is not actually testing reuse! (reusableTokenStream is not implemented)
     checkOneTermReuse(new Analyzer() {
       
       @Override
       public TokenStream tokenStream(String fieldName, Reader reader) {
-        return new OffsetLimitTokenFilter(new MockTokenizer(reader,
-            MockTokenizer.WHITESPACE, false), 10);
+        MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+        tokenizer.setEnableChecks(false);
+        return new OffsetLimitTokenFilter(tokenizer, 10);
       }
     }, "llenges", "llenges");
   }

Modified: lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
(original)
+++ lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
Thu May 12 13:32:32 2011
@@ -192,6 +192,7 @@ public class FuzzyLikeThisQuery extends 
         int corpusNumDocs=reader.numDocs();
         Term internSavingTemplateTerm =new Term(f.fieldName); //optimization to avoid constructing
new Term() objects
         HashSet<String> processedTerms=new HashSet<String>();
+        ts.reset();
         while (ts.incrementToken()) 
         {
                 String term = termAtt.toString();
@@ -244,7 +245,9 @@ public class FuzzyLikeThisQuery extends 
 	                }                            
                 }
         	}
-        }     
+        }
+        ts.end();
+        ts.close();
     }
             
     @Override

Modified: lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
(original)
+++ lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
Thu May 12 13:32:32 2011
@@ -885,7 +885,7 @@ public final class MoreLikeThis {
 			int tokenCount=0;
 			// for every token
 			CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
-			
+			ts.reset();
 			while (ts.incrementToken()) {
 				String word = termAtt.toString();
 				tokenCount++;
@@ -906,6 +906,8 @@ public final class MoreLikeThis {
 					cnt.x++;
 				}
 			}
+			ts.end();
+			ts.close();
 	}
 	
 	

Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java
(original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java
Thu May 12 13:32:32 2011
@@ -110,6 +110,11 @@ public class AnalyzingQueryParser extend
     CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
     
     int countTokens = 0;
+    try {
+      source.reset();
+    } catch (IOException e1) {
+      throw new RuntimeException(e1);
+    }
     while (true) {
       try {
         if (!source.incrementToken()) break;
@@ -126,6 +131,7 @@ public class AnalyzingQueryParser extend
       }
     }
     try {
+      source.end();
       source.close();
     } catch (IOException e) {
       // ignore
@@ -191,7 +197,11 @@ public class AnalyzingQueryParser extend
     TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr));
     List<String> tlist = new ArrayList<String>();
     CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
-    
+    try {
+      source.reset();
+    } catch (IOException e1) {
+      throw new RuntimeException(e1);
+    }
     while (true) {
       try {
         if (!source.incrementToken()) break;
@@ -202,6 +212,7 @@ public class AnalyzingQueryParser extend
     }
 
     try {
+      source.end();
       source.close();
     } catch (IOException e) {
       // ignore
@@ -242,6 +253,7 @@ public class AnalyzingQueryParser extend
     boolean multipleTokens = false;
     
     try {
+      source.reset();
       if (source.incrementToken()) {
         nextToken = termAtt.toString();
       }
@@ -251,6 +263,7 @@ public class AnalyzingQueryParser extend
     }
 
     try {
+      source.end();
       source.close();
     } catch (IOException e) {
       // ignore
@@ -281,6 +294,7 @@ public class AnalyzingQueryParser extend
       try {
         source = getAnalyzer().tokenStream(field, new StringReader(part1));
         termAtt = source.addAttribute(CharTermAttribute.class);
+        source.reset();
         multipleTokens = false;
 
 
@@ -292,6 +306,7 @@ public class AnalyzingQueryParser extend
         // ignore
       }
       try {
+        source.end();
         source.close();
       } catch (IOException e) {
         // ignore
@@ -308,6 +323,7 @@ public class AnalyzingQueryParser extend
       termAtt = source.addAttribute(CharTermAttribute.class);
 
       try {
+        source.reset();
         if (source.incrementToken()) {
           part2 = termAtt.toString();
         }
@@ -316,6 +332,7 @@ public class AnalyzingQueryParser extend
         // ignore
       }
       try {
+        source.end();
         source.close();
       } catch (IOException e) {
         // ignore

Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java
(original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java
Thu May 12 13:32:32 2011
@@ -123,6 +123,11 @@ public class AnalyzerQueryNodeProcessor 
 
       TokenStream source = this.analyzer.tokenStream(field, new StringReader(
           text));
+      try {
+        source.reset();
+      } catch (IOException e1) {
+        throw new RuntimeException(e1);
+      }
       CachingTokenFilter buffer = new CachingTokenFilter(source);
 
       PositionIncrementAttribute posIncrAtt = null;

Modified: lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java
(original)
+++ lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java
Thu May 12 13:32:32 2011
@@ -118,12 +118,14 @@ public final class SynExpand {
 		// [1] Parse query into separate words so that when we expand we can avoid dups
 		TokenStream ts = a.tokenStream( field, new StringReader( query));
 		CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
-		
+		ts.reset();
 		while (ts.incrementToken()) {
 		  String word = termAtt.toString();
 			if ( already.add( word))
 				top.add( word);
 		}
+		ts.end();
+		ts.close();
 		final BooleanQuery tmp = new BooleanQuery();
 		
 		// [2] form query

Modified: lucene/dev/trunk/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java
(original)
+++ lucene/dev/trunk/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java
Thu May 12 13:32:32 2011
@@ -111,7 +111,6 @@ public class TestSynonymTokenFilter exte
         setPreviousTokenStream(streams);
       } else {
         streams.source.reset(reader);
-        streams.result.reset(); // reset the SynonymTokenFilter
       }
       return streams.result;
     }

Modified: lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java
(original)
+++ lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java
Thu May 12 13:32:32 2011
@@ -80,9 +80,12 @@ public class LikeThisQueryBuilder implem
                 CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
                 try
                 {
+                  ts.reset();
 	                while(ts.incrementToken()) {
 	                    stopWordsSet.add(termAtt.toString());
 	                }
+	                ts.end();
+	                ts.close();
                 }
                 catch(IOException ioe)
                 {

Modified: lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java
(original)
+++ lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java
Thu May 12 13:32:32 2011
@@ -59,11 +59,14 @@ public class SpanOrTermsBuilder extends 
 			TokenStream ts=analyzer.tokenStream(fieldName,new StringReader(value));
 			TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
       BytesRef bytes = termAtt.getBytesRef();
+      ts.reset();
 	    while (ts.incrementToken()) {
 	        termAtt.fillBytesRef();
 			    SpanTermQuery stq=new SpanTermQuery(new Term(fieldName, new BytesRef(bytes)));
 			    clausesList.add(stq);
 			}
+	    ts.end();
+	    ts.close();
 			SpanOrQuery soq=new SpanOrQuery(clausesList.toArray(new SpanQuery[clausesList.size()]));
 			soq.setBoost(DOMUtils.getAttribute(e,"boost",1.0f));
 			return soq;

Modified: lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java
(original)
+++ lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java
Thu May 12 13:32:32 2011
@@ -64,6 +64,7 @@ public class TermsFilterBuilder implemen
 		{
 			Term term = null;
       BytesRef bytes = termAtt.getBytesRef();
+      ts.reset();
 	      while (ts.incrementToken()) {
 	        termAtt.fillBytesRef();
 				if (term == null)
@@ -76,6 +77,8 @@ public class TermsFilterBuilder implemen
 				}
 				tf.addTerm(term);
 			}
+	    ts.end();
+	    ts.close();
 		} 
 		catch (IOException ioe)
 		{

Modified: lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java
(original)
+++ lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java
Thu May 12 13:32:32 2011
@@ -61,6 +61,7 @@ public class TermsQueryBuilder implement
 		  TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
 			Term term = null;
       BytesRef bytes = termAtt.getBytesRef();
+      ts.reset();
 			while (ts.incrementToken()) {
         termAtt.fillBytesRef();
 				if (term == null)
@@ -73,6 +74,8 @@ public class TermsQueryBuilder implement
 				}
 				bq.add(new BooleanClause(new TermQuery(term),BooleanClause.Occur.SHOULD));
 			}
+			ts.end();
+			ts.close();
 		} 
 		catch (IOException ioe)
 		{

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java Thu
May 12 13:32:32 2011
@@ -806,6 +806,7 @@ public abstract class QueryParserBase {
     }
       
     try {
+      source.end();
       source.close();
     } catch (IOException ignored) {}
     

Modified: lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
(original)
+++ lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
Thu May 12 13:32:32 2011
@@ -262,6 +262,7 @@ public abstract class BaseTokenStreamTes
         tokens.add(termAtt.toString());
         // TODO: we could collect offsets etc here for better checking that reset() really
works.
       }
+      ts.end();
       ts.close();
       // verify reusing is "reproducable" and also get the normal tokenstream sanity checks
       if (!tokens.isEmpty())

Modified: lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java
(original)
+++ lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java
Thu May 12 13:32:32 2011
@@ -36,6 +36,7 @@ public final class MockAnalyzer extends 
   private int positionIncrementGap;
   private final Random random;
   private Map<String,Integer> previousMappings = new HashMap<String,Integer>();
+  private boolean enableChecks = true;
 
   /**
    * Creates a new MockAnalyzer.
@@ -75,6 +76,7 @@ public final class MockAnalyzer extends 
   @Override
   public TokenStream tokenStream(String fieldName, Reader reader) {
     MockTokenizer tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase);
+    tokenizer.setEnableChecks(enableChecks);
     TokenFilter filt = new MockTokenFilter(tokenizer, filter, enablePositionIncrements);
     filt = maybePayload(filt, fieldName);
     return filt;
@@ -98,13 +100,13 @@ public final class MockAnalyzer extends 
     if (saved == null) {
       saved = new SavedStreams();
       saved.tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase);
+      saved.tokenizer.setEnableChecks(enableChecks);
       saved.filter = new MockTokenFilter(saved.tokenizer, filter, enablePositionIncrements);
       saved.filter = maybePayload(saved.filter, fieldName);
       map.put(fieldName, saved);
       return saved.filter;
     } else {
       saved.tokenizer.reset(reader);
-      saved.filter.reset();
       return saved.filter;
     }
   }
@@ -139,4 +141,12 @@ public final class MockAnalyzer extends 
   public int getPositionIncrementGap(String fieldName){
     return positionIncrementGap;
   }
+  
+  /** 
+   * Toggle consumer workflow checking: if your test consumes tokenstreams normally you
+   * should leave this enabled.
+   */
+  public void setEnableChecks(boolean enableChecks) {
+    this.enableChecks = enableChecks;
+  }
 }

Modified: lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java
(original)
+++ lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java
Thu May 12 13:32:32 2011
@@ -86,6 +86,7 @@ final class MockPayloadFilter extends To
 
   @Override
   public void reset() throws IOException {
+    super.reset();
     i = 0;
     pos = 0;
   }

Modified: lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java
(original)
+++ lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java
Thu May 12 13:32:32 2011
@@ -50,11 +50,27 @@ public class MockTokenizer extends Token
   private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
   int off = 0;
 
+  // TODO: "register" with LuceneTestCase to ensure all streams are closed() ?
+  // currently, we can only check that the lifecycle is correct if someone is reusing,
+  // but not for "one-offs".
+  private static enum State { 
+    SETREADER,       // consumer set a reader input either via ctor or via reset(Reader)
+    RESET,           // consumer has called reset()
+    INCREMENT,       // consumer is consuming, has called incrementToken() == true
+    INCREMENT_FALSE, // consumer has called incrementToken() which returned false
+    END,             // consumer has called end() to perform end of stream operations
+    CLOSE            // consumer has called close() to release any resources
+  };
+  
+  private State streamState = State.CLOSE;
+  private boolean enableChecks = true;
+  
   public MockTokenizer(AttributeFactory factory, Reader input, CharacterRunAutomaton runAutomaton,
boolean lowerCase) {
     super(factory, input);
     this.runAutomaton = runAutomaton;
     this.lowerCase = lowerCase;
     this.state = runAutomaton.getInitialState();
+    this.streamState = State.SETREADER;
   }
 
   public MockTokenizer(Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase)
{
@@ -62,10 +78,13 @@ public class MockTokenizer extends Token
     this.runAutomaton = runAutomaton;
     this.lowerCase = lowerCase;
     this.state = runAutomaton.getInitialState();
+    this.streamState = State.SETREADER;
   }
   
   @Override
   public final boolean incrementToken() throws IOException {
+    assert !enableChecks || (streamState == State.RESET || streamState == State.INCREMENT)

+                            : "incrementToken() called while in wrong state: " + streamState;
     clearAttributes();
     for (;;) {
       int startOffset = off;
@@ -82,9 +101,11 @@ public class MockTokenizer extends Token
           cp = readCodePoint();
         } while (cp >= 0 && isTokenChar(cp));
         offsetAtt.setOffset(startOffset, endOffset);
+        streamState = State.INCREMENT;
         return true;
       }
     }
+    streamState = State.INCREMENT_FALSE;
     return false;
   }
 
@@ -126,11 +147,42 @@ public class MockTokenizer extends Token
     super.reset();
     state = runAutomaton.getInitialState();
     off = 0;
+    assert !enableChecks || streamState != State.RESET : "double reset()";
+    streamState = State.RESET;
+  }
+  
+  @Override
+  public void close() throws IOException {
+    super.close();
+    // in some exceptional cases (e.g. TestIndexWriterExceptions) a test can prematurely
close()
+    // these tests should disable this check, by default we check the normal workflow.
+    // TODO: investigate the CachingTokenFilter "double-close"... for now we ignore this
+    assert !enableChecks || streamState == State.END || streamState == State.CLOSE : "close()
called in wrong state: " + streamState;
+    streamState = State.CLOSE;
+  }
+
+  @Override
+  public void reset(Reader input) throws IOException {
+    super.reset(input);
+    assert !enableChecks || streamState == State.CLOSE : "setReader() called in wrong state:
" + streamState;
+    streamState = State.SETREADER;
   }
 
   @Override
   public void end() throws IOException {
     int finalOffset = correctOffset(off);
     offsetAtt.setOffset(finalOffset, finalOffset);
+    // some tokenizers, such as limiting tokenizers, call end() before incrementToken() returns
false.
+    // these tests should disable this check (in general you should consume the entire stream)
+    assert !enableChecks || streamState == State.INCREMENT_FALSE : "end() called before incrementToken()
returned false!";
+    streamState = State.END;
+  }
+
+  /** 
+   * Toggle consumer workflow checking: if your test consumes tokenstreams normally you
+   * should leave this enabled.
+   */
+  public void setEnableChecks(boolean enableChecks) {
+    this.enableChecks = enableChecks;
   }
 }

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java Thu
May 12 13:32:32 2011
@@ -107,6 +107,7 @@ public class TestMockAnalyzer extends Ba
       // consume
     }
     stream.end();
+    stream.close();
     
     assertAnalyzesToReuse(analyzer, testString, new String[] { "t" });
   }

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
(original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
Thu May 12 13:32:32 2011
@@ -155,7 +155,9 @@ public class TestIndexWriterExceptions e
     }
     MockDirectoryWrapper dir = newDirectory();
 
-    MockIndexWriter writer  = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
new MockAnalyzer(random))
+    MockAnalyzer analyzer = new MockAnalyzer(random);
+    analyzer.setEnableChecks(false); // disable workflow checking as we forcefully close()
in exceptional cases.
+    MockIndexWriter writer  = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
analyzer)
         .setRAMBufferSizeMB(0.1).setMergeScheduler(new ConcurrentMergeScheduler()));
     ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions();
     //writer.setMaxBufferedDocs(10);
@@ -201,7 +203,9 @@ public class TestIndexWriterExceptions e
 
   public void testRandomExceptionsThreads() throws Throwable {
     MockDirectoryWrapper dir = newDirectory();
-    MockIndexWriter writer  = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
new MockAnalyzer(random))
+    MockAnalyzer analyzer = new MockAnalyzer(random);
+    analyzer.setEnableChecks(false); // disable workflow checking as we forcefully close()
in exceptional cases.
+    MockIndexWriter writer  = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
analyzer)
         .setRAMBufferSizeMB(0.2).setMergeScheduler(new ConcurrentMergeScheduler()));
     ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions();
     //writer.setMaxBufferedDocs(10);
@@ -321,7 +325,9 @@ public class TestIndexWriterExceptions e
     Analyzer analyzer = new Analyzer() {
       @Override
       public TokenStream tokenStream(String fieldName, Reader reader) {
-        return new CrashingFilter(fieldName, new MockTokenizer(reader, MockTokenizer.WHITESPACE,
false));
+        MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+        tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close()
in exceptional cases.
+        return new CrashingFilter(fieldName, tokenizer);
       }
     };
 
@@ -390,7 +396,9 @@ public class TestIndexWriterExceptions e
 
       @Override
       public TokenStream tokenStream(String fieldName, Reader reader) {
-        return new TokenFilter(new MockTokenizer(reader, MockTokenizer.SIMPLE, true)) {
+        MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+        tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close()
in exceptional cases.
+        return new TokenFilter(tokenizer) {
           private int count = 0;
 
           @Override
@@ -522,7 +530,9 @@ public class TestIndexWriterExceptions e
     Analyzer analyzer = new Analyzer() {
       @Override
       public TokenStream tokenStream(String fieldName, Reader reader) {
-        return new CrashingFilter(fieldName, new MockTokenizer(reader, MockTokenizer.WHITESPACE,
false));
+        MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+        tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close()
in exceptional cases.
+        return new CrashingFilter(fieldName, tokenizer);
       }
     };
 
@@ -621,7 +631,9 @@ public class TestIndexWriterExceptions e
     Analyzer analyzer = new Analyzer() {
       @Override
       public TokenStream tokenStream(String fieldName, Reader reader) {
-        return new CrashingFilter(fieldName, new MockTokenizer(reader, MockTokenizer.WHITESPACE,
false));
+        MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+        tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close()
in exceptional cases.
+        return new CrashingFilter(fieldName, tokenizer);
       }
     };
 

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLongPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLongPostings.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLongPostings.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLongPostings.java Thu May
12 13:32:32 2011
@@ -49,6 +49,7 @@ public class TestLongPostings extends Lu
       final TermToBytesRefAttribute termAtt = ts.getAttribute(TermToBytesRefAttribute.class);
       final BytesRef termBytes = termAtt.getBytesRef();
       int count = 0;
+      ts.reset();
       while(ts.incrementToken()) {
         termAtt.fillBytesRef();
         if (count == 0 && !termBytes.utf8ToString().equals(s)) {

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java Thu
May 12 13:32:32 2011
@@ -121,7 +121,9 @@ public class TestTermVectorsWriter exten
     Analyzer analyzer = new MockAnalyzer(random);
     IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer));
     Document doc = new Document();
-    TokenStream stream = new CachingTokenFilter(analyzer.tokenStream("field", new StringReader("abcd
  ")));
+    TokenStream stream = analyzer.tokenStream("field", new StringReader("abcd   "));
+    stream.reset(); // TODO: wierd to reset before wrapping with CachingTokenFilter... correct?
+    stream = new CachingTokenFilter(stream);
     Field f = new Field("field", stream, Field.TermVector.WITH_POSITIONS_OFFSETS);
     doc.add(f);
     doc.add(f);

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java Thu May
12 13:32:32 2011
@@ -626,11 +626,14 @@ public class TestPhraseQuery extends Luc
           }
           TokenStream ts = analyzer.reusableTokenStream("ignore", new StringReader(term));
           CharTermAttribute termAttr = ts.addAttribute(CharTermAttribute.class);
+          ts.reset();
           while(ts.incrementToken()) {
             String text = termAttr.toString();
             doc.add(text);
             sb.append(text).append(' ');
           }
+          ts.end();
+          ts.close();
         } else {
           // pick existing sub-phrase
           List<String> lastDoc = docs.get(r.nextInt(docs.size()));



Mime
View raw message