Return-Path: Delivered-To: apmail-lucene-java-commits-archive@www.apache.org Received: (qmail 85266 invoked from network); 23 Oct 2009 20:26:33 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 23 Oct 2009 20:26:33 -0000 Received: (qmail 32322 invoked by uid 500); 23 Oct 2009 20:26:33 -0000 Delivered-To: apmail-lucene-java-commits-archive@lucene.apache.org Received: (qmail 32292 invoked by uid 500); 23 Oct 2009 20:26:32 -0000 Mailing-List: contact java-commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: java-dev@lucene.apache.org Delivered-To: mailing list java-commits@lucene.apache.org Received: (qmail 32283 invoked by uid 99); 23 Oct 2009 20:26:32 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 23 Oct 2009 20:26:32 +0000 X-ASF-Spam-Status: No, hits=-2.5 required=5.0 tests=AWL,BAYES_00 X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 23 Oct 2009 20:26:26 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 66C5A23888E5; Fri, 23 Oct 2009 20:26:06 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: svn commit: r829206 [2/3] - in /lucene/java/trunk: ./ contrib/ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/ contrib/analyzers/common/src/java/org/apache/lucene/analysi... Date: Fri, 23 Oct 2009 20:25:25 -0000 To: java-commits@lucene.apache.org From: mikemccand@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20091023202606.66C5A23888E5@eris.apache.org> Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java (original) +++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java Fri Oct 23 20:25:17 2009 @@ -23,6 +23,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.WhitespaceTokenizer; +import org.apache.lucene.util.Version; /** * Test case for ThaiAnalyzer, modified from TestFrenchAnalyzer @@ -36,7 +37,7 @@ * testcase for offsets */ public void testOffsets() throws Exception { - assertAnalyzesTo(new ThaiAnalyzer(), "เดอะนิวยอร์กไทมส์", + assertAnalyzesTo(new ThaiAnalyzer(Version.LUCENE_CURRENT), "เดอะนิวยอร์กไทมส์", new String[] { "เด", "อะนิว", "ยอ", "ร์ก", "ไทมส์"}, new int[] { 0, 2, 7, 9, 12 }, new int[] { 2, 7, 9, 12, 17}); @@ -54,7 +55,7 @@ * Instead, allow the definition of alphanum to include relevant categories like nonspacing marks! */ public void testBuggyTokenType() throws Exception { - assertAnalyzesTo(new ThaiAnalyzer(), "เดอะนิวยอร์กไทมส์ ๑๒๓", + assertAnalyzesTo(new ThaiAnalyzer(Version.LUCENE_CURRENT), "เดอะนิวยอร์กไทมส์ ๑๒๓", new String[] { "เด", "อะนิว", "ยอ", "ร์ก", "ไทมส์", "๑๒๓" }, new String[] { "", "", "", "", "", "" }); } @@ -68,7 +69,7 @@ */ public void testAnalyzer() throws Exception { - ThaiAnalyzer analyzer = new ThaiAnalyzer(); + ThaiAnalyzer analyzer = new ThaiAnalyzer(Version.LUCENE_CURRENT); assertAnalyzesTo(analyzer, "", new String[] {}); @@ -90,7 +91,7 @@ } public void testReusableTokenStream() throws Exception { - ThaiAnalyzer analyzer = new ThaiAnalyzer(); + ThaiAnalyzer analyzer = new ThaiAnalyzer(Version.LUCENE_CURRENT); assertAnalyzesToReuse(analyzer, "", new String[] {}); assertAnalyzesToReuse( @@ -108,13 +109,16 @@ * subclass that acts just like whitespace analyzer for testing */ private class ThaiSubclassAnalyzer extends ThaiAnalyzer { + public ThaiSubclassAnalyzer(Version matchVersion) { + super(matchVersion); + } public TokenStream tokenStream(String fieldName, Reader reader) { return new WhitespaceTokenizer(reader); } } public void testLUCENE1678BWComp() throws Exception { - ThaiSubclassAnalyzer a = new ThaiSubclassAnalyzer(); + ThaiSubclassAnalyzer a = new ThaiSubclassAnalyzer(Version.LUCENE_CURRENT); assertAnalyzesToReuse(a, "การที่ได้ต้องแสดงว่างานดี", new String[] { "การที่ได้ต้องแสดงว่างานดี" }); } } Modified: lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java (original) +++ lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java Fri Oct 23 20:25:17 2009 @@ -32,6 +32,7 @@ import org.apache.lucene.analysis.WordlistLoader; import org.apache.lucene.analysis.cn.smart.SentenceTokenizer; import org.apache.lucene.analysis.cn.smart.WordTokenFilter; +import org.apache.lucene.util.Version; /** *

@@ -103,11 +104,13 @@ } } + private final Version matchVersion; + /** * Create a new SmartChineseAnalyzer, using the default stopword list. */ - public SmartChineseAnalyzer() { - this(true); + public SmartChineseAnalyzer(Version matchVersion) { + this(matchVersion, true); } /** @@ -121,9 +124,10 @@ * * @param useDefaultStopWords true to use the default stopword list. */ - public SmartChineseAnalyzer(boolean useDefaultStopWords) { + public SmartChineseAnalyzer(Version matchVersion, boolean useDefaultStopWords) { stopWords = useDefaultStopWords ? DefaultSetHolder.DEFAULT_STOP_SET - : Collections.EMPTY_SET; + : Collections.EMPTY_SET; + this.matchVersion = matchVersion; } /** @@ -135,8 +139,9 @@ *

* @param stopWords {@link Set} of stopwords to use. */ - public SmartChineseAnalyzer(Set stopWords) { + public SmartChineseAnalyzer(Version matchVersion, Set stopWords) { this.stopWords = stopWords==null?Collections.EMPTY_SET:stopWords; + this.matchVersion = matchVersion; } public TokenStream tokenStream(String fieldName, Reader reader) { @@ -147,7 +152,8 @@ // The porter stemming is too strict, this is not a bug, this is a feature:) result = new PorterStemFilter(result); if (!stopWords.isEmpty()) { - result = new StopFilter(false,result, stopWords, false); + result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), + result, stopWords, false); } return result; } @@ -167,7 +173,8 @@ streams.filteredTokenStream = new WordTokenFilter(streams.tokenStream); streams.filteredTokenStream = new PorterStemFilter(streams.filteredTokenStream); if (!stopWords.isEmpty()) { - streams.filteredTokenStream = new StopFilter(false, streams.filteredTokenStream, stopWords, false); + streams.filteredTokenStream = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), + streams.filteredTokenStream, stopWords, false); } } else { streams.tokenStream.reset(reader); Modified: lucene/java/trunk/contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java (original) +++ lucene/java/trunk/contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java Fri Oct 23 20:25:17 2009 @@ -26,16 +26,17 @@ import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.util.Version; public class TestSmartChineseAnalyzer extends BaseTokenStreamTestCase { public void testChineseStopWordsDefault() throws Exception { - Analyzer ca = new SmartChineseAnalyzer(); /* will load stopwords */ + Analyzer ca = new SmartChineseAnalyzer(Version.LUCENE_CURRENT); /* will load stopwords */ String sentence = "我购买了道具和服装。"; String result[] = { "我", "购买", "了", "道具", "和", "服装" }; assertAnalyzesTo(ca, sentence, result); // set stop-words from the outer world - must yield same behavior - ca = new SmartChineseAnalyzer(SmartChineseAnalyzer.getDefaultStopSet()); + ca = new SmartChineseAnalyzer(Version.LUCENE_CURRENT, SmartChineseAnalyzer.getDefaultStopSet()); assertAnalyzesTo(ca, sentence, result); } @@ -44,7 +45,7 @@ * This tests to ensure the SentenceTokenizer->WordTokenFilter chain works correctly. */ public void testChineseStopWordsDefaultTwoPhrases() throws Exception { - Analyzer ca = new SmartChineseAnalyzer(); /* will load stopwords */ + Analyzer ca = new SmartChineseAnalyzer(Version.LUCENE_CURRENT); /* will load stopwords */ String sentence = "我购买了道具和服装。 我购买了道具和服装。"; String result[] = { "我", "购买", "了", "道具", "和", "服装", "我", "购买", "了", "道具", "和", "服装" }; assertAnalyzesTo(ca, sentence, result); @@ -55,7 +56,7 @@ * This tests to ensure the stopwords are working correctly. */ public void testChineseStopWordsDefaultTwoPhrasesIdeoSpace() throws Exception { - Analyzer ca = new SmartChineseAnalyzer(); /* will load stopwords */ + Analyzer ca = new SmartChineseAnalyzer(Version.LUCENE_CURRENT); /* will load stopwords */ String sentence = "我购买了道具和服装　我购买了道具和服装。"; String result[] = { "我", "购买", "了", "道具", "和", "服装", "我", "购买", "了", "道具", "和", "服装" }; assertAnalyzesTo(ca, sentence, result); @@ -69,8 +70,8 @@ */ public void testChineseStopWordsOff() throws Exception { Analyzer[] analyzers = new Analyzer[] { - new SmartChineseAnalyzer(false),/* doesn't load stopwords */ - new SmartChineseAnalyzer(null) /* sets stopwords to empty set */}; + new SmartChineseAnalyzer(Version.LUCENE_CURRENT, false),/* doesn't load stopwords */ + new SmartChineseAnalyzer(Version.LUCENE_CURRENT, null) /* sets stopwords to empty set */}; String sentence = "我购买了道具和服装。"; String result[] = { "我", "购买", "了", "道具", "和", "服装", "," }; for (Analyzer analyzer : analyzers) { @@ -80,7 +81,7 @@ } public void testChineseAnalyzer() throws Exception { - Analyzer ca = new SmartChineseAnalyzer(true); + Analyzer ca = new SmartChineseAnalyzer(Version.LUCENE_CURRENT, true); String sentence = "我购买了道具和服装。"; String[] result = { "我", "购买", "了", "道具", "和", "服装" }; assertAnalyzesTo(ca, sentence, result); @@ -90,7 +91,7 @@ * English words are lowercased and porter-stemmed. */ public void testMixedLatinChinese() throws Exception { - assertAnalyzesTo(new SmartChineseAnalyzer(true), "我购买 Tests 了道具和服装", + assertAnalyzesTo(new SmartChineseAnalyzer(Version.LUCENE_CURRENT, true), "我购买 Tests 了道具和服装", new String[] { "我", "购买", "test", "了", "道具", "和", "服装"}); } @@ -98,7 +99,7 @@ * Numerics are parsed as their own tokens */ public void testNumerics() throws Exception { - assertAnalyzesTo(new SmartChineseAnalyzer(true), "我购买 Tests 了道具和服装1234", + assertAnalyzesTo(new SmartChineseAnalyzer(Version.LUCENE_CURRENT, true), "我购买 Tests 了道具和服装1234", new String[] { "我", "购买", "test", "了", "道具", "和", "服装", "1234"}); } @@ -106,7 +107,7 @@ * Full width alphas and numerics are folded to half-width */ public void testFullWidth() throws Exception { - assertAnalyzesTo(new SmartChineseAnalyzer(true), "我购买 Ｔｅｓｔｓ 了道具和服装１２３４", + assertAnalyzesTo(new SmartChineseAnalyzer(Version.LUCENE_CURRENT, true), "我购买 Ｔｅｓｔｓ 了道具和服装１２３４", new String[] { "我", "购买", "test", "了", "道具", "和", "服装", "1234"}); } @@ -114,7 +115,7 @@ * Presentation form delimiters are removed */ public void testDelimiters() throws Exception { - assertAnalyzesTo(new SmartChineseAnalyzer(true), "我购买︱ Tests 了道具和服装", + assertAnalyzesTo(new SmartChineseAnalyzer(Version.LUCENE_CURRENT, true), "我购买︱ Tests 了道具和服装", new String[] { "我", "购买", "test", "了", "道具", "和", "服装"}); } @@ -123,7 +124,7 @@ * (regardless of Unicode category) */ public void testNonChinese() throws Exception { - assertAnalyzesTo(new SmartChineseAnalyzer(true), "我购买 روبرتTests 了道具和服装", + assertAnalyzesTo(new SmartChineseAnalyzer(Version.LUCENE_CURRENT, true), "我购买 روبرتTests 了道具和服装", new String[] { "我", "购买", "ر", "و", "ب", "ر", "ت", "test", "了", "道具", "和", "服装"}); } @@ -133,22 +134,22 @@ * Currently it is being analyzed into single characters... */ public void testOOV() throws Exception { - assertAnalyzesTo(new SmartChineseAnalyzer(true), "优素福·拉扎·吉拉尼", + assertAnalyzesTo(new SmartChineseAnalyzer(Version.LUCENE_CURRENT, true), "优素福·拉扎·吉拉尼", new String[] { "优", "素", "福", "拉", "扎", "吉", "拉", "尼" }); - assertAnalyzesTo(new SmartChineseAnalyzer(true), "优素福拉扎吉拉尼", + assertAnalyzesTo(new SmartChineseAnalyzer(Version.LUCENE_CURRENT, true), "优素福拉扎吉拉尼", new String[] { "优", "素", "福", "拉", "扎", "吉", "拉", "尼" }); } public void testOffsets() throws Exception { - assertAnalyzesTo(new SmartChineseAnalyzer(true), "我购买了道具和服装", + assertAnalyzesTo(new SmartChineseAnalyzer(Version.LUCENE_CURRENT, true), "我购买了道具和服装", new String[] { "我", "购买", "了", "道具", "和", "服装" }, new int[] { 0, 1, 3, 4, 6, 7 }, new int[] { 1, 3, 4, 6, 7, 9 }); } public void testReusableTokenStream() throws Exception { - Analyzer a = new SmartChineseAnalyzer(); + Analyzer a = new SmartChineseAnalyzer(Version.LUCENE_CURRENT); assertAnalyzesToReuse(a, "我购买 Tests 了道具和服装", new String[] { "我", "购买", "test", "了", "道具", "和", "服装"}, new int[] { 0, 1, 4, 10, 11, 13, 14 }, Modified: lucene/java/trunk/contrib/ant/src/test/org/apache/lucene/ant/IndexTaskTest.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/ant/src/test/org/apache/lucene/ant/IndexTaskTest.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/ant/src/test/org/apache/lucene/ant/IndexTaskTest.java (original) +++ lucene/java/trunk/contrib/ant/src/test/org/apache/lucene/ant/IndexTaskTest.java Fri Oct 23 20:25:17 2009 @@ -31,6 +31,7 @@ import org.apache.lucene.store.FSDirectory; import org.apache.tools.ant.Project; import org.apache.tools.ant.types.FileSet; +import org.apache.lucene.util.Version; /** * Test cases for index task @@ -69,12 +70,12 @@ dir = FSDirectory.open(indexDir); searcher = new IndexSearcher(dir, true); - analyzer = new StopAnalyzer(false); + analyzer = new StopAnalyzer(Version.LUCENE_CURRENT); } public void testSearch() throws Exception { - Query query = new QueryParser("contents",analyzer).parse("test"); + Query query = new QueryParser(Version.LUCENE_CURRENT, "contents",analyzer).parse("test"); int numHits = searcher.search(query, null, 1000).totalHits; Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiQueryMaker.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiQueryMaker.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiQueryMaker.java (original) +++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiQueryMaker.java Fri Oct 23 20:25:17 2009 @@ -33,6 +33,7 @@ import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.benchmark.byTask.tasks.NewAnalyzerTask; +import org.apache.lucene.util.Version; /** * A QueryMaker that uses common and uncommon actual Wikipedia queries for @@ -92,7 +93,7 @@ * @return array of Lucene queries */ private static Query[] createQueries(List qs, Analyzer a) { - QueryParser qp = new QueryParser(DocMaker.BODY_FIELD, a); + QueryParser qp = new QueryParser(Version.LUCENE_CURRENT, DocMaker.BODY_FIELD, a); List queries = new ArrayList(); for (int i = 0; i < qs.size(); i++) { try { Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/FileBasedQueryMaker.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/FileBasedQueryMaker.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/FileBasedQueryMaker.java (original) +++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/FileBasedQueryMaker.java Fri Oct 23 20:25:17 2009 @@ -5,6 +5,7 @@ import org.apache.lucene.queryParser.ParseException; import org.apache.lucene.search.Query; import org.apache.lucene.benchmark.byTask.tasks.NewAnalyzerTask; +import org.apache.lucene.util.Version; import java.io.*; import java.util.ArrayList; @@ -48,7 +49,7 @@ Analyzer anlzr = NewAnalyzerTask.createAnalyzer(config.get("analyzer", "org.apache.lucene.analysis.standard.StandardAnalyzer")); String defaultField = config.get("file.query.maker.default.field", DocMaker.BODY_FIELD); - QueryParser qp = new QueryParser(defaultField, anlzr); + QueryParser qp = new QueryParser(Version.LUCENE_CURRENT, defaultField, anlzr); List qq = new ArrayList(); String fileName = config.get("file.query.maker.file", null); Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ReutersQueryMaker.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ReutersQueryMaker.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ReutersQueryMaker.java (original) +++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ReutersQueryMaker.java Fri Oct 23 20:25:17 2009 @@ -27,6 +27,7 @@ import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.benchmark.byTask.tasks.NewAnalyzerTask; +import org.apache.lucene.util.Version; import java.util.ArrayList; import java.util.Arrays; @@ -72,7 +73,7 @@ * @return array of Lucene queries */ private static Query[] createQueries(List qs, Analyzer a) { - QueryParser qp = new QueryParser(DocMaker.BODY_FIELD, a); + QueryParser qp = new QueryParser(Version.LUCENE_CURRENT, DocMaker.BODY_FIELD, a); List queries = new ArrayList(); for (int i = 0; i < qs.size(); i++) { try { Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SimpleQueryMaker.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SimpleQueryMaker.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SimpleQueryMaker.java (original) +++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SimpleQueryMaker.java Fri Oct 23 20:25:17 2009 @@ -25,6 +25,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.benchmark.byTask.tasks.NewAnalyzerTask; +import org.apache.lucene.util.Version; import java.util.ArrayList; @@ -46,7 +47,7 @@ Analyzer anlzr= NewAnalyzerTask.createAnalyzer(config.get("analyzer", "org.apache.lucene.analysis.standard.StandardAnalyzer")); - QueryParser qp = new QueryParser(DocMaker.BODY_FIELD,anlzr); + QueryParser qp = new QueryParser(Version.LUCENE_CURRENT, DocMaker.BODY_FIELD,anlzr); ArrayList qq = new ArrayList(); Query q1 = new TermQuery(new Term(DocMaker.ID_FIELD,"doc2")); qq.add(q1); Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/SimpleQQParser.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/SimpleQQParser.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/SimpleQQParser.java (original) +++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/SimpleQQParser.java Fri Oct 23 20:25:17 2009 @@ -50,7 +50,7 @@ public Query parse(QualityQuery qq) throws ParseException { QueryParser qp = queryParser.get(); if (qp==null) { - qp = new QueryParser(indexField, new StandardAnalyzer(Version.LUCENE_CURRENT)); + qp = new QueryParser(Version.LUCENE_CURRENT, indexField, new StandardAnalyzer(Version.LUCENE_CURRENT)); queryParser.set(qp); } return qp.parse(qq.getValue(qqName)); Modified: lucene/java/trunk/contrib/collation/src/test/org/apache/lucene/collation/CollationTestBase.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/collation/src/test/org/apache/lucene/collation/CollationTestBase.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/collation/src/test/org/apache/lucene/collation/CollationTestBase.java (original) +++ lucene/java/trunk/contrib/collation/src/test/org/apache/lucene/collation/CollationTestBase.java Fri Oct 23 20:25:17 2009 @@ -39,6 +39,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.util.IndexableBinaryStringTools; import org.apache.lucene.queryParser.analyzing.AnalyzingQueryParser; +import org.apache.lucene.util.Version; import java.io.IOException; import java.nio.CharBuffer; @@ -83,7 +84,7 @@ writer.close(); IndexSearcher is = new IndexSearcher(ramDir, true); - AnalyzingQueryParser aqp = new AnalyzingQueryParser("content", analyzer); + AnalyzingQueryParser aqp = new AnalyzingQueryParser(Version.LUCENE_CURRENT, "content", analyzer); aqp.setLowercaseExpandedTerms(false); // Unicode order would include U+0633 in [ U+062F - U+0698 ], but Farsi Modified: lucene/java/trunk/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java (original) +++ lucene/java/trunk/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java Fri Oct 23 20:25:17 2009 @@ -38,6 +38,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.Version; /** * FieldTermStack is a stack that keeps query terms in the specified field @@ -50,7 +51,7 @@ public static void main( String[] args ) throws Exception { Analyzer analyzer = new WhitespaceAnalyzer(); - QueryParser parser = new QueryParser( "f", analyzer ); + QueryParser parser = new QueryParser(Version.LUCENE_CURRENT, "f", analyzer ); Query query = parser.parse( "a x:b" ); FieldQuery fieldQuery = new FieldQuery( query, true, false ); Modified: lucene/java/trunk/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java (original) +++ lucene/java/trunk/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java Fri Oct 23 20:25:17 2009 @@ -45,6 +45,7 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.Version; public abstract class AbstractTestCase extends TestCase { @@ -78,8 +79,8 @@ protected void setUp() throws Exception { analyzerW = new WhitespaceAnalyzer(); analyzerB = new BigramAnalyzer(); - paW = new QueryParser( F, analyzerW ); - paB = new QueryParser( F, analyzerB ); + paW = new QueryParser(Version.LUCENE_CURRENT, F, analyzerW ); + paB = new QueryParser(Version.LUCENE_CURRENT, F, analyzerB ); dir = new RAMDirectory(); } Modified: lucene/java/trunk/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original) +++ lucene/java/trunk/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Fri Oct 23 20:25:17 2009 @@ -113,7 +113,7 @@ public void testQueryScorerHits() throws Exception { Analyzer analyzer = new SimpleAnalyzer(); - QueryParser qp = new QueryParser(FIELD_NAME, analyzer); + QueryParser qp = new QueryParser(Version.LUCENE_CURRENT, FIELD_NAME, analyzer); query = qp.parse("\"very long\""); searcher = new IndexSearcher(ramDir, true); TopDocs hits = searcher.search(query, 10); @@ -143,7 +143,7 @@ String s1 = "I call our world Flatland, not because we call it so,"; - QueryParser parser = new QueryParser(FIELD_NAME, new StandardAnalyzer(TEST_VERSION)); + QueryParser parser = new QueryParser(Version.LUCENE_CURRENT, FIELD_NAME, new StandardAnalyzer(TEST_VERSION)); // Verify that a query against the default field results in text being // highlighted @@ -221,7 +221,7 @@ String q = "(" + f1c + ph1 + " OR " + f2c + ph1 + ") AND (" + f1c + ph2 + " OR " + f2c + ph2 + ")"; Analyzer analyzer = new WhitespaceAnalyzer(); - QueryParser qp = new QueryParser(f1, analyzer); + QueryParser qp = new QueryParser(Version.LUCENE_CURRENT, f1, analyzer); Query query = qp.parse(q); QueryScorer scorer = new QueryScorer(query, f1); @@ -590,7 +590,7 @@ // Need to explicitly set the QueryParser property to use TermRangeQuery // rather // than RangeFilters - QueryParser parser = new QueryParser(FIELD_NAME, analyzer); + QueryParser parser = new QueryParser(Version.LUCENE_CURRENT, FIELD_NAME, analyzer); parser.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE); query = parser.parse(queryString); doSearching(query); @@ -930,7 +930,7 @@ String srchkey = "football"; String s = "football-soccer in the euro 2004 footie competition"; - QueryParser parser = new QueryParser("bookid", analyzer); + QueryParser parser = new QueryParser(Version.LUCENE_CURRENT, "bookid", analyzer); Query query = parser.parse(srchkey); TokenStream tokenStream = analyzer.tokenStream(null, new StringReader(s)); @@ -1111,7 +1111,7 @@ searcher = new IndexSearcher(ramDir, true); Analyzer analyzer = new StandardAnalyzer(TEST_VERSION); - QueryParser parser = new QueryParser(FIELD_NAME, analyzer); + QueryParser parser = new QueryParser(Version.LUCENE_CURRENT, FIELD_NAME, analyzer); Query query = parser.parse("JF? or Kenned*"); System.out.println("Searching with primitive query"); // forget to set this and... @@ -1245,7 +1245,7 @@ searchers[0] = new IndexSearcher(ramDir1, true); searchers[1] = new IndexSearcher(ramDir2, true); MultiSearcher multiSearcher = new MultiSearcher(searchers); - QueryParser parser = new QueryParser(FIELD_NAME, new StandardAnalyzer(TEST_VERSION)); + QueryParser parser = new QueryParser(Version.LUCENE_CURRENT, FIELD_NAME, new StandardAnalyzer(TEST_VERSION)); parser.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE); query = parser.parse("multi*"); System.out.println("Searching for: " + query.toString(FIELD_NAME)); @@ -1278,7 +1278,7 @@ public void run() throws Exception { String docMainText = "fred is one of the people"; - QueryParser parser = new QueryParser(FIELD_NAME, analyzer); + QueryParser parser = new QueryParser(Version.LUCENE_CURRENT, FIELD_NAME, analyzer); Query query = parser.parse("fred category:people"); // highlighting respects fieldnames used in query @@ -1419,64 +1419,64 @@ Highlighter highlighter; String result; - query = new QueryParser("text", new WhitespaceAnalyzer()).parse("foo"); + query = new QueryParser(Version.LUCENE_CURRENT, "text", new WhitespaceAnalyzer()).parse("foo"); highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2(), s, 3, "..."); assertEquals("Hi-Speed10 foo", result); - query = new QueryParser("text", new WhitespaceAnalyzer()).parse("10"); + query = new QueryParser(Version.LUCENE_CURRENT, "text", new WhitespaceAnalyzer()).parse("10"); highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2(), s, 3, "..."); assertEquals("Hi-Speed10 foo", result); - query = new QueryParser("text", new WhitespaceAnalyzer()).parse("hi"); + query = new QueryParser(Version.LUCENE_CURRENT, "text", new WhitespaceAnalyzer()).parse("hi"); highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2(), s, 3, "..."); assertEquals("Hi-Speed10 foo", result); - query = new QueryParser("text", new WhitespaceAnalyzer()).parse("speed"); + query = new QueryParser(Version.LUCENE_CURRENT, "text", new WhitespaceAnalyzer()).parse("speed"); highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2(), s, 3, "..."); assertEquals("Hi-Speed10 foo", result); - query = new QueryParser("text", new WhitespaceAnalyzer()).parse("hispeed"); + query = new QueryParser(Version.LUCENE_CURRENT, "text", new WhitespaceAnalyzer()).parse("hispeed"); highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2(), s, 3, "..."); assertEquals("Hi-Speed10 foo", result); - query = new QueryParser("text", new WhitespaceAnalyzer()).parse("hi speed"); + query = new QueryParser(Version.LUCENE_CURRENT, "text", new WhitespaceAnalyzer()).parse("hi speed"); highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2(), s, 3, "..."); assertEquals("Hi-Speed10 foo", result); // ///////////////// same tests, just put the bigger overlapping token // first - query = new QueryParser("text", new WhitespaceAnalyzer()).parse("foo"); + query = new QueryParser(Version.LUCENE_CURRENT, "text", new WhitespaceAnalyzer()).parse("foo"); highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2a(), s, 3, "..."); assertEquals("Hi-Speed10 foo", result); - query = new QueryParser("text", new WhitespaceAnalyzer()).parse("10"); + query = new QueryParser(Version.LUCENE_CURRENT, "text", new WhitespaceAnalyzer()).parse("10"); highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2a(), s, 3, "..."); assertEquals("Hi-Speed10 foo", result); - query = new QueryParser("text", new WhitespaceAnalyzer()).parse("hi"); + query = new QueryParser(Version.LUCENE_CURRENT, "text", new WhitespaceAnalyzer()).parse("hi"); highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2a(), s, 3, "..."); assertEquals("Hi-Speed10 foo", result); - query = new QueryParser("text", new WhitespaceAnalyzer()).parse("speed"); + query = new QueryParser(Version.LUCENE_CURRENT, "text", new WhitespaceAnalyzer()).parse("speed"); highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2a(), s, 3, "..."); assertEquals("Hi-Speed10 foo", result); - query = new QueryParser("text", new WhitespaceAnalyzer()).parse("hispeed"); + query = new QueryParser(Version.LUCENE_CURRENT, "text", new WhitespaceAnalyzer()).parse("hispeed"); highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2a(), s, 3, "..."); assertEquals("Hi-Speed10 foo", result); - query = new QueryParser("text", new WhitespaceAnalyzer()).parse("hi speed"); + query = new QueryParser(Version.LUCENE_CURRENT, "text", new WhitespaceAnalyzer()).parse("hi speed"); highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2a(), s, 3, "..."); assertEquals("Hi-Speed10 foo", result); @@ -1521,7 +1521,7 @@ private void searchIndex() throws IOException, ParseException, InvalidTokenOffsetsException { String q = "t_text1:random"; - QueryParser parser = new QueryParser( "t_text1", a ); + QueryParser parser = new QueryParser(Version.LUCENE_CURRENT, "t_text1", a ); Query query = parser.parse( q ); IndexSearcher searcher = new IndexSearcher( dir, true ); // This scorer can return negative idf -> null fragment @@ -1575,7 +1575,7 @@ } public void doSearching(String queryString) throws Exception { - QueryParser parser = new QueryParser(FIELD_NAME, analyzer); + QueryParser parser = new QueryParser(Version.LUCENE_CURRENT, FIELD_NAME, analyzer); parser.setEnablePositionIncrements(true); parser.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE); query = parser.parse(queryString); Modified: lucene/java/trunk/contrib/lucli/src/java/lucli/LuceneMethods.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/lucli/src/java/lucli/LuceneMethods.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/lucli/src/java/lucli/LuceneMethods.java (original) +++ lucene/java/trunk/contrib/lucli/src/java/lucli/LuceneMethods.java Fri Oct 23 20:25:17 2009 @@ -195,7 +195,7 @@ for (int ii = 0; ii < arraySize; ii++) { indexedArray[ii] = (String) indexedFields.get(ii); } - MultiFieldQueryParser parser = new MultiFieldQueryParser(indexedArray, analyzer); + MultiFieldQueryParser parser = new MultiFieldQueryParser(Version.LUCENE_CURRENT, indexedArray, analyzer); query = parser.parse(queryString); System.out.println("Searching for: " + query.toString()); return (query); @@ -216,7 +216,7 @@ for (int ii = 0; ii < arraySize; ii++) { fieldsArray[ii] = (String) fields.get(ii); } - MultiFieldQueryParser parser = new MultiFieldQueryParser(fieldsArray, analyzer); + MultiFieldQueryParser parser = new MultiFieldQueryParser(Version.LUCENE_CURRENT, fieldsArray, analyzer); query = parser.parse(queryString); System.out.println("Searching for: " + query.toString()); } Modified: lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/PatternAnalyzer.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/PatternAnalyzer.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/PatternAnalyzer.java (original) +++ lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/PatternAnalyzer.java Fri Oct 23 20:25:17 2009 @@ -33,6 +33,7 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.TermAttribute; +import org.apache.lucene.util.Version; /** * Efficient Lucene analyzer/tokenizer that preferably operates on a String rather than a @@ -124,7 +125,7 @@ * freely across threads without harm); global per class loader. */ public static final PatternAnalyzer DEFAULT_ANALYZER = new PatternAnalyzer( - NON_WORD_PATTERN, true, StopAnalyzer.ENGLISH_STOP_WORDS_SET); + Version.LUCENE_CURRENT, NON_WORD_PATTERN, true, StopAnalyzer.ENGLISH_STOP_WORDS_SET); /** * A lower-casing word analyzer with extended English stop words @@ -134,15 +135,18 @@ * http://thomas.loc.gov/home/all.about.inquery.html */ public static final PatternAnalyzer EXTENDED_ANALYZER = new PatternAnalyzer( - NON_WORD_PATTERN, true, EXTENDED_ENGLISH_STOP_WORDS); + Version.LUCENE_CURRENT, NON_WORD_PATTERN, true, EXTENDED_ENGLISH_STOP_WORDS); private final Pattern pattern; private final boolean toLowerCase; private final Set stopWords; + + private final Version matchVersion; /** * Constructs a new instance with the given parameters. * + * @param matchVersion If >= {@link Version#LUCENE_29}, StopFilter.enablePositionIncrement is set to true * @param pattern * a regular expression delimiting tokens * @param toLowerCase @@ -158,7 +162,7 @@ * or other stop words * lists . */ - public PatternAnalyzer(Pattern pattern, boolean toLowerCase, Set stopWords) { + public PatternAnalyzer(Version matchVersion, Pattern pattern, boolean toLowerCase, Set stopWords) { if (pattern == null) throw new IllegalArgumentException("pattern must not be null"); @@ -170,6 +174,7 @@ this.pattern = pattern; this.toLowerCase = toLowerCase; this.stopWords = stopWords; + this.matchVersion = matchVersion; } /** @@ -197,7 +202,7 @@ } else { stream = new PatternTokenizer(text, pattern, toLowerCase); - if (stopWords != null) stream = new StopFilter(false, stream, stopWords); + if (stopWords != null) stream = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), stream, stopWords); } return stream; Modified: lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (original) +++ lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java Fri Oct 23 20:25:17 2009 @@ -53,6 +53,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.index.TermDocs; +import org.apache.lucene.util.Version; /** Verifies that Lucene MemoryIndex and RAMDirectory have the same behaviour, @@ -277,7 +278,7 @@ Analyzer[] analyzers = new Analyzer[] { new SimpleAnalyzer(), - new StopAnalyzer(true), + new StopAnalyzer(Version.LUCENE_CURRENT), new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), PatternAnalyzer.DEFAULT_ANALYZER, // new WhitespaceAnalyzer(), @@ -480,7 +481,7 @@ } private Query parseQuery(String expression) throws ParseException { - QueryParser parser = new QueryParser(FIELD_NAME, analyzer); + QueryParser parser = new QueryParser(Version.LUCENE_CURRENT, FIELD_NAME, analyzer); // parser.setPhraseSlop(0); return parser.parse(expression); } Modified: lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/PatternAnalyzerTest.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/PatternAnalyzerTest.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/PatternAnalyzerTest.java (original) +++ lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/PatternAnalyzerTest.java Fri Oct 23 20:25:17 2009 @@ -24,6 +24,7 @@ import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.StopAnalyzer; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.util.Version; /** * Verifies the behavior of PatternAnalyzer. @@ -36,13 +37,13 @@ */ public void testNonWordPattern() throws IOException { // Split on non-letter pattern, do not lowercase, no stopwords - PatternAnalyzer a = new PatternAnalyzer(PatternAnalyzer.NON_WORD_PATTERN, + PatternAnalyzer a = new PatternAnalyzer(Version.LUCENE_CURRENT, PatternAnalyzer.NON_WORD_PATTERN, false, null); check(a, "The quick brown Fox,the abcd1234 (56.78) dc.", new String[] { "The", "quick", "brown", "Fox", "the", "abcd", "dc" }); // split on non-letter pattern, lowercase, english stopwords - PatternAnalyzer b = new PatternAnalyzer(PatternAnalyzer.NON_WORD_PATTERN, + PatternAnalyzer b = new PatternAnalyzer(Version.LUCENE_CURRENT, PatternAnalyzer.NON_WORD_PATTERN, true, StopAnalyzer.ENGLISH_STOP_WORDS_SET); check(b, "The quick brown Fox,the abcd1234 (56.78) dc.", new String[] { "quick", "brown", "fox", "abcd", "dc" }); @@ -54,13 +55,13 @@ */ public void testWhitespacePattern() throws IOException { // Split on whitespace patterns, do not lowercase, no stopwords - PatternAnalyzer a = new PatternAnalyzer(PatternAnalyzer.WHITESPACE_PATTERN, + PatternAnalyzer a = new PatternAnalyzer(Version.LUCENE_CURRENT, PatternAnalyzer.WHITESPACE_PATTERN, false, null); check(a, "The quick brown Fox,the abcd1234 (56.78) dc.", new String[] { "The", "quick", "brown", "Fox,the", "abcd1234", "(56.78)", "dc." }); // Split on whitespace patterns, lowercase, english stopwords - PatternAnalyzer b = new PatternAnalyzer(PatternAnalyzer.WHITESPACE_PATTERN, + PatternAnalyzer b = new PatternAnalyzer(Version.LUCENE_CURRENT, PatternAnalyzer.WHITESPACE_PATTERN, true, StopAnalyzer.ENGLISH_STOP_WORDS_SET); check(b, "The quick brown Fox,the abcd1234 (56.78) dc.", new String[] { "quick", "brown", "fox,the", "abcd1234", "(56.78)", "dc." }); @@ -72,12 +73,12 @@ */ public void testCustomPattern() throws IOException { // Split on comma, do not lowercase, no stopwords - PatternAnalyzer a = new PatternAnalyzer(Pattern.compile(","), false, null); + PatternAnalyzer a = new PatternAnalyzer(Version.LUCENE_CURRENT, Pattern.compile(","), false, null); check(a, "Here,Are,some,Comma,separated,words,", new String[] { "Here", "Are", "some", "Comma", "separated", "words" }); // split on comma, lowercase, english stopwords - PatternAnalyzer b = new PatternAnalyzer(Pattern.compile(","), true, + PatternAnalyzer b = new PatternAnalyzer(Version.LUCENE_CURRENT, Pattern.compile(","), true, StopAnalyzer.ENGLISH_STOP_WORDS_SET); check(b, "Here,Are,some,Comma,separated,words,", new String[] { "here", "some", "comma", "separated", "words" }); @@ -102,7 +103,7 @@ document.append(largeWord2); // Split on whitespace patterns, do not lowercase, no stopwords - PatternAnalyzer a = new PatternAnalyzer(PatternAnalyzer.WHITESPACE_PATTERN, + PatternAnalyzer a = new PatternAnalyzer(Version.LUCENE_CURRENT, PatternAnalyzer.WHITESPACE_PATTERN, false, null); check(a, document.toString(), new String[] { new String(largeWord), new String(largeWord2) }); Modified: lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java (original) +++ lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java Fri Oct 23 20:25:17 2009 @@ -28,6 +28,7 @@ import org.apache.lucene.analysis.tokenattributes.TermAttribute; import org.apache.lucene.queryParser.ParseException; import org.apache.lucene.search.Query; +import org.apache.lucene.util.Version; /** * Overrides Lucene's default QueryParser so that Fuzzy-, Prefix-, Range-, and WildcardQuerys @@ -49,8 +50,8 @@ * @param field the default field for query terms. * @param analyzer used to find terms in the query text. */ - public AnalyzingQueryParser(String field, Analyzer analyzer) { - super(field, analyzer); + public AnalyzingQueryParser(Version matchVersion, String field, Analyzer analyzer) { + super(matchVersion, field, analyzer); } /** Modified: lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/complexPhrase/ComplexPhraseQueryParser.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/complexPhrase/ComplexPhraseQueryParser.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/complexPhrase/ComplexPhraseQueryParser.java (original) +++ lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/complexPhrase/ComplexPhraseQueryParser.java Fri Oct 23 20:25:17 2009 @@ -38,6 +38,7 @@ import org.apache.lucene.search.spans.SpanOrQuery; import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.search.spans.SpanTermQuery; +import org.apache.lucene.util.Version; /** * QueryParser which permits complex phrase query syntax eg "(john jon @@ -67,8 +68,8 @@ private ComplexPhraseQuery currentPhraseQuery = null; - public ComplexPhraseQueryParser(String f, Analyzer a) { - super(f, a); + public ComplexPhraseQueryParser(Version matchVersion, String f, Analyzer a) { + super(matchVersion, f, a); } protected Query getFieldQuery(String field, String queryText, int slop) { Modified: lucene/java/trunk/contrib/misc/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/misc/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/misc/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java (original) +++ lucene/java/trunk/contrib/misc/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java Fri Oct 23 20:25:17 2009 @@ -28,6 +28,7 @@ import org.apache.lucene.analysis.standard.StandardFilter; import org.apache.lucene.analysis.standard.StandardTokenizer; import org.apache.lucene.queryParser.ParseException; +import org.apache.lucene.util.Version; /** * @version $Revision$, $Date$ @@ -97,7 +98,7 @@ } private String parseWithAnalyzingQueryParser(String s, Analyzer a) throws ParseException { - AnalyzingQueryParser qp = new AnalyzingQueryParser("field", a); + AnalyzingQueryParser qp = new AnalyzingQueryParser(Version.LUCENE_CURRENT, "field", a); org.apache.lucene.search.Query q = qp.parse(s); return q.toString("field"); } @@ -109,7 +110,7 @@ } public TokenStream tokenStream(String fieldName, Reader reader) { - TokenStream result = new StandardTokenizer(reader); + TokenStream result = new StandardTokenizer(Version.LUCENE_CURRENT, reader); result = new StandardFilter(result); result = new ASCIIFoldingFilter(result); result = new LowerCaseFilter(result); Modified: lucene/java/trunk/contrib/misc/src/test/org/apache/lucene/queryParser/complexPhrase/TestComplexPhraseQuery.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/misc/src/test/org/apache/lucene/queryParser/complexPhrase/TestComplexPhraseQuery.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/misc/src/test/org/apache/lucene/queryParser/complexPhrase/TestComplexPhraseQuery.java (original) +++ lucene/java/trunk/contrib/misc/src/test/org/apache/lucene/queryParser/complexPhrase/TestComplexPhraseQuery.java Fri Oct 23 20:25:17 2009 @@ -33,6 +33,7 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.Version; public class TestComplexPhraseQuery extends TestCase { @@ -71,7 +72,7 @@ } private void checkBadQuery(String qString) { - QueryParser qp = new ComplexPhraseQueryParser(defaultFieldName, analyzer); + QueryParser qp = new ComplexPhraseQueryParser(Version.LUCENE_CURRENT, defaultFieldName, analyzer); Throwable expected = null; try { qp.parse(qString); @@ -84,7 +85,7 @@ private void checkMatches(String qString, String expectedVals) throws Exception { - QueryParser qp = new ComplexPhraseQueryParser(defaultFieldName, analyzer); + QueryParser qp = new ComplexPhraseQueryParser(Version.LUCENE_CURRENT, defaultFieldName, analyzer); qp.setFuzzyPrefixLength(1); // usually a good idea Query q = qp.parse(qString); Modified: lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java (original) +++ lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java Fri Oct 23 20:25:17 2009 @@ -34,6 +34,7 @@ import org.apache.lucene.queryParser.standard.StandardQueryParser; import org.apache.lucene.queryParser.standard.config.DefaultOperatorAttribute.Operator; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.Version; /** * This test case is a copy of the core Lucene query parser test, it was adapted @@ -154,7 +155,7 @@ } public TokenStream tokenStream(String fieldName, Reader reader) { - TokenStream result = new StandardTokenizer(reader); + TokenStream result = new StandardTokenizer(Version.LUCENE_CURRENT, reader); result = new TestFilter(result); result = new LowerCaseFilter(result); return result; @@ -222,7 +223,7 @@ } public TokenStream tokenStream(String fieldName, Reader reader) { - TokenStream result = new StandardTokenizer(reader); + TokenStream result = new StandardTokenizer(Version.LUCENE_CURRENT, reader); result = new TestPosIncrementFilter(result); result = new LowerCaseFilter(result); return result; Modified: lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java (original) +++ lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java Fri Oct 23 20:25:17 2009 @@ -33,6 +33,7 @@ import org.apache.lucene.queryParser.ParseException; import org.apache.lucene.queryParser.standard.QueryParserWrapper; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.Version; /** * This test case is a copy of the core Lucene query parser test, it was adapted @@ -148,7 +149,7 @@ } public TokenStream tokenStream(String fieldName, Reader reader) { - TokenStream result = new StandardTokenizer(reader); + TokenStream result = new StandardTokenizer(Version.LUCENE_CURRENT, reader); result = new TestFilter(result); result = new LowerCaseFilter(result); return result; @@ -216,7 +217,7 @@ } public TokenStream tokenStream(String fieldName, Reader reader) { - TokenStream result = new StandardTokenizer(reader); + TokenStream result = new StandardTokenizer(Version.LUCENE_CURRENT, reader); result = new TestPosIncrementFilter(result); result = new LowerCaseFilter(result); return result; Modified: lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java (original) +++ lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java Fri Oct 23 20:25:17 2009 @@ -1070,7 +1070,7 @@ public void testStopwords() throws Exception { StandardQueryParser qp = new StandardQueryParser(); qp.setAnalyzer( - new StopAnalyzer(StopFilter.makeStopSet("the", "foo" ), true)); + new StopAnalyzer(Version.LUCENE_CURRENT, StopFilter.makeStopSet("the", "foo" ))); Query result = qp.parse("a:the OR a:foo", "a"); assertNotNull("result is null and it shouldn't be", result); @@ -1093,7 +1093,7 @@ public void testPositionIncrement() throws Exception { StandardQueryParser qp = new StandardQueryParser(); qp.setAnalyzer( - new StopAnalyzer(StopFilter.makeStopSet("the", "in", "are", "this" ), true)); + new StopAnalyzer(Version.LUCENE_CURRENT, StopFilter.makeStopSet("the", "in", "are", "this" ))); qp.setEnablePositionIncrements(true); Modified: lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java (original) +++ lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java Fri Oct 23 20:25:17 2009 @@ -1048,7 +1048,7 @@ } public void testStopwords() throws Exception { - QueryParserWrapper qp = new QueryParserWrapper("a", new StopAnalyzer(StopFilter.makeStopSet("the", "foo"), false)); + QueryParserWrapper qp = new QueryParserWrapper("a", new StopAnalyzer(Version.LUCENE_CURRENT, StopFilter.makeStopSet("the", "foo"))); Query result = qp.parse("a:the OR a:foo"); assertNotNull("result is null and it shouldn't be", result); assertTrue("result is not a BooleanQuery", result instanceof BooleanQuery); @@ -1067,7 +1067,7 @@ } public void testPositionIncrement() throws Exception { - QueryParserWrapper qp = new QueryParserWrapper("a", new StopAnalyzer(StopFilter.makeStopSet("the", "in", "are", "this"), true)); + QueryParserWrapper qp = new QueryParserWrapper("a", new StopAnalyzer(Version.LUCENE_CURRENT, StopFilter.makeStopSet("the", "in", "are", "this"))); qp.setEnablePositionIncrements(true); String qtxt = "\"the words in poisitions pos02578 are stopped in this phrasequery\""; // 0 2 5 7 8 Modified: lucene/java/trunk/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java (original) +++ lucene/java/trunk/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java Fri Oct 23 20:25:17 2009 @@ -19,6 +19,7 @@ import org.apache.lucene.analysis.*; import org.apache.lucene.analysis.standard.*; +import org.apache.lucene.util.Version; import java.io.IOException; import java.io.Reader; @@ -30,20 +31,25 @@ * Available stemmers are listed in org.tartarus.snowball.ext. The name of a * stemmer is the part of the class name before "Stemmer", e.g., the stemmer in * {@link org.tartarus.snowball.ext.EnglishStemmer} is named "English". + * + *

NOTE: This class uses the same {@link Version} + * dependent settings as {@link StandardAnalyzer}.

*/ public class SnowballAnalyzer extends Analyzer { private String name; private Set stopSet; + private final Version matchVersion; /** Builds the named analyzer with no stop words. */ - public SnowballAnalyzer(String name) { + public SnowballAnalyzer(Version matchVersion, String name) { this.name = name; setOverridesTokenStreamMethod(SnowballAnalyzer.class); + this.matchVersion = matchVersion; } /** Builds the named analyzer with the given stop words. */ - public SnowballAnalyzer(String name, String[] stopWords) { - this(name); + public SnowballAnalyzer(Version matchVersion, String name, String[] stopWords) { + this(matchVersion, name); stopSet = StopFilter.makeStopSet(stopWords); } @@ -51,11 +57,12 @@ StandardFilter}, a {@link LowerCaseFilter}, a {@link StopFilter}, and a {@link SnowballFilter} */ public TokenStream tokenStream(String fieldName, Reader reader) { - TokenStream result = new StandardTokenizer(reader); + TokenStream result = new StandardTokenizer(matchVersion, reader); result = new StandardFilter(result); result = new LowerCaseFilter(result); if (stopSet != null) - result = new StopFilter(false, result, stopSet); + result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), + result, stopSet); result = new SnowballFilter(result, name); return result; } @@ -80,11 +87,12 @@ SavedStreams streams = (SavedStreams) getPreviousTokenStream(); if (streams == null) { streams = new SavedStreams(); - streams.source = new StandardTokenizer(reader); + streams.source = new StandardTokenizer(matchVersion, reader); streams.result = new StandardFilter(streams.source); streams.result = new LowerCaseFilter(streams.result); if (stopSet != null) - streams.result = new StopFilter(false, streams.result, stopSet); + streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), + streams.result, stopSet); streams.result = new SnowballFilter(streams.result, name); setPreviousTokenStream(streams); } else { Modified: lucene/java/trunk/contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java (original) +++ lucene/java/trunk/contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java Fri Oct 23 20:25:17 2009 @@ -31,17 +31,18 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.analysis.tokenattributes.TermAttribute; import org.apache.lucene.analysis.tokenattributes.TypeAttribute; +import org.apache.lucene.util.Version; public class TestSnowball extends BaseTokenStreamTestCase { public void testEnglish() throws Exception { - Analyzer a = new SnowballAnalyzer("English"); + Analyzer a = new SnowballAnalyzer(Version.LUCENE_CURRENT, "English"); assertAnalyzesTo(a, "he abhorred accents", new String[]{"he", "abhor", "accent"}); } public void testReusableTokenStream() throws Exception { - Analyzer a = new SnowballAnalyzer("English"); + Analyzer a = new SnowballAnalyzer(Version.LUCENE_CURRENT, "English"); assertAnalyzesToReuse(a, "he abhorred accents", new String[]{"he", "abhor", "accent"}); assertAnalyzesToReuse(a, "she abhorred him", @@ -53,7 +54,7 @@ */ private class SnowballSubclassAnalyzer extends SnowballAnalyzer { public SnowballSubclassAnalyzer(String name) { - super(name); + super(Version.LUCENE_CURRENT, name); } public TokenStream tokenStream(String fieldName, Reader reader) { Modified: lucene/java/trunk/contrib/swing/src/java/org/apache/lucene/swing/models/ListSearcher.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/swing/src/java/org/apache/lucene/swing/models/ListSearcher.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/swing/src/java/org/apache/lucene/swing/models/ListSearcher.java (original) +++ lucene/java/trunk/contrib/swing/src/java/org/apache/lucene/swing/models/ListSearcher.java Fri Oct 23 20:25:17 2009 @@ -38,6 +38,7 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Scorer; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.Version; /** * See table searcher explanation. @@ -167,7 +168,7 @@ //build a query based on the fields, searchString and cached analyzer //NOTE: This is an area for improvement since the MultiFieldQueryParser // has some weirdness. - MultiFieldQueryParser parser = new MultiFieldQueryParser(fields, analyzer); + MultiFieldQueryParser parser = new MultiFieldQueryParser(Version.LUCENE_CURRENT, fields, analyzer); Query query =parser.parse(searchString); //reset this list model with the new results resetSearchResults(is, query); Modified: lucene/java/trunk/contrib/swing/src/java/org/apache/lucene/swing/models/TableSearcher.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/swing/src/java/org/apache/lucene/swing/models/TableSearcher.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/swing/src/java/org/apache/lucene/swing/models/TableSearcher.java (original) +++ lucene/java/trunk/contrib/swing/src/java/org/apache/lucene/swing/models/TableSearcher.java Fri Oct 23 20:25:17 2009 @@ -35,7 +35,7 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.swing.models.ListSearcher.CountingCollector; - +import org.apache.lucene.util.Version; /** * This is a TableModel that encapsulates Lucene @@ -244,7 +244,7 @@ //build a query based on the fields, searchString and cached analyzer //NOTE: This is an area for improvement since the MultiFieldQueryParser // has some weirdness. - MultiFieldQueryParser parser = new MultiFieldQueryParser(fields, analyzer); + MultiFieldQueryParser parser = new MultiFieldQueryParser(Version.LUCENE_CURRENT, fields, analyzer); Query query = parser.parse(searchString); //reset this table model with the new results resetSearchResults(is, query); Modified: lucene/java/trunk/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/UserInputQueryBuilder.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/UserInputQueryBuilder.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/UserInputQueryBuilder.java (original) +++ lucene/java/trunk/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/UserInputQueryBuilder.java Fri Oct 23 20:25:17 2009 @@ -8,6 +8,7 @@ import org.apache.lucene.xmlparser.ParserException; import org.apache.lucene.xmlparser.QueryBuilder; import org.w3c.dom.Element; +import org.apache.lucene.util.Version; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -88,7 +89,7 @@ */ protected QueryParser createQueryParser(String fieldName, Analyzer analyzer) { - return new QueryParser(fieldName,analyzer); + return new QueryParser(Version.LUCENE_CURRENT, fieldName,analyzer); } } Modified: lucene/java/trunk/src/demo/org/apache/lucene/demo/SearchFiles.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/demo/org/apache/lucene/demo/SearchFiles.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/src/demo/org/apache/lucene/demo/SearchFiles.java (original) +++ lucene/java/trunk/src/demo/org/apache/lucene/demo/SearchFiles.java Fri Oct 23 20:25:17 2009 @@ -127,7 +127,7 @@ } else { in = new BufferedReader(new InputStreamReader(System.in, "UTF-8")); } - QueryParser parser = new QueryParser(field, analyzer); + QueryParser parser = new QueryParser(Version.LUCENE_CURRENT, field, analyzer); while (true) { if (queries == null) // prompt the user System.out.println("Enter query: "); Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/StopAnalyzer.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/StopAnalyzer.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/src/java/org/apache/lucene/analysis/StopAnalyzer.java (original) +++ lucene/java/trunk/src/java/org/apache/lucene/analysis/StopAnalyzer.java Fri Oct 23 20:25:17 2009 @@ -24,7 +24,17 @@ import java.util.Set; import java.util.List; -/** Filters {@link LetterTokenizer} with {@link LowerCaseFilter} and {@link StopFilter}. */ +import org.apache.lucene.util.Version; + +/** Filters {@link LetterTokenizer} with {@link LowerCaseFilter} and {@link StopFilter}. + * + * + *

You must specify the required {@link Version} + * compatibility when creating StopAnalyzer: + *

    + *
  • As of 2.9, position increments are preserved + *
+*/ public final class StopAnalyzer extends Analyzer { private final Set stopWords; @@ -49,40 +59,39 @@ /** Builds an analyzer which removes words in * {@link #ENGLISH_STOP_WORDS}. - * @param enablePositionIncrements See {@link - * StopFilter#setEnablePositionIncrements} */ - public StopAnalyzer(boolean enablePositionIncrements) { + * @param matchVersion See above + */ + public StopAnalyzer(Version matchVersion) { stopWords = ENGLISH_STOP_WORDS_SET; - this.enablePositionIncrements = enablePositionIncrements; + enablePositionIncrements = StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion); } /** Builds an analyzer with the stop words from the given set. + * @param matchVersion See above * @param stopWords Set of stop words * @param enablePositionIncrements See {@link * StopFilter#setEnablePositionIncrements} */ - public StopAnalyzer(Set stopWords, boolean enablePositionIncrements) { + public StopAnalyzer(Version matchVersion, Set stopWords) { this.stopWords = stopWords; - this.enablePositionIncrements = enablePositionIncrements; + enablePositionIncrements = StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion); } /** Builds an analyzer with the stop words from the given file. * @see WordlistLoader#getWordSet(File) - * @param stopwordsFile File to load stop words from - * @param enablePositionIncrements See {@link - * StopFilter#setEnablePositionIncrements} */ - public StopAnalyzer(File stopwordsFile, boolean enablePositionIncrements) throws IOException { + * @param matchVersion See above + * @param stopwordsFile File to load stop words from */ + public StopAnalyzer(Version matchVersion, File stopwordsFile) throws IOException { stopWords = WordlistLoader.getWordSet(stopwordsFile); - this.enablePositionIncrements = enablePositionIncrements; + this.enablePositionIncrements = StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion); } /** Builds an analyzer with the stop words from the given reader. * @see WordlistLoader#getWordSet(Reader) - * @param stopwords Reader to load stop words from - * @param enablePositionIncrements See {@link - * StopFilter#setEnablePositionIncrements} */ - public StopAnalyzer(Reader stopwords, boolean enablePositionIncrements) throws IOException { + * @param matchVersion See above + * @param stopwords Reader to load stop words from */ + public StopAnalyzer(Version matchVersion, Reader stopwords) throws IOException { stopWords = WordlistLoader.getWordSet(stopwords); - this.enablePositionIncrements = enablePositionIncrements; + this.enablePositionIncrements = StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion); } /** Filters LowerCaseTokenizer with StopFilter. */ Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/StopFilter.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/StopFilter.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/src/java/org/apache/lucene/analysis/StopFilter.java (original) +++ lucene/java/trunk/src/java/org/apache/lucene/analysis/StopFilter.java Fri Oct 23 20:25:17 2009 @@ -25,6 +25,7 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.analysis.tokenattributes.TermAttribute; import org.apache.lucene.queryParser.QueryParser; // for javadoc +import org.apache.lucene.util.Version; /** * Removes stop words from a token stream. @@ -151,6 +152,21 @@ } /** + * Returns version-dependent default for + * enablePositionIncrements. Analyzers that embed + * StopFilter use this method when creating the + * StopFilter. Prior to 2.9, this returns false. On 2.9 + * or later, it returns true. + */ + public static boolean getEnablePositionIncrementsVersionDefault(Version matchVersion) { + if (matchVersion.onOrAfter(Version.LUCENE_29)) { + return true; + } else { + return false; + } + } + + /** * @see #setEnablePositionIncrements(boolean). */ public boolean getEnablePositionIncrements() { Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java (original) +++ lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java Fri Oct 23 20:25:17 2009 @@ -35,7 +35,7 @@ * compatibility when creating StandardAnalyzer: *
    *
  • As of 2.9, StopFilter preserves position - * increments by default + * increments *
  • As of 2.4, Tokens incorrectly identified as acronyms * are corrected (see LUCENE-1608 *
@@ -52,6 +52,7 @@ /** An unmodifiable set containing some common English words that are usually not useful for searching. */ public static final Set STOP_WORDS_SET = StopAnalyzer.ENGLISH_STOP_WORDS_SET; + private final Version matchVersion; /** Builds an analyzer with the default stop words ({@link * #STOP_WORDS_SET}). @@ -71,6 +72,7 @@ setOverridesTokenStreamMethod(StandardAnalyzer.class); enableStopPositionIncrements = matchVersion.onOrAfter(Version.LUCENE_29); replaceInvalidAcronym = matchVersion.onOrAfter(Version.LUCENE_24); + this.matchVersion = matchVersion; } /** Builds an analyzer with the stop words from the given file. @@ -94,11 +96,12 @@ /** Constructs a {@link StandardTokenizer} filtered by a {@link StandardFilter}, a {@link LowerCaseFilter} and a {@link StopFilter}. */ public TokenStream tokenStream(String fieldName, Reader reader) { - StandardTokenizer tokenStream = new StandardTokenizer(reader, replaceInvalidAcronym); + StandardTokenizer tokenStream = new StandardTokenizer(matchVersion, reader); tokenStream.setMaxTokenLength(maxTokenLength); TokenStream result = new StandardFilter(tokenStream); result = new LowerCaseFilter(result); - result = new StopFilter(enableStopPositionIncrements, result, stopSet); + result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), + result, stopSet); return result; } @@ -140,10 +143,11 @@ if (streams == null) { streams = new SavedStreams(); setPreviousTokenStream(streams); - streams.tokenStream = new StandardTokenizer(reader); + streams.tokenStream = new StandardTokenizer(matchVersion, reader); streams.filteredTokenStream = new StandardFilter(streams.tokenStream); streams.filteredTokenStream = new LowerCaseFilter(streams.filteredTokenStream); - streams.filteredTokenStream = new StopFilter(enableStopPositionIncrements, streams.filteredTokenStream, stopSet); + streams.filteredTokenStream = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), + streams.filteredTokenStream, stopSet); } else { streams.tokenStream.reset(reader); } Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (original) +++ lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java Fri Oct 23 20:25:17 2009 @@ -27,6 +27,7 @@ import org.apache.lucene.analysis.tokenattributes.TermAttribute; import org.apache.lucene.analysis.tokenattributes.TypeAttribute; import org.apache.lucene.util.AttributeSource; +import org.apache.lucene.util.Version; /** A grammar-based tokenizer constructed with JFlex * @@ -43,6 +44,14 @@ *

Many applications have specific tokenizer needs. If this tokenizer does * not suit your application, please consider copying this source code * directory to your project and maintaining your own grammar-based tokenizer. + * + * + *

You must specify the required {@link Version} + * compatibility when creating StandardAnalyzer: + *

*/ public final class StandardTokenizer extends Tokenizer { @@ -105,55 +114,50 @@ } /** - * Creates a new instance of the {@link StandardTokenizer}. Attaches the - * input to a newly created JFlex scanner. - */ - public StandardTokenizer(Reader input) { - this(input, false); - } - - /** * Creates a new instance of the {@link org.apache.lucene.analysis.standard.StandardTokenizer}. Attaches * the input to the newly created JFlex scanner. * * @param input The input reader - * @param replaceInvalidAcronym Set to true to replace mischaracterized acronyms with HOST. * * See http://issues.apache.org/jira/browse/LUCENE-1068 */ - public StandardTokenizer(Reader input, boolean replaceInvalidAcronym) { + public StandardTokenizer(Version matchVersion, Reader input) { super(); this.scanner = new StandardTokenizerImpl(input); - init(input, replaceInvalidAcronym); + init(input, matchVersion); } /** * Creates a new StandardTokenizer with a given {@link AttributeSource}. */ - public StandardTokenizer(AttributeSource source, Reader input, boolean replaceInvalidAcronym) { + public StandardTokenizer(Version matchVersion, AttributeSource source, Reader input) { super(source); this.scanner = new StandardTokenizerImpl(input); - init(input, replaceInvalidAcronym); + init(input, matchVersion); } /** * Creates a new StandardTokenizer with a given {@link org.apache.lucene.util.AttributeSource.AttributeFactory} */ - public StandardTokenizer(AttributeFactory factory, Reader input, boolean replaceInvalidAcronym) { + public StandardTokenizer(Version matchVersion, AttributeFactory factory, Reader input) { super(factory); this.scanner = new StandardTokenizerImpl(input); - init(input, replaceInvalidAcronym); + init(input, matchVersion); } - private void init(Reader input, boolean replaceInvalidAcronym) { - this.replaceInvalidAcronym = replaceInvalidAcronym; + private void init(Reader input, Version matchVersion) { + if (matchVersion.onOrAfter(Version.LUCENE_24)) { + replaceInvalidAcronym = true; + } else { + replaceInvalidAcronym = false; + } this.input = input; termAtt = addAttribute(TermAttribute.class); offsetAtt = addAttribute(OffsetAttribute.class); posIncrAtt = addAttribute(PositionIncrementAttribute.class); typeAtt = addAttribute(TypeAttribute.class); } - + // this tokenizer generates three attributes: // offset, positionIncrement and type private TermAttribute termAtt; Modified: lucene/java/trunk/src/java/org/apache/lucene/queryParser/MultiFieldQueryParser.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/queryParser/MultiFieldQueryParser.java?rev=829206&r1=829205&r2=829206&view=diff ============================================================================== --- lucene/java/trunk/src/java/org/apache/lucene/queryParser/MultiFieldQueryParser.java (original) +++ lucene/java/trunk/src/java/org/apache/lucene/queryParser/MultiFieldQueryParser.java Fri Oct 23 20:25:17 2009 @@ -27,6 +27,7 @@ import org.apache.lucene.search.MultiPhraseQuery; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.util.Version; /** * A QueryParser which constructs queries to search multiple fields. @@ -65,8 +66,8 @@ *

In other words, all the query's terms must appear, but it doesn't matter in * what fields they appear.

*/ - public MultiFieldQueryParser(String[] fields, Analyzer analyzer, Map boosts) { - this(fields,analyzer); + public MultiFieldQueryParser(Version matchVersion, String[] fields, Analyzer analyzer, Map boosts) { + this(matchVersion, fields, analyzer); this.boosts = boosts; } @@ -90,8 +91,8 @@ *

In other words, all the query's terms must appear, but it doesn't matter in * what fields they appear.

*/ - public MultiFieldQueryParser(String[] fields, Analyzer analyzer) { - super(null, analyzer); + public MultiFieldQueryParser(Version matchVersion, String[] fields, Analyzer analyzer) { + super(matchVersion, null, analyzer); this.fields = fields; } @@ -196,6 +197,7 @@ * (field1:query1) (field2:query2) (field3:query3)...(fieldx:queryx) * * + * @param matchVersion Lucene version to match; this is passed through to QueryParser. * @param queries Queries strings to parse * @param fields Fields to search on * @param analyzer Analyzer to use @@ -203,7 +205,7 @@ * @throws IllegalArgumentException if the length of the queries array differs * from the length of the fields array */ - public static Query parse(String[] queries, String[] fields, + public static Query parse(Version matchVersion, String[] queries, String[] fields, Analyzer analyzer) throws ParseException { if (queries.length != fields.length) @@ -211,7 +213,7 @@ BooleanQuery bQuery = new BooleanQuery(); for (int i = 0; i < fields.length; i++) { - QueryParser qp = new QueryParser(fields[i], analyzer); + QueryParser qp = new QueryParser(matchVersion, fields[i], analyzer); Query q = qp.parse(queries[i]); if (q!=null && // q never null, just being defensive (!(q instanceof BooleanQuery) || ((BooleanQuery)q).getClauses().length>0)) { @@ -243,6 +245,7 @@ * * * + * @param matchVersion Lucene version to match; this is passed through to QueryParser. * @param query Query string to parse * @param fields Fields to search on * @param flags Flags describing the fields @@ -251,13 +254,13 @@ * @throws IllegalArgumentException if the length of the fields array differs * from the length of the flags array */ - public static Query parse(String query, String[] fields, + public static Query parse(Version matchVersion, String query, String[] fields, BooleanClause.Occur[] flags, Analyzer analyzer) throws ParseException { if (fields.length != flags.length) throw new IllegalArgumentException("fields.length != flags.length"); BooleanQuery bQuery = new BooleanQuery(); for (int i = 0; i < fields.length; i++) { - QueryParser qp = new QueryParser(fields[i], analyzer); + QueryParser qp = new QueryParser(matchVersion, fields[i], analyzer); Query q = qp.parse(query); if (q!=null && // q never null, just being defensive (!(q instanceof BooleanQuery) || ((BooleanQuery)q).getClauses().length>0)) { @@ -290,6 +293,7 @@ * * * + * @param matchVersion Lucene version to match; this is passed through to QueryParser. * @param queries Queries string to parse * @param fields Fields to search on * @param flags Flags describing the fields @@ -298,7 +302,7 @@ * @throws IllegalArgumentException if the length of the queries, fields, * and flags array differ */ - public static Query parse(String[] queries, String[] fields, BooleanClause.Occur[] flags, + public static Query parse(Version matchVersion, String[] queries, String[] fields, BooleanClause.Occur[] flags, Analyzer analyzer) throws ParseException { if (!(queries.length == fields.length && queries.length == flags.length)) @@ -306,7 +310,7 @@ BooleanQuery bQuery = new BooleanQuery(); for (int i = 0; i < fields.length; i++) { - QueryParser qp = new QueryParser(fields[i], analyzer); + QueryParser qp = new QueryParser(matchVersion, fields[i], analyzer); Query q = qp.parse(queries[i]); if (q!=null && // q never null, just being defensive (!(q instanceof BooleanQuery) || ((BooleanQuery)q).getClauses().length>0)) {