Return-Path: Delivered-To: apmail-lucene-java-commits-archive@www.apache.org Received: (qmail 57664 invoked from network); 26 Feb 2010 13:11:13 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 26 Feb 2010 13:11:13 -0000 Received: (qmail 13665 invoked by uid 500); 26 Feb 2010 13:11:13 -0000 Delivered-To: apmail-lucene-java-commits-archive@lucene.apache.org Received: (qmail 13627 invoked by uid 500); 26 Feb 2010 13:11:13 -0000 Mailing-List: contact java-commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: java-dev@lucene.apache.org Delivered-To: mailing list java-commits@lucene.apache.org Received: (qmail 13618 invoked by uid 99); 26 Feb 2010 13:11:13 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 26 Feb 2010 13:11:13 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 26 Feb 2010 13:11:01 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id C36C92388B34; Fri, 26 Feb 2010 13:10:16 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: svn commit: r916666 [6/16] - in /lucene/java/branches/flex_1458: ./ contrib/ contrib/analyzers/common/ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ contrib/analyzers/common/src/java/org/apache/lucene/analysis/bg/ contrib/analyzers/c... Date: Fri, 26 Feb 2010 13:10:08 -0000 To: java-commits@lucene.apache.org From: rmuir@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20100226131016.C36C92388B34@eris.apache.org> Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (original) +++ lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java Fri Feb 26 13:09:54 2010 @@ -979,6 +979,79 @@ return algLines; } + /** + * Test that we can create ShingleAnalyzerWrappers. + */ + public void testShingleAnalyzer() throws Exception { + String text = "one,two,three, four five six"; + + // Default analyzer, maxShingleSize, and outputUnigrams + Benchmark benchmark = execBenchmark(getShingleConfig("")); + TokenStream stream = benchmark.getRunData().getAnalyzer().tokenStream + ("bogus", new StringReader(text)); + assertEqualShingle(benchmark.getRunData().getAnalyzer(), text, + new String[] {"one", "one two", "two", "two three", + "three", "three four", "four", "four five", + "five", "five six", "six"}); + // Default analyzer, maxShingleSize = 3, and outputUnigrams = false + benchmark = execBenchmark + (getShingleConfig("maxShingleSize:3,outputUnigrams:false")); + assertEqualShingle(benchmark.getRunData().getAnalyzer(), text, + new String[] { "one two", "one two three", "two three", + "two three four", "three four", + "three four five", "four five", + "four five six", "five six" }); + // WhitespaceAnalyzer, default maxShingleSize and outputUnigrams + benchmark = execBenchmark + (getShingleConfig("analyzer:WhitespaceAnalyzer")); + assertEqualShingle(benchmark.getRunData().getAnalyzer(), text, + new String[] { "one,two,three,", "one,two,three, four", + "four", "four five", "five", "five six", + "six" }); + + // WhitespaceAnalyzer, maxShingleSize=3 and outputUnigrams=false + benchmark = execBenchmark + (getShingleConfig + ("outputUnigrams:false,maxShingleSize:3,analyzer:WhitespaceAnalyzer")); + assertEqualShingle(benchmark.getRunData().getAnalyzer(), text, + new String[] { "one,two,three, four", + "one,two,three, four five", + "four five", "four five six", + "five six" }); + } + + private void assertEqualShingle + (Analyzer analyzer, String text, String[] expected) throws Exception { + TokenStream stream = analyzer.tokenStream("bogus", new StringReader(text)); + stream.reset(); + TermAttribute termAtt = stream.addAttribute(TermAttribute.class); + int termNum = 0; + while (stream.incrementToken()) { + assertTrue("Extra output term(s), starting with '" + + new String(termAtt.termBuffer(), 0, termAtt.termLength()) + "'", + termNum < expected.length); + assertEquals("Mismatch in output term # " + termNum + " - ", + expected[termNum], + new String(termAtt.termBuffer(), 0, termAtt.termLength())); + ++termNum; + } + assertEquals("Too few output terms", expected.length, termNum); + stream.close(); + } + + private static String[] getShingleConfig(String params) { + String algLines[] = { + "content.source=org.apache.lucene.benchmark.byTask.feeds.LineDocSource", + "docs.file=" + getReuters20LinesFile(), + "content.source.forever=false", + "directory=RAMDirectory", + "NewShingleAnalyzer(" + params + ")", + "CreateIndex", + "{ \"AddDocs\" AddDoc > : * " + }; + return algLines; + } + private static String getReuters20LinesFile() { return System.getProperty("lucene.common.dir").replace('\\','/') + "/contrib/benchmark/src/test/org/apache/lucene/benchmark/reuters.first20.lines.txt"; Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/TestQualityRun.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/TestQualityRun.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/TestQualityRun.java (original) +++ lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/TestQualityRun.java Fri Feb 26 13:09:54 2010 @@ -20,10 +20,11 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileReader; +import java.io.InputStream; +import java.io.InputStreamReader; import java.io.PrintWriter; import org.apache.lucene.benchmark.byTask.TestPerfTasksLogic; -import org.apache.lucene.benchmark.byTask.feeds.ReutersContentSource; import org.apache.lucene.benchmark.quality.Judge; import org.apache.lucene.benchmark.quality.QualityQuery; import org.apache.lucene.benchmark.quality.QualityQueryParser; @@ -39,6 +40,10 @@ /** * Test that quality run does its job. + *

+ * NOTE: if the default scoring or StandardAnalyzer is changed, then + * this test will not work correctly, as it does not dynamically + * generate its test trec topics/qrels! */ public class TestQualityRun extends TestCase { @@ -52,14 +57,14 @@ } public void testTrecQuality() throws Exception { - // first create the complete reuters index + // first create the partial reuters index createReutersIndex(); File workDir = new File(System.getProperty("benchmark.work.dir","work")); assertTrue("Bad workDir: "+workDir, workDir.exists()&& workDir.isDirectory()); int maxResults = 1000; - String docNameField = "docid"; + String docNameField = "doctitle"; // orig docID is in the linedoc format title PrintWriter logger = DEBUG ? new PrintWriter(System.out,true) : null; @@ -105,13 +110,13 @@ assertTrue("avg-p should be hurt: "+s.getAvp(), 1.0 > s.getAvp()); assertTrue("recall should be hurt: "+s.getRecall(), 1.0 > s.getRecall()); for (int j = 1; j <= QualityStats.MAX_POINTS; j++) { - assertEquals("p_at_"+j+" should be perfect: "+s.getPrecisionAt(j), 1.0, s.getPrecisionAt(j), 1E-9); + assertEquals("p_at_"+j+" should be perfect: "+s.getPrecisionAt(j), 1.0, s.getPrecisionAt(j), 1E-2); } break; case 1: assertTrue("avg-p should be hurt", 1.0 > s.getAvp()); - assertEquals("recall should be perfect: "+s.getRecall(), 1.0, s.getRecall(), 1E-9); + assertEquals("recall should be perfect: "+s.getRecall(), 1.0, s.getRecall(), 1E-2); for (int j = 1; j <= QualityStats.MAX_POINTS; j++) { assertTrue("p_at_"+j+" should be hurt: "+s.getPrecisionAt(j), 1.0 > s.getPrecisionAt(j)); } @@ -126,10 +131,10 @@ break; default: { - assertEquals("avg-p should be perfect: "+s.getAvp(), 1.0, s.getAvp(), 1E-9); - assertEquals("recall should be perfect: "+s.getRecall(), 1.0, s.getRecall(), 1E-9); + assertEquals("avg-p should be perfect: "+s.getAvp(), 1.0, s.getAvp(), 1E-2); + assertEquals("recall should be perfect: "+s.getRecall(), 1.0, s.getRecall(), 1E-2); for (int j = 1; j <= QualityStats.MAX_POINTS; j++) { - assertEquals("p_at_"+j+" should be perfect: "+s.getPrecisionAt(j), 1.0, s.getPrecisionAt(j), 1E-9); + assertEquals("p_at_"+j+" should be perfect: "+s.getPrecisionAt(j), 1.0, s.getPrecisionAt(j), 1E-2); } } @@ -149,13 +154,45 @@ } + + public void testTrecTopicsReader() throws Exception { + // prepare topics + InputStream topicsFile = getClass().getResourceAsStream("trecTopics.txt"); + TrecTopicsReader qReader = new TrecTopicsReader(); + QualityQuery qqs[] = qReader.readQueries( + new BufferedReader(new InputStreamReader(topicsFile, "UTF-8"))); + + assertEquals(20, qqs.length); + + QualityQuery qq = qqs[0]; + assertEquals("statement months total 1987", qq.getValue("title")); + assertEquals("Topic 0 Description Line 1 Topic 0 Description Line 2", + qq.getValue("description")); + assertEquals("Topic 0 Narrative Line 1 Topic 0 Narrative Line 2", + qq.getValue("narrative")); + + qq = qqs[1]; + assertEquals("agreed 15 against five", qq.getValue("title")); + assertEquals("Topic 1 Description Line 1 Topic 1 Description Line 2", + qq.getValue("description")); + assertEquals("Topic 1 Narrative Line 1 Topic 1 Narrative Line 2", + qq.getValue("narrative")); + + qq = qqs[19]; + assertEquals("20 while common week", qq.getValue("title")); + assertEquals("Topic 19 Description Line 1 Topic 19 Description Line 2", + qq.getValue("description")); + assertEquals("Topic 19 Narrative Line 1 Topic 19 Narrative Line 2", + qq.getValue("narrative")); + } - // use benchmark logic to create the full Reuters index + // use benchmark logic to create the mini Reuters index private void createReutersIndex() throws Exception { // 1. alg definition String algLines[] = { "# ----- properties ", - "content.source="+ReutersContentSource.class.getName(), + "content.source=org.apache.lucene.benchmark.byTask.feeds.LineDocSource", + "docs.file=" + getReuters578LinesFile(), "content.source.log.step=2500", "doc.term.vector=false", "content.source.forever=false", @@ -172,4 +209,9 @@ // 2. execute the algorithm (required in every "logic" test) TestPerfTasksLogic.execBenchmark(algLines); } + + private static String getReuters578LinesFile() { + return System.getProperty("lucene.common.dir").replace('\\','/') + + "/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/reuters.578.lines.txt.bz2"; + } } Modified: lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/trecTopics.txt URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/trecTopics.txt?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/trecTopics.txt (original) +++ lucene/java/branches/flex_1458/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/trecTopics.txt Fri Feb 26 13:09:54 2010 @@ -26,10 +26,12 @@ statement months total 1987 <desc> Description: - +Topic 0 Description Line 1 +Topic 0 Description Line 2 <narr> Narrative: - +Topic 0 Narrative Line 1 +Topic 0 Narrative Line 2 </top> @@ -39,10 +41,12 @@ <title> agreed 15 against five <desc> Description: - +Topic 1 Description Line 1 +Topic 1 Description Line 2 <narr> Narrative: - +Topic 1 Narrative Line 1 +Topic 1 Narrative Line 2 </top> @@ -273,9 +277,11 @@ <title> 20 while common week <desc> Description: - +Topic 19 Description Line 1 +Topic 19 Description Line 2 <narr> Narrative: - +Topic 19 Narrative Line 1 +Topic 19 Narrative Line 2 </top> Modified: lucene/java/branches/flex_1458/contrib/db/bdb-je/build.xml URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/db/bdb-je/build.xml?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/db/bdb-je/build.xml (original) +++ lucene/java/branches/flex_1458/contrib/db/bdb-je/build.xml Fri Feb 26 13:09:54 2010 @@ -21,10 +21,10 @@ Lucene Berkeley DB Java Edition integration </description> - <property name="je.version" value="3.3.69" /> + <property name="je.version" value="3.3.93" /> <path id="je.jar"> - <pathelement location="lib/je-${je.version}/lib/je-${je.version}.jar" /> + <pathelement location="lib/je-${je.version}.jar" /> </path> <available classname="com.sleepycat.je.Database" property="je.jar.exists"> @@ -48,13 +48,8 @@ <target name="get-je-jar" unless="je.jar.exists"> <mkdir dir="lib" /> - <get src="http://download.oracle.com/berkeley-db/je-${je.version}.zip" - dest="lib/je-${je.version}.zip" /> - <unzip src="lib/je-${je.version}.zip" dest="lib"> - <patternset> - <include name="je-${je.version}/lib/je-${je.version}.jar" /> - </patternset> - </unzip> + <get src="http://download.oracle.com/maven/com/sleepycat/je/${je.version}/je-${je.version}.jar" + dest="lib/je-${je.version}.jar" /> </target> <target name="check-and-get-je-jar" depends="get-je-jar" /> Modified: lucene/java/branches/flex_1458/contrib/db/bdb-je/pom.xml.template URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/db/bdb-je/pom.xml.template?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/db/bdb-je/pom.xml.template (original) +++ lucene/java/branches/flex_1458/contrib/db/bdb-je/pom.xml.template Fri Feb 26 13:09:54 2010 @@ -36,7 +36,7 @@ <dependency> <groupId>sleepycat</groupId> <artifactId>je</artifactId> - <version>1.7.0</version> + <version>${sleepycat-je-version}</version> </dependency> </dependencies> </project> Modified: lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java (original) +++ lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java Fri Feb 26 13:09:54 2010 @@ -22,6 +22,7 @@ import java.util.List; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.MapFieldSelector; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo; @@ -72,7 +73,7 @@ List<WeightedFragInfo> fragInfos = getWeightedFragInfoList( fieldFragList.fragInfos ); List<String> fragments = new ArrayList<String>( maxNumFragments ); - String[] values = getFieldValues( reader, docId, fieldName ); + Field[] values = getFields( reader, docId, fieldName ); if( values.length == 0 ) return null; StringBuilder buffer = new StringBuilder(); int[] nextValueIndex = { 0 }; @@ -83,15 +84,31 @@ return fragments.toArray( new String[fragments.size()] ); } + @Deprecated protected String[] getFieldValues( IndexReader reader, int docId, String fieldName) throws IOException { Document doc = reader.document( docId, new MapFieldSelector( new String[]{ fieldName } ) ); return doc.getValues( fieldName ); // according to Document class javadoc, this never returns null } + + protected Field[] getFields( IndexReader reader, int docId, String fieldName) throws IOException { + // according to javadoc, doc.getFields(fieldName) cannot be used with lazy loaded field??? + Document doc = reader.document( docId, new MapFieldSelector( new String[]{ fieldName } ) ); + return doc.getFields( fieldName ); // according to Document class javadoc, this never returns null + } + @Deprecated protected String makeFragment( StringBuilder buffer, int[] index, String[] values, WeightedFragInfo fragInfo ){ - StringBuilder fragment = new StringBuilder(); final int s = fragInfo.startOffset; - String src = getFragmentSource( buffer, index, values, s, fragInfo.endOffset ); + return makeFragment( fragInfo, getFragmentSource( buffer, index, values, s, fragInfo.endOffset ), s ); + } + + protected String makeFragment( StringBuilder buffer, int[] index, Field[] values, WeightedFragInfo fragInfo ){ + final int s = fragInfo.startOffset; + return makeFragment( fragInfo, getFragmentSource( buffer, index, values, s, fragInfo.endOffset ), s ); + } + + private String makeFragment( WeightedFragInfo fragInfo, String src, int s ){ + StringBuilder fragment = new StringBuilder(); int srcIndex = 0; for( SubInfo subInfo : fragInfo.subInfos ){ for( Toffs to : subInfo.termsOffsets ){ @@ -104,6 +121,7 @@ return fragment.toString(); } + @Deprecated protected String getFragmentSource( StringBuilder buffer, int[] index, String[] values, int startOffset, int endOffset ){ while( buffer.length() < endOffset && index[0] < values.length ){ @@ -114,6 +132,17 @@ int eo = buffer.length() < endOffset ? buffer.length() : endOffset; return buffer.substring( startOffset, eo ); } + + protected String getFragmentSource( StringBuilder buffer, int[] index, Field[] values, + int startOffset, int endOffset ){ + while( buffer.length() < endOffset && index[0] < values.length ){ + if( index[0] > 0 && values[index[0]].isTokenized() && values[index[0]].stringValue().length() > 0 ) + buffer.append( ' ' ); + buffer.append( values[index[0]++].stringValue() ); + } + int eo = buffer.length() < endOffset ? buffer.length() : endOffset; + return buffer.substring( startOffset, eo ); + } protected String getPreTag( int num ){ return preTags.length > num ? preTags[num] : preTags[0]; Modified: lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java (original) +++ lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java Fri Feb 26 13:09:54 2010 @@ -27,6 +27,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; @@ -78,6 +79,12 @@ flatten( clause.getQuery(), flatQueries ); } } + else if( sourceQuery instanceof DisjunctionMaxQuery ){ + DisjunctionMaxQuery dmq = (DisjunctionMaxQuery)sourceQuery; + for( Query query : dmq ){ + flatten( query, flatQueries ); + } + } else if( sourceQuery instanceof TermQuery ){ if( !flatQueries.contains( sourceQuery ) ) flatQueries.add( sourceQuery ); Modified: lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java (original) +++ lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java Fri Feb 26 13:09:54 2010 @@ -24,6 +24,7 @@ import junit.framework.TestCase; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.KeywordAnalyzer; import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; @@ -40,6 +41,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexWriter.MaxFieldLength; import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; @@ -55,6 +57,7 @@ protected Directory dir; protected Analyzer analyzerW; protected Analyzer analyzerB; + protected Analyzer analyzerK; protected IndexReader reader; protected QueryParser paW; protected QueryParser paB; @@ -75,11 +78,18 @@ "\nLucene/Solr does not require such additional hardware.", "\nWhen you talk about processing speed, the" }; + + protected static final String[] strMVValues = { + "abc", + "defg", + "hijkl" + }; @Override protected void setUp() throws Exception { - analyzerW = new WhitespaceAnalyzer(); + analyzerW = new WhitespaceAnalyzer(Version.LUCENE_CURRENT); analyzerB = new BigramAnalyzer(); + analyzerK = new KeywordAnalyzer(); paW = new QueryParser(Version.LUCENE_CURRENT, F, analyzerW ); paB = new QueryParser(Version.LUCENE_CURRENT, F, analyzerB ); dir = new RAMDirectory(); @@ -141,6 +151,18 @@ return query; } + protected Query dmq( Query... queries ){ + return dmq( 0.0F, queries ); + } + + protected Query dmq( float tieBreakerMultiplier, Query... queries ){ + DisjunctionMaxQuery query = new DisjunctionMaxQuery( tieBreakerMultiplier ); + for( Query q : queries ){ + query.add( q ); + } + return query; + } + protected void assertCollectionQueries( Collection<Query> actual, Query... expected ){ assertEquals( expected.length, actual.size() ); for( Query query : expected ){ @@ -205,7 +227,7 @@ public boolean incrementToken() throws IOException { if( !getNextPartialSnippet() ) return false; - + clearAttributes(); termAtt.setTermBuffer(snippet, startTerm, lenTerm); offsetAtt.setOffset(correctOffset(startOffset), correctOffset(startOffset + lenTerm)); return true; @@ -301,6 +323,7 @@ make1dmfIndex( analyzerB, values ); } + // make 1 doc with multi valued field protected void make1dmfIndex( Analyzer analyzer, String... values ) throws Exception { IndexWriter writer = new IndexWriter( dir, analyzer, true, MaxFieldLength.LIMITED ); Document doc = new Document(); @@ -312,6 +335,18 @@ reader = IndexReader.open( dir, true ); } + // make 1 doc with multi valued & not analyzed field + protected void make1dmfIndexNA( String... values ) throws Exception { + IndexWriter writer = new IndexWriter( dir, analyzerK, true, MaxFieldLength.LIMITED ); + Document doc = new Document(); + for( String value: values ) + doc.add( new Field( F, value, Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS ) ); + writer.addDocument( doc ); + writer.close(); + + reader = IndexReader.open( dir, true ); + } + protected void makeIndexShortMV() throws Exception { // 012345 @@ -373,4 +408,18 @@ make1dmfIndexB( biMVValues ); } + + protected void makeIndexStrMV() throws Exception { + + // 0123 + // "abc" + + // 34567 + // "defg" + + // 111 + // 789012 + // "hijkl" + make1dmfIndexNA( strMVValues ); + } } Modified: lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java (original) +++ lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/FieldQueryTest.java Fri Feb 26 13:09:54 2010 @@ -38,6 +38,14 @@ assertCollectionQueries( flatQueries, tq( "A" ), tq( "B" ), tq( "C" ) ); } + public void testFlattenDisjunctionMaxQuery() throws Exception { + Query query = dmq( tq( "A" ), tq( "B" ), pqF( "C", "D" ) ); + FieldQuery fq = new FieldQuery( query, true, true ); + Set<Query> flatQueries = new HashSet<Query>(); + fq.flatten( query, flatQueries ); + assertCollectionQueries( flatQueries, tq( "A" ), tq( "B" ), pqF( "C", "D" ) ); + } + public void testFlattenTermAndPhrase() throws Exception { Query query = paW.parse( "A AND \"B C\"" ); FieldQuery fq = new FieldQuery( query, true, true ); Modified: lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java (original) +++ lucene/java/branches/flex_1458/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java Fri Feb 26 13:09:54 2010 @@ -127,4 +127,16 @@ reader = IndexReader.open( dir, true ); } + + public void test1StrMV() throws Exception { + makeIndexStrMV(); + + FieldQuery fq = new FieldQuery( tq( "defg" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + SimpleFragListBuilder sflb = new SimpleFragListBuilder(); + FieldFragList ffl = sflb.createFieldFragList( fpl, 100 ); + SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder(); + assertEquals( "abc<b>defg</b>hijkl", sfb.createFragment( reader, 0, F, ffl ) ); + } } Modified: lucene/java/branches/flex_1458/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java (original) +++ lucene/java/branches/flex_1458/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java Fri Feb 26 13:09:54 2010 @@ -169,6 +169,7 @@ return false; } Token token = tokens[currentToken++]; + clearAttributes(); termAtt.setTermBuffer(token.term()); offsetAtt.setOffset(token.startOffset(), token.endOffset()); return true; Modified: lucene/java/branches/flex_1458/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java (original) +++ lucene/java/branches/flex_1458/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java Fri Feb 26 13:09:54 2010 @@ -102,6 +102,7 @@ public boolean incrementToken() throws IOException { if (this.tokensAtCurrentPosition.hasNext()) { final Token next = this.tokensAtCurrentPosition.next(); + clearAttributes(); termAttribute.setTermBuffer(next.term()); positionIncrementAttribute.setPositionIncrement(next .getPositionIncrement()); Propchange: lucene/java/branches/flex_1458/contrib/highlighter/src/test/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Feb 26 13:09:54 2010 @@ -1,5 +1,5 @@ /lucene/java/branches/lucene_2_4/contrib/highlighter/src/test:748824 -/lucene/java/branches/lucene_2_9/contrib/highlighter/src/test:817269-818600,825998,826775,829134,829816,829881,831036,896850 +/lucene/java/branches/lucene_2_9/contrib/highlighter/src/test:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib/highlighter/src/test:818601-821336 /lucene/java/branches/lucene_3_0/contrib/highlighter/src/test:880793,896906 -/lucene/java/trunk/contrib/highlighter/src/test:829439-833960,880727-886190,889185,889622,889667,889866-899001 +/lucene/java/trunk/contrib/highlighter/src/test:829439-833960,880727-886190,889185,889614-916543 Modified: lucene/java/branches/flex_1458/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java (original) +++ lucene/java/branches/flex_1458/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java Fri Feb 26 13:09:54 2010 @@ -49,6 +49,7 @@ import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.Version; import junit.framework.TestCase; @@ -60,7 +61,7 @@ final String TEXT = "the fox jumped"; final Directory directory = new RAMDirectory(); final IndexWriter indexWriter = new IndexWriter(directory, - new WhitespaceAnalyzer(), MaxFieldLength.UNLIMITED); + new WhitespaceAnalyzer(Version.LUCENE_CURRENT), MaxFieldLength.UNLIMITED); try { final Document document = new Document(); document.add(new Field(FIELD, new TokenStreamConcurrent(), @@ -103,7 +104,7 @@ final String TEXT = "the fox jumped"; final Directory directory = new RAMDirectory(); final IndexWriter indexWriter = new IndexWriter(directory, - new WhitespaceAnalyzer(), MaxFieldLength.UNLIMITED); + new WhitespaceAnalyzer(Version.LUCENE_CURRENT), MaxFieldLength.UNLIMITED); try { final Document document = new Document(); document.add(new Field(FIELD, new TokenStreamConcurrent(), @@ -168,7 +169,7 @@ final String TEXT = "the fox did not jump"; final Directory directory = new RAMDirectory(); final IndexWriter indexWriter = new IndexWriter(directory, - new WhitespaceAnalyzer(), MaxFieldLength.UNLIMITED); + new WhitespaceAnalyzer(Version.LUCENE_CURRENT), MaxFieldLength.UNLIMITED); try { final Document document = new Document(); document.add(new Field(FIELD, new TokenStreamSparse(), @@ -210,7 +211,7 @@ final String TEXT = "the fox did not jump"; final Directory directory = new RAMDirectory(); final IndexWriter indexWriter = new IndexWriter(directory, - new WhitespaceAnalyzer(), MaxFieldLength.UNLIMITED); + new WhitespaceAnalyzer(Version.LUCENE_CURRENT), MaxFieldLength.UNLIMITED); try { final Document document = new Document(); document.add(new Field(FIELD, TEXT, Store.YES, Index.ANALYZED, @@ -250,7 +251,7 @@ final String TEXT = "the fox did not jump"; final Directory directory = new RAMDirectory(); final IndexWriter indexWriter = new IndexWriter(directory, - new WhitespaceAnalyzer(), MaxFieldLength.UNLIMITED); + new WhitespaceAnalyzer(Version.LUCENE_CURRENT), MaxFieldLength.UNLIMITED); try { final Document document = new Document(); document.add(new Field(FIELD, new TokenStreamSparse(), @@ -311,6 +312,7 @@ if (this.i >= this.tokens.length) { return false; } + clearAttributes(); termAttribute.setTermBuffer(this.tokens[i].term(), 0, this.tokens[i] .term().length()); offsetAttribute.setOffset(this.tokens[i].startOffset(), this.tokens[i] @@ -355,6 +357,7 @@ if (this.i >= this.tokens.length) { return false; } + clearAttributes(); termAttribute.setTermBuffer(this.tokens[i].term(), 0, this.tokens[i] .term().length()); offsetAttribute.setOffset(this.tokens[i].startOffset(), this.tokens[i] Modified: lucene/java/branches/flex_1458/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original) +++ lucene/java/branches/flex_1458/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Fri Feb 26 13:09:54 2010 @@ -118,7 +118,7 @@ } public void testQueryScorerHits() throws Exception { - Analyzer analyzer = new SimpleAnalyzer(); + Analyzer analyzer = new SimpleAnalyzer(Version.LUCENE_CURRENT); QueryParser qp = new QueryParser(TEST_VERSION, FIELD_NAME, analyzer); query = qp.parse("\"very long\""); searcher = new IndexSearcher(ramDir, true); @@ -226,7 +226,7 @@ String f2c = f2 + ":"; String q = "(" + f1c + ph1 + " OR " + f2c + ph1 + ") AND (" + f1c + ph2 + " OR " + f2c + ph2 + ")"; - Analyzer analyzer = new WhitespaceAnalyzer(); + Analyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_CURRENT); QueryParser qp = new QueryParser(TEST_VERSION, f1, analyzer); Query query = qp.parse(q); @@ -1458,6 +1458,7 @@ public boolean incrementToken() throws IOException { if(iter.hasNext()) { Token token = iter.next(); + clearAttributes(); termAtt.setTermBuffer(token.term()); posIncrAtt.setPositionIncrement(token.getPositionIncrement()); offsetAtt.setOffset(token.startOffset(), token.endOffset()); @@ -1506,6 +1507,7 @@ public boolean incrementToken() throws IOException { if(iter.hasNext()) { Token token = iter.next(); + clearAttributes(); termAtt.setTermBuffer(token.term()); posIncrAtt.setPositionIncrement(token.getPositionIncrement()); offsetAtt.setOffset(token.startOffset(), token.endOffset()); @@ -1527,64 +1529,64 @@ Highlighter highlighter; String result; - query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("foo"); + query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("foo"); highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2(), s, 3, "..."); assertEquals("Hi-Speed10 <B>foo</B>", result); - query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("10"); + query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("10"); highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2(), s, 3, "..."); assertEquals("Hi-Speed<B>10</B> foo", result); - query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("hi"); + query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("hi"); highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2(), s, 3, "..."); assertEquals("<B>Hi</B>-Speed10 foo", result); - query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("speed"); + query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("speed"); highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2(), s, 3, "..."); assertEquals("Hi-<B>Speed</B>10 foo", result); - query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("hispeed"); + query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("hispeed"); highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2(), s, 3, "..."); assertEquals("<B>Hi-Speed</B>10 foo", result); - query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("hi speed"); + query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("hi speed"); highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2(), s, 3, "..."); assertEquals("<B>Hi-Speed</B>10 foo", result); // ///////////////// same tests, just put the bigger overlapping token // first - query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("foo"); + query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("foo"); highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2a(), s, 3, "..."); assertEquals("Hi-Speed10 <B>foo</B>", result); - query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("10"); + query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("10"); highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2a(), s, 3, "..."); assertEquals("Hi-Speed<B>10</B> foo", result); - query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("hi"); + query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("hi"); highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2a(), s, 3, "..."); assertEquals("<B>Hi</B>-Speed10 foo", result); - query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("speed"); + query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("speed"); highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2a(), s, 3, "..."); assertEquals("Hi-<B>Speed</B>10 foo", result); - query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("hispeed"); + query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("hispeed"); highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2a(), s, 3, "..."); assertEquals("<B>Hi-Speed</B>10 foo", result); - query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer()).parse("hi speed"); + query = new QueryParser(TEST_VERSION, "text", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("hi speed"); highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this); result = highlighter.getBestFragments(getTS2a(), s, 3, "..."); assertEquals("<B>Hi-Speed</B>10 foo", result); @@ -1595,7 +1597,7 @@ } private Directory dir = new RAMDirectory(); - private Analyzer a = new WhitespaceAnalyzer(); + private Analyzer a = new WhitespaceAnalyzer(Version.LUCENE_CURRENT); public void testWeightedTermsWithDeletes() throws IOException, ParseException, InvalidTokenOffsetsException { makeIndex(); @@ -1799,7 +1801,7 @@ */ @Override public TokenStream tokenStream(String arg0, Reader arg1) { - LowerCaseTokenizer stream = new LowerCaseTokenizer(arg1); + LowerCaseTokenizer stream = new LowerCaseTokenizer(Version.LUCENE_CURRENT, arg1); stream.addAttribute(TermAttribute.class); stream.addAttribute(PositionIncrementAttribute.class); stream.addAttribute(OffsetAttribute.class); @@ -1845,6 +1847,7 @@ return false; } //Token nextRealToken = new Token(, offsetAtt.startOffset(), offsetAtt.endOffset()); + clearAttributes(); termAtt.setTermBuffer(realTermAtt.term()); offsetAtt.setOffset(realOffsetAtt.startOffset(), realOffsetAtt.endOffset()); posIncrAtt.setPositionIncrement(realPosIncrAtt.getPositionIncrement()); @@ -1862,6 +1865,7 @@ return true; } else { String tok = st.nextToken(); + clearAttributes(); termAtt.setTermBuffer(tok); offsetAtt.setOffset(currentRealToken.startOffset(), currentRealToken.endOffset()); posIncrAtt.setPositionIncrement(0); Modified: lucene/java/branches/flex_1458/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java (original) +++ lucene/java/branches/flex_1458/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java Fri Feb 26 13:09:54 2010 @@ -43,7 +43,7 @@ * results up to a 100 times faster than the file-centric RAMDirectory * at the cost of greater RAM consumption. * <p> - * WARNING: This contrib is experimental and the APIs may change without warning. + * @lucene.experimental * <p> * There are no read and write locks in this store. * {@link InstantiatedIndexReader} {@link InstantiatedIndexReader#isCurrent()} all the time Modified: lucene/java/branches/flex_1458/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/package.html URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/package.html?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/package.html (original) +++ lucene/java/branches/flex_1458/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/package.html Fri Feb 26 13:09:54 2010 @@ -22,7 +22,7 @@ <body> <p>InstantiatedIndex, alternative RAM store for small corpora.</p> -<p>WARNING: This contrib is experimental and the APIs may change without warning.</p> +<p>@lucene.experimental</p> <h2>Abstract</h2> <p> Propchange: lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java ------------------------------------------------------------------------------ --- svn:mergeinfo (added) +++ svn:mergeinfo Fri Feb 26 13:09:54 2010 @@ -0,0 +1 @@ +/lucene/java/branches/lucene_2_9/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:909334 Modified: lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestSerialization.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestSerialization.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestSerialization.java (original) +++ lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestSerialization.java Fri Feb 26 13:09:54 2010 @@ -20,6 +20,7 @@ import junit.framework.TestCase; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.Version; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexReader; import org.apache.lucene.analysis.WhitespaceAnalyzer; @@ -35,7 +36,7 @@ Directory dir = new RAMDirectory(); - IndexWriter iw = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); + IndexWriter iw = new IndexWriter(dir, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.UNLIMITED); Document doc = new Document(); doc.add(new Field("foo", "bar rab abr bra rba", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); doc.add(new Field("moo", "bar rab abr bra rba", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); Modified: lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestUnoptimizedReaderOnConstructor.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestUnoptimizedReaderOnConstructor.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestUnoptimizedReaderOnConstructor.java (original) +++ lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestUnoptimizedReaderOnConstructor.java Fri Feb 26 13:09:54 2010 @@ -24,6 +24,7 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.Version; import org.apache.lucene.analysis.WhitespaceAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -35,17 +36,17 @@ public void test() throws Exception { Directory dir = new RAMDirectory(); - IndexWriter iw = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); + IndexWriter iw = new IndexWriter(dir, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.UNLIMITED); addDocument(iw, "Hello, world!"); addDocument(iw, "All work and no play makes jack a dull boy"); iw.close(); - iw = new IndexWriter(dir, new WhitespaceAnalyzer(), false, IndexWriter.MaxFieldLength.UNLIMITED); + iw = new IndexWriter(dir, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), false, IndexWriter.MaxFieldLength.UNLIMITED); addDocument(iw, "Hello, tellus!"); addDocument(iw, "All work and no play makes danny a dull boy"); iw.close(); - iw = new IndexWriter(dir, new WhitespaceAnalyzer(), false, IndexWriter.MaxFieldLength.UNLIMITED); + iw = new IndexWriter(dir, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), false, IndexWriter.MaxFieldLength.UNLIMITED); addDocument(iw, "Hello, earth!"); addDocument(iw, "All work and no play makes wendy a dull girl"); iw.close(); Modified: lucene/java/branches/flex_1458/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original) +++ lucene/java/branches/flex_1458/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Fri Feb 26 13:09:54 2010 @@ -287,6 +287,7 @@ throw new IllegalArgumentException("keyword must not be null"); String term = obj.toString(); + clearAttributes(); termAtt.setTermBuffer(term); offsetAtt.setOffset(start, start+termAtt.termLength()); start += term.length() + 1; // separate words by 1 (blank) character Modified: lucene/java/branches/flex_1458/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (original) +++ lucene/java/branches/flex_1458/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java Fri Feb 26 13:09:54 2010 @@ -277,10 +277,10 @@ Set<?> stopWords = StopAnalyzer.ENGLISH_STOP_WORDS_SET; Analyzer[] analyzers = new Analyzer[] { - new SimpleAnalyzer(), + new SimpleAnalyzer(Version.LUCENE_CURRENT), new StopAnalyzer(Version.LUCENE_CURRENT), new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), -// new WhitespaceAnalyzer(), +// new WhitespaceAnalyzer(Version.LUCENE_CURRENT), // new PatternAnalyzer(PatternAnalyzer.NON_WORD_PATTERN, false, null), // new PatternAnalyzer(PatternAnalyzer.NON_WORD_PATTERN, true, stopWords), // new SnowballAnalyzer("English", StopAnalyzer.ENGLISH_STOP_WORDS), Modified: lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java (original) +++ lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java Fri Feb 26 13:09:54 2010 @@ -41,8 +41,7 @@ * File{In,Out}putStream) so it will not work with non * FSDirectory Directory impls.</p> * - * <p><b>NOTE</b>: The tool is experimental and might change - * in incompatible ways in the next release. You can easily + * @lucene.experimental You can easily * accidentally remove segments from your index so be * careful! */ Modified: lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/misc/IndexMergeTool.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/misc/IndexMergeTool.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/misc/IndexMergeTool.java (original) +++ lucene/java/branches/flex_1458/contrib/misc/src/java/org/apache/lucene/misc/IndexMergeTool.java Fri Feb 26 13:09:54 2010 @@ -27,7 +27,6 @@ /** * Merges indices specified on the command line into the index * specified as the first command line argument. - * @version $Id$ */ public class IndexMergeTool { public static void main(String[] args) throws IOException { Modified: lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java (original) +++ lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java Fri Feb 26 13:09:54 2010 @@ -34,11 +34,10 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.Version; /** * Tests changing of field norms with a custom similarity and with fake norms. - * - * @version $Id$ */ public class TestFieldNormModifier extends TestCase { public TestFieldNormModifier(String name) { @@ -59,7 +58,7 @@ @Override public void setUp() throws Exception { - IndexWriter writer = new IndexWriter(store, new SimpleAnalyzer(), true, MaxFieldLength.UNLIMITED); + IndexWriter writer = new IndexWriter(store, new SimpleAnalyzer(Version.LUCENE_CURRENT), true, MaxFieldLength.UNLIMITED); for (int i = 0; i < NUM_DOCS; i++) { Document d = new Document(); Modified: lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java (original) +++ lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java Fri Feb 26 13:09:54 2010 @@ -23,6 +23,7 @@ import org.apache.lucene.index.IndexWriter.MaxFieldLength; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.Version; import org.apache.lucene.util._TestUtil; public class TestIndexSplitter extends LuceneTestCase { @@ -35,7 +36,7 @@ _TestUtil.rmDir(destDir); destDir.mkdirs(); FSDirectory fsDir = FSDirectory.open(dir); - IndexWriter iw = new IndexWriter(fsDir, new WhitespaceAnalyzer(), true, MaxFieldLength.UNLIMITED); + IndexWriter iw = new IndexWriter(fsDir, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), true, MaxFieldLength.UNLIMITED); for (int x=0; x < 100; x++) { Document doc = TestIndexWriterReader.createDocument(x, "index", 5); iw.addDocument(doc); Modified: lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java (original) +++ lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java Fri Feb 26 13:09:54 2010 @@ -22,6 +22,7 @@ import org.apache.lucene.index.IndexWriter.MaxFieldLength; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.Version; import junit.framework.TestCase; @@ -32,7 +33,7 @@ @Override public void setUp() throws Exception { RAMDirectory dir = new RAMDirectory(); - IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), true, + IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), true, MaxFieldLength.LIMITED); Document doc; for (int i = 0; i < NUM_DOCS; i++) { Modified: lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/misc/ChainedFilterTest.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/misc/ChainedFilterTest.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/misc/ChainedFilterTest.java (original) +++ lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/misc/ChainedFilterTest.java Fri Feb 26 13:09:54 2010 @@ -43,6 +43,7 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.Version; public class ChainedFilterTest extends TestCase { public static final int MAX = 500; @@ -59,7 +60,7 @@ public void setUp() throws Exception { directory = new RAMDirectory(); IndexWriter writer = - new IndexWriter(directory, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); + new IndexWriter(directory, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.UNLIMITED); Calendar cal = new GregorianCalendar(); cal.clear(); @@ -187,7 +188,7 @@ public void testWithCachingFilter() throws Exception { Directory dir = new RAMDirectory(); - Analyzer analyzer = new WhitespaceAnalyzer(); + Analyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_CURRENT); IndexWriter writer = new IndexWriter(dir, analyzer, true, MaxFieldLength.LIMITED); writer.close(); Modified: lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java (original) +++ lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java Fri Feb 26 13:09:54 2010 @@ -37,11 +37,10 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.Version; /** * Tests changing the norms after changing the simularity - * - * @version $Id:$ */ public class TestLengthNormModifier extends TestCase { public TestLengthNormModifier(String name) { @@ -62,7 +61,7 @@ @Override public void setUp() throws Exception { - IndexWriter writer = new IndexWriter(store, new SimpleAnalyzer(), true, MaxFieldLength.UNLIMITED); + IndexWriter writer = new IndexWriter(store, new SimpleAnalyzer(Version.LUCENE_CURRENT), true, MaxFieldLength.UNLIMITED); for (int i = 0; i < NUM_DOCS; i++) { Document d = new Document(); Modified: lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/queryParser/ext/TestExtendableQueryParser.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/queryParser/ext/TestExtendableQueryParser.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/queryParser/ext/TestExtendableQueryParser.java (original) +++ lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/queryParser/ext/TestExtendableQueryParser.java Fri Feb 26 13:09:54 2010 @@ -47,7 +47,7 @@ public QueryParser getParser(Analyzer a, Extensions extensions) throws Exception { if (a == null) - a = new SimpleAnalyzer(); + a = new SimpleAnalyzer(Version.LUCENE_CURRENT); QueryParser qp = extensions == null ? new ExtendableQueryParser( Version.LUCENE_CURRENT, "field", a) : new ExtendableQueryParser( Version.LUCENE_CURRENT, "field", a, extensions); Modified: lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java (original) +++ lucene/java/branches/flex_1458/contrib/misc/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java Fri Feb 26 13:09:54 2010 @@ -36,6 +36,7 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.util.LocalizedTestCase; +import org.apache.lucene.util.Version; import java.io.IOException; import java.io.Reader; @@ -73,6 +74,7 @@ @Override public boolean incrementToken() throws IOException { + clearAttributes(); if (inPhrase) { inPhrase = false; termAtt.setTermBuffer("phrase2"); @@ -98,7 +100,7 @@ /** Filters LowerCaseTokenizer with StopFilter. */ @Override public final TokenStream tokenStream(String fieldName, Reader reader) { - return new QPTestFilter(new LowerCaseTokenizer(reader)); + return new QPTestFilter(new LowerCaseTokenizer(Version.LUCENE_CURRENT, reader)); } } @@ -128,7 +130,7 @@ public PrecedenceQueryParser getParser(Analyzer a) throws Exception { if (a == null) - a = new SimpleAnalyzer(); + a = new SimpleAnalyzer(Version.LUCENE_CURRENT); PrecedenceQueryParser qp = new PrecedenceQueryParser("field", a); qp.setDefaultOperator(PrecedenceQueryParser.OR_OPERATOR); return qp; @@ -173,7 +175,7 @@ public Query getQueryDOA(String query, Analyzer a) throws Exception { if (a == null) - a = new SimpleAnalyzer(); + a = new SimpleAnalyzer(Version.LUCENE_CURRENT); PrecedenceQueryParser qp = new PrecedenceQueryParser("field", a); qp.setDefaultOperator(PrecedenceQueryParser.AND_OPERATOR); return qp.parse(query); @@ -253,7 +255,7 @@ } public void testPunct() throws Exception { - Analyzer a = new WhitespaceAnalyzer(); + Analyzer a = new WhitespaceAnalyzer(Version.LUCENE_CURRENT); assertQueryEquals("a&b", a, "a&b"); assertQueryEquals("a&&b", a, "a&&b"); assertQueryEquals(".NET", a, ".NET"); @@ -411,7 +413,7 @@ } public void testEscaped() throws Exception { - Analyzer a = new WhitespaceAnalyzer(); + Analyzer a = new WhitespaceAnalyzer(Version.LUCENE_CURRENT); /*assertQueryEquals("\\[brackets", a, "\\[brackets"); assertQueryEquals("\\[brackets", null, "brackets"); @@ -543,7 +545,7 @@ public void testCustomQueryParserWildcard() { try { - new QPTestParser("contents", new WhitespaceAnalyzer()).parse("a?t"); + new QPTestParser("contents", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("a?t"); } catch (ParseException expected) { return; } @@ -552,7 +554,7 @@ public void testCustomQueryParserFuzzy() throws Exception { try { - new QPTestParser("contents", new WhitespaceAnalyzer()).parse("xunit~"); + new QPTestParser("contents", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("xunit~"); } catch (ParseException expected) { return; } @@ -562,7 +564,7 @@ public void testBooleanQuery() throws Exception { BooleanQuery.setMaxClauseCount(2); try { - getParser(new WhitespaceAnalyzer()).parse("one two three"); + getParser(new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("one two three"); fail("ParseException expected due to too many boolean clauses"); } catch (ParseException expected) { // too many boolean clauses, so ParseException is expected @@ -576,7 +578,7 @@ // failing tests disabled since PrecedenceQueryParser // is currently unmaintained public void _testPrecedence() throws Exception { - PrecedenceQueryParser parser = getParser(new WhitespaceAnalyzer()); + PrecedenceQueryParser parser = getParser(new WhitespaceAnalyzer(Version.LUCENE_CURRENT)); Query query1 = parser.parse("A AND B OR C AND D"); Query query2 = parser.parse("(A AND B) OR (C AND D)"); assertEquals(query1, query2); Modified: lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/BooleanFilterTest.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/BooleanFilterTest.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/BooleanFilterTest.java (original) +++ lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/BooleanFilterTest.java Fri Feb 26 13:09:54 2010 @@ -28,6 +28,7 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.Version; public class BooleanFilterTest extends TestCase { @@ -38,7 +39,7 @@ protected void setUp() throws Exception { directory = new RAMDirectory(); - IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); + IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.UNLIMITED); //Add series of docs with filterable fields : acces rights, prices, dates and "in-stock" flags addDoc(writer, "admin guest", "010", "20040101","Y"); Modified: lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java (original) +++ lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java Fri Feb 26 13:09:54 2010 @@ -30,12 +30,13 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexWriter.MaxFieldLength; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.Version; public class FuzzyLikeThisQueryTest extends TestCase { private RAMDirectory directory; private IndexSearcher searcher; - private Analyzer analyzer=new WhitespaceAnalyzer(); + private Analyzer analyzer=new WhitespaceAnalyzer(Version.LUCENE_CURRENT); @Override protected void setUp() throws Exception @@ -114,7 +115,7 @@ } public void testFuzzyLikeThisQueryEquals() { - Analyzer analyzer = new WhitespaceAnalyzer(); + Analyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_CURRENT); FuzzyLikeThisQuery fltq1 = new FuzzyLikeThisQuery(10, analyzer); fltq1.addTerms("javi", "subject", 0.5f, 2); FuzzyLikeThisQuery fltq2 = new FuzzyLikeThisQuery(10, analyzer); Modified: lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/TermsFilterTest.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/TermsFilterTest.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/TermsFilterTest.java (original) +++ lucene/java/branches/flex_1458/contrib/queries/src/test/org/apache/lucene/search/TermsFilterTest.java Fri Feb 26 13:09:54 2010 @@ -30,6 +30,7 @@ import org.apache.lucene.index.IndexWriter.MaxFieldLength; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.Version; public class TermsFilterTest extends TestCase { @@ -55,7 +56,7 @@ { String fieldName="field1"; RAMDirectory rd=new RAMDirectory(); - IndexWriter w=new IndexWriter(rd,new WhitespaceAnalyzer(),MaxFieldLength.UNLIMITED); + IndexWriter w=new IndexWriter(rd,new WhitespaceAnalyzer(Version.LUCENE_CURRENT),MaxFieldLength.UNLIMITED); for (int i = 0; i < 100; i++) { Document doc=new Document(); Modified: lucene/java/branches/flex_1458/contrib/queryparser/src/java/org/apache/lucene/queryParser/core/nodes/MatchAllDocsQueryNode.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/queryparser/src/java/org/apache/lucene/queryParser/core/nodes/MatchAllDocsQueryNode.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/queryparser/src/java/org/apache/lucene/queryParser/core/nodes/MatchAllDocsQueryNode.java (original) +++ lucene/java/branches/flex_1458/contrib/queryparser/src/java/org/apache/lucene/queryParser/core/nodes/MatchAllDocsQueryNode.java Fri Feb 26 13:09:54 2010 @@ -33,7 +33,7 @@ @Override public String toString() { - return "<matchAllDocs field='*' term='*'>"; + return "<matchAllDocs field='*' term='*'/>"; } public CharSequence toQueryString(EscapeQuerySyntax escapeSyntaxParser) { Modified: lucene/java/branches/flex_1458/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java (original) +++ lucene/java/branches/flex_1458/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java Fri Feb 26 13:09:54 2010 @@ -119,6 +119,7 @@ public boolean incrementToken() throws IOException { if (inPhrase) { inPhrase = false; + clearAttributes(); termAtt.setTermBuffer("phrase2"); offsetAtt.setOffset(savedStart, savedEnd); return true; @@ -143,7 +144,7 @@ /** Filters LowerCaseTokenizer with StopFilter. */ @Override public final TokenStream tokenStream(String fieldName, Reader reader) { - return new QPTestFilter(new LowerCaseTokenizer(reader)); + return new QPTestFilter(new LowerCaseTokenizer(Version.LUCENE_CURRENT, reader)); } } @@ -203,7 +204,7 @@ public StandardQueryParser getParser(Analyzer a) throws Exception { if (a == null) - a = new SimpleAnalyzer(); + a = new SimpleAnalyzer(Version.LUCENE_CURRENT); StandardQueryParser qp = new StandardQueryParser(); qp.setAnalyzer(a); @@ -293,7 +294,7 @@ public Query getQueryDOA(String query, Analyzer a) throws Exception { if (a == null) - a = new SimpleAnalyzer(); + a = new SimpleAnalyzer(Version.LUCENE_CURRENT); StandardQueryParser qp = new StandardQueryParser(); qp.setAnalyzer(a); qp.setDefaultOperator(Operator.AND); @@ -313,7 +314,7 @@ } public void testConstantScoreAutoRewrite() throws Exception { - StandardQueryParser qp = new StandardQueryParser(new WhitespaceAnalyzer()); + StandardQueryParser qp = new StandardQueryParser(new WhitespaceAnalyzer(Version.LUCENE_CURRENT)); Query q = qp.parse("foo*bar", "field"); assertTrue(q instanceof WildcardQuery); assertEquals(MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT, ((MultiTermQuery) q).getRewriteMethod()); @@ -338,9 +339,9 @@ public void testSimple() throws Exception { assertQueryEquals("\"term germ\"~2", null, "\"term germ\"~2"); assertQueryEquals("term term term", null, "term term term"); - assertQueryEquals("t�rm term term", new WhitespaceAnalyzer(), + assertQueryEquals("t�rm term term", new WhitespaceAnalyzer(Version.LUCENE_CURRENT), "t�rm term term"); - assertQueryEquals("�mlaut", new WhitespaceAnalyzer(), "�mlaut"); + assertQueryEquals("�mlaut", new WhitespaceAnalyzer(Version.LUCENE_CURRENT), "�mlaut"); assertQueryEquals("\"\"", new KeywordAnalyzer(), ""); assertQueryEquals("foo:\"\"", new KeywordAnalyzer(), "foo:"); @@ -397,7 +398,7 @@ } public void testPunct() throws Exception { - Analyzer a = new WhitespaceAnalyzer(); + Analyzer a = new WhitespaceAnalyzer(Version.LUCENE_CURRENT); assertQueryEquals("a&b", a, "a&b"); assertQueryEquals("a&&b", a, "a&&b"); assertQueryEquals(".NET", a, ".NET"); @@ -572,7 +573,7 @@ public void testFarsiRangeCollating() throws Exception { RAMDirectory ramDir = new RAMDirectory(); - IndexWriter iw = new IndexWriter(ramDir, new WhitespaceAnalyzer(), true, + IndexWriter iw = new IndexWriter(ramDir, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED); Document doc = new Document(); doc.add(new Field("content", "\u0633\u0627\u0628", Field.Store.YES, @@ -582,7 +583,7 @@ IndexSearcher is = new IndexSearcher(ramDir, true); StandardQueryParser qp = new StandardQueryParser(); - qp.setAnalyzer(new WhitespaceAnalyzer()); + qp.setAnalyzer(new WhitespaceAnalyzer(Version.LUCENE_CURRENT)); // Neither Java 1.4.2 nor 1.5.0 has Farsi Locale collation available in // RuleBasedCollator. However, the Arabic Locale seems to order the @@ -736,7 +737,7 @@ } public void testEscaped() throws Exception { - Analyzer a = new WhitespaceAnalyzer(); + Analyzer a = new WhitespaceAnalyzer(Version.LUCENE_CURRENT); /* * assertQueryEquals("\\[brackets", a, "\\[brackets"); @@ -835,7 +836,7 @@ } public void testQueryStringEscaping() throws Exception { - Analyzer a = new WhitespaceAnalyzer(); + Analyzer a = new WhitespaceAnalyzer(Version.LUCENE_CURRENT); assertEscapedQueryEquals("a-b:c", a, "a\\-b\\:c"); assertEscapedQueryEquals("a+b:c", a, "a\\+b\\:c"); @@ -950,7 +951,7 @@ public void testCustomQueryParserWildcard() { try { - new QPTestParser(new WhitespaceAnalyzer()).parse("a?t", "contents"); + new QPTestParser(new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("a?t", "contents"); fail("Wildcard queries should not be allowed"); } catch (QueryNodeException expected) { // expected exception @@ -959,7 +960,7 @@ public void testCustomQueryParserFuzzy() throws Exception { try { - new QPTestParser(new WhitespaceAnalyzer()).parse("xunit~", "contents"); + new QPTestParser(new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("xunit~", "contents"); fail("Fuzzy queries should not be allowed"); } catch (QueryNodeException expected) { // expected exception @@ -970,7 +971,7 @@ BooleanQuery.setMaxClauseCount(2); try { StandardQueryParser qp = new StandardQueryParser(); - qp.setAnalyzer(new WhitespaceAnalyzer()); + qp.setAnalyzer(new WhitespaceAnalyzer(Version.LUCENE_CURRENT)); qp.parse("one two three", "field"); fail("ParseException expected due to too many boolean clauses"); @@ -984,7 +985,7 @@ */ public void testPrecedence() throws Exception { StandardQueryParser qp = new StandardQueryParser(); - qp.setAnalyzer(new WhitespaceAnalyzer()); + qp.setAnalyzer(new WhitespaceAnalyzer(Version.LUCENE_CURRENT)); Query query1 = qp.parse("A AND B OR C AND D", "field"); Query query2 = qp.parse("+A +B +C +D", "field"); @@ -995,7 +996,7 @@ public void testLocalDateFormat() throws IOException, QueryNodeException { RAMDirectory ramDir = new RAMDirectory(); - IndexWriter iw = new IndexWriter(ramDir, new WhitespaceAnalyzer(), true, + IndexWriter iw = new IndexWriter(ramDir, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED); addDateDoc("a", 2005, 12, 2, 10, 15, 33, iw); addDateDoc("b", 2005, 12, 4, 22, 15, 00, iw); @@ -1120,7 +1121,7 @@ public void testMatchAllDocs() throws Exception { StandardQueryParser qp = new StandardQueryParser(); - qp.setAnalyzer(new WhitespaceAnalyzer()); + qp.setAnalyzer(new WhitespaceAnalyzer(Version.LUCENE_CURRENT)); assertEquals(new MatchAllDocsQuery(), qp.parse("*:*", "field")); assertEquals(new MatchAllDocsQuery(), qp.parse("(*:*)", "field")); @@ -1132,7 +1133,7 @@ private void assertHits(int expected, String query, IndexSearcher is) throws IOException, QueryNodeException { StandardQueryParser qp = new StandardQueryParser(); - qp.setAnalyzer(new WhitespaceAnalyzer()); + qp.setAnalyzer(new WhitespaceAnalyzer(Version.LUCENE_CURRENT)); qp.setLocale(Locale.ENGLISH); Query q = qp.parse(query, "date"); @@ -1163,6 +1164,7 @@ final TermAttribute term = addAttribute(TermAttribute.class); @Override public boolean incrementToken() { + clearAttributes(); if (upto == 4) { return false; } Modified: lucene/java/branches/flex_1458/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java (original) +++ lucene/java/branches/flex_1458/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java Fri Feb 26 13:09:54 2010 @@ -114,6 +114,7 @@ public boolean incrementToken() throws IOException { if (inPhrase) { inPhrase = false; + clearAttributes(); termAtt.setTermBuffer("phrase2"); offsetAtt.setOffset(savedStart, savedEnd); return true; @@ -138,7 +139,7 @@ /** Filters LowerCaseTokenizer with StopFilter. */ @Override public final TokenStream tokenStream(String fieldName, Reader reader) { - return new QPTestFilter(new LowerCaseTokenizer(reader)); + return new QPTestFilter(new LowerCaseTokenizer(Version.LUCENE_CURRENT, reader)); } } @@ -216,7 +217,7 @@ public QueryParserWrapper getParser(Analyzer a) throws Exception { if (a == null) - a = new SimpleAnalyzer(); + a = new SimpleAnalyzer(Version.LUCENE_CURRENT); QueryParserWrapper qp = new QueryParserWrapper("field", a); qp.setDefaultOperator(QueryParserWrapper.OR_OPERATOR); return qp; @@ -301,7 +302,7 @@ public Query getQueryDOA(String query, Analyzer a) throws Exception { if (a == null) - a = new SimpleAnalyzer(); + a = new SimpleAnalyzer(Version.LUCENE_CURRENT); QueryParserWrapper qp = new QueryParserWrapper("field", a); qp.setDefaultOperator(QueryParserWrapper.AND_OPERATOR); return qp.parse(query); @@ -328,9 +329,9 @@ public void testSimple() throws Exception { assertQueryEquals("\"term germ\"~2", null, "\"term germ\"~2"); assertQueryEquals("term term term", null, "term term term"); - assertQueryEquals("t�rm term term", new WhitespaceAnalyzer(), + assertQueryEquals("t�rm term term", new WhitespaceAnalyzer(Version.LUCENE_CURRENT), "t�rm term term"); - assertQueryEquals("�mlaut", new WhitespaceAnalyzer(), "�mlaut"); + assertQueryEquals("�mlaut", new WhitespaceAnalyzer(Version.LUCENE_CURRENT), "�mlaut"); assertQueryEquals("\"\"", new KeywordAnalyzer(), ""); assertQueryEquals("foo:\"\"", new KeywordAnalyzer(), "foo:"); @@ -395,7 +396,7 @@ } public void testPunct() throws Exception { - Analyzer a = new WhitespaceAnalyzer(); + Analyzer a = new WhitespaceAnalyzer(Version.LUCENE_CURRENT); assertQueryEquals("a&b", a, "a&b"); assertQueryEquals("a&&b", a, "a&&b"); assertQueryEquals(".NET", a, ".NET"); @@ -551,7 +552,7 @@ assertEquals(MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT, ((TermRangeQuery)getQuery("[ a TO z]", null)).getRewriteMethod()); QueryParserWrapper qp = new QueryParserWrapper("field", - new SimpleAnalyzer()); + new SimpleAnalyzer(Version.LUCENE_CURRENT)); qp.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE); assertEquals(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE,((TermRangeQuery)qp.parse("[ a TO z]")).getRewriteMethod()); @@ -570,7 +571,7 @@ public void testFarsiRangeCollating() throws Exception { RAMDirectory ramDir = new RAMDirectory(); - IndexWriter iw = new IndexWriter(ramDir, new WhitespaceAnalyzer(), true, + IndexWriter iw = new IndexWriter(ramDir, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED); Document doc = new Document(); doc.add(new Field("content", "\u0633\u0627\u0628", Field.Store.YES, @@ -580,7 +581,7 @@ IndexSearcher is = new IndexSearcher(ramDir, true); QueryParserWrapper qp = new QueryParserWrapper("content", - new WhitespaceAnalyzer()); + new WhitespaceAnalyzer(Version.LUCENE_CURRENT)); // Neither Java 1.4.2 nor 1.5.0 has Farsi Locale collation available in // RuleBasedCollator. However, the Arabic Locale seems to order the Farsi @@ -682,7 +683,7 @@ final String monthField = "month"; final String hourField = "hour"; QueryParserWrapper qp = new QueryParserWrapper("field", - new SimpleAnalyzer()); + new SimpleAnalyzer(Version.LUCENE_CURRENT)); // Don't set any date resolution and verify if DateField is used assertDateRangeQueryEquals(qp, defaultField, startDate, endDate, @@ -726,7 +727,7 @@ } public void testEscaped() throws Exception { - Analyzer a = new WhitespaceAnalyzer(); + Analyzer a = new WhitespaceAnalyzer(Version.LUCENE_CURRENT); /* * assertQueryEquals("\\[brackets", a, "\\[brackets"); @@ -823,7 +824,7 @@ } public void testQueryStringEscaping() throws Exception { - Analyzer a = new WhitespaceAnalyzer(); + Analyzer a = new WhitespaceAnalyzer(Version.LUCENE_CURRENT); assertEscapedQueryEquals("a-b:c", a, "a\\-b\\:c"); assertEscapedQueryEquals("a+b:c", a, "a\\+b\\:c"); @@ -934,7 +935,7 @@ public void testCustomQueryParserWildcard() { try { - new QPTestParser("contents", new WhitespaceAnalyzer()).parse("a?t"); + new QPTestParser("contents", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("a?t"); fail("Wildcard queries should not be allowed"); } catch (ParseException expected) { // expected exception @@ -943,7 +944,7 @@ public void testCustomQueryParserFuzzy() throws Exception { try { - new QPTestParser("contents", new WhitespaceAnalyzer()).parse("xunit~"); + new QPTestParser("contents", new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).parse("xunit~"); fail("Fuzzy queries should not be allowed"); } catch (ParseException expected) { // expected exception @@ -954,7 +955,7 @@ BooleanQuery.setMaxClauseCount(2); try { QueryParserWrapper qp = new QueryParserWrapper("field", - new WhitespaceAnalyzer()); + new WhitespaceAnalyzer(Version.LUCENE_CURRENT)); qp.parse("one two three"); fail("ParseException expected due to too many boolean clauses"); } catch (ParseException expected) { @@ -967,7 +968,7 @@ */ public void testPrecedence() throws Exception { QueryParserWrapper qp = new QueryParserWrapper("field", - new WhitespaceAnalyzer()); + new WhitespaceAnalyzer(Version.LUCENE_CURRENT)); Query query1 = qp.parse("A AND B OR C AND D"); Query query2 = qp.parse("+A +B +C +D"); @@ -977,7 +978,7 @@ public void testLocalDateFormat() throws IOException, ParseException { RAMDirectory ramDir = new RAMDirectory(); - IndexWriter iw = new IndexWriter(ramDir, new WhitespaceAnalyzer(), true, + IndexWriter iw = new IndexWriter(ramDir, new WhitespaceAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED); addDateDoc("a", 2005, 12, 2, 10, 15, 33, iw); addDateDoc("b", 2005, 12, 4, 22, 15, 00, iw); @@ -1094,7 +1095,7 @@ public void testMatchAllDocs() throws Exception { QueryParserWrapper qp = new QueryParserWrapper("field", - new WhitespaceAnalyzer()); + new WhitespaceAnalyzer(Version.LUCENE_CURRENT)); assertEquals(new MatchAllDocsQuery(), qp.parse("*:*")); assertEquals(new MatchAllDocsQuery(), qp.parse("(*:*)")); BooleanQuery bq = (BooleanQuery) qp.parse("+*:* -*:*"); @@ -1105,7 +1106,7 @@ private void assertHits(int expected, String query, IndexSearcher is) throws ParseException, IOException { QueryParserWrapper qp = new QueryParserWrapper("date", - new WhitespaceAnalyzer()); + new WhitespaceAnalyzer(Version.LUCENE_CURRENT)); qp.setLocale(Locale.ENGLISH); Query q = qp.parse(query); ScoreDoc[] hits = is.search(q, null, 1000).scoreDocs; Modified: lucene/java/branches/flex_1458/contrib/regex/src/test/org/apache/lucene/search/regex/TestRegexQuery.java URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/regex/src/test/org/apache/lucene/search/regex/TestRegexQuery.java?rev=916666&r1=916665&r2=916666&view=diff ============================================================================== --- lucene/java/branches/flex_1458/contrib/regex/src/test/org/apache/lucene/search/regex/TestRegexQuery.java (original) +++ lucene/java/branches/flex_1458/contrib/regex/src/test/org/apache/lucene/search/regex/TestRegexQuery.java Fri Feb 26 13:09:54 2010 @@ -29,6 +29,7 @@ import org.apache.lucene.search.spans.SpanNearQuery; import org.apache.lucene.search.spans.SpanQuery; +import org.apache.lucene.util.Version; public class TestRegexQuery extends TestCase { private IndexSearcher searcher; @@ -39,7 +40,7 @@ public void setUp() { RAMDirectory directory = new RAMDirectory(); try { - IndexWriter writer = new IndexWriter(directory, new SimpleAnalyzer(), true, + IndexWriter writer = new IndexWriter(directory, new SimpleAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED); Document doc = new Document(); doc.add(new Field(FN, "the quick brown fox jumps over the lazy dog", Field.Store.NO, Field.Index.ANALYZED));