Return-Path: Delivered-To: apmail-lucene-java-commits-archive@www.apache.org Received: (qmail 96307 invoked from network); 22 Aug 2009 12:40:14 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 22 Aug 2009 12:40:14 -0000 Received: (qmail 11064 invoked by uid 500); 22 Aug 2009 12:40:35 -0000 Delivered-To: apmail-lucene-java-commits-archive@lucene.apache.org Received: (qmail 11001 invoked by uid 500); 22 Aug 2009 12:40:35 -0000 Mailing-List: contact java-commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: java-dev@lucene.apache.org Delivered-To: mailing list java-commits@lucene.apache.org Received: (qmail 10991 invoked by uid 99); 22 Aug 2009 12:40:35 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 22 Aug 2009 12:40:35 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 22 Aug 2009 12:40:31 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 15A9D23888D2; Sat, 22 Aug 2009 12:40:10 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r806847 - in /lucene/java/trunk/src: java/org/apache/lucene/analysis/ java/org/apache/lucene/util/ test/org/apache/lucene/analysis/ test/org/apache/lucene/index/ Date: Sat, 22 Aug 2009 12:40:09 -0000 To: java-commits@lucene.apache.org From: uschindler@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20090822124010.15A9D23888D2@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: uschindler Date: Sat Aug 22 12:40:09 2009 New Revision: 806847 URL: http://svn.apache.org/viewvc?rev=806847&view=rev Log: LUCENE-1843: Convert some tests to new TokenStream API, better support of cross-impl AttributeImpl.copyTo() Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenWrapper.java lucene/java/trunk/src/java/org/apache/lucene/util/AttributeImpl.java lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenTestCase.java lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java?rev=806847&r1=806846&r2=806847&view=diff ============================================================================== --- lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java (original) +++ lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java Sat Aug 22 12:40:09 2009 @@ -859,11 +859,20 @@ } public void copyTo(AttributeImpl target) { - Token to = (Token) target; - to.reinit(this); - // reinit shares the payload, so clone it: - if (payload !=null) { - to.payload = (Payload) payload.clone(); + if (target instanceof Token) { + final Token to = (Token) target; + to.reinit(this); + // reinit shares the payload, so clone it: + if (payload !=null) { + to.payload = (Payload) payload.clone(); + } + } else { + ((TermAttribute) target).setTermBuffer(termBuffer, 0, termLength); + ((OffsetAttribute) target).setOffset(startOffset, endOffset); + ((PositionIncrementAttribute) target).setPositionIncrement(positionIncrement); + ((PayloadAttribute) target).setPayload((payload == null) ? null : (Payload) payload.clone()); + ((FlagsAttribute) target).setFlags(flags); + ((TypeAttribute) target).setType(type); } } } Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenWrapper.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenWrapper.java?rev=806847&r1=806846&r2=806847&view=diff ============================================================================== --- lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenWrapper.java (original) +++ lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenWrapper.java Sat Aug 22 12:40:09 2009 @@ -157,6 +157,10 @@ } public void copyTo(AttributeImpl target) { - ((TokenWrapper) target).delegate = (Token) this.delegate.clone(); + if (target instanceof TokenWrapper) { + ((TokenWrapper) target).delegate = (Token) this.delegate.clone(); + } else { + this.delegate.copyTo(target); + } } } Modified: lucene/java/trunk/src/java/org/apache/lucene/util/AttributeImpl.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/util/AttributeImpl.java?rev=806847&r1=806846&r2=806847&view=diff ============================================================================== --- lucene/java/trunk/src/java/org/apache/lucene/util/AttributeImpl.java (original) +++ lucene/java/trunk/src/java/org/apache/lucene/util/AttributeImpl.java Sat Aug 22 12:40:09 2009 @@ -101,8 +101,8 @@ /** * Copies the values from this Attribute into the passed-in - * target attribute. The type of the target must match the type - * of this attribute. + * target attribute. The target implementation must support all the + * Attributes this implementation supports. */ public abstract void copyTo(AttributeImpl target); Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenTestCase.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenTestCase.java?rev=806847&r1=806846&r2=806847&view=diff ============================================================================== --- lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenTestCase.java (original) +++ lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenTestCase.java Sat Aug 22 12:40:09 2009 @@ -19,35 +19,25 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Iterator; import java.util.List; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.AttributeImpl; +import org.apache.lucene.analysis.tokenattributes.TermAttribute; + +public abstract class BaseTokenTestCase extends BaseTokenStreamTestCase { -/* TODO: Convert to new TokenStream API. Token instances must be removed for that to work */ -public abstract class BaseTokenTestCase extends LuceneTestCase { public static String tsToString(TokenStream in) throws IOException { - StringBuffer out = new StringBuffer(); - Token t = in.next(); - if (null != t) - out.append(new String(t.termBuffer(), 0, t.termLength())); - - for (t = in.next(); null != t; t = in.next()) { - out.append(" ").append(new String(t.termBuffer(), 0, t.termLength())); + final TermAttribute termAtt = (TermAttribute) in.addAttribute(TermAttribute.class); + final StringBuffer out = new StringBuffer(); + in.reset(); + while (in.incrementToken()) { + if (out.length()>0) out.append(' '); + out.append(termAtt.term()); } in.close(); return out.toString(); } -/* - public List tok2str(Iterable tokLst) { - ArrayList lst = new ArrayList(); - for ( Token t : tokLst ) { - lst.add( new String(t.termBuffer(), 0, t.termLength())); - } - return lst; - } -*/ public void assertTokEqual(List/**/ a, List/**/ b) { assertTokEq(a,b,false); @@ -64,7 +54,7 @@ for (Iterator iter = a.iterator(); iter.hasNext();) { Token tok = (Token)iter.next(); pos += tok.getPositionIncrement(); - if (!tokAt(b, new String(tok.termBuffer(), 0, tok.termLength()), pos + if (!tokAt(b, tok.term(), pos , checkOff ? tok.startOffset() : -1 , checkOff ? tok.endOffset() : -1 )) @@ -79,7 +69,7 @@ for (Iterator iter = lst.iterator(); iter.hasNext();) { Token tok = (Token)iter.next(); pos += tok.getPositionIncrement(); - if (pos==tokPos && new String(tok.termBuffer(), 0, tok.termLength()).equals(val) + if (pos==tokPos && tok.term().equals(val) && (startOff==-1 || tok.startOffset()==startOff) && (endOff ==-1 || tok.endOffset() ==endOff ) ) @@ -146,41 +136,22 @@ static List/**/ getTokens(TokenStream tstream) throws IOException { List/**/ tokens = new ArrayList/**/(); - while (true) { - Token t = tstream.next(); - if (t==null) break; + tstream.reset(); + while (tstream.incrementToken()) { + final Token t = new Token(); + for (Iterator it = tstream.getAttributeImplsIterator(); it.hasNext();) { + final AttributeImpl att = (AttributeImpl) it.next(); + try { + att.copyTo(t); + } catch (ClassCastException ce) { + // ignore Attributes unsupported by Token + } + } tokens.add(t); } + tstream.close(); + return tokens; } -/* - public static class IterTokenStream extends TokenStream { - Iterator toks; - public IterTokenStream(Token... toks) { - this.toks = Arrays.asList(toks).iterator(); - } - public IterTokenStream(Iterable toks) { - this.toks = toks.iterator(); - } - public IterTokenStream(Iterator toks) { - this.toks = toks; - } - public IterTokenStream(String ... text) { - int off = 0; - ArrayList t = new ArrayList( text.length ); - for( String txt : text ) { - t.add( new Token( txt, off, off+txt.length() ) ); - off += txt.length() + 2; - } - this.toks = t.iterator(); - } - @Override - public Token next() { - if (toks.hasNext()) { - return toks.next(); - } - return null; - } - } -*/ + } Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=806847&r1=806846&r2=806847&view=diff ============================================================================== --- lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java (original) +++ lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java Sat Aug 22 12:40:09 2009 @@ -29,21 +29,23 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.HashSet; import java.util.Random; +import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.CachingTokenFilter; import org.apache.lucene.analysis.SimpleAnalyzer; -import org.apache.lucene.analysis.SinkTokenizer; import org.apache.lucene.analysis.StopAnalyzer; import org.apache.lucene.analysis.TeeSinkTokenFilter; -import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.WhitespaceAnalyzer; import org.apache.lucene.analysis.WhitespaceTokenizer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.standard.StandardTokenizer; +import org.apache.lucene.analysis.tokenattributes.TermAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.search.IndexSearcher; @@ -62,7 +64,6 @@ import org.apache.lucene.store.MockRAMDirectory; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.store.SingleInstanceLockFactory; -import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.UnicodeUtil; import org.apache.lucene.util._TestUtil; @@ -70,8 +71,16 @@ * * @version $Id$ */ -public class TestIndexWriter extends LuceneTestCase -{ +public class TestIndexWriter extends BaseTokenStreamTestCase { + public TestIndexWriter(String name) { + super(name, new HashSet(Arrays.asList(new String[]{ + "testExceptionFromTokenStream", "testDocumentsWriterExceptions", "testNegativePositions", + "testEndOffsetPositionWithCachingTokenFilter", "testEndOffsetPositionWithTeeSinkTokenFilter", + "testEndOffsetPositionStandard", "testEndOffsetPositionStandardEmptyField", + "testEndOffsetPositionStandardEmptyField2" + }))); + } + public void testDocCount() throws IOException { Directory dir = new RAMDirectory(); @@ -3530,16 +3539,22 @@ // LUCENE-1255 public void testNegativePositions() throws Throwable { - SinkTokenizer tokens = new SinkTokenizer(); - Token t = new Token(); - t.setTermBuffer("a"); - t.setPositionIncrement(0); - tokens.add(t); - t.setTermBuffer("b"); - t.setPositionIncrement(1); - tokens.add(t); - t.setTermBuffer("c"); - tokens.add(t); + final TokenStream tokens = new TokenStream() { + final TermAttribute termAtt = (TermAttribute) addAttribute(TermAttribute.class); + final PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class); + + final Iterator tokens = Arrays.asList(new String[]{"a","b","c"}).iterator(); + boolean first = true; + + public boolean incrementToken() { + if (!tokens.hasNext()) return false; + clearAttributes(); + termAtt.setTermBuffer((String) tokens.next()); + posIncrAtt.setPositionIncrement(first ? 0 : 1); + first = false; + return true; + } + }; MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);