lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From uschind...@apache.org
Subject svn commit: r806847 - in /lucene/java/trunk/src: java/org/apache/lucene/analysis/ java/org/apache/lucene/util/ test/org/apache/lucene/analysis/ test/org/apache/lucene/index/
Date Sat, 22 Aug 2009 12:40:09 GMT
Author: uschindler
Date: Sat Aug 22 12:40:09 2009
New Revision: 806847

URL: http://svn.apache.org/viewvc?rev=806847&view=rev
Log:
LUCENE-1843: Convert some tests to new TokenStream API, better support of cross-impl AttributeImpl.copyTo()

Modified:
    lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java
    lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenWrapper.java
    lucene/java/trunk/src/java/org/apache/lucene/util/AttributeImpl.java
    lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenTestCase.java
    lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java

Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java?rev=806847&r1=806846&r2=806847&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java Sat Aug 22 12:40:09 2009
@@ -859,11 +859,20 @@
   }
 
   public void copyTo(AttributeImpl target) {
-    Token to = (Token) target;
-    to.reinit(this);
-    // reinit shares the payload, so clone it:
-    if (payload !=null) {
-      to.payload = (Payload) payload.clone();
+    if (target instanceof Token) {
+      final Token to = (Token) target;
+      to.reinit(this);
+      // reinit shares the payload, so clone it:
+      if (payload !=null) {
+        to.payload = (Payload) payload.clone();
+      }
+    } else {
+      ((TermAttribute) target).setTermBuffer(termBuffer, 0, termLength);
+      ((OffsetAttribute) target).setOffset(startOffset, endOffset);
+      ((PositionIncrementAttribute) target).setPositionIncrement(positionIncrement);
+      ((PayloadAttribute) target).setPayload((payload == null) ? null : (Payload) payload.clone());
+      ((FlagsAttribute) target).setFlags(flags);
+      ((TypeAttribute) target).setType(type);
     }
   }
 }

Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenWrapper.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenWrapper.java?rev=806847&r1=806846&r2=806847&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenWrapper.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenWrapper.java Sat Aug 22 12:40:09
2009
@@ -157,6 +157,10 @@
   }
 
   public void copyTo(AttributeImpl target) {
-    ((TokenWrapper) target).delegate = (Token) this.delegate.clone();
+    if (target instanceof TokenWrapper) {
+      ((TokenWrapper) target).delegate = (Token) this.delegate.clone();
+    } else {
+      this.delegate.copyTo(target);
+    }
   }
 }

Modified: lucene/java/trunk/src/java/org/apache/lucene/util/AttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/util/AttributeImpl.java?rev=806847&r1=806846&r2=806847&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/util/AttributeImpl.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/util/AttributeImpl.java Sat Aug 22 12:40:09
2009
@@ -101,8 +101,8 @@
   
   /**
    * Copies the values from this Attribute into the passed-in
-   * target attribute. The type of the target must match the type
-   * of this attribute. 
+   * target attribute. The target implementation must support all the
+   * Attributes this implementation supports.
    */
   public abstract void copyTo(AttributeImpl target);
     

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenTestCase.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenTestCase.java?rev=806847&r1=806846&r2=806847&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenTestCase.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenTestCase.java Sat Aug 22
12:40:09 2009
@@ -19,35 +19,25 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+
+public abstract class BaseTokenTestCase extends BaseTokenStreamTestCase {
 
-/* TODO: Convert to new TokenStream API. Token instances must be removed for that to work
*/
-public abstract class BaseTokenTestCase extends LuceneTestCase {
   public static String tsToString(TokenStream in) throws IOException {
-    StringBuffer out = new StringBuffer();
-    Token t = in.next();
-    if (null != t)
-      out.append(new String(t.termBuffer(), 0, t.termLength()));
-    
-    for (t = in.next(); null != t; t = in.next()) {
-      out.append(" ").append(new String(t.termBuffer(), 0, t.termLength()));
+    final TermAttribute termAtt = (TermAttribute) in.addAttribute(TermAttribute.class);
+    final StringBuffer out = new StringBuffer();
+    in.reset();
+    while (in.incrementToken()) {
+      if (out.length()>0) out.append(' ');
+      out.append(termAtt.term());
     }
     in.close();
     return out.toString();
   }
-/*
-  public List<String> tok2str(Iterable<Token> tokLst) {
-    ArrayList<String> lst = new ArrayList<String>();
-    for ( Token t : tokLst ) {
-      lst.add( new String(t.termBuffer(), 0, t.termLength()));
-    }
-    return lst;
-  }
-*/
 
   public void assertTokEqual(List/*<Token>*/ a, List/*<Token>*/ b) {
     assertTokEq(a,b,false);
@@ -64,7 +54,7 @@
     for (Iterator iter = a.iterator(); iter.hasNext();) {
       Token tok = (Token)iter.next();
       pos += tok.getPositionIncrement();
-      if (!tokAt(b, new String(tok.termBuffer(), 0, tok.termLength()), pos
+      if (!tokAt(b, tok.term(), pos
               , checkOff ? tok.startOffset() : -1
               , checkOff ? tok.endOffset() : -1
               )) 
@@ -79,7 +69,7 @@
     for (Iterator iter = lst.iterator(); iter.hasNext();) {
       Token tok = (Token)iter.next();
       pos += tok.getPositionIncrement();
-      if (pos==tokPos && new String(tok.termBuffer(), 0, tok.termLength()).equals(val)
+      if (pos==tokPos && tok.term().equals(val)
           && (startOff==-1 || tok.startOffset()==startOff)
           && (endOff  ==-1 || tok.endOffset()  ==endOff  )
            )
@@ -146,41 +136,22 @@
 
   static List/*<Token>*/ getTokens(TokenStream tstream) throws IOException {
     List/*<Token>*/ tokens = new ArrayList/*<Token>*/();
-    while (true) {
-      Token t = tstream.next();
-      if (t==null) break;
+    tstream.reset();
+    while (tstream.incrementToken()) {
+      final Token t = new Token();
+      for (Iterator it = tstream.getAttributeImplsIterator(); it.hasNext();) {
+        final AttributeImpl att = (AttributeImpl) it.next();
+        try {
+          att.copyTo(t);
+        } catch (ClassCastException ce) {
+          // ignore Attributes unsupported by Token
+        }
+      }
       tokens.add(t);
     }
+    tstream.close();
+    
     return tokens;
   }
-/*
-  public static class IterTokenStream extends TokenStream {
-    Iterator<Token> toks;
-    public IterTokenStream(Token... toks) {
-      this.toks = Arrays.asList(toks).iterator();
-    }
-    public IterTokenStream(Iterable<Token> toks) {
-      this.toks = toks.iterator();
-    }
-    public IterTokenStream(Iterator<Token> toks) {
-      this.toks = toks;
-    }
-    public IterTokenStream(String ... text) {
-      int off = 0;
-      ArrayList<Token> t = new ArrayList<Token>( text.length );
-      for( String txt : text ) {
-        t.add( new Token( txt, off, off+txt.length() ) );
-        off += txt.length() + 2;
-      }
-      this.toks = t.iterator();
-    }
-    @Override
-    public Token next() {
-      if (toks.hasNext()) {
-        return toks.next();
-      }
-      return null;
-    }
-  }
-*/
+
 }

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=806847&r1=806846&r2=806847&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java Sat Aug 22 12:40:09
2009
@@ -29,21 +29,23 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.HashSet;
 import java.util.Random;
 
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.CachingTokenFilter;
 import org.apache.lucene.analysis.SimpleAnalyzer;
-import org.apache.lucene.analysis.SinkTokenizer;
 import org.apache.lucene.analysis.StopAnalyzer;
 import org.apache.lucene.analysis.TeeSinkTokenFilter;
-import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.WhitespaceAnalyzer;
 import org.apache.lucene.analysis.WhitespaceTokenizer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.search.IndexSearcher;
@@ -62,7 +64,6 @@
 import org.apache.lucene.store.MockRAMDirectory;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.store.SingleInstanceLockFactory;
-import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.UnicodeUtil;
 import org.apache.lucene.util._TestUtil;
 
@@ -70,8 +71,16 @@
  *
  * @version $Id$
  */
-public class TestIndexWriter extends LuceneTestCase
-{
+public class TestIndexWriter extends BaseTokenStreamTestCase {
+    public TestIndexWriter(String name) {
+      super(name, new HashSet(Arrays.asList(new String[]{
+        "testExceptionFromTokenStream", "testDocumentsWriterExceptions", "testNegativePositions",
+        "testEndOffsetPositionWithCachingTokenFilter", "testEndOffsetPositionWithTeeSinkTokenFilter",
+        "testEndOffsetPositionStandard", "testEndOffsetPositionStandardEmptyField",
+        "testEndOffsetPositionStandardEmptyField2"
+      })));
+    }
+
     public void testDocCount() throws IOException
     {
         Directory dir = new RAMDirectory();
@@ -3530,16 +3539,22 @@
 
   // LUCENE-1255
   public void testNegativePositions() throws Throwable {
-    SinkTokenizer tokens = new SinkTokenizer();
-    Token t = new Token();
-    t.setTermBuffer("a");
-    t.setPositionIncrement(0);
-    tokens.add(t);
-    t.setTermBuffer("b");
-    t.setPositionIncrement(1);
-    tokens.add(t);
-    t.setTermBuffer("c");
-    tokens.add(t);
+    final TokenStream tokens = new TokenStream() {
+      final TermAttribute termAtt = (TermAttribute) addAttribute(TermAttribute.class);
+      final PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class);
+      
+      final Iterator tokens = Arrays.asList(new String[]{"a","b","c"}).iterator();
+      boolean first = true;
+      
+      public boolean incrementToken() {
+        if (!tokens.hasNext()) return false;
+        clearAttributes();
+        termAtt.setTermBuffer((String) tokens.next());
+        posIncrAtt.setPositionIncrement(first ? 0 : 1);
+        first = false;
+        return true;
+      }
+    };
 
     MockRAMDirectory dir = new MockRAMDirectory();
     IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);



Mime
View raw message