Return-Path: Delivered-To: apmail-lucene-solr-commits-archive@minotaur.apache.org Received: (qmail 90349 invoked from network); 15 Mar 2010 00:44:11 -0000 Received: from unknown (HELO mail.apache.org) (140.211.11.3) by 140.211.11.9 with SMTP; 15 Mar 2010 00:44:11 -0000 Received: (qmail 59380 invoked by uid 500); 15 Mar 2010 00:43:26 -0000 Delivered-To: apmail-lucene-solr-commits-archive@lucene.apache.org Received: (qmail 59345 invoked by uid 500); 15 Mar 2010 00:43:26 -0000 Mailing-List: contact solr-commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: solr-dev@lucene.apache.org Delivered-To: mailing list solr-commits@lucene.apache.org Received: (qmail 59338 invoked by uid 99); 15 Mar 2010 00:43:26 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 15 Mar 2010 00:43:26 +0000 X-ASF-Spam-Status: No, hits=-1109.4 required=10.0 tests=ALL_TRUSTED,AWL X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 15 Mar 2010 00:43:25 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id A836B23888CD; Mon, 15 Mar 2010 00:43:05 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r923020 - /lucene/solr/branches/solr/src/webapp/web/admin/analysis.jsp Date: Mon, 15 Mar 2010 00:43:05 -0000 To: solr-commits@lucene.apache.org From: rmuir@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20100315004305.A836B23888CD@eris.apache.org> Author: rmuir Date: Mon Mar 15 00:43:05 2010 New Revision: 923020 URL: http://svn.apache.org/viewvc?rev=923020&view=rev Log: convert uber-secret tokenstreams inside of jsp Modified: lucene/solr/branches/solr/src/webapp/web/admin/analysis.jsp Modified: lucene/solr/branches/solr/src/webapp/web/admin/analysis.jsp URL: http://svn.apache.org/viewvc/lucene/solr/branches/solr/src/webapp/web/admin/analysis.jsp?rev=923020&r1=923019&r2=923020&view=diff ============================================================================== --- lucene/solr/branches/solr/src/webapp/web/admin/analysis.jsp (original) +++ lucene/solr/branches/solr/src/webapp/web/admin/analysis.jsp Mon Mar 15 00:43:05 2010 @@ -21,6 +21,7 @@ org.apache.lucene.index.Payload, org.apache.lucene.analysis.CharReader, org.apache.lucene.analysis.CharStream, + org.apache.lucene.analysis.tokenattributes.*, org.apache.solr.analysis.CharFilterFactory, org.apache.solr.analysis.TokenFilterFactory, org.apache.solr.analysis.TokenizerChain, @@ -212,8 +213,26 @@ final Iterator iter = tokens.iterator(); tstream = filtfac.create( new TokenStream() { - public Token next() throws IOException { - return iter.hasNext() ? iter.next() : null; + TermAttribute termAtt = (TermAttribute) addAttribute(TermAttribute.class); + OffsetAttribute offsetAtt = (OffsetAttribute) addAttribute (OffsetAttribute.class); + TypeAttribute typeAtt = (TypeAttribute) addAttribute (TypeAttribute.class); + FlagsAttribute flagsAtt = (FlagsAttribute) addAttribute (FlagsAttribute.class); + PayloadAttribute payloadAtt = (PayloadAttribute) addAttribute (PayloadAttribute.class); + PositionIncrementAttribute posIncAtt = (PositionIncrementAttribute) addAttribute (PositionIncrementAttribute.class); + + public boolean incrementToken() throws IOException { + if (iter.hasNext()) { + Token token = iter.next(); + termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength()); + offsetAtt.setOffset(token.startOffset(), token.endOffset()); + typeAtt.setType(token.type()); + flagsAtt.setFlags(token.getFlags()); + posIncAtt.setPositionIncrement(token.getPositionIncrement()); + payloadAtt.setPayload(token.getPayload()); + return true; + } else { + return false; + } } } ); @@ -236,10 +255,26 @@ static List getTokens(TokenStream tstream) throws IOException { List tokens = new ArrayList(); + TermAttribute termAtt = (TermAttribute) tstream.addAttribute(TermAttribute.class); + OffsetAttribute offsetAtt = (OffsetAttribute) tstream.addAttribute (OffsetAttribute.class); + TypeAttribute typeAtt = (TypeAttribute) tstream.addAttribute (TypeAttribute.class); + FlagsAttribute flagsAtt = (FlagsAttribute) tstream.addAttribute (FlagsAttribute.class); + PayloadAttribute payloadAtt = (PayloadAttribute) tstream.addAttribute (PayloadAttribute.class); + PositionIncrementAttribute posIncAtt = (PositionIncrementAttribute) tstream.addAttribute (PositionIncrementAttribute.class); + while (true) { - Token t = tstream.next(); - if (t==null) break; - tokens.add(t); + if (!tstream.incrementToken()) + break; + else { + Token token = new Token(); + token.setTermBuffer(termAtt.termBuffer(), 0, termAtt.termLength()); + token.setType(typeAtt.type()); + token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset()); + token.setPayload(payloadAtt.getPayload()); + token.setFlags(flagsAtt.getFlags()); + token.setPositionIncrement(posIncAtt.getPositionIncrement()); + tokens.add(token); + } } return tokens; } @@ -254,13 +289,13 @@ } public boolean equals(Object o) { - return ((Tok)o).token.termText().equals(token.termText()); + return ((Tok)o).token.term().equals(token.term()); } public int hashCode() { - return token.termText().hashCode(); + return token.term().hashCode(); } public String toString() { - return token.termText(); + return token.term(); } } @@ -342,7 +377,7 @@ boolean needRaw=false; int pos=0; for (Token t : tokens) { - if (!t.termText().equals(ft.indexedToReadable(t.termText()))) { + if (!t.term().equals(ft.indexedToReadable(t.term()))) { needRaw=true; } @@ -391,7 +426,7 @@ printRow(out,"term text", arr, new ToStr() { public String toStr(Object o) { - return ft.indexedToReadable( ((Tok)o).token.termText() ); + return ft.indexedToReadable( ((Tok)o).token.term() ); } } ,true @@ -403,7 +438,7 @@ printRow(out,"raw text", arr, new ToStr() { public String toStr(Object o) { // page is UTF-8, so anything goes. - return ((Tok)o).token.termText(); + return ((Tok)o).token.term(); } } ,true