Return-Path: Delivered-To: apmail-lucene-java-commits-archive@www.apache.org Received: (qmail 69838 invoked from network); 15 Apr 2008 09:12:43 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2) by minotaur.apache.org with SMTP; 15 Apr 2008 09:12:43 -0000 Received: (qmail 47374 invoked by uid 500); 15 Apr 2008 09:12:43 -0000 Delivered-To: apmail-lucene-java-commits-archive@lucene.apache.org Received: (qmail 47346 invoked by uid 500); 15 Apr 2008 09:12:43 -0000 Mailing-List: contact java-commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: java-dev@lucene.apache.org Delivered-To: mailing list java-commits@lucene.apache.org Received: (qmail 47335 invoked by uid 99); 15 Apr 2008 09:12:43 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 15 Apr 2008 02:12:43 -0700 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO eris.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 15 Apr 2008 09:11:49 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id BC5B91A9832; Tue, 15 Apr 2008 02:12:07 -0700 (PDT) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r648188 - in /lucene/java/branches/lucene_2_3: contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/ src/java/org/apache/lucene/analysis/standard/ src/test/org/apache/lucene/analysis/ Date: Tue, 15 Apr 2008 09:12:03 -0000 To: java-commits@lucene.apache.org From: mikemccand@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20080415091207.BC5B91A9832@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: mikemccand Date: Tue Apr 15 02:12:00 2008 New Revision: 648188 URL: http://svn.apache.org/viewvc?rev=648188&view=rev Log: LUCENE-1150: put back public tokenImage/TOKEN_TYPES in StandardTokenizer and WikipediaTokenizer Modified: lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex lucene/java/branches/lucene_2_3/src/test/org/apache/lucene/analysis/TestAnalyzers.java Modified: lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java?rev=648188&r1=648187&r2=648188&view=diff ============================================================================== --- lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java (original) +++ lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java Tue Apr 15 02:12:00 2008 @@ -63,6 +63,31 @@ public static final int SUB_HEADING_ID = 16; public static final int EXTERNAL_LINK_URL_ID = 17; + /** String token types that correspond to token type int constants */ + public static final String [] TOKEN_TYPES = new String [] { + "", + "", + "", + "", + "", + "", + "", + "", + INTERNAL_LINK, + EXTERNAL_LINK, + CITATION, + CATEGORY, + BOLD, + ITALICS, + BOLD_ITALICS, + HEADING, + SUB_HEADING, + EXTERNAL_LINK_URL + }; + + /** @deprecated Please use {@link #TOKEN_TYPES} instead */ + public static final String [] tokenImage = TOKEN_TYPES; + public static final int TOKENS_ONLY = 0; public static final int UNTOKENIZED_ONLY = 1; public static final int BOTH = 2; Modified: lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java?rev=648188&r1=648187&r2=648188&view=diff ============================================================================== --- lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java (original) +++ lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java Tue Apr 15 02:12:00 2008 @@ -1,5 +1,4 @@ - -/* The following code was generated by JFlex 1.4.1 on 1/4/08 3:30 PM */ +/* The following code was generated by JFlex 1.4.1 on 4/15/08 8:53 AM */ package org.apache.lucene.wikipedia.analysis; @@ -26,8 +25,8 @@ /** * This class is a scanner generated by * JFlex 1.4.1 - * on 1/4/08 3:30 PM from the specification file - * /Volumes/User/grantingersoll/projects/lucene/Lucene-Trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex + * on 4/15/08 8:53 AM from the specification file + * /tango/mike/src/lucene.23.clean/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex */ class WikipediaTokenizerImpl { @@ -448,26 +447,7 @@ private int positionInc = 1; private int numLinkToks = 0; -public static final String [] TOKEN_TYPES = new String [] { - "", - "", - "", - "", - "", - "", - "", - "", - WikipediaTokenizer.INTERNAL_LINK, - WikipediaTokenizer.EXTERNAL_LINK, - WikipediaTokenizer.CITATION, - WikipediaTokenizer.CATEGORY, - WikipediaTokenizer.BOLD, - WikipediaTokenizer.ITALICS, - WikipediaTokenizer.BOLD_ITALICS, - WikipediaTokenizer.HEADING, - WikipediaTokenizer.SUB_HEADING, - WikipediaTokenizer.EXTERNAL_LINK_URL -}; +public static final String [] TOKEN_TYPES = WikipediaTokenizer.TOKEN_TYPES; public final int yychar() { Modified: lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex?rev=648188&r1=648187&r2=648188&view=diff ============================================================================== --- lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex (original) +++ lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex Tue Apr 15 02:12:00 2008 @@ -55,26 +55,7 @@ private int positionInc = 1; private int numLinkToks = 0; -public static final String [] TOKEN_TYPES = new String [] { - "", - "", - "", - "", - "", - "", - "", - "", - WikipediaTokenizer.INTERNAL_LINK, - WikipediaTokenizer.EXTERNAL_LINK, - WikipediaTokenizer.CITATION, - WikipediaTokenizer.CATEGORY, - WikipediaTokenizer.BOLD, - WikipediaTokenizer.ITALICS, - WikipediaTokenizer.BOLD_ITALICS, - WikipediaTokenizer.HEADING, - WikipediaTokenizer.SUB_HEADING, - WikipediaTokenizer.EXTERNAL_LINK_URL -}; +public static final String [] TOKEN_TYPES = WikipediaTokenizer.TOKEN_TYPES; public final int yychar() { @@ -326,4 +307,4 @@ //ITALICS = ''({ALPHANUM}+{WHITESPACE}*)+'' //BOLD_ITALICS = '''''({ALPHANUM}+{WHITESPACE}*)+''''' //HEADING = "="{2}({ALPHANUM}+{WHITESPACE}*)+"="{2} -//SUB_HEADING ="="{3}({ALPHANUM}+{WHITESPACE}*)+"="{3} \ No newline at end of file +//SUB_HEADING ="="{3}({ALPHANUM}+{WHITESPACE}*)+"="{3} Modified: lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=648188&r1=648187&r2=648188&view=diff ============================================================================== --- lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (original) +++ lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java Tue Apr 15 02:12:00 2008 @@ -59,7 +59,23 @@ * release. */ public static final int ACRONYM_DEP = 8; - + + /** String token types that correspond to token type int constants */ + public static final String [] TOKEN_TYPES = new String [] { + "", + "", + "", + "", + "", + "", + "", + "", + "" + }; + + /** @deprecated Please use {@link #TOKEN_TYPES} instead */ + public static final String [] tokenImage = TOKEN_TYPES; + /** * Specifies whether deprecated acronyms should be replaced with HOST type. * This is false by default to support backward compatibility. Modified: lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java?rev=648188&r1=648187&r2=648188&view=diff ============================================================================== --- lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java (original) +++ lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java Tue Apr 15 02:12:00 2008 @@ -1,4 +1,4 @@ -/* The following code was generated by JFlex 1.4.1 on 12/18/07 9:22 PM */ +/* The following code was generated by JFlex 1.4.1 on 4/15/08 8:53 AM */ package org.apache.lucene.analysis.standard; @@ -25,8 +25,8 @@ /** * This class is a scanner generated by * JFlex 1.4.1 - * on 12/18/07 9:22 PM from the specification file - * /Volumes/User/grantingersoll/projects/lucene/java/lucene-clean/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex + * on 4/15/08 8:53 AM from the specification file + * /tango/mike/src/lucene.23.clean/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex */ class StandardTokenizerImpl { @@ -298,17 +298,7 @@ */ public static final int ACRONYM_DEP = StandardTokenizer.ACRONYM_DEP; -public static final String [] TOKEN_TYPES = new String [] { - "", - "", - "", - "", - "", - "", - "", - "", - "" -}; +public static final String [] TOKEN_TYPES = StandardTokenizer.TOKEN_TYPES; public final int yychar() { Modified: lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex?rev=648188&r1=648187&r2=648188&view=diff ============================================================================== --- lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex (original) +++ lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex Tue Apr 15 02:12:00 2008 @@ -45,17 +45,7 @@ */ public static final int ACRONYM_DEP = StandardTokenizer.ACRONYM_DEP; -public static final String [] TOKEN_TYPES = new String [] { - "", - "", - "", - "", - "", - "", - "", - "", - "" -}; +public static final String [] TOKEN_TYPES = StandardTokenizer.TOKEN_TYPES; public final int yychar() { Modified: lucene/java/branches/lucene_2_3/src/test/org/apache/lucene/analysis/TestAnalyzers.java URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/src/test/org/apache/lucene/analysis/TestAnalyzers.java?rev=648188&r1=648187&r2=648188&view=diff ============================================================================== --- lucene/java/branches/lucene_2_3/src/test/org/apache/lucene/analysis/TestAnalyzers.java (original) +++ lucene/java/branches/lucene_2_3/src/test/org/apache/lucene/analysis/TestAnalyzers.java Tue Apr 15 02:12:00 2008 @@ -119,7 +119,7 @@ verifyPayload(ts); } - // Just a compile time test, to ensure the + // LUCENE-1150: Just a compile time test, to ensure the // StandardAnalyzer constants remain publicly accessible public void _testStandardConstants() { int x = StandardTokenizer.ALPHANUM; @@ -130,6 +130,7 @@ x = StandardTokenizer.HOST; x = StandardTokenizer.NUM; x = StandardTokenizer.CJ; + String[] y = StandardTokenizer.TOKEN_TYPES; } }