Return-Path: Delivered-To: apmail-lucene-java-commits-archive@www.apache.org Received: (qmail 54839 invoked from network); 15 Apr 2008 08:49:15 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2) by minotaur.apache.org with SMTP; 15 Apr 2008 08:49:15 -0000 Received: (qmail 22737 invoked by uid 500); 15 Apr 2008 08:49:15 -0000 Delivered-To: apmail-lucene-java-commits-archive@lucene.apache.org Received: (qmail 22716 invoked by uid 500); 15 Apr 2008 08:49:15 -0000 Mailing-List: contact java-commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: java-dev@lucene.apache.org Delivered-To: mailing list java-commits@lucene.apache.org Received: (qmail 22664 invoked by uid 99); 15 Apr 2008 08:49:15 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 15 Apr 2008 01:49:15 -0700 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO eris.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 15 Apr 2008 08:48:30 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 94EA21A983A; Tue, 15 Apr 2008 01:48:43 -0700 (PDT) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r648183 - in /lucene/java/trunk: contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/ src/java/org/apache/lucene/analysis/standard/ src/test/org/apache/lucene/analysis/ Date: Tue, 15 Apr 2008 08:48:42 -0000 To: java-commits@lucene.apache.org From: mikemccand@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20080415084843.94EA21A983A@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: mikemccand Date: Tue Apr 15 01:48:41 2008 New Revision: 648183 URL: http://svn.apache.org/viewvc?rev=648183&view=rev Log: LUCENE-1150: put back public tokenImage/TOKEN_TYPES in StandardTokenizer and WikipediaTokenizer Modified: lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java Modified: lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java?rev=648183&r1=648182&r2=648183&view=diff ============================================================================== --- lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java (original) +++ lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java Tue Apr 15 01:48:41 2008 @@ -65,6 +65,31 @@ public static final int SUB_HEADING_ID = 16; public static final int EXTERNAL_LINK_URL_ID = 17; + /** String token types that correspond to token type int constants */ + public static final String [] TOKEN_TYPES = new String [] { + "", + "", + "", + "", + "", + "", + "", + "", + INTERNAL_LINK, + EXTERNAL_LINK, + CITATION, + CATEGORY, + BOLD, + ITALICS, + BOLD_ITALICS, + HEADING, + SUB_HEADING, + EXTERNAL_LINK_URL + }; + + /** @deprecated Please use {@link #TOKEN_TYPES} instead */ + public static final String [] tokenImage = TOKEN_TYPES; + public static final int TOKENS_ONLY = 0; public static final int UNTOKENIZED_ONLY = 1; public static final int BOTH = 2; Modified: lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java?rev=648183&r1=648182&r2=648183&view=diff ============================================================================== --- lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java (original) +++ lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java Tue Apr 15 01:48:41 2008 @@ -1,4 +1,4 @@ -/* The following code was generated by JFlex 1.4.1 on 1/25/08 6:54 PM */ +/* The following code was generated by JFlex 1.4.1 on 4/15/08 4:31 AM */ package org.apache.lucene.wikipedia.analysis; @@ -25,8 +25,8 @@ /** * This class is a scanner generated by * JFlex 1.4.1 - * on 1/25/08 6:54 PM from the specification file - * /tango/mike/src/lucene.clean/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex + * on 4/15/08 4:31 AM from the specification file + * /mnt2/mike/src/lucene.clean/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex */ class WikipediaTokenizerImpl { @@ -454,26 +454,7 @@ //see https://issues.apache.org/jira/browse/LUCENE-1133 private int numWikiTokensSeen = 0; -public static final String [] TOKEN_TYPES = new String [] { - "", - "", - "", - "", - "", - "", - "", - "", - WikipediaTokenizer.INTERNAL_LINK, - WikipediaTokenizer.EXTERNAL_LINK, - WikipediaTokenizer.CITATION, - WikipediaTokenizer.CATEGORY, - WikipediaTokenizer.BOLD, - WikipediaTokenizer.ITALICS, - WikipediaTokenizer.BOLD_ITALICS, - WikipediaTokenizer.HEADING, - WikipediaTokenizer.SUB_HEADING, - WikipediaTokenizer.EXTERNAL_LINK_URL -}; +public static final String [] TOKEN_TYPES = WikipediaTokenizer.TOKEN_TYPES; /** Returns the number of tokens seen inside a category or link, etc. Modified: lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex?rev=648183&r1=648182&r2=648183&view=diff ============================================================================== --- lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex (original) +++ lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex Tue Apr 15 01:48:41 2008 @@ -59,26 +59,7 @@ //see https://issues.apache.org/jira/browse/LUCENE-1133 private int numWikiTokensSeen = 0; -public static final String [] TOKEN_TYPES = new String [] { - "", - "", - "", - "", - "", - "", - "", - "", - WikipediaTokenizer.INTERNAL_LINK, - WikipediaTokenizer.EXTERNAL_LINK, - WikipediaTokenizer.CITATION, - WikipediaTokenizer.CATEGORY, - WikipediaTokenizer.BOLD, - WikipediaTokenizer.ITALICS, - WikipediaTokenizer.BOLD_ITALICS, - WikipediaTokenizer.HEADING, - WikipediaTokenizer.SUB_HEADING, - WikipediaTokenizer.EXTERNAL_LINK_URL -}; +public static final String [] TOKEN_TYPES = WikipediaTokenizer.TOKEN_TYPES; /** Returns the number of tokens seen inside a category or link, etc. @@ -346,4 +327,4 @@ //ITALICS = ''({ALPHANUM}+{WHITESPACE}*)+'' //BOLD_ITALICS = '''''({ALPHANUM}+{WHITESPACE}*)+''''' //HEADING = "="{2}({ALPHANUM}+{WHITESPACE}*)+"="{2} -//SUB_HEADING ="="{3}({ALPHANUM}+{WHITESPACE}*)+"="{3} \ No newline at end of file +//SUB_HEADING ="="{3}({ALPHANUM}+{WHITESPACE}*)+"="{3} Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=648183&r1=648182&r2=648183&view=diff ============================================================================== --- lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (original) +++ lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java Tue Apr 15 01:48:41 2008 @@ -59,7 +59,23 @@ * release. */ public static final int ACRONYM_DEP = 8; - + + /** String token types that correspond to token type int constants */ + public static final String [] TOKEN_TYPES = new String [] { + "", + "", + "", + "", + "", + "", + "", + "", + "" + }; + + /** @deprecated Please use {@link #TOKEN_TYPES} instead */ + public static final String [] tokenImage = TOKEN_TYPES; + /** * Specifies whether deprecated acronyms should be replaced with HOST type. * This is false by default to support backward compatibility. Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java?rev=648183&r1=648182&r2=648183&view=diff ============================================================================== --- lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java (original) +++ lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java Tue Apr 15 01:48:41 2008 @@ -1,4 +1,4 @@ -/* The following code was generated by JFlex 1.4.1 on 1/25/08 6:54 PM */ +/* The following code was generated by JFlex 1.4.1 on 4/15/08 4:31 AM */ package org.apache.lucene.analysis.standard; @@ -25,8 +25,8 @@ /** * This class is a scanner generated by * JFlex 1.4.1 - * on 1/25/08 6:54 PM from the specification file - * /tango/mike/src/lucene.clean/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex + * on 4/15/08 4:31 AM from the specification file + * /mnt2/mike/src/lucene.clean/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex */ class StandardTokenizerImpl { @@ -298,17 +298,7 @@ */ public static final int ACRONYM_DEP = StandardTokenizer.ACRONYM_DEP; -public static final String [] TOKEN_TYPES = new String [] { - "", - "", - "", - "", - "", - "", - "", - "", - "" -}; +public static final String [] TOKEN_TYPES = StandardTokenizer.TOKEN_TYPES; public final int yychar() { Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex?rev=648183&r1=648182&r2=648183&view=diff ============================================================================== --- lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex (original) +++ lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex Tue Apr 15 01:48:41 2008 @@ -45,17 +45,7 @@ */ public static final int ACRONYM_DEP = StandardTokenizer.ACRONYM_DEP; -public static final String [] TOKEN_TYPES = new String [] { - "", - "", - "", - "", - "", - "", - "", - "", - "" -}; +public static final String [] TOKEN_TYPES = StandardTokenizer.TOKEN_TYPES; public final int yychar() { Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java?rev=648183&r1=648182&r2=648183&view=diff ============================================================================== --- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java (original) +++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java Tue Apr 15 01:48:41 2008 @@ -119,7 +119,7 @@ verifyPayload(ts); } - // Just a compile time test, to ensure the + // LUCENE-1150: Just a compile time test, to ensure the // StandardAnalyzer constants remain publicly accessible public void _testStandardConstants() { int x = StandardTokenizer.ALPHANUM; @@ -130,6 +130,7 @@ x = StandardTokenizer.HOST; x = StandardTokenizer.NUM; x = StandardTokenizer.CJ; + String[] y = StandardTokenizer.TOKEN_TYPES; } }