lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r648188 - in /lucene/java/branches/lucene_2_3: contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/ src/java/org/apache/lucene/analysis/standard/ src/test/org/apache/lucene/analysis/
Date Tue, 15 Apr 2008 09:12:03 GMT
Author: mikemccand
Date: Tue Apr 15 02:12:00 2008
New Revision: 648188

URL: http://svn.apache.org/viewvc?rev=648188&view=rev
Log:
LUCENE-1150: put back public tokenImage/TOKEN_TYPES in StandardTokenizer and WikipediaTokenizer

Modified:
    lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java
    lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java
    lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex
    lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
    lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
    lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
    lucene/java/branches/lucene_2_3/src/test/org/apache/lucene/analysis/TestAnalyzers.java

Modified: lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java?rev=648188&r1=648187&r2=648188&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java
(original)
+++ lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java
Tue Apr 15 02:12:00 2008
@@ -63,6 +63,31 @@
   public static final int SUB_HEADING_ID       = 16;
   public static final int EXTERNAL_LINK_URL_ID = 17;
 
+  /** String token types that correspond to token type int constants */
+  public static final String [] TOKEN_TYPES = new String [] {
+    "<ALPHANUM>",
+    "<APOSTROPHE>",
+    "<ACRONYM>",
+    "<COMPANY>",
+    "<EMAIL>",
+    "<HOST>",
+    "<NUM>",
+    "<CJ>",
+    INTERNAL_LINK,
+    EXTERNAL_LINK,
+    CITATION,
+    CATEGORY,
+    BOLD,
+    ITALICS,
+    BOLD_ITALICS,
+    HEADING,
+    SUB_HEADING,
+    EXTERNAL_LINK_URL
+  };
+
+  /** @deprecated Please use {@link #TOKEN_TYPES} instead */
+  public static final String [] tokenImage = TOKEN_TYPES;
+
   public static final int TOKENS_ONLY = 0;
   public static final int UNTOKENIZED_ONLY = 1;
   public static final int BOTH = 2;

Modified: lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java?rev=648188&r1=648187&r2=648188&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java
(original)
+++ lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java
Tue Apr 15 02:12:00 2008
@@ -1,5 +1,4 @@
-
-/* The following code was generated by JFlex 1.4.1 on 1/4/08 3:30 PM */
+/* The following code was generated by JFlex 1.4.1 on 4/15/08 8:53 AM */
 
 package org.apache.lucene.wikipedia.analysis;
 
@@ -26,8 +25,8 @@
 /**
  * This class is a scanner generated by 
  * <a href="http://www.jflex.de/">JFlex</a> 1.4.1
- * on 1/4/08 3:30 PM from the specification file
- * <tt>/Volumes/User/grantingersoll/projects/lucene/Lucene-Trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex</tt>
+ * on 4/15/08 8:53 AM from the specification file
+ * <tt>/tango/mike/src/lucene.23.clean/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex</tt>
  */
 class WikipediaTokenizerImpl {
 
@@ -448,26 +447,7 @@
 private int positionInc = 1;
 private int numLinkToks = 0;
 
-public static final String [] TOKEN_TYPES = new String [] {
-    "<ALPHANUM>",
-    "<APOSTROPHE>",
-    "<ACRONYM>",
-    "<COMPANY>",
-    "<EMAIL>",
-    "<HOST>",
-    "<NUM>",
-    "<CJ>",
-    WikipediaTokenizer.INTERNAL_LINK,
-    WikipediaTokenizer.EXTERNAL_LINK,
-    WikipediaTokenizer.CITATION,
-    WikipediaTokenizer.CATEGORY,
-    WikipediaTokenizer.BOLD,
-    WikipediaTokenizer.ITALICS,
-    WikipediaTokenizer.BOLD_ITALICS,
-    WikipediaTokenizer.HEADING,
-    WikipediaTokenizer.SUB_HEADING,
-    WikipediaTokenizer.EXTERNAL_LINK_URL
-};
+public static final String [] TOKEN_TYPES = WikipediaTokenizer.TOKEN_TYPES;
 
 public final int yychar()
 {

Modified: lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex?rev=648188&r1=648187&r2=648188&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex
(original)
+++ lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex
Tue Apr 15 02:12:00 2008
@@ -55,26 +55,7 @@
 private int positionInc = 1;
 private int numLinkToks = 0;
 
-public static final String [] TOKEN_TYPES = new String [] {
-    "<ALPHANUM>",
-    "<APOSTROPHE>",
-    "<ACRONYM>",
-    "<COMPANY>",
-    "<EMAIL>",
-    "<HOST>",
-    "<NUM>",
-    "<CJ>",
-    WikipediaTokenizer.INTERNAL_LINK,
-    WikipediaTokenizer.EXTERNAL_LINK,
-    WikipediaTokenizer.CITATION,
-    WikipediaTokenizer.CATEGORY,
-    WikipediaTokenizer.BOLD,
-    WikipediaTokenizer.ITALICS,
-    WikipediaTokenizer.BOLD_ITALICS,
-    WikipediaTokenizer.HEADING,
-    WikipediaTokenizer.SUB_HEADING,
-    WikipediaTokenizer.EXTERNAL_LINK_URL
-};
+public static final String [] TOKEN_TYPES = WikipediaTokenizer.TOKEN_TYPES;
 
 public final int yychar()
 {
@@ -326,4 +307,4 @@
 //ITALICS = ''({ALPHANUM}+{WHITESPACE}*)+''
 //BOLD_ITALICS = '''''({ALPHANUM}+{WHITESPACE}*)+'''''
 //HEADING = "="{2}({ALPHANUM}+{WHITESPACE}*)+"="{2}
-//SUB_HEADING ="="{3}({ALPHANUM}+{WHITESPACE}*)+"="{3}
\ No newline at end of file
+//SUB_HEADING ="="{3}({ALPHANUM}+{WHITESPACE}*)+"="{3}

Modified: lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=648188&r1=648187&r2=648188&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
(original)
+++ lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
Tue Apr 15 02:12:00 2008
@@ -59,7 +59,23 @@
    *             release.
    */
   public static final int ACRONYM_DEP       = 8;
-    
+
+  /** String token types that correspond to token type int constants */
+  public static final String [] TOKEN_TYPES = new String [] {
+    "<ALPHANUM>",
+    "<APOSTROPHE>",
+    "<ACRONYM>",
+    "<COMPANY>",
+    "<EMAIL>",
+    "<HOST>",
+    "<NUM>",
+    "<CJ>",
+    "<ACRONYM_DEP>"
+  };
+
+  /** @deprecated Please use {@link #TOKEN_TYPES} instead */
+  public static final String [] tokenImage = TOKEN_TYPES;
+
   /**
    * Specifies whether deprecated acronyms should be replaced with HOST type.
    * This is false by default to support backward compatibility.

Modified: lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java?rev=648188&r1=648187&r2=648188&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
(original)
+++ lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
Tue Apr 15 02:12:00 2008
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.4.1 on 12/18/07 9:22 PM */
+/* The following code was generated by JFlex 1.4.1 on 4/15/08 8:53 AM */
 
 package org.apache.lucene.analysis.standard;
 
@@ -25,8 +25,8 @@
 /**
  * This class is a scanner generated by 
  * <a href="http://www.jflex.de/">JFlex</a> 1.4.1
- * on 12/18/07 9:22 PM from the specification file
- * <tt>/Volumes/User/grantingersoll/projects/lucene/java/lucene-clean/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex</tt>
+ * on 4/15/08 8:53 AM from the specification file
+ * <tt>/tango/mike/src/lucene.23.clean/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex</tt>
  */
 class StandardTokenizerImpl {
 
@@ -298,17 +298,7 @@
  */
 public static final int ACRONYM_DEP       = StandardTokenizer.ACRONYM_DEP;
 
-public static final String [] TOKEN_TYPES = new String [] {
-    "<ALPHANUM>",
-    "<APOSTROPHE>",
-    "<ACRONYM>",
-    "<COMPANY>",
-    "<EMAIL>",
-    "<HOST>",
-    "<NUM>",
-    "<CJ>",
-    "<ACRONYM_DEP>"
-};
+public static final String [] TOKEN_TYPES = StandardTokenizer.TOKEN_TYPES;
 
 public final int yychar()
 {

Modified: lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex?rev=648188&r1=648187&r2=648188&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
(original)
+++ lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
Tue Apr 15 02:12:00 2008
@@ -45,17 +45,7 @@
  */
 public static final int ACRONYM_DEP       = StandardTokenizer.ACRONYM_DEP;
 
-public static final String [] TOKEN_TYPES = new String [] {
-    "<ALPHANUM>",
-    "<APOSTROPHE>",
-    "<ACRONYM>",
-    "<COMPANY>",
-    "<EMAIL>",
-    "<HOST>",
-    "<NUM>",
-    "<CJ>",
-    "<ACRONYM_DEP>"
-};
+public static final String [] TOKEN_TYPES = StandardTokenizer.TOKEN_TYPES;
 
 public final int yychar()
 {

Modified: lucene/java/branches/lucene_2_3/src/test/org/apache/lucene/analysis/TestAnalyzers.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/src/test/org/apache/lucene/analysis/TestAnalyzers.java?rev=648188&r1=648187&r2=648188&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_3/src/test/org/apache/lucene/analysis/TestAnalyzers.java
(original)
+++ lucene/java/branches/lucene_2_3/src/test/org/apache/lucene/analysis/TestAnalyzers.java
Tue Apr 15 02:12:00 2008
@@ -119,7 +119,7 @@
     verifyPayload(ts);
   }
 
-  // Just a compile time test, to ensure the
+  // LUCENE-1150: Just a compile time test, to ensure the
   // StandardAnalyzer constants remain publicly accessible
   public void _testStandardConstants() {
     int x = StandardTokenizer.ALPHANUM;
@@ -130,6 +130,7 @@
     x = StandardTokenizer.HOST;
     x = StandardTokenizer.NUM;
     x = StandardTokenizer.CJ;
+    String[] y = StandardTokenizer.TOKEN_TYPES;
   }
 }
 



Mime
View raw message