lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r648183 - in /lucene/java/trunk: contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/ src/java/org/apache/lucene/analysis/standard/ src/test/org/apache/lucene/analysis/
Date Tue, 15 Apr 2008 08:48:42 GMT
Author: mikemccand
Date: Tue Apr 15 01:48:41 2008
New Revision: 648183

URL: http://svn.apache.org/viewvc?rev=648183&view=rev
Log:
LUCENE-1150: put back public tokenImage/TOKEN_TYPES in StandardTokenizer and WikipediaTokenizer

Modified:
    lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java
    lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java
    lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex
    lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
    lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
    lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
    lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java

Modified: lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java?rev=648183&r1=648182&r2=648183&view=diff
==============================================================================
--- lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java
(original)
+++ lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java
Tue Apr 15 01:48:41 2008
@@ -65,6 +65,31 @@
   public static final int SUB_HEADING_ID       = 16;
   public static final int EXTERNAL_LINK_URL_ID = 17;
 
+  /** String token types that correspond to token type int constants */
+  public static final String [] TOKEN_TYPES = new String [] {
+    "<ALPHANUM>",
+    "<APOSTROPHE>",
+    "<ACRONYM>",
+    "<COMPANY>",
+    "<EMAIL>",
+    "<HOST>",
+    "<NUM>",
+    "<CJ>",
+    INTERNAL_LINK,
+    EXTERNAL_LINK,
+    CITATION,
+    CATEGORY,
+    BOLD,
+    ITALICS,
+    BOLD_ITALICS,
+    HEADING,
+    SUB_HEADING,
+    EXTERNAL_LINK_URL
+  };
+
+  /** @deprecated Please use {@link #TOKEN_TYPES} instead */
+  public static final String [] tokenImage = TOKEN_TYPES;
+
   public static final int TOKENS_ONLY = 0;
   public static final int UNTOKENIZED_ONLY = 1;
   public static final int BOTH = 2;

Modified: lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java?rev=648183&r1=648182&r2=648183&view=diff
==============================================================================
--- lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java
(original)
+++ lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java
Tue Apr 15 01:48:41 2008
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.4.1 on 1/25/08 6:54 PM */
+/* The following code was generated by JFlex 1.4.1 on 4/15/08 4:31 AM */
 
 package org.apache.lucene.wikipedia.analysis;
 
@@ -25,8 +25,8 @@
 /**
  * This class is a scanner generated by 
  * <a href="http://www.jflex.de/">JFlex</a> 1.4.1
- * on 1/25/08 6:54 PM from the specification file
- * <tt>/tango/mike/src/lucene.clean/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex</tt>
+ * on 4/15/08 4:31 AM from the specification file
+ * <tt>/mnt2/mike/src/lucene.clean/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex</tt>
  */
 class WikipediaTokenizerImpl {
 
@@ -454,26 +454,7 @@
 //see https://issues.apache.org/jira/browse/LUCENE-1133
 private int numWikiTokensSeen = 0;
 
-public static final String [] TOKEN_TYPES = new String [] {
-    "<ALPHANUM>",
-    "<APOSTROPHE>",
-    "<ACRONYM>",
-    "<COMPANY>",
-    "<EMAIL>",
-    "<HOST>",
-    "<NUM>",
-    "<CJ>",
-    WikipediaTokenizer.INTERNAL_LINK,
-    WikipediaTokenizer.EXTERNAL_LINK,
-    WikipediaTokenizer.CITATION,
-    WikipediaTokenizer.CATEGORY,
-    WikipediaTokenizer.BOLD,
-    WikipediaTokenizer.ITALICS,
-    WikipediaTokenizer.BOLD_ITALICS,
-    WikipediaTokenizer.HEADING,
-    WikipediaTokenizer.SUB_HEADING,
-    WikipediaTokenizer.EXTERNAL_LINK_URL
-};
+public static final String [] TOKEN_TYPES = WikipediaTokenizer.TOKEN_TYPES;
 
 /**
 Returns the number of tokens seen inside a category or link, etc.

Modified: lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex?rev=648183&r1=648182&r2=648183&view=diff
==============================================================================
--- lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex
(original)
+++ lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex
Tue Apr 15 01:48:41 2008
@@ -59,26 +59,7 @@
 //see https://issues.apache.org/jira/browse/LUCENE-1133
 private int numWikiTokensSeen = 0;
 
-public static final String [] TOKEN_TYPES = new String [] {
-    "<ALPHANUM>",
-    "<APOSTROPHE>",
-    "<ACRONYM>",
-    "<COMPANY>",
-    "<EMAIL>",
-    "<HOST>",
-    "<NUM>",
-    "<CJ>",
-    WikipediaTokenizer.INTERNAL_LINK,
-    WikipediaTokenizer.EXTERNAL_LINK,
-    WikipediaTokenizer.CITATION,
-    WikipediaTokenizer.CATEGORY,
-    WikipediaTokenizer.BOLD,
-    WikipediaTokenizer.ITALICS,
-    WikipediaTokenizer.BOLD_ITALICS,
-    WikipediaTokenizer.HEADING,
-    WikipediaTokenizer.SUB_HEADING,
-    WikipediaTokenizer.EXTERNAL_LINK_URL
-};
+public static final String [] TOKEN_TYPES = WikipediaTokenizer.TOKEN_TYPES;
 
 /**
 Returns the number of tokens seen inside a category or link, etc.
@@ -346,4 +327,4 @@
 //ITALICS = ''({ALPHANUM}+{WHITESPACE}*)+''
 //BOLD_ITALICS = '''''({ALPHANUM}+{WHITESPACE}*)+'''''
 //HEADING = "="{2}({ALPHANUM}+{WHITESPACE}*)+"="{2}
-//SUB_HEADING ="="{3}({ALPHANUM}+{WHITESPACE}*)+"="{3}
\ No newline at end of file
+//SUB_HEADING ="="{3}({ALPHANUM}+{WHITESPACE}*)+"="{3}

Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=648183&r1=648182&r2=648183&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
(original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
Tue Apr 15 01:48:41 2008
@@ -59,7 +59,23 @@
    *             release.
    */
   public static final int ACRONYM_DEP       = 8;
-    
+
+  /** String token types that correspond to token type int constants */
+  public static final String [] TOKEN_TYPES = new String [] {
+    "<ALPHANUM>",
+    "<APOSTROPHE>",
+    "<ACRONYM>",
+    "<COMPANY>",
+    "<EMAIL>",
+    "<HOST>",
+    "<NUM>",
+    "<CJ>",
+    "<ACRONYM_DEP>"
+  };
+
+  /** @deprecated Please use {@link #TOKEN_TYPES} instead */
+  public static final String [] tokenImage = TOKEN_TYPES;
+
   /**
    * Specifies whether deprecated acronyms should be replaced with HOST type.
    * This is false by default to support backward compatibility.

Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java?rev=648183&r1=648182&r2=648183&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
(original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
Tue Apr 15 01:48:41 2008
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.4.1 on 1/25/08 6:54 PM */
+/* The following code was generated by JFlex 1.4.1 on 4/15/08 4:31 AM */
 
 package org.apache.lucene.analysis.standard;
 
@@ -25,8 +25,8 @@
 /**
  * This class is a scanner generated by 
  * <a href="http://www.jflex.de/">JFlex</a> 1.4.1
- * on 1/25/08 6:54 PM from the specification file
- * <tt>/tango/mike/src/lucene.clean/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex</tt>
+ * on 4/15/08 4:31 AM from the specification file
+ * <tt>/mnt2/mike/src/lucene.clean/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex</tt>
  */
 class StandardTokenizerImpl {
 
@@ -298,17 +298,7 @@
  */
 public static final int ACRONYM_DEP       = StandardTokenizer.ACRONYM_DEP;
 
-public static final String [] TOKEN_TYPES = new String [] {
-    "<ALPHANUM>",
-    "<APOSTROPHE>",
-    "<ACRONYM>",
-    "<COMPANY>",
-    "<EMAIL>",
-    "<HOST>",
-    "<NUM>",
-    "<CJ>",
-    "<ACRONYM_DEP>"
-};
+public static final String [] TOKEN_TYPES = StandardTokenizer.TOKEN_TYPES;
 
 public final int yychar()
 {

Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex?rev=648183&r1=648182&r2=648183&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
(original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
Tue Apr 15 01:48:41 2008
@@ -45,17 +45,7 @@
  */
 public static final int ACRONYM_DEP       = StandardTokenizer.ACRONYM_DEP;
 
-public static final String [] TOKEN_TYPES = new String [] {
-    "<ALPHANUM>",
-    "<APOSTROPHE>",
-    "<ACRONYM>",
-    "<COMPANY>",
-    "<EMAIL>",
-    "<HOST>",
-    "<NUM>",
-    "<CJ>",
-    "<ACRONYM_DEP>"
-};
+public static final String [] TOKEN_TYPES = StandardTokenizer.TOKEN_TYPES;
 
 public final int yychar()
 {

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java?rev=648183&r1=648182&r2=648183&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestAnalyzers.java Tue Apr 15 01:48:41
2008
@@ -119,7 +119,7 @@
     verifyPayload(ts);
   }
 
-  // Just a compile time test, to ensure the
+  // LUCENE-1150: Just a compile time test, to ensure the
   // StandardAnalyzer constants remain publicly accessible
   public void _testStandardConstants() {
     int x = StandardTokenizer.ALPHANUM;
@@ -130,6 +130,7 @@
     x = StandardTokenizer.HOST;
     x = StandardTokenizer.NUM;
     x = StandardTokenizer.CJ;
+    String[] y = StandardTokenizer.TOKEN_TYPES;
   }
 }
 



Mime
View raw message