lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rjer...@apache.org
Subject svn commit: r1630292 - in /lucene/dev/trunk/lucene: ./ analysis/common/src/java/org/apache/lucene/analysis/standard/
Date Thu, 09 Oct 2014 05:58:08 GMT
Author: rjernst
Date: Thu Oct  9 05:58:07 2014
New Revision: 1630292

URL: http://svn.apache.org/r1630292
Log:
LUCENE-6000: Removed StandardTokenizerInterface

Removed:
    lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerInterface.java
Modified:
    lucene/dev/trunk/lucene/CHANGES.txt
    lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java
    lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java
    lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex
    lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
    lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
    lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
    lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java
    lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java
    lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex

Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1630292&r1=1630291&r2=1630292&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Thu Oct  9 05:58:07 2014
@@ -156,6 +156,10 @@ API Changes
 * LUCENE-5998: Simplify Field/SegmentInfoFormat to read+write methods.
   (Robert Muir)
 
+* LUCENE-6000: Removed StandardTokenizerInterface.  Tokenizers now use
+  their jflex impl directly.
+  (Ryan Ernst)
+
 Bug Fixes
 
 * LUCENE-5650: Enforce read-only access to any path outside the temporary

Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java?rev=1630292&r1=1630291&r2=1630292&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java
(original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java
Thu Oct  9 05:58:07 2014
@@ -49,7 +49,7 @@ import org.apache.lucene.util.AttributeF
 
 public final class ClassicTokenizer extends Tokenizer {
   /** A private instance of the JFlex-constructed scanner */
-  private StandardTokenizerInterface scanner;
+  private ClassicTokenizerImpl scanner;
 
   public static final int ALPHANUM          = 0;
   public static final int APOSTROPHE        = 1;
@@ -135,7 +135,7 @@ public final class ClassicTokenizer exte
     while(true) {
       int tokenType = scanner.getNextToken();
 
-      if (tokenType == StandardTokenizerInterface.YYEOF) {
+      if (tokenType == ClassicTokenizerImpl.YYEOF) {
         return false;
       }
 

Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java?rev=1630292&r1=1630291&r2=1630292&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java
(original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java
Thu Oct  9 05:58:07 2014
@@ -33,7 +33,7 @@ import org.apache.lucene.analysis.tokena
  * This class implements the classic lucene StandardTokenizer up until 3.0 
  */
 
-class ClassicTokenizerImpl implements StandardTokenizerInterface {
+class ClassicTokenizerImpl {
 
   /** This character denotes the end of file */
   public static final int YYEOF = -1;

Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex?rev=1630292&r1=1630291&r2=1630292&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex
(original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex
Thu Oct  9 05:58:07 2014
@@ -33,7 +33,6 @@ import org.apache.lucene.analysis.tokena
 %%
 
 %class ClassicTokenizerImpl
-%implements StandardTokenizerInterface
 %unicode 3.0
 %integer
 %function getNextToken

Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=1630292&r1=1630291&r2=1630292&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
(original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
Thu Oct  9 05:58:07 2014
@@ -39,7 +39,7 @@ import org.apache.lucene.util.AttributeF
 
 public final class StandardTokenizer extends Tokenizer {
   /** A private instance of the JFlex-constructed scanner */
-  private StandardTokenizerInterface scanner;
+  private StandardTokenizerImpl scanner;
 
   // TODO: how can we remove these old types?!
   public static final int ALPHANUM          = 0;
@@ -150,7 +150,7 @@ public final class StandardTokenizer ext
     while(true) {
       int tokenType = scanner.getNextToken();
 
-      if (tokenType == StandardTokenizerInterface.YYEOF) {
+      if (tokenType == StandardTokenizerImpl.YYEOF) {
         return false;
       }
 

Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java?rev=1630292&r1=1630291&r2=1630292&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
(original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
Thu Oct  9 05:58:07 2014
@@ -39,7 +39,7 @@ import org.apache.lucene.analysis.tokena
  * </ul>
  */
 
-public final class StandardTokenizerImpl implements StandardTokenizerInterface {
+public final class StandardTokenizerImpl {
 
   /** This character denotes the end of file */
   public static final int YYEOF = -1;
@@ -804,7 +804,7 @@ public final class StandardTokenizerImpl
           if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
             zzAtEOF = true;
               {
-                return StandardTokenizerInterface.YYEOF;
+                return YYEOF;
               }
           } 
           else {

Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex?rev=1630292&r1=1630291&r2=1630292&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
(original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
Thu Oct  9 05:58:07 2014
@@ -43,7 +43,6 @@ import org.apache.lucene.analysis.tokena
 %final
 %public
 %class StandardTokenizerImpl
-%implements StandardTokenizerInterface
 %function getNextToken
 %char
 %buffer 255
@@ -118,7 +117,7 @@ ComplexContextEx    = \p{LB:Complex_Cont
 // UAX#29 WB1.   sot   ÷
 //        WB2.     ÷   eot
 //
-<<EOF>> { return StandardTokenizerInterface.YYEOF; }
+<<EOF>> { return YYEOF; }
 
 // UAX#29 WB8.   Numeric × Numeric
 //        WB11.  Numeric (MidNum | MidNumLet | Single_Quote) × Numeric

Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java?rev=1630292&r1=1630291&r2=1630292&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java
(original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java
Thu Oct  9 05:58:07 2014
@@ -47,7 +47,7 @@ import org.apache.lucene.util.AttributeF
 
 public final class UAX29URLEmailTokenizer extends Tokenizer {
   /** A private instance of the JFlex-constructed scanner */
-  private final StandardTokenizerInterface scanner;
+  private final UAX29URLEmailTokenizerImpl scanner;
   
   public static final int ALPHANUM          = 0;
   public static final int NUM               = 1;
@@ -108,7 +108,7 @@ public final class UAX29URLEmailTokenize
     this.scanner = getScanner();
   }
 
-  private StandardTokenizerInterface getScanner() {
+  private UAX29URLEmailTokenizerImpl getScanner() {
     return new UAX29URLEmailTokenizerImpl(input);
   }
 
@@ -127,7 +127,7 @@ public final class UAX29URLEmailTokenize
     while(true) {
       int tokenType = scanner.getNextToken();
 
-      if (tokenType == StandardTokenizerInterface.YYEOF) {
+      if (tokenType == UAX29URLEmailTokenizerImpl.YYEOF) {
         return false;
       }
 

Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java?rev=1630292&r1=1630291&r2=1630292&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java
(original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java
Thu Oct  9 05:58:07 2014
@@ -42,7 +42,7 @@ import org.apache.lucene.analysis.tokena
  * </ul>
  */
 
-public final class UAX29URLEmailTokenizerImpl implements StandardTokenizerInterface {
+public final class UAX29URLEmailTokenizerImpl {
 
   /** This character denotes the end of file */
   public static final int YYEOF = -1;
@@ -7204,11 +7204,11 @@ public final class UAX29URLEmailTokenize
             zzAtEOF = true;
             switch (zzLexicalState) {
             case YYINITIAL: {
-              return StandardTokenizerInterface.YYEOF;
+              return YYEOF;
             }
             case 2910: break;
             case AVOID_BAD_URL: {
-              return StandardTokenizerInterface.YYEOF;
+              return YYEOF;
             }
             case 2911: break;
             default:

Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex?rev=1630292&r1=1630291&r2=1630292&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex
(original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex
Thu Oct  9 05:58:07 2014
@@ -46,7 +46,6 @@ import org.apache.lucene.analysis.tokena
 %final
 %public
 %class UAX29URLEmailTokenizerImpl
-%implements StandardTokenizerInterface
 %function getNextToken
 %char
 %xstate AVOID_BAD_URL
@@ -208,7 +207,7 @@ EMAIL = {EMAILlocalPart} "@" ({DomainNam
 // UAX#29 WB1.   sot   ÷
 //        WB2.     ÷   eot
 //
-  <<EOF>> { return StandardTokenizerInterface.YYEOF; }
+  <<EOF>> { return YYEOF; }
 
   {URL}   { yybegin(YYINITIAL); return URL_TYPE; }
 



Mime
View raw message