lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r646243 - in /lucene/java/branches/lucene_2_3: ./ contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/ src/java/org/apache/lucene/analysis/standard/ src/java/org/apache/lucene/store/ src/test/org/apache/lucene/analysis/
Date Wed, 09 Apr 2008 09:31:41 GMT
Author: mikemccand
Date: Wed Apr  9 02:31:37 2008
New Revision: 646243

URL: http://svn.apache.org/viewvc?rev=646243&view=rev
Log:
LUCENE-1150: re-instate constants in StandardTokenizer

Modified:
    lucene/java/branches/lucene_2_3/CHANGES.txt
    lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java
    lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java
    lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex
    lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
    lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
    lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
    lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/store/FSDirectory.java
    lucene/java/branches/lucene_2_3/src/test/org/apache/lucene/analysis/TestAnalyzers.java

Modified: lucene/java/branches/lucene_2_3/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/CHANGES.txt?rev=646243&r1=646242&r2=646243&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_3/CHANGES.txt (original)
+++ lucene/java/branches/lucene_2_3/CHANGES.txt Wed Apr  9 02:31:37 2008
@@ -31,6 +31,10 @@
  9. LUCENE-1226: Fixed IndexWriter.addIndexes(IndexReader[]) to commit
     successfully created compound files. (Michael Busch)
 
+10. LUCENE-1150: Re-expose StandardTokenizer's constants publicly;
+     this was accidentally lost with LUCENE-966.  (Nicolas Lalevée via
+     Mike McCandless)
+
 Build
 
  1. LUCENE-1230: Include *pom.xml* in source release files. (Michael Busch)

Modified: lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java?rev=646243&r1=646242&r2=646243&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java
(original)
+++ lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java
Wed Apr  9 02:31:37 2008
@@ -43,6 +43,30 @@
   public static final String BOLD_ITALICS = "bi";
   public static final String HEADING = "h";
   public static final String SUB_HEADING = "sh";
+
+  public static final int ALPHANUM_ID          = 0;
+  public static final int APOSTROPHE_ID        = 1;
+  public static final int ACRONYM_ID           = 2;
+  public static final int COMPANY_ID           = 3;
+  public static final int EMAIL_ID             = 4;
+  public static final int HOST_ID              = 5;
+  public static final int NUM_ID               = 6;
+  public static final int CJ_ID                = 7;
+  public static final int INTERNAL_LINK_ID     = 8;
+  public static final int EXTERNAL_LINK_ID     = 9;
+  public static final int CITATION_ID          = 10;
+  public static final int CATEGORY_ID          = 11;
+  public static final int BOLD_ID              = 12;
+  public static final int ITALICS_ID           = 13;
+  public static final int BOLD_ITALICS_ID      = 14;
+  public static final int HEADING_ID           = 15;
+  public static final int SUB_HEADING_ID       = 16;
+  public static final int EXTERNAL_LINK_URL_ID = 17;
+
+  public static final int TOKENS_ONLY = 0;
+  public static final int UNTOKENIZED_ONLY = 1;
+  public static final int BOTH = 2;
+
   /**
    * A private instance of the JFlex-constructed scanner
    */

Modified: lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java?rev=646243&r1=646242&r2=646243&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java
(original)
+++ lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.java
Wed Apr  9 02:31:37 2008
@@ -1,3 +1,4 @@
+
 /* The following code was generated by JFlex 1.4.1 on 1/4/08 3:30 PM */
 
 package org.apache.lucene.wikipedia.analysis;
@@ -422,24 +423,24 @@
 
   /* user code: */
 
-public static final int ALPHANUM          = 0;
-public static final int APOSTROPHE        = 1;
-public static final int ACRONYM           = 2;
-public static final int COMPANY           = 3;
-public static final int EMAIL             = 4;
-public static final int HOST              = 5;
-public static final int NUM               = 6;
-public static final int CJ                = 7;
-public static final int INTERNAL_LINK     = 8;
-public static final int EXTERNAL_LINK     = 9;
-public static final int CITATION          = 10;
-public static final int CATEGORY         = 11;
-public static final int BOLD     = 12;
-public static final int ITALICS     = 13;
-public static final int BOLD_ITALICS     = 14;
-public static final int HEADING     = 15;
-public static final int SUB_HEADING     = 16;
-public static final int EXTERNAL_LINK_URL = 17;
+public static final int ALPHANUM          = WikipediaTokenizer.ALPHANUM_ID;
+public static final int APOSTROPHE        = WikipediaTokenizer.APOSTROPHE_ID;
+public static final int ACRONYM           = WikipediaTokenizer.ACRONYM_ID;
+public static final int COMPANY           = WikipediaTokenizer.COMPANY_ID;
+public static final int EMAIL             = WikipediaTokenizer.EMAIL_ID;
+public static final int HOST              = WikipediaTokenizer.HOST_ID;
+public static final int NUM               = WikipediaTokenizer.NUM_ID;
+public static final int CJ                = WikipediaTokenizer.CJ_ID;
+public static final int INTERNAL_LINK     = WikipediaTokenizer.INTERNAL_LINK_ID;
+public static final int EXTERNAL_LINK     = WikipediaTokenizer.EXTERNAL_LINK_ID;
+public static final int CITATION          = WikipediaTokenizer.CITATION_ID;
+public static final int CATEGORY          = WikipediaTokenizer.CATEGORY_ID;
+public static final int BOLD              = WikipediaTokenizer.BOLD_ID;
+public static final int ITALICS           = WikipediaTokenizer.ITALICS_ID;
+public static final int BOLD_ITALICS      = WikipediaTokenizer.BOLD_ITALICS_ID;
+public static final int HEADING           = WikipediaTokenizer.HEADING_ID;
+public static final int SUB_HEADING       = WikipediaTokenizer.SUB_HEADING_ID;
+public static final int EXTERNAL_LINK_URL = WikipediaTokenizer.EXTERNAL_LINK_URL_ID;
 
 
 private int currentTokType;

Modified: lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex?rev=646243&r1=646242&r2=646243&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex
(original)
+++ lucene/java/branches/lucene_2_3/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex
Wed Apr  9 02:31:37 2008
@@ -30,24 +30,24 @@
 
 %{
 
-public static final int ALPHANUM          = 0;
-public static final int APOSTROPHE        = 1;
-public static final int ACRONYM           = 2;
-public static final int COMPANY           = 3;
-public static final int EMAIL             = 4;
-public static final int HOST              = 5;
-public static final int NUM               = 6;
-public static final int CJ                = 7;
-public static final int INTERNAL_LINK     = 8;
-public static final int EXTERNAL_LINK     = 9;
-public static final int CITATION          = 10;
-public static final int CATEGORY         = 11;
-public static final int BOLD     = 12;
-public static final int ITALICS     = 13;
-public static final int BOLD_ITALICS     = 14;
-public static final int HEADING     = 15;
-public static final int SUB_HEADING     = 16;
-public static final int EXTERNAL_LINK_URL = 17;
+public static final int ALPHANUM          = WikipediaTokenizer.ALPHANUM_ID;
+public static final int APOSTROPHE        = WikipediaTokenizer.APOSTROPHE_ID;
+public static final int ACRONYM           = WikipediaTokenizer.ACRONYM_ID;
+public static final int COMPANY           = WikipediaTokenizer.COMPANY_ID;
+public static final int EMAIL             = WikipediaTokenizer.EMAIL_ID;
+public static final int HOST              = WikipediaTokenizer.HOST_ID;
+public static final int NUM               = WikipediaTokenizer.NUM_ID;
+public static final int CJ                = WikipediaTokenizer.CJ_ID;
+public static final int INTERNAL_LINK     = WikipediaTokenizer.INTERNAL_LINK_ID;
+public static final int EXTERNAL_LINK     = WikipediaTokenizer.EXTERNAL_LINK_ID;
+public static final int CITATION          = WikipediaTokenizer.CITATION_ID;
+public static final int CATEGORY          = WikipediaTokenizer.CATEGORY_ID;
+public static final int BOLD              = WikipediaTokenizer.BOLD_ID;
+public static final int ITALICS           = WikipediaTokenizer.ITALICS_ID;
+public static final int BOLD_ITALICS      = WikipediaTokenizer.BOLD_ITALICS_ID;
+public static final int HEADING           = WikipediaTokenizer.HEADING_ID;
+public static final int SUB_HEADING       = WikipediaTokenizer.SUB_HEADING_ID;
+public static final int EXTERNAL_LINK_URL = WikipediaTokenizer.EXTERNAL_LINK_URL_ID;
 
 
 private int currentTokType;

Modified: lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=646243&r1=646242&r2=646243&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
(original)
+++ lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
Wed Apr  9 02:31:37 2008
@@ -41,8 +41,24 @@
  */
 
 public class StandardTokenizer extends Tokenizer {
-    /** A private instance of the JFlex-constructed scanner */
-    private final StandardTokenizerImpl scanner;
+  /** A private instance of the JFlex-constructed scanner */
+  private final StandardTokenizerImpl scanner;
+
+  public static final int ALPHANUM          = 0;
+  public static final int APOSTROPHE        = 1;
+  public static final int ACRONYM           = 2;
+  public static final int COMPANY           = 3;
+  public static final int EMAIL             = 4;
+  public static final int HOST              = 5;
+  public static final int NUM               = 6;
+  public static final int CJ                = 7;
+
+  /**
+   * @deprecated this solves a bug where HOSTs that end with '.' are identified
+   *             as ACRONYMs. It is deprecated and will be removed in the next
+   *             release.
+   */
+  public static final int ACRONYM_DEP       = 8;
     
   /**
    * Specifies whether deprecated acronyms should be replaced with HOST type.

Modified: lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java?rev=646243&r1=646242&r2=646243&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
(original)
+++ lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
Wed Apr  9 02:31:37 2008
@@ -283,20 +283,20 @@
 
   /* user code: */
 
-public static final int ALPHANUM          = 0;
-public static final int APOSTROPHE        = 1;
-public static final int ACRONYM           = 2;
-public static final int COMPANY           = 3;
-public static final int EMAIL             = 4;
-public static final int HOST              = 5;
-public static final int NUM               = 6;
-public static final int CJ                = 7;
+public static final int ALPHANUM          = StandardTokenizer.ALPHANUM;
+public static final int APOSTROPHE        = StandardTokenizer.APOSTROPHE;
+public static final int ACRONYM           = StandardTokenizer.ACRONYM;
+public static final int COMPANY           = StandardTokenizer.COMPANY;
+public static final int EMAIL             = StandardTokenizer.EMAIL;
+public static final int HOST              = StandardTokenizer.HOST;
+public static final int NUM               = StandardTokenizer.NUM;
+public static final int CJ                = StandardTokenizer.CJ;
 /**
  * @deprecated this solves a bug where HOSTs that end with '.' are identified
  *             as ACRONYMs. It is deprecated and will be removed in the next
  *             release.
  */
-public static final int ACRONYM_DEP       = 8;
+public static final int ACRONYM_DEP       = StandardTokenizer.ACRONYM_DEP;
 
 public static final String [] TOKEN_TYPES = new String [] {
     "<ALPHANUM>",

Modified: lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex?rev=646243&r1=646242&r2=646243&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
(original)
+++ lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
Wed Apr  9 02:31:37 2008
@@ -30,20 +30,20 @@
 
 %{
 
-public static final int ALPHANUM          = 0;
-public static final int APOSTROPHE        = 1;
-public static final int ACRONYM           = 2;
-public static final int COMPANY           = 3;
-public static final int EMAIL             = 4;
-public static final int HOST              = 5;
-public static final int NUM               = 6;
-public static final int CJ                = 7;
+public static final int ALPHANUM          = StandardTokenizer.ALPHANUM;
+public static final int APOSTROPHE        = StandardTokenizer.APOSTROPHE;
+public static final int ACRONYM           = StandardTokenizer.ACRONYM;
+public static final int COMPANY           = StandardTokenizer.COMPANY;
+public static final int EMAIL             = StandardTokenizer.EMAIL;
+public static final int HOST              = StandardTokenizer.HOST;
+public static final int NUM               = StandardTokenizer.NUM;
+public static final int CJ                = StandardTokenizer.CJ;
 /**
  * @deprecated this solves a bug where HOSTs that end with '.' are identified
  *             as ACRONYMs. It is deprecated and will be removed in the next
  *             release.
  */
-public static final int ACRONYM_DEP       = 8;
+public static final int ACRONYM_DEP       = StandardTokenizer.ACRONYM_DEP;
 
 public static final String [] TOKEN_TYPES = new String [] {
     "<ALPHANUM>",

Modified: lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/store/FSDirectory.java?rev=646243&r1=646242&r2=646243&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/java/branches/lucene_2_3/src/java/org/apache/lucene/store/FSDirectory.java Wed
Apr  9 02:31:37 2008
@@ -437,7 +437,7 @@
 
   // Inherit javadoc
   public IndexInput openInput(String name) throws IOException {
-    return new FSIndexInput(new File(directory, name));
+    return openInput(name, BufferedIndexInput.BUFFER_SIZE);
   }
 
   // Inherit javadoc

Modified: lucene/java/branches/lucene_2_3/src/test/org/apache/lucene/analysis/TestAnalyzers.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_3/src/test/org/apache/lucene/analysis/TestAnalyzers.java?rev=646243&r1=646242&r2=646243&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_3/src/test/org/apache/lucene/analysis/TestAnalyzers.java
(original)
+++ lucene/java/branches/lucene_2_3/src/test/org/apache/lucene/analysis/TestAnalyzers.java
Wed Apr  9 02:31:37 2008
@@ -23,6 +23,7 @@
 
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.index.Payload;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
 
 public class TestAnalyzers extends LuceneTestCase {
 
@@ -118,6 +119,18 @@
     verifyPayload(ts);
   }
 
+  // Just a compile time test, to ensure the
+  // StandardAnalyzer constants remain publicly accessible
+  public void _testStandardConstants() {
+    int x = StandardTokenizer.ALPHANUM;
+    x = StandardTokenizer.APOSTROPHE;
+    x = StandardTokenizer.ACRONYM;
+    x = StandardTokenizer.COMPANY;
+    x = StandardTokenizer.EMAIL;
+    x = StandardTokenizer.HOST;
+    x = StandardTokenizer.NUM;
+    x = StandardTokenizer.CJ;
+  }
 }
 
 class BuffTokenFilter extends TokenFilter {



Mime
View raw message