lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From uschind...@apache.org
Subject svn commit: r932698 [1/4] - in /lucene/dev/trunk/lucene: ./ contrib/analyzers/common/ contrib/analyzers/common/src/java/org/apache/lucene/analysis/wikipedia/ contrib/queryparser/ contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parse...
Date Sat, 10 Apr 2010 11:42:54 GMT
Author: uschindler
Date: Sat Apr 10 11:42:53 2010
New Revision: 932698

URL: http://svn.apache.org/viewvc?rev=932698&view=rev
Log:
Fix javacc and jflex builds, speed up contrib build by removing un related checks. Also updated wikipedia tokenizer to jeflex 1.5 and hardcoded unicode 3.0 (which is java 1.4 for backwards compatibility). Also rebuilt all javacc and jflex parsers.

Modified:
    lucene/dev/trunk/lucene/build.xml
    lucene/dev/trunk/lucene/common-build.xml
    lucene/dev/trunk/lucene/contrib/analyzers/common/build.xml
    lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java
    lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex
    lucene/dev/trunk/lucene/contrib/queryparser/build.xml
    lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/JavaCharStream.java   (contents, props changed)
    lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/ParseException.java   (contents, props changed)
    lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/StandardSyntaxParser.java   (contents, props changed)
    lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/StandardSyntaxParserTokenManager.java   (contents, props changed)
    lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/Token.java   (contents, props changed)
    lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/TokenMgrError.java   (contents, props changed)
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.jj
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java

Modified: lucene/dev/trunk/lucene/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/build.xml?rev=932698&r1=932697&r2=932698&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/build.xml (original)
+++ lucene/dev/trunk/lucene/build.xml Sat Apr 10 11:42:53 2010
@@ -232,7 +232,8 @@ The source distribution does not contain
     </sequential>
   </target>
 
-
+  <target name="compile-core" depends="jflex-notice, javacc-notice, common.compile-core"/>
+  
   <!-- ================================================================== -->
   <!-- B U I L D  D E M O                                                 -->
   <!-- ================================================================== -->
@@ -601,26 +602,6 @@ The source distribution does not contain
   <!-- ================================================================== -->
   <!-- Build the JavaCC files into the source tree                        -->
   <!-- ================================================================== -->
-  <target name="jjdoc">
-    <mkdir dir="${build.dir}/docs/grammars"/>
-    <jjdoc target="src/java/org/apache/lucene/queryParser/QueryParser.jj"
-           outputfile="${build.dir}/docs/grammars/QueryParser.html"
-           javacchome="${javacc.home}"
-    />
-    <jjdoc target="src/demo/org/apache/lucene/demo/html/HTMLParser.jj"
-           outputfile="${build.dir}/docs/grammars/HTMLParser.html"
-           javacchome="${javacc.home}"
-    />
-    <jjdoc target="contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/QueryParser.jj"
-           outputfile="${build.dir}/docs/grammars/Surround_QueryParser.html"
-           javacchome="${javacc.home}"
-    />
-    <jjdoc target="contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/StandardSyntaxParser.jj"
-           outputfile="${build.dir}/docs/grammars/StandardSyntaxParser.html"
-           javacchome="${javacc.home}"
-    />
-  </target>
-
   <target name="clean-javacc">
     <delete>
       <fileset dir="src/java/org/apache/lucene/analysis/standard" includes="*.java">
@@ -642,7 +623,7 @@ The source distribution does not contain
     </delete>
   </target>
 
-  <target name="javacc" depends="clean-javacc,javacc-QueryParser,javacc-HTMLParser,javacc-contrib-queryparser, javacc-contrib-surround, javacc-contrib-precedence"/>
+  <target name="javacc" depends="init,javacc-check,clean-javacc,javacc-QueryParser,javacc-HTMLParser,javacc-contrib-queryparser,javacc-contrib-surround"/>
 
   <target name="javacc-QueryParser" depends="init,javacc-check" if="javacc.present">
     <sequential>
@@ -682,18 +663,11 @@ The source distribution does not contain
     />
   </target>
   
-  <target name="javacc-contrib-precedence" depends="init,javacc-check" if="javacc.present">
-    <ant target="javacc"
-      dir="contrib/misc"
-      antfile="build.xml" 
-    />
-  </target>
-  
   <!-- ================================================================== -->
   <!-- Build the JFlex files into the source tree                         -->
   <!-- ================================================================== -->
 
-  <target name="jflex" depends="clean-jflex,jflex-StandardAnalyzer" />
+  <target name="jflex" depends="jflex-check, clean-jflex,jflex-StandardAnalyzer" />
 
   <target name="jflex-StandardAnalyzer" depends="init,jflex-check" if="jflex.present">
     <taskdef classname="jflex.anttask.JFlexTask" name="jflex">

Modified: lucene/dev/trunk/lucene/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/common-build.xml?rev=932698&r1=932697&r2=932698&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/common-build.xml (original)
+++ lucene/dev/trunk/lucene/common-build.xml Sat Apr 10 11:42:53 2010
@@ -153,16 +153,6 @@
    </condition>
 
   <available
-    property="javacc.present"
-    classname="org.javacc.parser.Main"
-    classpath="${javacc.home}/bin/lib/javacc.jar"
-    />
-
-  <available property="jflex.present" classname="jflex.anttask.JFlexTask">
-		<classpath refid="jflex.classpath"/>
-	</available>
-
-  <available
     property="maven.ant.tasks.present"
     classname="org.apache.maven.artifact.ant.Pom"
   />
@@ -179,29 +169,30 @@
   -->
   <target name="javacc-uptodate-check">
     <uptodate property="javacc.files.uptodate">
-      <srcfiles dir="src" includes="**/*.jj" />
+      <srcfiles dir="src/java" includes="**/*.jj" />
       <mapper type="glob" from="*.jj" to="*.java"/>
     </uptodate>
   </target>
 
-  <target name="javacc-notice" unless="javacc.files.uptodate">
+  <target name="javacc-notice" depends="javacc-uptodate-check" unless="javacc.files.uptodate">
     <echo>
       One or more of the JavaCC .jj files is newer than its corresponding
       .java file.  Run the "javacc" target to regenerate the artifacts.
     </echo>
   </target>
 
-  <target name="init" depends="javacc-uptodate-check, javacc-notice, jflex-uptodate-check, jflex-notice">
+  <target name="init">
+    <!-- currently empty -->
   </target>
 
   <target name="jflex-uptodate-check">
     <uptodate property="jflex.files.uptodate">
-      <srcfiles dir="src" includes="**/*.jflex" />
+      <srcfiles dir="src/java" includes="**/*.jflex" />
       <mapper type="glob" from="*.jflex" to="*.java"/>
     </uptodate>
   </target>
  
-  <target name="jflex-notice" unless="jflex.files.uptodate">
+  <target name="jflex-notice" depends="jflex-uptodate-check" unless="jflex.files.uptodate">
     <echo>
       One or more of the JFlex .jflex files is newer than its corresponding
       .java file.  Run the "jflex" target to regenerate the artifacts.
@@ -209,13 +200,18 @@
   </target>
 
   <target name="javacc-check">
+    <available
+      property="javacc.present"
+      classname="org.javacc.parser.Main"
+      classpath="${javacc.home}/bin/lib/javacc.jar"
+      />
     <fail unless="javacc.present">
       ##################################################################
       JavaCC not found.
       JavaCC Home: ${javacc.home}
       JavaCC JAR: ${javacc.jar}
 
-      Please download and install JavaCC from:
+      Please download and install JavaCC 4.1 from:
 
       &lt;http://javacc.dev.java.net&gt;
 
@@ -237,6 +233,9 @@
   </target>
 	
   <target name="jflex-check">
+    <available property="jflex.present" classname="jflex.anttask.JFlexTask">
+      <classpath refid="jflex.classpath"/>
+    </available>
     <fail unless="jflex.present">
       ##################################################################
       JFlex not found.

Modified: lucene/dev/trunk/lucene/contrib/analyzers/common/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/analyzers/common/build.xml?rev=932698&r1=932697&r2=932698&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/analyzers/common/build.xml (original)
+++ lucene/dev/trunk/lucene/contrib/analyzers/common/build.xml Sat Apr 10 11:42:53 2010
@@ -36,11 +36,13 @@
     <pathelement location="${build.dir}/classes/java"/>
   </path>	
 
-  <target name="jflex" depends="clean-jflex,jflex-wiki-tokenizer"/>
+  <target name="compile-core" depends="jflex-notice, common.compile-core"/>
+
+  <target name="jflex" depends="jflex-check,clean-jflex,jflex-wiki-tokenizer"/>
 
   <target name="jflex-wiki-tokenizer" depends="init,jflex-check" if="jflex.present">
-    <taskdef classname="JFlex.anttask.JFlexTask" name="jflex">
-      <classpath location="${jflex.home}/lib/JFlex.jar"/>
+    <taskdef classname="jflex.anttask.JFlexTask" name="jflex">
+      <classpath refid="jflex.classpath"/>
     </taskdef>
     <jflex file="src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex"
            outdir="src/java/org/apache/lucene/analysis/wikipedia"

Modified: lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java?rev=932698&r1=932697&r2=932698&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java (original)
+++ lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java Sat Apr 10 11:42:53 2010
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.4.1 on 4/15/08 4:31 AM */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 10.04.10 12:56 */
 
 package org.apache.lucene.analysis.wikipedia;
 
@@ -24,9 +24,9 @@ import org.apache.lucene.analysis.tokena
 
 /**
  * This class is a scanner generated by 
- * <a href="http://www.jflex.de/">JFlex</a> 1.4.1
- * on 4/15/08 4:31 AM from the specification file
- * <tt>/mnt2/mike/src/lucene.clean/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerImpl.jflex</tt>
+ * <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
+ * on 10.04.10 12:56 from the specification file
+ * <tt>C:/Users/Uwe Schindler/Projects/lucene/trunk-full1/contrib/analyzers/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex</tt>
  */
 class WikipediaTokenizerImpl {
 
@@ -37,16 +37,27 @@ class WikipediaTokenizerImpl {
   private static final int ZZ_BUFFERSIZE = 16384;
 
   /** lexical states */
-  public static final int DOUBLE_BRACE_STATE = 8;
-  public static final int INTERNAL_LINK_STATE = 2;
-  public static final int TWO_SINGLE_QUOTES_STATE = 4;
-  public static final int CATEGORY_STATE = 1;
-  public static final int FIVE_SINGLE_QUOTES_STATE = 6;
-  public static final int STRING = 9;
+  public static final int CATEGORY_STATE = 2;
+  public static final int DOUBLE_EQUALS_STATE = 14;
+  public static final int EXTERNAL_LINK_STATE = 6;
+  public static final int INTERNAL_LINK_STATE = 4;
+  public static final int DOUBLE_BRACE_STATE = 16;
+  public static final int FIVE_SINGLE_QUOTES_STATE = 12;
+  public static final int STRING = 18;
+  public static final int TWO_SINGLE_QUOTES_STATE = 8;
   public static final int YYINITIAL = 0;
-  public static final int DOUBLE_EQUALS_STATE = 7;
-  public static final int THREE_SINGLE_QUOTES_STATE = 5;
-  public static final int EXTERNAL_LINK_STATE = 3;
+  public static final int THREE_SINGLE_QUOTES_STATE = 10;
+
+  /**
+   * ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l
+   * ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l
+   *                  at the beginning of a line
+   * l is of the form l = 2*k, k a non negative integer
+   */
+  private static final int ZZ_LEXSTATE[] = { 
+     0,  0,  1,  1,  2,  2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7, 
+     8,  8,  9, 9
+  };
 
   /** 
    * Translates characters to character classes
@@ -390,9 +401,6 @@ class WikipediaTokenizerImpl {
   /** the textposition at the last accepting state */
   private int zzMarkedPos;
 
-  /** the textposition at the last state to be included in yytext */
-  private int zzPushbackPos;
-
   /** the current text position in the buffer */
   private int zzCurrentPos;
 
@@ -423,6 +431,9 @@ class WikipediaTokenizerImpl {
   /** zzAtEOF == true <=> the scanner is at the EOF */
   private boolean zzAtEOF;
 
+  /** denotes if the user-EOF-code has already been executed */
+  private boolean zzEOFDone;
+
   /* user code: */
 
 public static final int ALPHANUM          = WikipediaTokenizer.ALPHANUM_ID;
@@ -547,7 +558,6 @@ final int setText(StringBuilder buffer){
       zzEndRead-= zzStartRead;
       zzCurrentPos-= zzStartRead;
       zzMarkedPos-= zzStartRead;
-      zzPushbackPos-= zzStartRead;
       zzStartRead = 0;
     }
 
@@ -563,13 +573,23 @@ final int setText(StringBuilder buffer){
     int numRead = zzReader.read(zzBuffer, zzEndRead,
                                             zzBuffer.length-zzEndRead);
 
-    if (numRead < 0) {
-      return true;
-    }
-    else {
+    if (numRead > 0) {
       zzEndRead+= numRead;
       return false;
     }
+    // unlikely but not impossible: read 0 characters, but not at end of stream    
+    if (numRead == 0) {
+      int c = zzReader.read();
+      if (c == -1) {
+        return true;
+      } else {
+        zzBuffer[zzEndRead++] = (char) c;
+        return false;
+      }     
+    }
+
+	// numRead < 0
+    return true;
   }
 
     
@@ -599,8 +619,9 @@ final int setText(StringBuilder buffer){
     zzReader = reader;
     zzAtBOL  = true;
     zzAtEOF  = false;
+    zzEOFDone = false;
     zzEndRead = zzStartRead = 0;
-    zzCurrentPos = zzMarkedPos = zzPushbackPos = 0;
+    zzCurrentPos = zzMarkedPos = 0;
     yyline = yychar = yycolumn = 0;
     zzLexicalState = YYINITIAL;
   }
@@ -730,7 +751,7 @@ final int setText(StringBuilder buffer){
 
       zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
   
-      zzState = zzLexicalState;
+      zzState = ZZ_LEXSTATE[zzLexicalState];
 
 
       zzForAction: {
@@ -778,184 +799,184 @@ final int setText(StringBuilder buffer){
       zzMarkedPos = zzMarkedPosL;
 
       switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
-        case 8: 
-          { /* ignore */
+        case 25: 
+          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = CITATION; yybegin(DOUBLE_BRACE_STATE);
           }
         case 46: break;
-        case 28: 
-          { currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0; yybegin(INTERNAL_LINK_STATE);
+        case 30: 
+          { numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end italics*/
           }
         case 47: break;
-        case 3: 
-          { positionInc = 1; return CJ;
+        case 41: 
+          { numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end bold italics*/
           }
         case 48: break;
-        case 30: 
-          { numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end italics*/
+        case 14: 
+          { yybegin(STRING); numWikiTokensSeen++; return currentTokType;
           }
         case 49: break;
-        case 10: 
-          { numLinkToks = 0; positionInc = 0; yybegin(YYINITIAL);
+        case 23: 
+          { numWikiTokensSeen = 0; positionInc = 1; yybegin(DOUBLE_EQUALS_STATE);
           }
         case 50: break;
-        case 41: 
-          { numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end bold italics*/
+        case 34: 
+          { positionInc = 1; return NUM;
           }
         case 51: break;
-        case 7: 
-          { yybegin(INTERNAL_LINK_STATE); numWikiTokensSeen++; return currentTokType;
+        case 18: 
+          { /* ignore STRING */
           }
         case 52: break;
-        case 23: 
-          { numWikiTokensSeen = 0; positionInc = 1; yybegin(DOUBLE_EQUALS_STATE);
+        case 12: 
+          { currentTokType = ITALICS; numWikiTokensSeen++;  yybegin(STRING); return currentTokType;/*italics*/
           }
         case 53: break;
-        case 38: 
-          { numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end sub header*/
+        case 37: 
+          { numBalanced = 0;currentTokType = ALPHANUM;yybegin(YYINITIAL);/*end bold*/
           }
         case 54: break;
-        case 17: 
-          { yybegin(DOUBLE_BRACE_STATE); numWikiTokensSeen = 0; return currentTokType;
+        case 31: 
+          { numBalanced = 0; numWikiTokensSeen = 0; currentTokType = INTERNAL_LINK;yybegin(INTERNAL_LINK_STATE);
           }
         case 55: break;
-        case 24: 
-          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = INTERNAL_LINK; yybegin(INTERNAL_LINK_STATE);
+        case 10: 
+          { numLinkToks = 0; positionInc = 0; yybegin(YYINITIAL);
           }
         case 56: break;
-        case 14: 
-          { yybegin(STRING); numWikiTokensSeen++; return currentTokType;
+        case 38: 
+          { numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end sub header*/
           }
         case 57: break;
-        case 5: 
-          { positionInc = 1;
+        case 19: 
+          { yybegin(STRING); numWikiTokensSeen++; return currentTokType;/* STRING ALPHANUM*/
           }
         case 58: break;
-        case 43: 
-          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = CATEGORY; yybegin(CATEGORY_STATE);
+        case 11: 
+          { currentTokType = BOLD;  yybegin(THREE_SINGLE_QUOTES_STATE);
           }
         case 59: break;
-        case 26: 
-          { yybegin(YYINITIAL);
+        case 1: 
+          { numWikiTokensSeen = 0;  positionInc = 1;
           }
         case 60: break;
-        case 20: 
-          { numBalanced = 0; numWikiTokensSeen = 0; currentTokType = EXTERNAL_LINK;yybegin(EXTERNAL_LINK_STATE);
+        case 33: 
+          { positionInc = 1; return HOST;
           }
         case 61: break;
-        case 1: 
-          { numWikiTokensSeen = 0;  positionInc = 1;
+        case 3: 
+          { positionInc = 1; return CJ;
           }
         case 62: break;
-        case 40: 
-          { positionInc = 1; return EMAIL;
+        case 17: 
+          { yybegin(DOUBLE_BRACE_STATE); numWikiTokensSeen = 0; return currentTokType;
           }
         case 63: break;
-        case 25: 
-          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = CITATION; yybegin(DOUBLE_BRACE_STATE);
+        case 32: 
+          { positionInc = 1; return APOSTROPHE;
           }
         case 64: break;
-        case 39: 
-          { positionInc = 1; return ACRONYM;
+        case 8: 
+          { /* ignore */
           }
         case 65: break;
-        case 9: 
-          { if (numLinkToks == 0){positionInc = 0;} else{positionInc = 1;} numWikiTokensSeen++; currentTokType = EXTERNAL_LINK; yybegin(EXTERNAL_LINK_STATE); numLinkToks++; return currentTokType;
+        case 4: 
+          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = EXTERNAL_LINK_URL; yybegin(EXTERNAL_LINK_STATE);
           }
         case 66: break;
-        case 22: 
-          { numWikiTokensSeen = 0; positionInc = 1; if (numBalanced == 0){numBalanced++;yybegin(TWO_SINGLE_QUOTES_STATE);} else{numBalanced = 0;}
+        case 2: 
+          { positionInc = 1; return ALPHANUM;
           }
         case 67: break;
-        case 31: 
-          { numBalanced = 0; numWikiTokensSeen = 0; currentTokType = INTERNAL_LINK;yybegin(INTERNAL_LINK_STATE);
+        case 26: 
+          { yybegin(YYINITIAL);
           }
         case 68: break;
-        case 15: 
-          { currentTokType = SUB_HEADING; numWikiTokensSeen = 0; yybegin(STRING);
+        case 43: 
+          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = CATEGORY; yybegin(CATEGORY_STATE);
           }
         case 69: break;
-        case 18: 
-          { /* ignore STRING */
+        case 36: 
+          { currentTokType = BOLD_ITALICS;  yybegin(FIVE_SINGLE_QUOTES_STATE);
           }
         case 70: break;
-        case 42: 
-          { positionInc = 1; numWikiTokensSeen++; yybegin(EXTERNAL_LINK_STATE); return currentTokType;
+        case 13: 
+          { currentTokType = EXTERNAL_LINK; numWikiTokensSeen = 0; yybegin(EXTERNAL_LINK_STATE);
           }
         case 71: break;
-        case 21: 
-          { yybegin(STRING); return currentTokType;/*pipe*/
+        case 24: 
+          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = INTERNAL_LINK; yybegin(INTERNAL_LINK_STATE);
           }
         case 72: break;
-        case 37: 
-          { numBalanced = 0;currentTokType = ALPHANUM;yybegin(YYINITIAL);/*end bold*/
+        case 27: 
+          { numLinkToks = 0; yybegin(YYINITIAL);
           }
         case 73: break;
-        case 33: 
-          { positionInc = 1; return HOST;
+        case 15: 
+          { currentTokType = SUB_HEADING; numWikiTokensSeen = 0; yybegin(STRING);
           }
         case 74: break;
-        case 45: 
-          { numBalanced = 0; numWikiTokensSeen = 0; currentTokType = CATEGORY;yybegin(CATEGORY_STATE);
+        case 28: 
+          { currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0; yybegin(INTERNAL_LINK_STATE);
           }
         case 75: break;
-        case 36: 
-          { currentTokType = BOLD_ITALICS;  yybegin(FIVE_SINGLE_QUOTES_STATE);
+        case 39: 
+          { positionInc = 1; return ACRONYM;
           }
         case 76: break;
-        case 13: 
-          { currentTokType = EXTERNAL_LINK; numWikiTokensSeen = 0; yybegin(EXTERNAL_LINK_STATE);
+        case 29: 
+          { currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0;  yybegin(INTERNAL_LINK_STATE);
           }
         case 77: break;
-        case 16: 
-          { currentTokType = HEADING; yybegin(DOUBLE_EQUALS_STATE); numWikiTokensSeen++; return currentTokType;
+        case 7: 
+          { yybegin(INTERNAL_LINK_STATE); numWikiTokensSeen++; return currentTokType;
           }
         case 78: break;
-        case 12: 
-          { currentTokType = ITALICS; numWikiTokensSeen++;  yybegin(STRING); return currentTokType;/*italics*/
+        case 16: 
+          { currentTokType = HEADING; yybegin(DOUBLE_EQUALS_STATE); numWikiTokensSeen++; return currentTokType;
           }
         case 79: break;
-        case 6: 
-          { yybegin(CATEGORY_STATE); numWikiTokensSeen++; return currentTokType;
+        case 20: 
+          { numBalanced = 0; numWikiTokensSeen = 0; currentTokType = EXTERNAL_LINK;yybegin(EXTERNAL_LINK_STATE);
           }
         case 80: break;
-        case 32: 
-          { positionInc = 1; return APOSTROPHE;
+        case 35: 
+          { positionInc = 1; return COMPANY;
           }
         case 81: break;
-        case 19: 
-          { yybegin(STRING); numWikiTokensSeen++; return currentTokType;/* STRING ALPHANUM*/
+        case 40: 
+          { positionInc = 1; return EMAIL;
           }
         case 82: break;
-        case 34: 
-          { positionInc = 1; return NUM;
+        case 42: 
+          { positionInc = 1; numWikiTokensSeen++; yybegin(EXTERNAL_LINK_STATE); return currentTokType;
           }
         case 83: break;
-        case 44: 
-          { currentTokType = CATEGORY; numWikiTokensSeen = 0; yybegin(CATEGORY_STATE);
+        case 6: 
+          { yybegin(CATEGORY_STATE); numWikiTokensSeen++; return currentTokType;
           }
         case 84: break;
-        case 2: 
-          { positionInc = 1; return ALPHANUM;
+        case 44: 
+          { currentTokType = CATEGORY; numWikiTokensSeen = 0; yybegin(CATEGORY_STATE);
           }
         case 85: break;
-        case 35: 
-          { positionInc = 1; return COMPANY;
+        case 5: 
+          { positionInc = 1;
           }
         case 86: break;
-        case 11: 
-          { currentTokType = BOLD;  yybegin(THREE_SINGLE_QUOTES_STATE);
+        case 9: 
+          { if (numLinkToks == 0){positionInc = 0;} else{positionInc = 1;} numWikiTokensSeen++; currentTokType = EXTERNAL_LINK; yybegin(EXTERNAL_LINK_STATE); numLinkToks++; return currentTokType;
           }
         case 87: break;
-        case 29: 
-          { currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0;  yybegin(INTERNAL_LINK_STATE);
+        case 45: 
+          { numBalanced = 0; numWikiTokensSeen = 0; currentTokType = CATEGORY;yybegin(CATEGORY_STATE);
           }
         case 88: break;
-        case 4: 
-          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = EXTERNAL_LINK_URL; yybegin(EXTERNAL_LINK_STATE);
+        case 22: 
+          { numWikiTokensSeen = 0; positionInc = 1; if (numBalanced == 0){numBalanced++;yybegin(TWO_SINGLE_QUOTES_STATE);} else{numBalanced = 0;}
           }
         case 89: break;
-        case 27: 
-          { numLinkToks = 0; yybegin(YYINITIAL);
+        case 21: 
+          { yybegin(STRING); return currentTokType;/*pipe*/
           }
         case 90: break;
         default: 

Modified: lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex?rev=932698&r1=932697&r2=932698&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex (original)
+++ lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex Sat Apr 10 11:42:53 2010
@@ -17,12 +17,12 @@ package org.apache.lucene.analysis.wikip
  * limitations under the License.
  */
 
-import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 
 %%
 
 %class WikipediaTokenizerImpl
-%unicode
+%unicode 3.0
 %integer
 %function getNextToken
 %pack
@@ -81,7 +81,7 @@ public final int getPositionIncrement(){
 /**
  * Fills Lucene token with the current token text.
  */
-final void getText(Token t) {
+final void getText(TermAttribute t) {
   t.setTermBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
 }
 

Modified: lucene/dev/trunk/lucene/contrib/queryparser/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/build.xml?rev=932698&r1=932697&r2=932698&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/build.xml (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/build.xml Sat Apr 10 11:42:53 2010
@@ -30,68 +30,72 @@
   -->
   <target name="javacc" depends="javacc-flexible,javacc-precedence"/>
 
-  <target name="javacc-flexible" depends="init,javacc-check" if="javacc.present">
+  <target name="javacc-flexible" depends="javacc-check">
+    <delete>
+      <fileset dir="src/java/org/apache/lucene/queryParser/standard/parser" includes="*.java">
+        <containsregexp expression="Generated.*By.*JavaCC"/>
+      </fileset>
+    </delete>
     <invoke-javacc target="src/java/org/apache/lucene/queryParser/standard/parser/StandardSyntaxParser.jj"
                    outputDir="src/java/org/apache/lucene/queryParser/standard/parser"
     />
-  	    <replaceregexp file="src/java/org/apache/lucene/queryParser/standard/parser/ParseException.java"
-  	                         match="public class ParseException extends Exception"
-  	                         replace="public class ParseException extends QueryNodeParseException"
-  	                         flags="g"
-  	                         byline="false"/>
-  	    <replaceregexp file="src/java/org/apache/lucene/queryParser/standard/parser/ParseException.java"
-  	                         match="package org.apache.lucene.queryParser.standard.parser;"
-  	                         replace="package org.apache.lucene.queryParser.standard.parser;&#10;
-  	&#10;
-  	import org.apache.lucene.messages.Message;&#10;
-  	import org.apache.lucene.messages.MessageImpl;&#10;
-  	import org.apache.lucene.queryParser.core.*;&#10;
-  	import org.apache.lucene.queryParser.core.messages.*;"
-  	                         flags="g"
-  	                         byline="false"/>
-  	    <replaceregexp file="src/java/org/apache/lucene/queryParser/standard/parser/ParseException.java"
-  	                         match="^  public ParseException\(Token currentTokenVal.*$(\s\s[^}].*\n)*  \}"
-  	                         replace="  public ParseException(Token currentTokenVal,&#10;
-  	        int[][] expectedTokenSequencesVal, String[] tokenImageVal) {&#10;
-  	    super(new MessageImpl(QueryParserMessages.INVALID_SYNTAX, initialise(&#10;
-  	      currentTokenVal, expectedTokenSequencesVal, tokenImageVal)));&#10;
-  	    this.currentToken = currentTokenVal;&#10;
-  	    this.expectedTokenSequences = expectedTokenSequencesVal;&#10;
-  	    this.tokenImage = tokenImageVal;&#10;
-  	  }"
-  	                         flags="gm"
-  	                         byline="false"/>
-  	    <replaceregexp file="src/java/org/apache/lucene/queryParser/standard/parser/ParseException.java"
-  	                         match="^  public ParseException\(String message.*$(\s\s[^}].*\n)*  \}"
-  	                         replace="  public ParseException(Message message) {&#10;
-  	    super(message);&#10;                      
-  	  }"
-  	                         flags="gm"
-  	                         byline="false"/>
-  	    <replaceregexp file="src/java/org/apache/lucene/queryParser/standard/parser/ParseException.java"
-  	                         match="^  public ParseException\(\).*$(\s\s[^}].*\n)*  \}"
-  	                         replace="  public ParseException() {&#10;
-  	    super(new MessageImpl(QueryParserMessages.INVALID_SYNTAX, &quot;Error&quot;));&#10;
-  	  }"
-  	                         flags="gm"
-  	                         byline="false"/>
-  	    <replaceregexp file="src/java/org/apache/lucene/queryParser/standard/parser/ParseException.java"
-  	                         match="^  public String getMessage\(\).*$(\s\s\s\s[^}].*\n)*    \}"
-  	                         replace="  private static String initialise(Token currentToken, &#10;
-  	      int[][] expectedTokenSequences, String[] tokenImage) {&#10;
-  	    String eol = System.getProperty(&quot;line.separator&quot;, &quot;\n&quot;);"
-  	                         flags="gm"
-  	                         byline="false"/>
-  	    <replaceregexp file="src/java/org/apache/lucene/queryParser/standard/parser/ParseException.java"
-  	                         match="\s*protected String add_escapes.*"
-  	                         replace="  static private String add_escapes(String str) {"
-  	                         flags="g"
-  	                         byline="false"/>
+        <replaceregexp file="src/java/org/apache/lucene/queryParser/standard/parser/ParseException.java"
+                             match="public class ParseException extends Exception"
+                             replace="public class ParseException extends QueryNodeParseException"
+                             flags="g"
+                             byline="false"/>
+        <replaceregexp file="src/java/org/apache/lucene/queryParser/standard/parser/ParseException.java"
+                             match="package org.apache.lucene.queryParser.standard.parser;"
+                             replace="package org.apache.lucene.queryParser.standard.parser;${line.separator}
+${line.separator}
+import org.apache.lucene.messages.Message;${line.separator}
+import org.apache.lucene.messages.MessageImpl;${line.separator}
+import org.apache.lucene.queryParser.core.*;${line.separator}
+import org.apache.lucene.queryParser.core.messages.*;"
+                             flags="g"
+                             byline="false"/>
+        <replaceregexp file="src/java/org/apache/lucene/queryParser/standard/parser/ParseException.java"
+                             match="^  public ParseException\(Token currentTokenVal.*$(\s\s[^}].*\n)*  \}"
+                             replace="  public ParseException(Token currentTokenVal,${line.separator}
+    int[][] expectedTokenSequencesVal, String[] tokenImageVal) {${line.separator}
+    super(new MessageImpl(QueryParserMessages.INVALID_SYNTAX, initialise(${line.separator}
+    currentTokenVal, expectedTokenSequencesVal, tokenImageVal)));${line.separator}
+    this.currentToken = currentTokenVal;${line.separator}
+    this.expectedTokenSequences = expectedTokenSequencesVal;${line.separator}
+    this.tokenImage = tokenImageVal;${line.separator}
+  }"
+                             flags="gm"
+                             byline="false"/>
+        <replaceregexp file="src/java/org/apache/lucene/queryParser/standard/parser/ParseException.java"
+                             match="^  public ParseException\(String message.*$(\s\s[^}].*\n)*  \}"
+                             replace="  public ParseException(Message message) {${line.separator}
+    super(message);${line.separator}                      
+  }"
+                             flags="gm"
+                             byline="false"/>
+        <replaceregexp file="src/java/org/apache/lucene/queryParser/standard/parser/ParseException.java"
+                             match="^  public ParseException\(\).*$(\s\s[^}].*\n)*  \}"
+                             replace="  public ParseException() {${line.separator}
+    super(new MessageImpl(QueryParserMessages.INVALID_SYNTAX, &quot;Error&quot;));${line.separator}
+  }"
+                             flags="gm"
+                             byline="false"/>
+        <replaceregexp file="src/java/org/apache/lucene/queryParser/standard/parser/ParseException.java"
+                             match="^  public String getMessage\(\).*$(\s\s\s\s[^}].*\n)*    \}"
+                             replace="  private static String initialise(Token currentToken, int[][] expectedTokenSequences, String[] tokenImage) {${line.separator}
+    String eol = System.getProperty(&quot;line.separator&quot;, &quot;\n&quot;);"
+                             flags="gm"
+                             byline="false"/>
+        <replaceregexp file="src/java/org/apache/lucene/queryParser/standard/parser/ParseException.java"
+                             match="\s*protected String add_escapes.*"
+                             replace="  static private String add_escapes(String str) {"
+                             flags="g"
+                             byline="false"/>
   </target>
   
   <property name="javacc.precedence.path" location="src/java/org/apache/lucene/queryParser/precedence"/>
 
-  <target name="javacc-precedence" depends="javacc-check" description="generate precedence query parser from jj (requires javacc 3.2)">
+  <target name="javacc-precedence" depends="javacc-check" description="generate precedence query parser from jj (requires javacc 4.1)">
     <delete>
       <fileset dir="${javacc.precedence.path}" includes="*.java">
         <containsregexp expression="Generated.*By.*JavaCC"/>

Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/JavaCharStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/JavaCharStream.java?rev=932698&r1=932697&r2=932698&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/JavaCharStream.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/JavaCharStream.java Sat Apr 10 11:42:53 2010
@@ -1,617 +1,616 @@
-/* Generated By:JavaCC: Do not edit this line. JavaCharStream.java Version 4.1 */
-/* JavaCCOptions:STATIC=false,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */
-package org.apache.lucene.queryParser.standard.parser;
-
-/**
- * An implementation of interface CharStream, where the stream is assumed to
- * contain only ASCII characters (with java-like unicode escape processing).
- */
-
-public
-class JavaCharStream
-{
-  /** Whether parser is static. */
-  public static final boolean staticFlag = false;
-
-  static final int hexval(char c) throws java.io.IOException {
-    switch(c)
-    {
-       case '0' :
-          return 0;
-       case '1' :
-          return 1;
-       case '2' :
-          return 2;
-       case '3' :
-          return 3;
-       case '4' :
-          return 4;
-       case '5' :
-          return 5;
-       case '6' :
-          return 6;
-       case '7' :
-          return 7;
-       case '8' :
-          return 8;
-       case '9' :
-          return 9;
-
-       case 'a' :
-       case 'A' :
-          return 10;
-       case 'b' :
-       case 'B' :
-          return 11;
-       case 'c' :
-       case 'C' :
-          return 12;
-       case 'd' :
-       case 'D' :
-          return 13;
-       case 'e' :
-       case 'E' :
-          return 14;
-       case 'f' :
-       case 'F' :
-          return 15;
-    }
-
-    throw new java.io.IOException(); // Should never come here
-  }
-
-/** Position in buffer. */
-  public int bufpos = -1;
-  int bufsize;
-  int available;
-  int tokenBegin;
-  protected int bufline[];
-  protected int bufcolumn[];
-
-  protected int column = 0;
-  protected int line = 1;
-
-  protected boolean prevCharIsCR = false;
-  protected boolean prevCharIsLF = false;
-
-  protected java.io.Reader inputStream;
-
-  protected char[] nextCharBuf;
-  protected char[] buffer;
-  protected int maxNextCharInd = 0;
-  protected int nextCharInd = -1;
-  protected int inBuf = 0;
-  protected int tabSize = 8;
-
-  protected void setTabSize(int i) { tabSize = i; }
-  protected int getTabSize(int i) { return tabSize; }
-
-  protected void ExpandBuff(boolean wrapAround)
-  {
-    char[] newbuffer = new char[bufsize + 2048];
-    int newbufline[] = new int[bufsize + 2048];
-    int newbufcolumn[] = new int[bufsize + 2048];
-
-    try
-    {
-      if (wrapAround)
-      {
-        System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin);
-        System.arraycopy(buffer, 0, newbuffer, bufsize - tokenBegin, bufpos);
-        buffer = newbuffer;
-
-        System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin);
-        System.arraycopy(bufline, 0, newbufline, bufsize - tokenBegin, bufpos);
-        bufline = newbufline;
-
-        System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin);
-        System.arraycopy(bufcolumn, 0, newbufcolumn, bufsize - tokenBegin, bufpos);
-        bufcolumn = newbufcolumn;
-
-        bufpos += (bufsize - tokenBegin);
-    }
-    else
-    {
-        System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin);
-        buffer = newbuffer;
-
-        System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin);
-        bufline = newbufline;
-
-        System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin);
-        bufcolumn = newbufcolumn;
-
-        bufpos -= tokenBegin;
-      }
-    }
-    catch (Throwable t)
-    {
-      throw new Error(t.getMessage());
-    }
-
-    available = (bufsize += 2048);
-    tokenBegin = 0;
-  }
-
-  protected void FillBuff() throws java.io.IOException
-  {
-    int i;
-    if (maxNextCharInd == 4096)
-      maxNextCharInd = nextCharInd = 0;
-
-    try {
-      if ((i = inputStream.read(nextCharBuf, maxNextCharInd,
-                                          4096 - maxNextCharInd)) == -1)
-      {
-        inputStream.close();
-        throw new java.io.IOException();
-      }
-      else
-         maxNextCharInd += i;
-      return;
-    }
-    catch(java.io.IOException e) {
-      if (bufpos != 0)
-      {
-        --bufpos;
-        backup(0);
-      }
-      else
-      {
-        bufline[bufpos] = line;
-        bufcolumn[bufpos] = column;
-      }
-      throw e;
-    }
-  }
-
-  protected char ReadByte() throws java.io.IOException
-  {
-    if (++nextCharInd >= maxNextCharInd)
-      FillBuff();
-
-    return nextCharBuf[nextCharInd];
-  }
-
-/** @return starting character for token. */
-  public char BeginToken() throws java.io.IOException
-  {
-    if (inBuf > 0)
-    {
-      --inBuf;
-
-      if (++bufpos == bufsize)
-        bufpos = 0;
-
-      tokenBegin = bufpos;
-      return buffer[bufpos];
-    }
-
-    tokenBegin = 0;
-    bufpos = -1;
-
-    return readChar();
-  }
-
-  protected void AdjustBuffSize()
-  {
-    if (available == bufsize)
-    {
-      if (tokenBegin > 2048)
-      {
-        bufpos = 0;
-        available = tokenBegin;
-      }
-      else
-        ExpandBuff(false);
-    }
-    else if (available > tokenBegin)
-      available = bufsize;
-    else if ((tokenBegin - available) < 2048)
-      ExpandBuff(true);
-    else
-      available = tokenBegin;
-  }
-
-  protected void UpdateLineColumn(char c)
-  {
-    column++;
-
-    if (prevCharIsLF)
-    {
-      prevCharIsLF = false;
-      line += (column = 1);
-    }
-    else if (prevCharIsCR)
-    {
-      prevCharIsCR = false;
-      if (c == '\n')
-      {
-        prevCharIsLF = true;
-      }
-      else
-        line += (column = 1);
-    }
-
-    switch (c)
-    {
-      case '\r' :
-        prevCharIsCR = true;
-        break;
-      case '\n' :
-        prevCharIsLF = true;
-        break;
-      case '\t' :
-        column--;
-        column += (tabSize - (column % tabSize));
-        break;
-      default :
-        break;
-    }
-
-    bufline[bufpos] = line;
-    bufcolumn[bufpos] = column;
-  }
-
-/** Read a character. */
-  public char readChar() throws java.io.IOException
-  {
-    if (inBuf > 0)
-    {
-      --inBuf;
-
-      if (++bufpos == bufsize)
-        bufpos = 0;
-
-      return buffer[bufpos];
-    }
-
-    char c;
-
-    if (++bufpos == available)
-      AdjustBuffSize();
-
-    if ((buffer[bufpos] = c = ReadByte()) == '\\')
-    {
-      UpdateLineColumn(c);
-
-      int backSlashCnt = 1;
-
-      for (;;) // Read all the backslashes
-      {
-        if (++bufpos == available)
-          AdjustBuffSize();
-
-        try
-        {
-          if ((buffer[bufpos] = c = ReadByte()) != '\\')
-          {
-            UpdateLineColumn(c);
-            // found a non-backslash char.
-            if ((c == 'u') && ((backSlashCnt & 1) == 1))
-            {
-              if (--bufpos < 0)
-                bufpos = bufsize - 1;
-
-              break;
-            }
-
-            backup(backSlashCnt);
-            return '\\';
-          }
-        }
-        catch(java.io.IOException e)
-        {
-	  // We are returning one backslash so we should only backup (count-1)
-          if (backSlashCnt > 1)
-            backup(backSlashCnt-1);
-
-          return '\\';
-        }
-
-        UpdateLineColumn(c);
-        backSlashCnt++;
-      }
-
-      // Here, we have seen an odd number of backslash's followed by a 'u'
-      try
-      {
-        while ((c = ReadByte()) == 'u')
-          ++column;
-
-        buffer[bufpos] = c = (char)(hexval(c) << 12 |
-                                    hexval(ReadByte()) << 8 |
-                                    hexval(ReadByte()) << 4 |
-                                    hexval(ReadByte()));
-
-        column += 4;
-      }
-      catch(java.io.IOException e)
-      {
-        throw new Error("Invalid escape character at line " + line +
-                                         " column " + column + ".");
-      }
-
-      if (backSlashCnt == 1)
-        return c;
-      else
-      {
-        backup(backSlashCnt - 1);
-        return '\\';
-      }
-    }
-    else
-    {
-      UpdateLineColumn(c);
-      return c;
-    }
-  }
-
-  @Deprecated
-  /**
-   * @deprecated
-   * @see #getEndColumn
-   */
-  public int getColumn() {
-    return bufcolumn[bufpos];
-  }
-
-  @Deprecated
-  /**
-   * @deprecated
-   * @see #getEndLine
-   */
-  public int getLine() {
-    return bufline[bufpos];
-  }
-
-/** Get end column. */
-  public int getEndColumn() {
-    return bufcolumn[bufpos];
-  }
-
-/** Get end line. */
-  public int getEndLine() {
-    return bufline[bufpos];
-  }
-
-/** @return column of token start */
-  public int getBeginColumn() {
-    return bufcolumn[tokenBegin];
-  }
-
-/** @return line number of token start */
-  public int getBeginLine() {
-    return bufline[tokenBegin];
-  }
-
-/** Retreat. */
-  public void backup(int amount) {
-
-    inBuf += amount;
-    if ((bufpos -= amount) < 0)
-      bufpos += bufsize;
-  }
-
-/** Constructor. */
-  public JavaCharStream(java.io.Reader dstream,
-                 int startline, int startcolumn, int buffersize)
-  {
-    inputStream = dstream;
-    line = startline;
-    column = startcolumn - 1;
-
-    available = bufsize = buffersize;
-    buffer = new char[buffersize];
-    bufline = new int[buffersize];
-    bufcolumn = new int[buffersize];
-    nextCharBuf = new char[4096];
-  }
-
-/** Constructor. */
-  public JavaCharStream(java.io.Reader dstream,
-                                        int startline, int startcolumn)
-  {
-    this(dstream, startline, startcolumn, 4096);
-  }
-
-/** Constructor. */
-  public JavaCharStream(java.io.Reader dstream)
-  {
-    this(dstream, 1, 1, 4096);
-  }
-/** Reinitialise. */
-  public void ReInit(java.io.Reader dstream,
-                 int startline, int startcolumn, int buffersize)
-  {
-    inputStream = dstream;
-    line = startline;
-    column = startcolumn - 1;
-
-    if (buffer == null || buffersize != buffer.length)
-    {
-      available = bufsize = buffersize;
-      buffer = new char[buffersize];
-      bufline = new int[buffersize];
-      bufcolumn = new int[buffersize];
-      nextCharBuf = new char[4096];
-    }
-    prevCharIsLF = prevCharIsCR = false;
-    tokenBegin = inBuf = maxNextCharInd = 0;
-    nextCharInd = bufpos = -1;
-  }
-
-/** Reinitialise. */
-  public void ReInit(java.io.Reader dstream,
-                                        int startline, int startcolumn)
-  {
-    ReInit(dstream, startline, startcolumn, 4096);
-  }
-
-/** Reinitialise. */
-  public void ReInit(java.io.Reader dstream)
-  {
-    ReInit(dstream, 1, 1, 4096);
-  }
-/** Constructor. */
-  public JavaCharStream(java.io.InputStream dstream, String encoding, int startline,
-  int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException
-  {
-    this(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize);
-  }
-
-/** Constructor. */
-  public JavaCharStream(java.io.InputStream dstream, int startline,
-  int startcolumn, int buffersize)
-  {
-    this(new java.io.InputStreamReader(dstream), startline, startcolumn, 4096);
-  }
-
-/** Constructor. */
-  public JavaCharStream(java.io.InputStream dstream, String encoding, int startline,
-                        int startcolumn) throws java.io.UnsupportedEncodingException
-  {
-    this(dstream, encoding, startline, startcolumn, 4096);
-  }
-
-/** Constructor. */
-  public JavaCharStream(java.io.InputStream dstream, int startline,
-                        int startcolumn)
-  {
-    this(dstream, startline, startcolumn, 4096);
-  }
-
-/** Constructor. */
-  public JavaCharStream(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException
-  {
-    this(dstream, encoding, 1, 1, 4096);
-  }
-
-/** Constructor. */
-  public JavaCharStream(java.io.InputStream dstream)
-  {
-    this(dstream, 1, 1, 4096);
-  }
-
-/** Reinitialise. */
-  public void ReInit(java.io.InputStream dstream, String encoding, int startline,
-  int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException
-  {
-    ReInit(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize);
-  }
-
-/** Reinitialise. */
-  public void ReInit(java.io.InputStream dstream, int startline,
-  int startcolumn, int buffersize)
-  {
-    ReInit(new java.io.InputStreamReader(dstream), startline, startcolumn, buffersize);
-  }
-/** Reinitialise. */
-  public void ReInit(java.io.InputStream dstream, String encoding, int startline,
-                     int startcolumn) throws java.io.UnsupportedEncodingException
-  {
-    ReInit(dstream, encoding, startline, startcolumn, 4096);
-  }
-/** Reinitialise. */
-  public void ReInit(java.io.InputStream dstream, int startline,
-                     int startcolumn)
-  {
-    ReInit(dstream, startline, startcolumn, 4096);
-  }
-/** Reinitialise. */
-  public void ReInit(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException
-  {
-    ReInit(dstream, encoding, 1, 1, 4096);
-  }
-
-/** Reinitialise. */
-  public void ReInit(java.io.InputStream dstream)
-  {
-    ReInit(dstream, 1, 1, 4096);
-  }
-
-  /** @return token image as String */
-  public String GetImage()
-  {
-    if (bufpos >= tokenBegin)
-      return new String(buffer, tokenBegin, bufpos - tokenBegin + 1);
-    else
-      return new String(buffer, tokenBegin, bufsize - tokenBegin) +
-                              new String(buffer, 0, bufpos + 1);
-  }
-
-  /** @return suffix */
-  public char[] GetSuffix(int len)
-  {
-    char[] ret = new char[len];
-
-    if ((bufpos + 1) >= len)
-      System.arraycopy(buffer, bufpos - len + 1, ret, 0, len);
-    else
-    {
-      System.arraycopy(buffer, bufsize - (len - bufpos - 1), ret, 0,
-                                                        len - bufpos - 1);
-      System.arraycopy(buffer, 0, ret, len - bufpos - 1, bufpos + 1);
-    }
-
-    return ret;
-  }
-
-  /** Set buffers back to null when finished. */
-  public void Done()
-  {
-    nextCharBuf = null;
-    buffer = null;
-    bufline = null;
-    bufcolumn = null;
-  }
-
-  /**
-   * Method to adjust line and column numbers for the start of a token.
-   */
-  public void adjustBeginLineColumn(int newLine, int newCol)
-  {
-    int start = tokenBegin;
-    int len;
-
-    if (bufpos >= tokenBegin)
-    {
-      len = bufpos - tokenBegin + inBuf + 1;
-    }
-    else
-    {
-      len = bufsize - tokenBegin + bufpos + 1 + inBuf;
-    }
-
-    int i = 0, j = 0, k = 0;
-    int nextColDiff = 0, columnDiff = 0;
-
-    while (i < len && bufline[j = start % bufsize] == bufline[k = ++start % bufsize])
-    {
-      bufline[j] = newLine;
-      nextColDiff = columnDiff + bufcolumn[k] - bufcolumn[j];
-      bufcolumn[j] = newCol + columnDiff;
-      columnDiff = nextColDiff;
-      i++;
-    }
-
-    if (i < len)
-    {
-      bufline[j] = newLine++;
-      bufcolumn[j] = newCol + columnDiff;
-
-      while (i++ < len)
-      {
-        if (bufline[j = start % bufsize] != bufline[++start % bufsize])
-          bufline[j] = newLine++;
-        else
-          bufline[j] = newLine;
-      }
-    }
-
-    line = bufline[j];
-    column = bufcolumn[j];
-  }
-
-}
-/* JavaCC - OriginalChecksum=d665eff1df49d9f82f07f7dc863fcd22 (do not edit this line) */
+/* Generated By:JavaCC: Do not edit this line. JavaCharStream.java Version 4.1 */
+/* JavaCCOptions:STATIC=false */
+package org.apache.lucene.queryParser.standard.parser;
+
+/**
+ * An implementation of interface CharStream, where the stream is assumed to
+ * contain only ASCII characters (with java-like unicode escape processing).
+ */
+
+public class JavaCharStream
+{
+/** Whether parser is static. */
+  public static final boolean staticFlag = false;
+  static final int hexval(char c) throws java.io.IOException {
+    switch(c)
+    {
+       case '0' :
+          return 0;
+       case '1' :
+          return 1;
+       case '2' :
+          return 2;
+       case '3' :
+          return 3;
+       case '4' :
+          return 4;
+       case '5' :
+          return 5;
+       case '6' :
+          return 6;
+       case '7' :
+          return 7;
+       case '8' :
+          return 8;
+       case '9' :
+          return 9;
+
+       case 'a' :
+       case 'A' :
+          return 10;
+       case 'b' :
+       case 'B' :
+          return 11;
+       case 'c' :
+       case 'C' :
+          return 12;
+       case 'd' :
+       case 'D' :
+          return 13;
+       case 'e' :
+       case 'E' :
+          return 14;
+       case 'f' :
+       case 'F' :
+          return 15;
+    }
+
+    throw new java.io.IOException(); // Should never come here
+  }
+
+/** Position in buffer. */
+  public int bufpos = -1;
+  int bufsize;
+  int available;
+  int tokenBegin;
+  protected int bufline[];
+  protected int bufcolumn[];
+
+  protected int column = 0;
+  protected int line = 1;
+
+  protected boolean prevCharIsCR = false;
+  protected boolean prevCharIsLF = false;
+
+  protected java.io.Reader inputStream;
+
+  protected char[] nextCharBuf;
+  protected char[] buffer;
+  protected int maxNextCharInd = 0;
+  protected int nextCharInd = -1;
+  protected int inBuf = 0;
+  protected int tabSize = 8;
+
+  protected void setTabSize(int i) { tabSize = i; }
+  protected int getTabSize(int i) { return tabSize; }
+
+  protected void ExpandBuff(boolean wrapAround)
+  {
+     char[] newbuffer = new char[bufsize + 2048];
+     int newbufline[] = new int[bufsize + 2048];
+     int newbufcolumn[] = new int[bufsize + 2048];
+
+     try
+     {
+        if (wrapAround)
+        {
+           System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin);
+           System.arraycopy(buffer, 0, newbuffer,
+                                             bufsize - tokenBegin, bufpos);
+           buffer = newbuffer;
+
+           System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin);
+           System.arraycopy(bufline, 0, newbufline, bufsize - tokenBegin, bufpos);
+           bufline = newbufline;
+
+           System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin);
+           System.arraycopy(bufcolumn, 0, newbufcolumn, bufsize - tokenBegin, bufpos);
+           bufcolumn = newbufcolumn;
+
+           bufpos += (bufsize - tokenBegin);
+        }
+        else
+        {
+           System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin);
+           buffer = newbuffer;
+
+           System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin);
+           bufline = newbufline;
+
+           System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin);
+           bufcolumn = newbufcolumn;
+
+           bufpos -= tokenBegin;
+        }
+     }
+     catch (Throwable t)
+     {
+        throw new Error(t.getMessage());
+     }
+
+     available = (bufsize += 2048);
+     tokenBegin = 0;
+  }
+
+  protected void FillBuff() throws java.io.IOException
+  {
+     int i;
+     if (maxNextCharInd == 4096)
+        maxNextCharInd = nextCharInd = 0;
+
+     try {
+        if ((i = inputStream.read(nextCharBuf, maxNextCharInd,
+                                            4096 - maxNextCharInd)) == -1)
+        {
+           inputStream.close();
+           throw new java.io.IOException();
+        }
+        else
+           maxNextCharInd += i;
+        return;
+     }
+     catch(java.io.IOException e) {
+        if (bufpos != 0)
+        {
+           --bufpos;
+           backup(0);
+        }
+        else
+        {
+           bufline[bufpos] = line;
+           bufcolumn[bufpos] = column;
+        }
+        throw e;
+     }
+  }
+
+  protected char ReadByte() throws java.io.IOException
+  {
+     if (++nextCharInd >= maxNextCharInd)
+        FillBuff();
+
+     return nextCharBuf[nextCharInd];
+  }
+
+/** @return starting character for token. */
+  public char BeginToken() throws java.io.IOException
+  {
+     if (inBuf > 0)
+     {
+        --inBuf;
+
+        if (++bufpos == bufsize)
+           bufpos = 0;
+
+        tokenBegin = bufpos;
+        return buffer[bufpos];
+     }
+
+     tokenBegin = 0;
+     bufpos = -1;
+
+     return readChar();
+  }
+
+  protected void AdjustBuffSize()
+  {
+     if (available == bufsize)
+     {
+        if (tokenBegin > 2048)
+        {
+           bufpos = 0;
+           available = tokenBegin;
+        }
+        else
+           ExpandBuff(false);
+     }
+     else if (available > tokenBegin)
+        available = bufsize;
+     else if ((tokenBegin - available) < 2048)
+        ExpandBuff(true);
+     else
+        available = tokenBegin;
+  }
+
+  protected void UpdateLineColumn(char c)
+  {
+     column++;
+
+     if (prevCharIsLF)
+     {
+        prevCharIsLF = false;
+        line += (column = 1);
+     }
+     else if (prevCharIsCR)
+     {
+        prevCharIsCR = false;
+        if (c == '\n')
+        {
+           prevCharIsLF = true;
+        }
+        else
+           line += (column = 1);
+     }
+
+     switch (c)
+     {
+        case '\r' :
+           prevCharIsCR = true;
+           break;
+        case '\n' :
+           prevCharIsLF = true;
+           break;
+        case '\t' :
+           column--;
+           column += (tabSize - (column % tabSize));
+           break;
+        default :
+           break;
+     }
+
+     bufline[bufpos] = line;
+     bufcolumn[bufpos] = column;
+  }
+
+/** Read a character. */
+  public char readChar() throws java.io.IOException
+  {
+     if (inBuf > 0)
+     {
+        --inBuf;
+
+        if (++bufpos == bufsize)
+           bufpos = 0;
+
+        return buffer[bufpos];
+     }
+
+     char c;
+
+     if (++bufpos == available)
+        AdjustBuffSize();
+
+     if ((buffer[bufpos] = c = ReadByte()) == '\\')
+     {
+        UpdateLineColumn(c);
+
+        int backSlashCnt = 1;
+
+        for (;;) // Read all the backslashes
+        {
+           if (++bufpos == available)
+              AdjustBuffSize();
+
+           try
+           {
+              if ((buffer[bufpos] = c = ReadByte()) != '\\')
+              {
+                 UpdateLineColumn(c);
+                 // found a non-backslash char.
+                 if ((c == 'u') && ((backSlashCnt & 1) == 1))
+                 {
+                    if (--bufpos < 0)
+                       bufpos = bufsize - 1;
+
+                    break;
+                 }
+
+                 backup(backSlashCnt);
+                 return '\\';
+              }
+           }
+           catch(java.io.IOException e)
+           {
+              if (backSlashCnt > 1)
+                 backup(backSlashCnt-1);
+
+              return '\\';
+           }
+
+           UpdateLineColumn(c);
+           backSlashCnt++;
+        }
+
+        // Here, we have seen an odd number of backslash's followed by a 'u'
+        try
+        {
+           while ((c = ReadByte()) == 'u')
+              ++column;
+
+           buffer[bufpos] = c = (char)(hexval(c) << 12 |
+                                       hexval(ReadByte()) << 8 |
+                                       hexval(ReadByte()) << 4 |
+                                       hexval(ReadByte()));
+
+           column += 4;
+        }
+        catch(java.io.IOException e)
+        {
+           throw new Error("Invalid escape character at line " + line +
+                                         " column " + column + ".");
+        }
+
+        if (backSlashCnt == 1)
+           return c;
+        else
+        {
+           backup(backSlashCnt - 1);
+           return '\\';
+        }
+     }
+     else
+     {
+        UpdateLineColumn(c);
+        return c;
+     }
+  }
+
+  @Deprecated
+  /**
+   * @deprecated
+   * @see #getEndColumn
+   */
+  public int getColumn() {
+     return bufcolumn[bufpos];
+  }
+
+  @Deprecated
+  /**
+   * @deprecated
+   * @see #getEndLine
+   */
+  public int getLine() {
+     return bufline[bufpos];
+  }
+
+/** Get end column. */
+  public int getEndColumn() {
+     return bufcolumn[bufpos];
+  }
+
+/** Get end line. */
+  public int getEndLine() {
+     return bufline[bufpos];
+  }
+
+/** @return column of token start */
+  public int getBeginColumn() {
+     return bufcolumn[tokenBegin];
+  }
+
+/** @return line number of token start */
+  public int getBeginLine() {
+     return bufline[tokenBegin];
+  }
+
+/** Retreat. */
+  public void backup(int amount) {
+
+    inBuf += amount;
+    if ((bufpos -= amount) < 0)
+       bufpos += bufsize;
+  }
+
+/** Constructor. */
+  public JavaCharStream(java.io.Reader dstream,
+                 int startline, int startcolumn, int buffersize)
+  {
+    inputStream = dstream;
+    line = startline;
+    column = startcolumn - 1;
+
+    available = bufsize = buffersize;
+    buffer = new char[buffersize];
+    bufline = new int[buffersize];
+    bufcolumn = new int[buffersize];
+    nextCharBuf = new char[4096];
+  }
+
+/** Constructor. */
+  public JavaCharStream(java.io.Reader dstream,
+                                        int startline, int startcolumn)
+  {
+     this(dstream, startline, startcolumn, 4096);
+  }
+
+/** Constructor. */
+  public JavaCharStream(java.io.Reader dstream)
+  {
+     this(dstream, 1, 1, 4096);
+  }
+/** Reinitialise. */
+  public void ReInit(java.io.Reader dstream,
+                 int startline, int startcolumn, int buffersize)
+  {
+    inputStream = dstream;
+    line = startline;
+    column = startcolumn - 1;
+
+    if (buffer == null || buffersize != buffer.length)
+    {
+      available = bufsize = buffersize;
+      buffer = new char[buffersize];
+      bufline = new int[buffersize];
+      bufcolumn = new int[buffersize];
+      nextCharBuf = new char[4096];
+    }
+    prevCharIsLF = prevCharIsCR = false;
+    tokenBegin = inBuf = maxNextCharInd = 0;
+    nextCharInd = bufpos = -1;
+  }
+
+/** Reinitialise. */
+  public void ReInit(java.io.Reader dstream,
+                                        int startline, int startcolumn)
+  {
+     ReInit(dstream, startline, startcolumn, 4096);
+  }
+
+/** Reinitialise. */
+  public void ReInit(java.io.Reader dstream)
+  {
+     ReInit(dstream, 1, 1, 4096);
+  }
+/** Constructor. */
+  public JavaCharStream(java.io.InputStream dstream, String encoding, int startline,
+  int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException
+  {
+     this(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize);
+  }
+
+/** Constructor. */
+  public JavaCharStream(java.io.InputStream dstream, int startline,
+  int startcolumn, int buffersize)
+  {
+     this(new java.io.InputStreamReader(dstream), startline, startcolumn, 4096);
+  }
+
+/** Constructor. */
+  public JavaCharStream(java.io.InputStream dstream, String encoding, int startline,
+                        int startcolumn) throws java.io.UnsupportedEncodingException
+  {
+     this(dstream, encoding, startline, startcolumn, 4096);
+  }
+
+/** Constructor. */
+  public JavaCharStream(java.io.InputStream dstream, int startline,
+                        int startcolumn)
+  {
+     this(dstream, startline, startcolumn, 4096);
+  }
+
+/** Constructor. */
+  public JavaCharStream(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException
+  {
+     this(dstream, encoding, 1, 1, 4096);
+  }
+
+/** Constructor. */
+  public JavaCharStream(java.io.InputStream dstream)
+  {
+     this(dstream, 1, 1, 4096);
+  }
+
+/** Reinitialise. */
+  public void ReInit(java.io.InputStream dstream, String encoding, int startline,
+  int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException
+  {
+     ReInit(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize);
+  }
+
+/** Reinitialise. */
+  public void ReInit(java.io.InputStream dstream, int startline,
+  int startcolumn, int buffersize)
+  {
+     ReInit(new java.io.InputStreamReader(dstream), startline, startcolumn, buffersize);
+  }
+/** Reinitialise. */
+  public void ReInit(java.io.InputStream dstream, String encoding, int startline,
+                     int startcolumn) throws java.io.UnsupportedEncodingException
+  {
+     ReInit(dstream, encoding, startline, startcolumn, 4096);
+  }
+/** Reinitialise. */
+  public void ReInit(java.io.InputStream dstream, int startline,
+                     int startcolumn)
+  {
+     ReInit(dstream, startline, startcolumn, 4096);
+  }
+/** Reinitialise. */
+  public void ReInit(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException
+  {
+     ReInit(dstream, encoding, 1, 1, 4096);
+  }
+
+/** Reinitialise. */
+  public void ReInit(java.io.InputStream dstream)
+  {
+     ReInit(dstream, 1, 1, 4096);
+  }
+
+  /** @return token image as String */
+  public String GetImage()
+  {
+     if (bufpos >= tokenBegin)
+        return new String(buffer, tokenBegin, bufpos - tokenBegin + 1);
+     else
+        return new String(buffer, tokenBegin, bufsize - tokenBegin) +
+                              new String(buffer, 0, bufpos + 1);
+  }
+
+  /** @return suffix */
+  public char[] GetSuffix(int len)
+  {
+     char[] ret = new char[len];
+
+     if ((bufpos + 1) >= len)
+        System.arraycopy(buffer, bufpos - len + 1, ret, 0, len);
+     else
+     {
+        System.arraycopy(buffer, bufsize - (len - bufpos - 1), ret, 0,
+                                                          len - bufpos - 1);
+        System.arraycopy(buffer, 0, ret, len - bufpos - 1, bufpos + 1);
+     }
+
+     return ret;
+  }
+
+  /** Set buffers back to null when finished. */
+  public void Done()
+  {
+     nextCharBuf = null;
+     buffer = null;
+     bufline = null;
+     bufcolumn = null;
+  }
+
+  /**
+   * Method to adjust line and column numbers for the start of a token.
+   */
+  public void adjustBeginLineColumn(int newLine, int newCol)
+  {
+     int start = tokenBegin;
+     int len;
+
+     if (bufpos >= tokenBegin)
+     {
+        len = bufpos - tokenBegin + inBuf + 1;
+     }
+     else
+     {
+        len = bufsize - tokenBegin + bufpos + 1 + inBuf;
+     }
+
+     int i = 0, j = 0, k = 0;
+     int nextColDiff = 0, columnDiff = 0;
+
+     while (i < len &&
+            bufline[j = start % bufsize] == bufline[k = ++start % bufsize])
+     {
+        bufline[j] = newLine;
+        nextColDiff = columnDiff + bufcolumn[k] - bufcolumn[j];
+        bufcolumn[j] = newCol + columnDiff;
+        columnDiff = nextColDiff;
+        i++;
+     }
+
+     if (i < len)
+     {
+        bufline[j] = newLine++;
+        bufcolumn[j] = newCol + columnDiff;
+
+        while (i++ < len)
+        {
+           if (bufline[j = start % bufsize] != bufline[++start % bufsize])
+              bufline[j] = newLine++;
+           else
+              bufline[j] = newLine;
+        }
+     }
+
+     line = bufline[j];
+     column = bufcolumn[j];
+  }
+
+}
+/* JavaCC - OriginalChecksum=f19c73b8f7faf94cc4a581e7b2933cc6 (do not edit this line) */

Propchange: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/JavaCharStream.java
            ('svn:eol-style' removed)

Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/ParseException.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/ParseException.java?rev=932698&r1=932697&r2=932698&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/ParseException.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/ParseException.java Sat Apr 10 11:42:53 2010
@@ -1,197 +1,195 @@
-/* Generated By:JavaCC: Do not edit this line. ParseException.java Version 4.1 */
-/* JavaCCOptions:KEEP_LINE_COL=null */
-package org.apache.lucene.queryParser.standard.parser;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import org.apache.lucene.messages.Message;
-import org.apache.lucene.messages.MessageImpl;
-import org.apache.lucene.queryParser.core.QueryNodeParseException;
-import org.apache.lucene.queryParser.core.messages.QueryParserMessages;
-
-/**
- * This exception is thrown when parse errors are encountered. You can
- * explicitly create objects of this exception type by calling the method
- * generateParseException in the generated parser.
- * 
- * You can modify this class to customize your error reporting mechanisms so
- * long as you retain the public fields.
- */
-public class ParseException extends QueryNodeParseException {
-
-  /**
-   * The version identifier for this Serializable class. Increment only if the
-   * <i>serialized</i> form of the class changes.
-   */
-  private static final long serialVersionUID = 1L;
-
-  /**
-   * This constructor is used by the method "generateParseException" in the
-   * generated parser. Calling this constructor generates a new object of this
-   * type with the fields "currentToken", "expectedTokenSequences", and
-   * "tokenImage" set.
-   */
-  public ParseException(Token currentTokenVal,
-      int[][] expectedTokenSequencesVal, String[] tokenImageVal) {
-    super(new MessageImpl(QueryParserMessages.INVALID_SYNTAX, initialise(
-        currentTokenVal, expectedTokenSequencesVal, tokenImageVal)));
-    this.currentToken = currentTokenVal;
-    this.expectedTokenSequences = expectedTokenSequencesVal;
-    this.tokenImage = tokenImageVal;
-  }
-
-  public ParseException(Message message) {
-    super(message);
-  }
-
-  public ParseException() {
-    super(new MessageImpl(QueryParserMessages.INVALID_SYNTAX, "Error"));
-  }
-
-  /**
-   * This is the last token that has been consumed successfully. If this object
-   * has been created due to a parse error, the token followng this token will
-   * (therefore) be the first error token.
-   */
-  @SuppressWarnings("unused")
-  private Token currentToken;
-
-  /**
-   * Each entry in this array is an array of integers. Each array of integers
-   * represents a sequence of tokens (by their ordinal values) that is expected
-   * at this point of the parse.
-   */
-  @SuppressWarnings("unused")
-  private int[][] expectedTokenSequences;
-
-  /**
-   * This is a reference to the "tokenImage" array of the generated parser
-   * within which the parse error occurred. This array is defined in the
-   * generated ...Constants interface.
-   */
-  @SuppressWarnings("unused")
-  private String[] tokenImage;
-
-  /**
-   * It uses "currentToken" and "expectedTokenSequences" to generate a parse
-   * error message and returns it. If this object has been created due to a
-   * parse error, and you do not catch it (it gets thrown from the parser) the
-   * correct error message gets displayed.
-   */
-  private static String initialise(Token currentToken,
-      int[][] expectedTokenSequences, String[] tokenImage) {
-    String eol = System.getProperty("line.separator", "\n");
-    StringBuffer expected = new StringBuffer();
-    int maxSize = 0;
-    for (int i = 0; i < expectedTokenSequences.length; i++) {
-      if (maxSize < expectedTokenSequences[i].length) {
-        maxSize = expectedTokenSequences[i].length;
-      }
-      for (int j = 0; j < expectedTokenSequences[i].length; j++) {
-        expected.append(tokenImage[expectedTokenSequences[i][j]]).append(' ');
-      }
-      if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) {
-        expected.append("...");
-      }
-      expected.append(eol).append("    ");
-    }
-    String retval = "Encountered \"";
-    Token tok = currentToken.next;
-    for (int i = 0; i < maxSize; i++) {
-      if (i != 0)
-        retval += " ";
-      if (tok.kind == 0) {
-        retval += tokenImage[0];
-        break;
-      }
-      retval += " " + tokenImage[tok.kind];
-      retval += " \"";
-      retval += add_escapes(tok.image);
-      retval += " \"";
-      tok = tok.next;
-    }
-    retval += "\" at line " + currentToken.next.beginLine + ", column "
-        + currentToken.next.beginColumn;
-    retval += "." + eol;
-    if (expectedTokenSequences.length == 1) {
-      retval += "Was expecting:" + eol + "    ";
-    } else {
-      retval += "Was expecting one of:" + eol + "    ";
-    }
-    retval += expected.toString();
-    return retval;
-  }
-
-  /**
-   * The end of line string for this machine.
-   */
-  @SuppressWarnings("unused")
-  private String eol = System.getProperty("line.separator", "\n");
-
-  /**
-   * Used to convert raw characters to their escaped version when these raw
-   * version cannot be used as part of an ASCII string literal.
-   */
-  static private String add_escapes(String str) {
-    StringBuffer retval = new StringBuffer();
-    char ch;
-    for (int i = 0; i < str.length(); i++) {
-      switch (str.charAt(i)) {
-      case 0:
-        continue;
-      case '\b':
-        retval.append("\\b");
-        continue;
-      case '\t':
-        retval.append("\\t");
-        continue;
-      case '\n':
-        retval.append("\\n");
-        continue;
-      case '\f':
-        retval.append("\\f");
-        continue;
-      case '\r':
-        retval.append("\\r");
-        continue;
-      case '\"':
-        retval.append("\\\"");
-        continue;
-      case '\'':
-        retval.append("\\\'");
-        continue;
-      case '\\':
-        retval.append("\\\\");
-        continue;
-      default:
-        if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
-          String s = "0000" + Integer.toString(ch, 16);
-          retval.append("\\u" + s.substring(s.length() - 4, s.length()));
-        } else {
-          retval.append(ch);
-        }
-        continue;
-      }
-    }
-    return retval.toString();
-  }
-
-}
-/*
- * JavaCC - StandardChecksum=c04ac45b94787832e67e6d1b49d8774c (do not edit this
- * line)
- */
+/* Generated By:JavaCC: Do not edit this line. ParseException.java Version 4.1 */
+/* JavaCCOptions:KEEP_LINE_COL=null */
+package org.apache.lucene.queryParser.standard.parser;
+ 
+ import org.apache.lucene.messages.Message;
+ import org.apache.lucene.messages.MessageImpl;
+ import org.apache.lucene.queryParser.core.*;
+ import org.apache.lucene.queryParser.core.messages.*;
+
+/**
+ * This exception is thrown when parse errors are encountered.
+ * You can explicitly create objects of this exception type by
+ * calling the method generateParseException in the generated
+ * parser.
+ *
+ * You can modify this class to customize your error reporting
+ * mechanisms so long as you retain the public fields.
+ */
+public class ParseException extends QueryNodeParseException {
+
+  /**
+   * This constructor is used by the method "generateParseException"
+   * in the generated parser.  Calling this constructor generates
+   * a new object of this type with the fields "currentToken",
+   * "expectedTokenSequences", and "tokenImage" set.  The boolean
+   * flag "specialConstructor" is also set to true to indicate that
+   * this constructor was used to create this object.
+   * This constructor calls its super class with the empty string
+   * to force the "toString" method of parent class "Throwable" to
+   * print the error message in the form:
+   *     ParseException: <result of getMessage>
+   */
+  public ParseException(Token currentTokenVal,
+     int[][] expectedTokenSequencesVal, String[] tokenImageVal) {
+     super(new MessageImpl(QueryParserMessages.INVALID_SYNTAX, initialise(
+     currentTokenVal, expectedTokenSequencesVal, tokenImageVal)));
+     this.currentToken = currentTokenVal;
+     this.expectedTokenSequences = expectedTokenSequencesVal;
+     this.tokenImage = tokenImageVal;
+   }
+
+  /**
+   * The following constructors are for use by you for whatever
+   * purpose you can think of.  Constructing the exception in this
+   * manner makes the exception behave in the normal way - i.e., as
+   * documented in the class "Throwable".  The fields "errorToken",
+   * "expectedTokenSequences", and "tokenImage" do not contain
+   * relevant information.  The JavaCC generated code does not use
+   * these constructors.
+   */
+
+  public ParseException() {
+     super(new MessageImpl(QueryParserMessages.INVALID_SYNTAX, "Error"));
+   }
+
+  /** Constructor with message. */
+  public ParseException(Message message) {
+     super(message);
+                         }
+
+  /**
+   * This variable determines which constructor was used to create
+   * this object and thereby affects the semantics of the
+   * "getMessage" method (see below).
+   */
+  protected boolean specialConstructor;
+
+  /**
+   * This is the last token that has been consumed successfully.  If
+   * this object has been created due to a parse error, the token
+   * followng this token will (therefore) be the first error token.
+   */
+  public Token currentToken;
+
+  /**
+   * Each entry in this array is an array of integers.  Each array
+   * of integers represents a sequence of tokens (by their ordinal
+   * values) that is expected at this point of the parse.
+   */
+  public int[][] expectedTokenSequences;
+
+  /**
+   * This is a reference to the "tokenImage" array of the generated
+   * parser within which the parse error occurred.  This array is
+   * defined in the generated ...Constants interface.
+   */
+  public String[] tokenImage;
+
+  /**
+   * This method has the standard behavior when this object has been
+   * created using the standard constructors.  Otherwise, it uses
+   * "currentToken" and "expectedTokenSequences" to generate a parse
+   * error message and returns it.  If this object has been created
+   * due to a parse error, and you do not catch it (it gets thrown
+   * from the parser), then this method is called during the printing
+   * of the final stack trace, and hence the correct error message
+   * gets displayed.
+   */
+  private static String initialise(Token currentToken, int[][] expectedTokenSequences, String[] tokenImage) {
+     String eol = System.getProperty("line.separator", "n");
+    StringBuffer expected = new StringBuffer();
+    int maxSize = 0;
+    for (int i = 0; i < expectedTokenSequences.length; i++) {
+      if (maxSize < expectedTokenSequences[i].length) {
+        maxSize = expectedTokenSequences[i].length;
+      }
+      for (int j = 0; j < expectedTokenSequences[i].length; j++) {
+        expected.append(tokenImage[expectedTokenSequences[i][j]]).append(' ');
+      }
+      if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) {
+        expected.append("...");
+      }
+      expected.append(eol).append("    ");
+    }
+    String retval = "Encountered \"";
+    Token tok = currentToken.next;
+    for (int i = 0; i < maxSize; i++) {
+      if (i != 0) retval += " ";
+      if (tok.kind == 0) {
+        retval += tokenImage[0];
+        break;
+      }
+      retval += " " + tokenImage[tok.kind];
+      retval += " \"";
+      retval += add_escapes(tok.image);
+      retval += " \"";
+      tok = tok.next;
+    }
+    retval += "\" at line " + currentToken.next.beginLine + ", column " + currentToken.next.beginColumn;
+    retval += "." + eol;
+    if (expectedTokenSequences.length == 1) {
+      retval += "Was expecting:" + eol + "    ";
+    } else {
+      retval += "Was expecting one of:" + eol + "    ";
+    }
+    retval += expected.toString();
+    return retval;
+  }
+
+  /**
+   * The end of line string for this machine.
+   */
+  protected String eol = System.getProperty("line.separator", "\n");
+
+  /**
+   * Used to convert raw characters to their escaped version
+   * when these raw version cannot be used as part of an ASCII
+   * string literal.
+   */  static private String add_escapes(String str) {
+      StringBuffer retval = new StringBuffer();
+      char ch;
+      for (int i = 0; i < str.length(); i++) {
+        switch (str.charAt(i))
+        {
+           case 0 :
+              continue;
+           case '\b':
+              retval.append("\\b");
+              continue;
+           case '\t':
+              retval.append("\\t");
+              continue;
+           case '\n':
+              retval.append("\\n");
+              continue;
+           case '\f':
+              retval.append("\\f");
+              continue;
+           case '\r':
+              retval.append("\\r");
+              continue;
+           case '\"':
+              retval.append("\\\"");
+              continue;
+           case '\'':
+              retval.append("\\\'");
+              continue;
+           case '\\':
+              retval.append("\\\\");
+              continue;
+           default:
+              if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
+                 String s = "0000" + Integer.toString(ch, 16);
+                 retval.append("\\u" + s.substring(s.length() - 4, s.length()));
+              } else {
+                 retval.append(ch);
+              }
+              continue;
+        }
+      }
+      return retval.toString();
+   }
+
+}
+/* JavaCC - OriginalChecksum=38bce846fe6c8482993969f741c0323e (do not edit this line) */

Propchange: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/parser/ParseException.java
            ('svn:eol-style' removed)



Mime
View raw message