Return-Path: Delivered-To: apmail-lucene-java-commits-archive@www.apache.org Received: (qmail 35671 invoked from network); 18 Oct 2009 23:21:49 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 18 Oct 2009 23:21:49 -0000 Received: (qmail 78941 invoked by uid 500); 18 Oct 2009 23:21:48 -0000 Delivered-To: apmail-lucene-java-commits-archive@lucene.apache.org Received: (qmail 78868 invoked by uid 500); 18 Oct 2009 23:21:48 -0000 Mailing-List: contact java-commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: java-dev@lucene.apache.org Delivered-To: mailing list java-commits@lucene.apache.org Received: (qmail 78859 invoked by uid 99); 18 Oct 2009 23:21:48 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 18 Oct 2009 23:21:48 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 18 Oct 2009 23:21:37 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id E350723888FF; Sun, 18 Oct 2009 23:21:14 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r826549 - in /lucene/java/trunk: ./ contrib/misc/src/java/org/apache/lucene/queryParser/precedence/ contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/ src/demo/org/apache/lucene/demo/html/ src/java/org/apache/lucene/qu... Date: Sun, 18 Oct 2009 23:21:14 -0000 To: java-commits@lucene.apache.org From: uschindler@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20091018232114.E350723888FF@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: uschindler Date: Sun Oct 18 23:21:13 2009 New Revision: 826549 URL: http://svn.apache.org/viewvc?rev=826549&view=rev Log: Update JavaCC parser task to 2.9 and regenerate files. Modified: lucene/java/trunk/common-build.xml lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/TokenMgrError.java lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/CharStream.java lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/ParseException.java lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/QueryParser.java lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/QueryParserConstants.java lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/QueryParserTokenManager.java lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/Token.java lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/TokenMgrError.java lucene/java/trunk/src/demo/org/apache/lucene/demo/html/HTMLParser.java lucene/java/trunk/src/demo/org/apache/lucene/demo/html/HTMLParserConstants.java lucene/java/trunk/src/demo/org/apache/lucene/demo/html/HTMLParserTokenManager.java lucene/java/trunk/src/demo/org/apache/lucene/demo/html/ParseException.java lucene/java/trunk/src/demo/org/apache/lucene/demo/html/SimpleCharStream.java lucene/java/trunk/src/demo/org/apache/lucene/demo/html/Token.java lucene/java/trunk/src/demo/org/apache/lucene/demo/html/TokenMgrError.java lucene/java/trunk/src/java/org/apache/lucene/queryParser/CharStream.java lucene/java/trunk/src/java/org/apache/lucene/queryParser/ParseException.java lucene/java/trunk/src/java/org/apache/lucene/queryParser/QueryParser.java lucene/java/trunk/src/java/org/apache/lucene/queryParser/Token.java lucene/java/trunk/src/java/org/apache/lucene/queryParser/TokenMgrError.java Modified: lucene/java/trunk/common-build.xml URL: http://svn.apache.org/viewvc/lucene/java/trunk/common-build.xml?rev=826549&r1=826548&r2=826549&view=diff ============================================================================== --- lucene/java/trunk/common-build.xml (original) +++ lucene/java/trunk/common-build.xml Sun Oct 18 23:21:13 2009 @@ -575,6 +575,7 @@ debugParser="${javacc.debug.parser}" debuglookahead="${javacc.debug.lookahead}" javacchome="${javacc.home}" + jdkversion="${javac.source}" /> Modified: lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java?rev=826549&r1=826548&r2=826549&view=diff ============================================================================== --- lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java (original) +++ lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java Sun Oct 18 23:21:13 2009 @@ -1237,7 +1237,7 @@ return (jj_ntk = jj_nt.kind); } - private java.util.List jj_expentries = new java.util.ArrayList(); + private java.util.List jj_expentries = new java.util.ArrayList(); private int[] jj_expentry; private int jj_kind = -1; private int[] jj_lasttokens = new int[100]; @@ -1297,7 +1297,7 @@ jj_add_error_token(0, 0); int[][] exptokseq = new int[jj_expentries.size()][]; for (int i = 0; i < jj_expentries.size(); i++) { - exptokseq[i] = (int[])jj_expentries.get(i); + exptokseq[i] = jj_expentries.get(i); } return new ParseException(token, exptokseq, tokenImage); } Modified: lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/TokenMgrError.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/TokenMgrError.java?rev=826549&r1=826548&r2=826549&view=diff ============================================================================== --- lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/TokenMgrError.java (original) +++ lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/TokenMgrError.java Sun Oct 18 23:21:13 2009 @@ -3,6 +3,7 @@ package org.apache.lucene.queryParser.precedence; /** Token Manager Error. */ +@SuppressWarnings("serial") public class TokenMgrError extends Error { @@ -137,4 +138,4 @@ this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason); } } -/* JavaCC - OriginalChecksum=43f834e732189cec173ab002c94ae97f (do not edit this line) */ +/* JavaCC - OriginalChecksum=257b82f2650841e86289a309cb3dae76 (do not edit this line) */ Modified: lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/CharStream.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/CharStream.java?rev=826549&r1=826548&r2=826549&view=diff ============================================================================== --- lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/CharStream.java (original) +++ lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/CharStream.java Sun Oct 18 23:21:13 2009 @@ -1,4 +1,5 @@ -/* Generated By:JavaCC: Do not edit this line. CharStream.java Version 3.0 */ +/* Generated By:JavaCC: Do not edit this line. CharStream.java Version 4.1 */ +/* JavaCCOptions:STATIC=false */ package org.apache.lucene.queryParser.surround.parser; /** @@ -26,6 +27,20 @@ char readChar() throws java.io.IOException; /** + * Returns the column position of the character last read. + * @deprecated + * @see #getEndColumn + */ + int getColumn(); + + /** + * Returns the line number of the character last read. + * @deprecated + * @see #getEndLine + */ + int getLine(); + + /** * Returns the column number of the last character for current token (being * matched after the last call to BeginTOken). */ @@ -65,7 +80,7 @@ char BeginToken() throws java.io.IOException; /** - * Returns a string made up of characters from the marked token beginning + * Returns a string made up of characters from the marked token beginning * to the current buffer position. Implementations have the choice of returning * anything that they want to. For example, for efficiency, one might decide * to just return null, which is a valid implementation. @@ -94,3 +109,4 @@ void Done(); } +/* JavaCC - OriginalChecksum=872eafa2c180de2e08862f87f675b2dd (do not edit this line) */ Modified: lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/ParseException.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/ParseException.java?rev=826549&r1=826548&r2=826549&view=diff ============================================================================== --- lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/ParseException.java (original) +++ lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/ParseException.java Sun Oct 18 23:21:13 2009 @@ -1,5 +1,5 @@ -/* Generated By:JavaCC: Do not edit this line. ParseException.java Version 3.0 */ - +/* Generated By:JavaCC: Do not edit this line. ParseException.java Version 4.1 */ +/* JavaCCOptions:KEEP_LINE_COL=null */ package org.apache.lucene.queryParser.surround.parser; /** @@ -52,6 +52,7 @@ specialConstructor = false; } + /** Constructor with message. */ public ParseException(String message) { super(message); specialConstructor = false; @@ -99,19 +100,19 @@ if (!specialConstructor) { return super.getMessage(); } - String expected = ""; + StringBuffer expected = new StringBuffer(); int maxSize = 0; for (int i = 0; i < expectedTokenSequences.length; i++) { if (maxSize < expectedTokenSequences[i].length) { maxSize = expectedTokenSequences[i].length; } for (int j = 0; j < expectedTokenSequences[i].length; j++) { - expected += tokenImage[expectedTokenSequences[i][j]] + " "; + expected.append(tokenImage[expectedTokenSequences[i][j]]).append(' '); } if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) { - expected += "..."; + expected.append("..."); } - expected += eol + " "; + expected.append(eol).append(" "); } String retval = "Encountered \""; Token tok = currentToken.next; @@ -121,8 +122,11 @@ retval += tokenImage[0]; break; } + retval += " " + tokenImage[tok.kind]; + retval += " \""; retval += add_escapes(tok.image); - tok = tok.next; + retval += " \""; + tok = tok.next; } retval += "\" at line " + currentToken.next.beginLine + ", column " + currentToken.next.beginColumn; retval += "." + eol; @@ -131,7 +135,7 @@ } else { retval += "Was expecting one of:" + eol + " "; } - retval += expected; + retval += expected.toString(); return retval; } @@ -139,7 +143,7 @@ * The end of line string for this machine. */ protected String eol = System.getProperty("line.separator", "\n"); - + /** * Used to convert raw characters to their escaped version * when these raw version cannot be used as part of an ASCII @@ -191,3 +195,4 @@ } } +/* JavaCC - OriginalChecksum=2a154532f17c98e29d1671b8413edb60 (do not edit this line) */ Modified: lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/QueryParser.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/QueryParser.java?rev=826549&r1=826548&r2=826549&view=diff ============================================================================== --- lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/QueryParser.java (original) +++ lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/QueryParser.java Sun Oct 18 23:21:13 2009 @@ -35,7 +35,7 @@ final char quote = '\"'; final char fieldOperator = ':'; final char comma = ','; /* prefix list separator */ - final char carat = '^'; /* weight oparator */ + final char carat = '^'; /* weight operator */ static public SrndQuery parse(String query) throws ParseException { QueryParser parser = new QueryParser(); @@ -475,39 +475,42 @@ } } - final private boolean jj_2_1(int xla) { + private boolean jj_2_1(int xla) { jj_la = xla; jj_lastpos = jj_scanpos = token; try { return !jj_3_1(); } catch(LookaheadSuccess ls) { return true; } finally { jj_save(0, xla); } } - final private boolean jj_3_1() { + private boolean jj_3_1() { if (jj_scan_token(TERM)) return true; if (jj_scan_token(COLON)) return true; return false; } + /** Generated Token Manager. */ public QueryParserTokenManager token_source; - public Token token, jj_nt; + /** Current token. */ + public Token token; + /** Next token. */ + public Token jj_nt; private int jj_ntk; private Token jj_scanpos, jj_lastpos; private int jj_la; - public boolean lookingAhead = false; - private boolean jj_semLA; private int jj_gen; final private int[] jj_la1 = new int[10]; static private int[] jj_la1_0; static { - jj_la1_0(); + jj_la1_init_0(); } - private static void jj_la1_0() { + private static void jj_la1_init_0() { jj_la1_0 = new int[] {0x100,0x200,0x400,0x1000,0x800,0x7c3b00,0x1b00,0x8000,0x7c0000,0x20000,}; } final private JJCalls[] jj_2_rtns = new JJCalls[1]; private boolean jj_rescan = false; private int jj_gc = 0; + /** Constructor with user supplied CharStream. */ public QueryParser(CharStream stream) { token_source = new QueryParserTokenManager(stream); token = new Token(); @@ -517,6 +520,7 @@ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls(); } + /** Reinitialise. */ public void ReInit(CharStream stream) { token_source.ReInit(stream); token = new Token(); @@ -526,6 +530,7 @@ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls(); } + /** Constructor with generated Token Manager. */ public QueryParser(QueryParserTokenManager tm) { token_source = tm; token = new Token(); @@ -535,6 +540,7 @@ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls(); } + /** Reinitialise. */ public void ReInit(QueryParserTokenManager tm) { token_source = tm; token = new Token(); @@ -544,7 +550,7 @@ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls(); } - final private Token jj_consume_token(int kind) throws ParseException { + private Token jj_consume_token(int kind) throws ParseException { Token oldToken; if ((oldToken = token).next != null) token = token.next; else token = token.next = token_source.getNextToken(); @@ -570,7 +576,7 @@ static private final class LookaheadSuccess extends java.lang.Error { } final private LookaheadSuccess jj_ls = new LookaheadSuccess(); - final private boolean jj_scan_token(int kind) { + private boolean jj_scan_token(int kind) { if (jj_scanpos == jj_lastpos) { jj_la--; if (jj_scanpos.next == null) { @@ -591,6 +597,8 @@ return false; } + +/** Get the next Token. */ final public Token getNextToken() { if (token.next != null) token = token.next; else token = token.next = token_source.getNextToken(); @@ -599,8 +607,9 @@ return token; } +/** Get the specific Token. */ final public Token getToken(int index) { - Token t = lookingAhead ? jj_scanpos : token; + Token t = token; for (int i = 0; i < index; i++) { if (t.next != null) t = t.next; else t = t.next = token_source.getNextToken(); @@ -608,14 +617,14 @@ return t; } - final private int jj_ntk() { + private int jj_ntk() { if ((jj_nt=token.next) == null) return (jj_ntk = (token.next=token_source.getNextToken()).kind); else return (jj_ntk = jj_nt.kind); } - private java.util.Vector jj_expentries = new java.util.Vector(); + private java.util.List jj_expentries = new java.util.ArrayList(); private int[] jj_expentry; private int jj_kind = -1; private int[] jj_lasttokens = new int[100]; @@ -630,31 +639,26 @@ for (int i = 0; i < jj_endpos; i++) { jj_expentry[i] = jj_lasttokens[i]; } - boolean exists = false; - for (java.util.Enumeration e = jj_expentries.elements(); e.hasMoreElements();) { - int[] oldentry = (int[])(e.nextElement()); + jj_entries_loop: for (java.util.Iterator it = jj_expentries.iterator(); it.hasNext();) { + int[] oldentry = (int[])(it.next()); if (oldentry.length == jj_expentry.length) { - exists = true; for (int i = 0; i < jj_expentry.length; i++) { if (oldentry[i] != jj_expentry[i]) { - exists = false; - break; + continue jj_entries_loop; } } - if (exists) break; + jj_expentries.add(jj_expentry); + break jj_entries_loop; } } - if (!exists) jj_expentries.addElement(jj_expentry); if (pos != 0) jj_lasttokens[(jj_endpos = pos) - 1] = kind; } } + /** Generate ParseException. */ public ParseException generateParseException() { - jj_expentries.removeAllElements(); + jj_expentries.clear(); boolean[] la1tokens = new boolean[24]; - for (int i = 0; i < 24; i++) { - la1tokens[i] = false; - } if (jj_kind >= 0) { la1tokens[jj_kind] = true; jj_kind = -1; @@ -672,7 +676,7 @@ if (la1tokens[i]) { jj_expentry = new int[1]; jj_expentry[0] = i; - jj_expentries.addElement(jj_expentry); + jj_expentries.add(jj_expentry); } } jj_endpos = 0; @@ -680,20 +684,23 @@ jj_add_error_token(0, 0); int[][] exptokseq = new int[jj_expentries.size()][]; for (int i = 0; i < jj_expentries.size(); i++) { - exptokseq[i] = (int[])jj_expentries.elementAt(i); + exptokseq[i] = jj_expentries.get(i); } return new ParseException(token, exptokseq, tokenImage); } + /** Enable tracing. */ final public void enable_tracing() { } + /** Disable tracing. */ final public void disable_tracing() { } - final private void jj_rescan_token() { + private void jj_rescan_token() { jj_rescan = true; for (int i = 0; i < 1; i++) { + try { JJCalls p = jj_2_rtns[i]; do { if (p.gen > jj_gen) { @@ -704,11 +711,12 @@ } p = p.next; } while (p != null); + } catch(LookaheadSuccess ls) { } } jj_rescan = false; } - final private void jj_save(int index, int xla) { + private void jj_save(int index, int xla) { JJCalls p = jj_2_rtns[index]; while (p.gen > jj_gen) { if (p.next == null) { p = p.next = new JJCalls(); break; } Modified: lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/QueryParserConstants.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/QueryParserConstants.java?rev=826549&r1=826548&r2=826549&view=diff ============================================================================== --- lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/QueryParserConstants.java (original) +++ lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/QueryParserConstants.java Sun Oct 18 23:21:13 2009 @@ -1,35 +1,66 @@ /* Generated By:JavaCC: Do not edit this line. QueryParserConstants.java */ package org.apache.lucene.queryParser.surround.parser; + +/** + * Token literal values and constants. + * Generated by org.javacc.parser.OtherFilesGen#start() + */ public interface QueryParserConstants { + /** End of File. */ int EOF = 0; + /** RegularExpression Id. */ int _NUM_CHAR = 1; + /** RegularExpression Id. */ int _TERM_CHAR = 2; + /** RegularExpression Id. */ int _WHITESPACE = 3; + /** RegularExpression Id. */ int _STAR = 4; + /** RegularExpression Id. */ int _ONE_CHAR = 5; + /** RegularExpression Id. */ int _DISTOP_NUM = 6; + /** RegularExpression Id. */ int OR = 8; + /** RegularExpression Id. */ int AND = 9; + /** RegularExpression Id. */ int NOT = 10; + /** RegularExpression Id. */ int W = 11; + /** RegularExpression Id. */ int N = 12; + /** RegularExpression Id. */ int LPAREN = 13; + /** RegularExpression Id. */ int RPAREN = 14; + /** RegularExpression Id. */ int COMMA = 15; + /** RegularExpression Id. */ int COLON = 16; + /** RegularExpression Id. */ int CARAT = 17; + /** RegularExpression Id. */ int TRUNCQUOTED = 18; + /** RegularExpression Id. */ int QUOTED = 19; + /** RegularExpression Id. */ int SUFFIXTERM = 20; + /** RegularExpression Id. */ int TRUNCTERM = 21; + /** RegularExpression Id. */ int TERM = 22; + /** RegularExpression Id. */ int NUMBER = 23; + /** Lexical state. */ int Boost = 0; + /** Lexical state. */ int DEFAULT = 1; + /** Literal token values. */ String[] tokenImage = { "", "<_NUM_CHAR>", Modified: lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/QueryParserTokenManager.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/QueryParserTokenManager.java?rev=826549&r1=826548&r2=826549&view=diff ============================================================================== --- lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/QueryParserTokenManager.java (original) +++ lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/QueryParserTokenManager.java Sun Oct 18 23:21:13 2009 @@ -14,9 +14,13 @@ import org.apache.lucene.queryParser.surround.query.SrndPrefixQuery; import org.apache.lucene.queryParser.surround.query.SrndTruncQuery; +/** Token Manager. */ public class QueryParserTokenManager implements QueryParserConstants { + + /** Debug output. */ public java.io.PrintStream debugStream = System.out; + /** Set debug output. */ public void setDebugStream(java.io.PrintStream ds) { debugStream = ds; } private final int jjStopStringLiteralDfa_1(int pos, long active0) { @@ -30,21 +34,13 @@ { return jjMoveNfa_1(jjStopStringLiteralDfa_1(pos, active0), pos + 1); } -private final int jjStopAtPos(int pos, int kind) +private int jjStopAtPos(int pos, int kind) { jjmatchedKind = kind; jjmatchedPos = pos; return pos + 1; } -private final int jjStartNfaWithStates_1(int pos, int kind, int state) -{ - jjmatchedKind = kind; - jjmatchedPos = pos; - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { return pos + 1; } - return jjMoveNfa_1(state, pos + 1); -} -private final int jjMoveStringLiteralDfa0_1() +private int jjMoveStringLiteralDfa0_1() { switch(curChar) { @@ -62,50 +58,19 @@ return jjMoveNfa_1(0, 0); } } -private final void jjCheckNAdd(int state) -{ - if (jjrounds[state] != jjround) - { - jjstateSet[jjnewStateCnt++] = state; - jjrounds[state] = jjround; - } -} -private final void jjAddStates(int start, int end) -{ - do { - jjstateSet[jjnewStateCnt++] = jjnextStates[start]; - } while (start++ != end); -} -private final void jjCheckNAddTwoStates(int state1, int state2) -{ - jjCheckNAdd(state1); - jjCheckNAdd(state2); -} -private final void jjCheckNAddStates(int start, int end) -{ - do { - jjCheckNAdd(jjnextStates[start]); - } while (start++ != end); -} -private final void jjCheckNAddStates(int start) -{ - jjCheckNAdd(jjnextStates[start]); - jjCheckNAdd(jjnextStates[start + 1]); -} static final long[] jjbitVec0 = { 0xfffffffffffffffeL, 0xffffffffffffffffL, 0xffffffffffffffffL, 0xffffffffffffffffL }; static final long[] jjbitVec2 = { 0x0L, 0x0L, 0xffffffffffffffffL, 0xffffffffffffffffL }; -private final int jjMoveNfa_1(int startState, int curPos) +private int jjMoveNfa_1(int startState, int curPos) { - int[] nextStates; int startsAt = 0; jjnewStateCnt = 38; int i = 1; jjstateSet[0] = startState; - int j, kind = 0x7fffffff; + int kind = 0x7fffffff; for (;;) { if (++jjround == 0x7fffffff) @@ -113,7 +78,7 @@ if (curChar < 64) { long l = 1L << curChar; - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -227,7 +192,7 @@ else if (curChar < 128) { long l = 1L << (curChar & 077); - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -388,7 +353,7 @@ long l1 = 1L << (hiByte & 077); int i2 = (curChar & 0xff) >> 6; long l2 = 1L << (curChar & 077); - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -446,18 +411,17 @@ catch(java.io.IOException e) { return curPos; } } } -private final int jjMoveStringLiteralDfa0_0() +private int jjMoveStringLiteralDfa0_0() { return jjMoveNfa_0(0, 0); } -private final int jjMoveNfa_0(int startState, int curPos) +private int jjMoveNfa_0(int startState, int curPos) { - int[] nextStates; int startsAt = 0; jjnewStateCnt = 3; int i = 1; jjstateSet[0] = startState; - int j, kind = 0x7fffffff; + int kind = 0x7fffffff; for (;;) { if (++jjround == 0x7fffffff) @@ -465,7 +429,7 @@ if (curChar < 64) { long l = 1L << curChar; - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -494,7 +458,7 @@ else if (curChar < 128) { long l = 1L << (curChar & 077); - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -509,7 +473,7 @@ long l1 = 1L << (hiByte & 077); int i2 = (curChar & 0xff) >> 6; long l2 = 1L << (curChar & 077); - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -540,19 +504,25 @@ { case 0: return ((jjbitVec2[i2] & l2) != 0L); - default : + default : if ((jjbitVec0[i1] & l1) != 0L) return true; return false; } } + +/** Token literal values. */ public static final String[] jjstrLiteralImages = { "", null, null, null, null, null, null, null, null, null, null, null, null, "\50", "\51", "\54", "\72", "\136", null, null, null, null, null, null, }; + +/** Lexer state names. */ public static final String[] lexStateNames = { - "Boost", - "DEFAULT", + "Boost", + "DEFAULT", }; + +/** Lex State array. */ public static final int[] jjnewLexState = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, -1, -1, -1, -1, -1, 1, }; @@ -566,15 +536,18 @@ private final int[] jjrounds = new int[38]; private final int[] jjstateSet = new int[76]; protected char curChar; -public QueryParserTokenManager(CharStream stream) -{ +/** Constructor. */ +public QueryParserTokenManager(CharStream stream){ input_stream = stream; } -public QueryParserTokenManager(CharStream stream, int lexState) -{ + +/** Constructor. */ +public QueryParserTokenManager(CharStream stream, int lexState){ this(stream); SwitchTo(lexState); } + +/** Reinitialise parser. */ public void ReInit(CharStream stream) { jjmatchedPos = jjnewStateCnt = 0; @@ -582,18 +555,22 @@ input_stream = stream; ReInitRounds(); } -private final void ReInitRounds() +private void ReInitRounds() { int i; jjround = 0x80000001; for (i = 38; i-- > 0;) jjrounds[i] = 0x80000000; } + +/** Reinitialise parser. */ public void ReInit(CharStream stream, int lexState) { ReInit(stream); SwitchTo(lexState); } + +/** Switch to specified lex state. */ public void SwitchTo(int lexState) { if (lexState >= 2 || lexState < 0) @@ -604,14 +581,25 @@ protected Token jjFillToken() { - Token t = Token.newToken(jjmatchedKind); - t.kind = jjmatchedKind; + final Token t; + final String curTokenImage; + final int beginLine; + final int endLine; + final int beginColumn; + final int endColumn; String im = jjstrLiteralImages[jjmatchedKind]; - t.image = (im == null) ? input_stream.GetImage() : im; - t.beginLine = input_stream.getBeginLine(); - t.beginColumn = input_stream.getBeginColumn(); - t.endLine = input_stream.getEndLine(); - t.endColumn = input_stream.getEndColumn(); + curTokenImage = (im == null) ? input_stream.GetImage() : im; + beginLine = input_stream.getBeginLine(); + beginColumn = input_stream.getBeginColumn(); + endLine = input_stream.getEndLine(); + endColumn = input_stream.getEndColumn(); + t = Token.newToken(jjmatchedKind, curTokenImage); + + t.beginLine = beginLine; + t.endLine = endLine; + t.beginColumn = beginColumn; + t.endColumn = endColumn; + return t; } @@ -622,22 +610,21 @@ int jjmatchedPos; int jjmatchedKind; +/** Get the next Token. */ public Token getNextToken() { - int kind; - Token specialToken = null; Token matchedToken; int curPos = 0; EOFLoop : for (;;) - { - try - { + { + try + { curChar = input_stream.BeginToken(); - } + } catch(java.io.IOException e) - { + { jjmatchedKind = 0; matchedToken = jjFillToken(); return matchedToken; @@ -697,4 +684,31 @@ } } +private void jjCheckNAdd(int state) +{ + if (jjrounds[state] != jjround) + { + jjstateSet[jjnewStateCnt++] = state; + jjrounds[state] = jjround; + } +} +private void jjAddStates(int start, int end) +{ + do { + jjstateSet[jjnewStateCnt++] = jjnextStates[start]; + } while (start++ != end); +} +private void jjCheckNAddTwoStates(int state1, int state2) +{ + jjCheckNAdd(state1); + jjCheckNAdd(state2); +} + +private void jjCheckNAddStates(int start, int end) +{ + do { + jjCheckNAdd(jjnextStates[start]); + } while (start++ != end); +} + } Modified: lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/Token.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/Token.java?rev=826549&r1=826548&r2=826549&view=diff ============================================================================== --- lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/Token.java (original) +++ lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/Token.java Sun Oct 18 23:21:13 2009 @@ -1,4 +1,5 @@ -/* Generated By:JavaCC: Do not edit this line. Token.java Version 3.0 */ +/* Generated By:JavaCC: Do not edit this line. Token.java Version 4.1 */ +/* JavaCCOptions:TOKEN_EXTENDS=,KEEP_LINE_COL=null */ package org.apache.lucene.queryParser.surround.parser; /** @@ -14,12 +15,14 @@ */ public int kind; - /** - * beginLine and beginColumn describe the position of the first character - * of this token; endLine and endColumn describe the position of the - * last character of this token. - */ - public int beginLine, beginColumn, endLine, endColumn; + /** The line number of the first character of this Token. */ + public int beginLine; + /** The column number of the first character of this Token. */ + public int beginColumn; + /** The line number of the last character of this Token. */ + public int endLine; + /** The column number of the last character of this Token. */ + public int endColumn; /** * The string image of the token. @@ -51,6 +54,40 @@ public Token specialToken; /** + * An optional attribute value of the Token. + * Tokens which are not used as syntactic sugar will often contain + * meaningful values that will be used later on by the compiler or + * interpreter. This attribute value is often different from the image. + * Any subclass of Token that actually wants to return a non-null value can + * override this method as appropriate. + */ + public Object getValue() { + return null; + } + + /** + * No-argument constructor + */ + public Token() {} + + /** + * Constructs a new token for the specified Image. + */ + public Token(int kind) + { + this(kind, null); + } + + /** + * Constructs a new token for the specified Image and Kind. + */ + public Token(int kind, String image) + { + this.kind = kind; + this.image = image; + } + + /** * Returns the image. */ public String toString() @@ -63,19 +100,25 @@ * can create and return subclass objects based on the value of ofKind. * Simply add the cases to the switch for all those special cases. * For example, if you have a subclass of Token called IDToken that - * you want to create if ofKind is ID, simlpy add something like : + * you want to create if ofKind is ID, simply add something like : * - * case MyParserConstants.ID : return new IDToken(); + * case MyParserConstants.ID : return new IDToken(ofKind, image); * * to the following switch statement. Then you can cast matchedToken - * variable to the appropriate type and use it in your lexical actions. + * variable to the appropriate type and use sit in your lexical actions. */ - public static final Token newToken(int ofKind) + public static Token newToken(int ofKind, String image) { switch(ofKind) { - default : return new Token(); + default : return new Token(ofKind, image); } } + public static Token newToken(int ofKind) + { + return newToken(ofKind, null); + } + } +/* JavaCC - OriginalChecksum=a1d2d2c0d7f5810183a1438fb4d51c1c (do not edit this line) */ Modified: lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/TokenMgrError.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/TokenMgrError.java?rev=826549&r1=826548&r2=826549&view=diff ============================================================================== --- lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/TokenMgrError.java (original) +++ lucene/java/trunk/contrib/surround/src/java/org/apache/lucene/queryParser/surround/parser/TokenMgrError.java Sun Oct 18 23:21:13 2009 @@ -1,19 +1,23 @@ -/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 3.0 */ +/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 4.1 */ +/* JavaCCOptions: */ package org.apache.lucene.queryParser.surround.parser; +/** Token Manager Error. */ +@SuppressWarnings("serial") public class TokenMgrError extends Error { + /* * Ordinals for various reasons why an Error of this type can be thrown. */ /** - * Lexical error occured. + * Lexical error occurred. */ static final int LEXICAL_ERROR = 0; /** - * An attempt wass made to create a second instance of a static token manager. + * An attempt was made to create a second instance of a static token manager. */ static final int STATIC_LEXER_ERROR = 1; @@ -34,7 +38,7 @@ int errorCode; /** - * Replaces unprintable characters by their espaced (or unicode escaped) + * Replaces unprintable characters by their escaped (or unicode escaped) * equivalents in the given string */ protected static final String addEscapes(String str) { @@ -85,12 +89,12 @@ /** * Returns a detailed message for the Error when it is thrown by the * token manager to indicate a lexical error. - * Parameters : - * EOFSeen : indicates if EOF caused the lexicl error - * curLexState : lexical state in which this error occured - * errorLine : line number when the error occured - * errorColumn : column number when the error occured - * errorAfter : prefix that was seen before this error occured + * Parameters : + * EOFSeen : indicates if EOF caused the lexical error + * curLexState : lexical state in which this error occurred + * errorLine : line number when the error occurred + * errorColumn : column number when the error occurred + * errorAfter : prefix that was seen before this error occurred * curchar : the offending character * Note: You can customize the lexical error message by modifying this method. */ @@ -105,7 +109,7 @@ /** * You can also modify the body of this method to customize your error messages. * For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not - * of end-users concern, so you can return something like : + * of end-users concern, so you can return something like : * * "Internal Error : Please file a bug report .... " * @@ -119,15 +123,19 @@ * Constructors of various flavors follow. */ + /** No arg constructor. */ public TokenMgrError() { } + /** Constructor with message and reason. */ public TokenMgrError(String message, int reason) { super(message); errorCode = reason; } + /** Full Constructor. */ public TokenMgrError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar, int reason) { this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason); } } +/* JavaCC - OriginalChecksum=fecdd69c7d7b49407e441e312085db53 (do not edit this line) */ Modified: lucene/java/trunk/src/demo/org/apache/lucene/demo/html/HTMLParser.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/demo/org/apache/lucene/demo/html/HTMLParser.java?rev=826549&r1=826548&r2=826549&view=diff ============================================================================== --- lucene/java/trunk/src/demo/org/apache/lucene/demo/html/HTMLParser.java (original) +++ lucene/java/trunk/src/demo/org/apache/lucene/demo/html/HTMLParser.java Sun Oct 18 23:21:13 2009 @@ -439,56 +439,60 @@ jj_consume_token(ScriptEnd); } - final private boolean jj_2_1(int xla) { + private boolean jj_2_1(int xla) { jj_la = xla; jj_lastpos = jj_scanpos = token; try { return !jj_3_1(); } catch(LookaheadSuccess ls) { return true; } finally { jj_save(0, xla); } } - final private boolean jj_2_2(int xla) { + private boolean jj_2_2(int xla) { jj_la = xla; jj_lastpos = jj_scanpos = token; try { return !jj_3_2(); } catch(LookaheadSuccess ls) { return true; } finally { jj_save(1, xla); } } - final private boolean jj_3_1() { + private boolean jj_3_1() { if (jj_scan_token(ArgQuote1)) return true; if (jj_scan_token(CloseQuote1)) return true; return false; } - final private boolean jj_3_2() { + private boolean jj_3_2() { if (jj_scan_token(ArgQuote2)) return true; if (jj_scan_token(CloseQuote2)) return true; return false; } + /** Generated Token Manager. */ public HTMLParserTokenManager token_source; SimpleCharStream jj_input_stream; - public Token token, jj_nt; + /** Current token. */ + public Token token; + /** Next token. */ + public Token jj_nt; private int jj_ntk; private Token jj_scanpos, jj_lastpos; private int jj_la; - public boolean lookingAhead = false; - private boolean jj_semLA; private int jj_gen; final private int[] jj_la1 = new int[14]; static private int[] jj_la1_0; static { - jj_la1_0(); + jj_la1_init_0(); } - private static void jj_la1_0() { + private static void jj_la1_init_0() { jj_la1_0 = new int[] {0x2c7e,0x2c7e,0x10000,0x380000,0x20000,0x80000,0x100000,0x200000,0x3b0000,0x3b0000,0x8000000,0x20000000,0x30,0x4000,}; } final private JJCalls[] jj_2_rtns = new JJCalls[2]; private boolean jj_rescan = false; private int jj_gc = 0; + /** Constructor with InputStream. */ public HTMLParser(java.io.InputStream stream) { this(stream, null); } + /** Constructor with InputStream and supplied encoding */ public HTMLParser(java.io.InputStream stream, String encoding) { try { jj_input_stream = new SimpleCharStream(stream, encoding, 1, 1); } catch(java.io.UnsupportedEncodingException e) { throw new RuntimeException(e); } token_source = new HTMLParserTokenManager(jj_input_stream); @@ -499,9 +503,11 @@ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls(); } + /** Reinitialise. */ public void ReInit(java.io.InputStream stream) { ReInit(stream, null); } + /** Reinitialise. */ public void ReInit(java.io.InputStream stream, String encoding) { try { jj_input_stream.ReInit(stream, encoding, 1, 1); } catch(java.io.UnsupportedEncodingException e) { throw new RuntimeException(e); } token_source.ReInit(jj_input_stream); @@ -512,6 +518,7 @@ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls(); } + /** Constructor. */ public HTMLParser(java.io.Reader stream) { jj_input_stream = new SimpleCharStream(stream, 1, 1); token_source = new HTMLParserTokenManager(jj_input_stream); @@ -522,6 +529,7 @@ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls(); } + /** Reinitialise. */ public void ReInit(java.io.Reader stream) { jj_input_stream.ReInit(stream, 1, 1); token_source.ReInit(jj_input_stream); @@ -532,6 +540,7 @@ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls(); } + /** Constructor with generated Token Manager. */ public HTMLParser(HTMLParserTokenManager tm) { token_source = tm; token = new Token(); @@ -541,6 +550,7 @@ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls(); } + /** Reinitialise. */ public void ReInit(HTMLParserTokenManager tm) { token_source = tm; token = new Token(); @@ -550,7 +560,7 @@ for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls(); } - final private Token jj_consume_token(int kind) throws ParseException { + private Token jj_consume_token(int kind) throws ParseException { Token oldToken; if ((oldToken = token).next != null) token = token.next; else token = token.next = token_source.getNextToken(); @@ -576,7 +586,7 @@ static private final class LookaheadSuccess extends java.lang.Error { } final private LookaheadSuccess jj_ls = new LookaheadSuccess(); - final private boolean jj_scan_token(int kind) { + private boolean jj_scan_token(int kind) { if (jj_scanpos == jj_lastpos) { jj_la--; if (jj_scanpos.next == null) { @@ -597,6 +607,8 @@ return false; } + +/** Get the next Token. */ final public Token getNextToken() { if (token.next != null) token = token.next; else token = token.next = token_source.getNextToken(); @@ -605,8 +617,9 @@ return token; } +/** Get the specific Token. */ final public Token getToken(int index) { - Token t = lookingAhead ? jj_scanpos : token; + Token t = token; for (int i = 0; i < index; i++) { if (t.next != null) t = t.next; else t = t.next = token_source.getNextToken(); @@ -614,14 +627,14 @@ return t; } - final private int jj_ntk() { + private int jj_ntk() { if ((jj_nt=token.next) == null) return (jj_ntk = (token.next=token_source.getNextToken()).kind); else return (jj_ntk = jj_nt.kind); } - private java.util.Vector jj_expentries = new java.util.Vector(); + private java.util.List jj_expentries = new java.util.ArrayList(); private int[] jj_expentry; private int jj_kind = -1; private int[] jj_lasttokens = new int[100]; @@ -636,31 +649,26 @@ for (int i = 0; i < jj_endpos; i++) { jj_expentry[i] = jj_lasttokens[i]; } - boolean exists = false; - for (java.util.Enumeration e = jj_expentries.elements(); e.hasMoreElements();) { - int[] oldentry = (int[])(e.nextElement()); + jj_entries_loop: for (java.util.Iterator it = jj_expentries.iterator(); it.hasNext();) { + int[] oldentry = (int[])(it.next()); if (oldentry.length == jj_expentry.length) { - exists = true; for (int i = 0; i < jj_expentry.length; i++) { if (oldentry[i] != jj_expentry[i]) { - exists = false; - break; + continue jj_entries_loop; } } - if (exists) break; + jj_expentries.add(jj_expentry); + break jj_entries_loop; } } - if (!exists) jj_expentries.addElement(jj_expentry); if (pos != 0) jj_lasttokens[(jj_endpos = pos) - 1] = kind; } } + /** Generate ParseException. */ public ParseException generateParseException() { - jj_expentries.removeAllElements(); + jj_expentries.clear(); boolean[] la1tokens = new boolean[31]; - for (int i = 0; i < 31; i++) { - la1tokens[i] = false; - } if (jj_kind >= 0) { la1tokens[jj_kind] = true; jj_kind = -1; @@ -678,7 +686,7 @@ if (la1tokens[i]) { jj_expentry = new int[1]; jj_expentry[0] = i; - jj_expentries.addElement(jj_expentry); + jj_expentries.add(jj_expentry); } } jj_endpos = 0; @@ -686,18 +694,20 @@ jj_add_error_token(0, 0); int[][] exptokseq = new int[jj_expentries.size()][]; for (int i = 0; i < jj_expentries.size(); i++) { - exptokseq[i] = (int[])jj_expentries.elementAt(i); + exptokseq[i] = jj_expentries.get(i); } return new ParseException(token, exptokseq, tokenImage); } + /** Enable tracing. */ final public void enable_tracing() { } + /** Disable tracing. */ final public void disable_tracing() { } - final private void jj_rescan_token() { + private void jj_rescan_token() { jj_rescan = true; for (int i = 0; i < 2; i++) { try { @@ -717,7 +727,7 @@ jj_rescan = false; } - final private void jj_save(int index, int xla) { + private void jj_save(int index, int xla) { JJCalls p = jj_2_rtns[index]; while (p.gen > jj_gen) { if (p.next == null) { p = p.next = new JJCalls(); break; } Modified: lucene/java/trunk/src/demo/org/apache/lucene/demo/html/HTMLParserConstants.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/demo/org/apache/lucene/demo/html/HTMLParserConstants.java?rev=826549&r1=826548&r2=826549&view=diff ============================================================================== --- lucene/java/trunk/src/demo/org/apache/lucene/demo/html/HTMLParserConstants.java (original) +++ lucene/java/trunk/src/demo/org/apache/lucene/demo/html/HTMLParserConstants.java Sun Oct 18 23:21:13 2009 @@ -1,48 +1,92 @@ /* Generated By:JavaCC: Do not edit this line. HTMLParserConstants.java */ package org.apache.lucene.demo.html; + +/** + * Token literal values and constants. + * Generated by org.javacc.parser.OtherFilesGen#start() + */ public interface HTMLParserConstants { + /** End of File. */ int EOF = 0; + /** RegularExpression Id. */ int ScriptStart = 1; + /** RegularExpression Id. */ int TagName = 2; + /** RegularExpression Id. */ int DeclName = 3; + /** RegularExpression Id. */ int Comment1 = 4; + /** RegularExpression Id. */ int Comment2 = 5; + /** RegularExpression Id. */ int Word = 6; + /** RegularExpression Id. */ int LET = 7; + /** RegularExpression Id. */ int NUM = 8; + /** RegularExpression Id. */ int HEX = 9; + /** RegularExpression Id. */ int Entity = 10; + /** RegularExpression Id. */ int Space = 11; + /** RegularExpression Id. */ int SP = 12; + /** RegularExpression Id. */ int Punct = 13; + /** RegularExpression Id. */ int ScriptText = 14; + /** RegularExpression Id. */ int ScriptEnd = 15; + /** RegularExpression Id. */ int ArgName = 16; + /** RegularExpression Id. */ int ArgEquals = 17; + /** RegularExpression Id. */ int TagEnd = 18; + /** RegularExpression Id. */ int ArgValue = 19; + /** RegularExpression Id. */ int ArgQuote1 = 20; + /** RegularExpression Id. */ int ArgQuote2 = 21; + /** RegularExpression Id. */ int Quote1Text = 23; + /** RegularExpression Id. */ int CloseQuote1 = 24; + /** RegularExpression Id. */ int Quote2Text = 25; + /** RegularExpression Id. */ int CloseQuote2 = 26; + /** RegularExpression Id. */ int CommentText1 = 27; + /** RegularExpression Id. */ int CommentEnd1 = 28; + /** RegularExpression Id. */ int CommentText2 = 29; + /** RegularExpression Id. */ int CommentEnd2 = 30; + /** Lexical state. */ int DEFAULT = 0; + /** Lexical state. */ int WithinScript = 1; + /** Lexical state. */ int WithinTag = 2; + /** Lexical state. */ int AfterEquals = 3; + /** Lexical state. */ int WithinQuote1 = 4; + /** Lexical state. */ int WithinQuote2 = 5; + /** Lexical state. */ int WithinComment1 = 6; + /** Lexical state. */ int WithinComment2 = 7; + /** Literal token values. */ String[] tokenImage = { "", "\"> 6; long l2 = 1L << (curChar & 077); - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -526,18 +499,17 @@ catch(java.io.IOException e) { return curPos; } } } -private final int jjMoveStringLiteralDfa0_5() +private int jjMoveStringLiteralDfa0_5() { return jjMoveNfa_5(1, 0); } -private final int jjMoveNfa_5(int startState, int curPos) +private int jjMoveNfa_5(int startState, int curPos) { - int[] nextStates; int startsAt = 0; jjnewStateCnt = 2; int i = 1; jjstateSet[0] = startState; - int j, kind = 0x7fffffff; + int kind = 0x7fffffff; for (;;) { if (++jjround == 0x7fffffff) @@ -545,7 +517,7 @@ if (curChar < 64) { long l = 1L << curChar; - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -575,7 +547,7 @@ else if (curChar < 128) { long l = 1L << (curChar & 077); - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -592,7 +564,7 @@ { int i2 = (curChar & 0xff) >> 6; long l2 = 1L << (curChar & 077); - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -633,15 +605,7 @@ { return jjMoveNfa_7(jjStopStringLiteralDfa_7(pos, active0), pos + 1); } -private final int jjStartNfaWithStates_7(int pos, int kind, int state) -{ - jjmatchedKind = kind; - jjmatchedPos = pos; - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { return pos + 1; } - return jjMoveNfa_7(state, pos + 1); -} -private final int jjMoveStringLiteralDfa0_7() +private int jjMoveStringLiteralDfa0_7() { switch(curChar) { @@ -651,14 +615,13 @@ return jjMoveNfa_7(0, 0); } } -private final int jjMoveNfa_7(int startState, int curPos) +private int jjMoveNfa_7(int startState, int curPos) { - int[] nextStates; int startsAt = 0; jjnewStateCnt = 1; int i = 1; jjstateSet[0] = startState; - int j, kind = 0x7fffffff; + int kind = 0x7fffffff; for (;;) { if (++jjround == 0x7fffffff) @@ -666,7 +629,7 @@ if (curChar < 64) { long l = 1L << curChar; - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -683,7 +646,7 @@ else if (curChar < 128) { long l = 1L << (curChar & 077); - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -699,7 +662,7 @@ { int i2 = (curChar & 0xff) >> 6; long l2 = 1L << (curChar & 077); - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -727,18 +690,17 @@ catch(java.io.IOException e) { return curPos; } } } -private final int jjMoveStringLiteralDfa0_4() +private int jjMoveStringLiteralDfa0_4() { return jjMoveNfa_4(1, 0); } -private final int jjMoveNfa_4(int startState, int curPos) +private int jjMoveNfa_4(int startState, int curPos) { - int[] nextStates; int startsAt = 0; jjnewStateCnt = 2; int i = 1; jjstateSet[0] = startState; - int j, kind = 0x7fffffff; + int kind = 0x7fffffff; for (;;) { if (++jjround == 0x7fffffff) @@ -746,7 +708,7 @@ if (curChar < 64) { long l = 1L << curChar; - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -776,7 +738,7 @@ else if (curChar < 128) { long l = 1L << (curChar & 077); - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -793,7 +755,7 @@ { int i2 = (curChar & 0xff) >> 6; long l2 = 1L << (curChar & 077); - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -834,15 +796,7 @@ { return jjMoveNfa_3(jjStopStringLiteralDfa_3(pos, active0), pos + 1); } -private final int jjStartNfaWithStates_3(int pos, int kind, int state) -{ - jjmatchedKind = kind; - jjmatchedPos = pos; - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { return pos + 1; } - return jjMoveNfa_3(state, pos + 1); -} -private final int jjMoveStringLiteralDfa0_3() +private int jjMoveStringLiteralDfa0_3() { switch(curChar) { @@ -854,14 +808,13 @@ return jjMoveNfa_3(0, 0); } } -private final int jjMoveNfa_3(int startState, int curPos) +private int jjMoveNfa_3(int startState, int curPos) { - int[] nextStates; int startsAt = 0; jjnewStateCnt = 3; int i = 1; jjstateSet[0] = startState; - int j, kind = 0x7fffffff; + int kind = 0x7fffffff; for (;;) { if (++jjround == 0x7fffffff) @@ -869,7 +822,7 @@ if (curChar < 64) { long l = 1L << curChar; - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -907,7 +860,7 @@ else if (curChar < 128) { long l = 1L << (curChar & 077); - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -925,7 +878,7 @@ { int i2 = (curChar & 0xff) >> 6; long l2 = 1L << (curChar & 077); - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -984,15 +937,7 @@ { return jjMoveNfa_6(jjStopStringLiteralDfa_6(pos, active0), pos + 1); } -private final int jjStartNfaWithStates_6(int pos, int kind, int state) -{ - jjmatchedKind = kind; - jjmatchedPos = pos; - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { return pos + 1; } - return jjMoveNfa_6(state, pos + 1); -} -private final int jjMoveStringLiteralDfa0_6() +private int jjMoveStringLiteralDfa0_6() { switch(curChar) { @@ -1002,7 +947,7 @@ return jjMoveNfa_6(1, 0); } } -private final int jjMoveStringLiteralDfa1_6(long active0) +private int jjMoveStringLiteralDfa1_6(long active0) { try { curChar = input_stream.readChar(); } catch(java.io.IOException e) { @@ -1018,10 +963,10 @@ } return jjStartNfa_6(0, active0); } -private final int jjMoveStringLiteralDfa2_6(long old0, long active0) +private int jjMoveStringLiteralDfa2_6(long old0, long active0) { if (((active0 &= old0)) == 0L) - return jjStartNfa_6(0, old0); + return jjStartNfa_6(0, old0); try { curChar = input_stream.readChar(); } catch(java.io.IOException e) { jjStopStringLiteralDfa_6(1, active0); @@ -1038,14 +983,13 @@ } return jjStartNfa_6(1, active0); } -private final int jjMoveNfa_6(int startState, int curPos) +private int jjMoveNfa_6(int startState, int curPos) { - int[] nextStates; int startsAt = 0; jjnewStateCnt = 2; int i = 1; jjstateSet[0] = startState; - int j, kind = 0x7fffffff; + int kind = 0x7fffffff; for (;;) { if (++jjround == 0x7fffffff) @@ -1053,7 +997,7 @@ if (curChar < 64) { long l = 1L << curChar; - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -1083,7 +1027,7 @@ else if (curChar < 128) { long l = 1L << (curChar & 077); - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -1100,7 +1044,7 @@ { int i2 = (curChar & 0xff) >> 6; long l2 = 1L << (curChar & 077); - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -1129,18 +1073,17 @@ catch(java.io.IOException e) { return curPos; } } } -private final int jjMoveStringLiteralDfa0_1() +private int jjMoveStringLiteralDfa0_1() { return jjMoveNfa_1(1, 0); } -private final int jjMoveNfa_1(int startState, int curPos) +private int jjMoveNfa_1(int startState, int curPos) { - int[] nextStates; int startsAt = 0; jjnewStateCnt = 12; int i = 1; jjstateSet[0] = startState; - int j, kind = 0x7fffffff; + int kind = 0x7fffffff; for (;;) { if (++jjround == 0x7fffffff) @@ -1148,7 +1091,7 @@ if (curChar < 64) { long l = 1L << curChar; - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -1197,7 +1140,7 @@ else if (curChar < 128) { long l = 1L << (curChar & 077); - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -1242,7 +1185,7 @@ { int i2 = (curChar & 0xff) >> 6; long l2 = 1L << (curChar & 077); - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -1287,15 +1230,7 @@ { return jjMoveNfa_2(jjStopStringLiteralDfa_2(pos, active0), pos + 1); } -private final int jjStartNfaWithStates_2(int pos, int kind, int state) -{ - jjmatchedKind = kind; - jjmatchedPos = pos; - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { return pos + 1; } - return jjMoveNfa_2(state, pos + 1); -} -private final int jjMoveStringLiteralDfa0_2() +private int jjMoveStringLiteralDfa0_2() { switch(curChar) { @@ -1309,14 +1244,21 @@ return jjMoveNfa_2(0, 0); } } -private final int jjMoveNfa_2(int startState, int curPos) +private int jjStartNfaWithStates_2(int pos, int kind, int state) +{ + jjmatchedKind = kind; + jjmatchedPos = pos; + try { curChar = input_stream.readChar(); } + catch(java.io.IOException e) { return pos + 1; } + return jjMoveNfa_2(state, pos + 1); +} +private int jjMoveNfa_2(int startState, int curPos) { - int[] nextStates; int startsAt = 0; jjnewStateCnt = 6; int i = 1; jjstateSet[0] = startState; - int j, kind = 0x7fffffff; + int kind = 0x7fffffff; for (;;) { if (++jjround == 0x7fffffff) @@ -1324,7 +1266,7 @@ if (curChar < 64) { long l = 1L << curChar; - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -1378,7 +1320,7 @@ else if (curChar < 128) { long l = 1L << (curChar & 077); - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -1396,7 +1338,7 @@ { int i2 = (curChar & 0xff) >> 6; long l2 = 1L << (curChar & 077); - MatchLoop: do + do { switch(jjstateSet[--i]) { @@ -1429,20 +1371,26 @@ 20, 21, 24, 12, 14, 16, 5, 8, 0, 4, 6, 0, 4, 6, 5, 0, 4, 6, 3, 4, }; + +/** Token literal values. */ public static final String[] jjstrLiteralImages = { "", "\74\163\143\162\151\160\164", null, null, "\74\41\55\55", "\74\41", null, null, null, null, null, null, null, null, null, null, null, "\75", null, null, "\47", "\42", null, null, null, null, null, null, "\55\55\76", null, "\76", }; + +/** Lexer state names. */ public static final String[] lexStateNames = { - "DEFAULT", - "WithinScript", - "WithinTag", - "AfterEquals", - "WithinQuote1", - "WithinQuote2", - "WithinComment1", - "WithinComment2", + "DEFAULT", + "WithinScript", + "WithinTag", + "AfterEquals", + "WithinQuote1", + "WithinQuote2", + "WithinComment1", + "WithinComment2", }; + +/** Lex State array. */ public static final int[] jjnewLexState = { -1, 1, 2, 2, 6, 7, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, -1, 3, 0, 2, 4, 5, -1, -1, 2, -1, 2, -1, 0, -1, 0, @@ -1457,15 +1405,20 @@ private final int[] jjrounds = new int[28]; private final int[] jjstateSet = new int[56]; protected char curChar; +/** Constructor. */ public HTMLParserTokenManager(SimpleCharStream stream){ if (SimpleCharStream.staticFlag) throw new Error("ERROR: Cannot use a static CharStream class with a non-static lexical analyzer."); input_stream = stream; } + +/** Constructor. */ public HTMLParserTokenManager(SimpleCharStream stream, int lexState){ this(stream); SwitchTo(lexState); } + +/** Reinitialise parser. */ public void ReInit(SimpleCharStream stream) { jjmatchedPos = jjnewStateCnt = 0; @@ -1473,18 +1426,22 @@ input_stream = stream; ReInitRounds(); } -private final void ReInitRounds() +private void ReInitRounds() { int i; jjround = 0x80000001; for (i = 28; i-- > 0;) jjrounds[i] = 0x80000000; } + +/** Reinitialise parser. */ public void ReInit(SimpleCharStream stream, int lexState) { ReInit(stream); SwitchTo(lexState); } + +/** Switch to specified lex state. */ public void SwitchTo(int lexState) { if (lexState >= 8 || lexState < 0) @@ -1495,14 +1452,25 @@ protected Token jjFillToken() { - Token t = Token.newToken(jjmatchedKind); - t.kind = jjmatchedKind; + final Token t; + final String curTokenImage; + final int beginLine; + final int endLine; + final int beginColumn; + final int endColumn; String im = jjstrLiteralImages[jjmatchedKind]; - t.image = (im == null) ? input_stream.GetImage() : im; - t.beginLine = input_stream.getBeginLine(); - t.beginColumn = input_stream.getBeginColumn(); - t.endLine = input_stream.getEndLine(); - t.endColumn = input_stream.getEndColumn(); + curTokenImage = (im == null) ? input_stream.GetImage() : im; + beginLine = input_stream.getBeginLine(); + beginColumn = input_stream.getBeginColumn(); + endLine = input_stream.getEndLine(); + endColumn = input_stream.getEndColumn(); + t = Token.newToken(jjmatchedKind, curTokenImage); + + t.beginLine = beginLine; + t.endLine = endLine; + t.beginColumn = beginColumn; + t.endColumn = endColumn; + return t; } @@ -1513,22 +1481,21 @@ int jjmatchedPos; int jjmatchedKind; +/** Get the next Token. */ public Token getNextToken() { - int kind; - Token specialToken = null; Token matchedToken; int curPos = 0; EOFLoop : for (;;) - { - try - { + { + try + { curChar = input_stream.BeginToken(); - } + } catch(java.io.IOException e) - { + { jjmatchedKind = 0; matchedToken = jjFillToken(); return matchedToken; @@ -1622,4 +1589,31 @@ } } +private void jjCheckNAdd(int state) +{ + if (jjrounds[state] != jjround) + { + jjstateSet[jjnewStateCnt++] = state; + jjrounds[state] = jjround; + } +} +private void jjAddStates(int start, int end) +{ + do { + jjstateSet[jjnewStateCnt++] = jjnextStates[start]; + } while (start++ != end); +} +private void jjCheckNAddTwoStates(int state1, int state2) +{ + jjCheckNAdd(state1); + jjCheckNAdd(state2); +} + +private void jjCheckNAddStates(int start, int end) +{ + do { + jjCheckNAdd(jjnextStates[start]); + } while (start++ != end); +} + } Modified: lucene/java/trunk/src/demo/org/apache/lucene/demo/html/ParseException.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/demo/org/apache/lucene/demo/html/ParseException.java?rev=826549&r1=826548&r2=826549&view=diff ============================================================================== --- lucene/java/trunk/src/demo/org/apache/lucene/demo/html/ParseException.java (original) +++ lucene/java/trunk/src/demo/org/apache/lucene/demo/html/ParseException.java Sun Oct 18 23:21:13 2009 @@ -1,4 +1,5 @@ -/* Generated By:JavaCC: Do not edit this line. ParseException.java Version 3.0 */ +/* Generated By:JavaCC: Do not edit this line. ParseException.java Version 4.1 */ +/* JavaCCOptions:KEEP_LINE_COL=null */ package org.apache.lucene.demo.html; /** @@ -51,6 +52,7 @@ specialConstructor = false; } + /** Constructor with message. */ public ParseException(String message) { super(message); specialConstructor = false; @@ -105,7 +107,7 @@ maxSize = expectedTokenSequences[i].length; } for (int j = 0; j < expectedTokenSequences[i].length; j++) { - expected.append(tokenImage[expectedTokenSequences[i][j]]).append(" "); + expected.append(tokenImage[expectedTokenSequences[i][j]]).append(' '); } if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) { expected.append("..."); @@ -120,8 +122,11 @@ retval += tokenImage[0]; break; } + retval += " " + tokenImage[tok.kind]; + retval += " \""; retval += add_escapes(tok.image); - tok = tok.next; + retval += " \""; + tok = tok.next; } retval += "\" at line " + currentToken.next.beginLine + ", column " + currentToken.next.beginColumn; retval += "." + eol; @@ -138,7 +143,7 @@ * The end of line string for this machine. */ protected String eol = System.getProperty("line.separator", "\n"); - + /** * Used to convert raw characters to their escaped version * when these raw version cannot be used as part of an ASCII @@ -190,3 +195,4 @@ } } +/* JavaCC - OriginalChecksum=63b2008c66e199b79536447c26bee2ab (do not edit this line) */ Modified: lucene/java/trunk/src/demo/org/apache/lucene/demo/html/SimpleCharStream.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/demo/org/apache/lucene/demo/html/SimpleCharStream.java?rev=826549&r1=826548&r2=826549&view=diff ============================================================================== --- lucene/java/trunk/src/demo/org/apache/lucene/demo/html/SimpleCharStream.java (original) +++ lucene/java/trunk/src/demo/org/apache/lucene/demo/html/SimpleCharStream.java Sun Oct 18 23:21:13 2009 @@ -1,4 +1,5 @@ -/* Generated By:JavaCC: Do not edit this line. SimpleCharStream.java Version 4.0 */ +/* Generated By:JavaCC: Do not edit this line. SimpleCharStream.java Version 4.1 */ +/* JavaCCOptions:STATIC=false */ package org.apache.lucene.demo.html; /** @@ -8,10 +9,12 @@ public class SimpleCharStream { +/** Whether parser is static. */ public static final boolean staticFlag = false; int bufsize; int available; int tokenBegin; +/** Position in buffer. */ public int bufpos = -1; protected int bufline[]; protected int bufcolumn[]; @@ -128,6 +131,7 @@ } } +/** Start. */ public char BeginToken() throws java.io.IOException { tokenBegin = -1; @@ -177,6 +181,7 @@ bufcolumn[bufpos] = column; } +/** Read a character. */ public char readChar() throws java.io.IOException { if (inBuf > 0) @@ -195,11 +200,11 @@ char c = buffer[bufpos]; UpdateLineColumn(c); - return (c); + return c; } /** - * @deprecated + * @deprecated * @see #getEndColumn */ @@ -208,7 +213,7 @@ } /** - * @deprecated + * @deprecated * @see #getEndLine */ @@ -216,22 +221,27 @@ return bufline[bufpos]; } + /** Get token end column number. */ public int getEndColumn() { return bufcolumn[bufpos]; } + /** Get token end line number. */ public int getEndLine() { return bufline[bufpos]; } + /** Get token beginning column number. */ public int getBeginColumn() { return bufcolumn[tokenBegin]; } + /** Get token beginning line number. */ public int getBeginLine() { return bufline[tokenBegin]; } +/** Backup a number of characters. */ public void backup(int amount) { inBuf += amount; @@ -239,6 +249,7 @@ bufpos += bufsize; } + /** Constructor. */ public SimpleCharStream(java.io.Reader dstream, int startline, int startcolumn, int buffersize) { @@ -252,16 +263,20 @@ bufcolumn = new int[buffersize]; } + /** Constructor. */ public SimpleCharStream(java.io.Reader dstream, int startline, int startcolumn) { this(dstream, startline, startcolumn, 4096); } + /** Constructor. */ public SimpleCharStream(java.io.Reader dstream) { this(dstream, 1, 1, 4096); } + + /** Reinitialise. */ public void ReInit(java.io.Reader dstream, int startline, int startcolumn, int buffersize) { @@ -281,81 +296,96 @@ bufpos = -1; } + /** Reinitialise. */ public void ReInit(java.io.Reader dstream, int startline, int startcolumn) { ReInit(dstream, startline, startcolumn, 4096); } + /** Reinitialise. */ public void ReInit(java.io.Reader dstream) { ReInit(dstream, 1, 1, 4096); } + /** Constructor. */ public SimpleCharStream(java.io.InputStream dstream, String encoding, int startline, int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException { this(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize); } + /** Constructor. */ public SimpleCharStream(java.io.InputStream dstream, int startline, int startcolumn, int buffersize) { this(new java.io.InputStreamReader(dstream), startline, startcolumn, buffersize); } + /** Constructor. */ public SimpleCharStream(java.io.InputStream dstream, String encoding, int startline, int startcolumn) throws java.io.UnsupportedEncodingException { this(dstream, encoding, startline, startcolumn, 4096); } + /** Constructor. */ public SimpleCharStream(java.io.InputStream dstream, int startline, int startcolumn) { this(dstream, startline, startcolumn, 4096); } + /** Constructor. */ public SimpleCharStream(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException { this(dstream, encoding, 1, 1, 4096); } + /** Constructor. */ public SimpleCharStream(java.io.InputStream dstream) { this(dstream, 1, 1, 4096); } + /** Reinitialise. */ public void ReInit(java.io.InputStream dstream, String encoding, int startline, int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException { ReInit(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize); } + /** Reinitialise. */ public void ReInit(java.io.InputStream dstream, int startline, int startcolumn, int buffersize) { ReInit(new java.io.InputStreamReader(dstream), startline, startcolumn, buffersize); } + /** Reinitialise. */ public void ReInit(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException { ReInit(dstream, encoding, 1, 1, 4096); } + /** Reinitialise. */ public void ReInit(java.io.InputStream dstream) { ReInit(dstream, 1, 1, 4096); } + /** Reinitialise. */ public void ReInit(java.io.InputStream dstream, String encoding, int startline, int startcolumn) throws java.io.UnsupportedEncodingException { ReInit(dstream, encoding, startline, startcolumn, 4096); } + /** Reinitialise. */ public void ReInit(java.io.InputStream dstream, int startline, int startcolumn) { ReInit(dstream, startline, startcolumn, 4096); } + /** Get token literal value. */ public String GetImage() { if (bufpos >= tokenBegin) @@ -365,6 +395,7 @@ new String(buffer, 0, bufpos + 1); } + /** Get the suffix. */ public char[] GetSuffix(int len) { char[] ret = new char[len]; @@ -381,6 +412,7 @@ return ret; } + /** Reset buffer when finished. */ public void Done() { buffer = null; @@ -416,7 +448,7 @@ bufcolumn[j] = newCol + columnDiff; columnDiff = nextColDiff; i++; - } + } if (i < len) { @@ -437,3 +469,4 @@ } } +/* JavaCC - OriginalChecksum=7393ed4ac2709e2de22d164f9db78b65 (do not edit this line) */ Modified: lucene/java/trunk/src/demo/org/apache/lucene/demo/html/Token.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/demo/org/apache/lucene/demo/html/Token.java?rev=826549&r1=826548&r2=826549&view=diff ============================================================================== --- lucene/java/trunk/src/demo/org/apache/lucene/demo/html/Token.java (original) +++ lucene/java/trunk/src/demo/org/apache/lucene/demo/html/Token.java Sun Oct 18 23:21:13 2009 @@ -1,4 +1,5 @@ -/* Generated By:JavaCC: Do not edit this line. Token.java Version 3.0 */ +/* Generated By:JavaCC: Do not edit this line. Token.java Version 4.1 */ +/* JavaCCOptions:TOKEN_EXTENDS=,KEEP_LINE_COL=null */ package org.apache.lucene.demo.html; /** @@ -14,12 +15,14 @@ */ public int kind; - /** - * beginLine and beginColumn describe the position of the first character - * of this token; endLine and endColumn describe the position of the - * last character of this token. - */ - public int beginLine, beginColumn, endLine, endColumn; + /** The line number of the first character of this Token. */ + public int beginLine; + /** The column number of the first character of this Token. */ + public int beginColumn; + /** The line number of the last character of this Token. */ + public int endLine; + /** The column number of the last character of this Token. */ + public int endColumn; /** * The string image of the token. @@ -51,6 +54,40 @@ public Token specialToken; /** + * An optional attribute value of the Token. + * Tokens which are not used as syntactic sugar will often contain + * meaningful values that will be used later on by the compiler or + * interpreter. This attribute value is often different from the image. + * Any subclass of Token that actually wants to return a non-null value can + * override this method as appropriate. + */ + public Object getValue() { + return null; + } + + /** + * No-argument constructor + */ + public Token() {} + + /** + * Constructs a new token for the specified Image. + */ + public Token(int kind) + { + this(kind, null); + } + + /** + * Constructs a new token for the specified Image and Kind. + */ + public Token(int kind, String image) + { + this.kind = kind; + this.image = image; + } + + /** * Returns the image. */ public String toString() @@ -63,19 +100,25 @@ * can create and return subclass objects based on the value of ofKind. * Simply add the cases to the switch for all those special cases. * For example, if you have a subclass of Token called IDToken that - * you want to create if ofKind is ID, simlpy add something like : + * you want to create if ofKind is ID, simply add something like : * - * case MyParserConstants.ID : return new IDToken(); + * case MyParserConstants.ID : return new IDToken(ofKind, image); * * to the following switch statement. Then you can cast matchedToken - * variable to the appropriate type and use it in your lexical actions. + * variable to the appropriate type and use sit in your lexical actions. */ - public static final Token newToken(int ofKind) + public static Token newToken(int ofKind, String image) { switch(ofKind) { - default : return new Token(); + default : return new Token(ofKind, image); } } + public static Token newToken(int ofKind) + { + return newToken(ofKind, null); + } + } +/* JavaCC - OriginalChecksum=7bf8bdbb1c45bccd8162cdd48316d5e0 (do not edit this line) */ Modified: lucene/java/trunk/src/demo/org/apache/lucene/demo/html/TokenMgrError.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/demo/org/apache/lucene/demo/html/TokenMgrError.java?rev=826549&r1=826548&r2=826549&view=diff ============================================================================== --- lucene/java/trunk/src/demo/org/apache/lucene/demo/html/TokenMgrError.java (original) +++ lucene/java/trunk/src/demo/org/apache/lucene/demo/html/TokenMgrError.java Sun Oct 18 23:21:13 2009 @@ -1,19 +1,23 @@ -/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 3.0 */ +/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 4.1 */ +/* JavaCCOptions: */ package org.apache.lucene.demo.html; +/** Token Manager Error. */ +@SuppressWarnings("serial") public class TokenMgrError extends Error { + /* * Ordinals for various reasons why an Error of this type can be thrown. */ /** - * Lexical error occured. + * Lexical error occurred. */ static final int LEXICAL_ERROR = 0; /** - * An attempt wass made to create a second instance of a static token manager. + * An attempt was made to create a second instance of a static token manager. */ static final int STATIC_LEXER_ERROR = 1; @@ -34,7 +38,7 @@ int errorCode; /** - * Replaces unprintable characters by their espaced (or unicode escaped) + * Replaces unprintable characters by their escaped (or unicode escaped) * equivalents in the given string */ protected static final String addEscapes(String str) { @@ -85,12 +89,12 @@ /** * Returns a detailed message for the Error when it is thrown by the * token manager to indicate a lexical error. - * Parameters : - * EOFSeen : indicates if EOF caused the lexicl error - * curLexState : lexical state in which this error occured - * errorLine : line number when the error occured - * errorColumn : column number when the error occured - * errorAfter : prefix that was seen before this error occured + * Parameters : + * EOFSeen : indicates if EOF caused the lexical error + * curLexState : lexical state in which this error occurred + * errorLine : line number when the error occurred + * errorColumn : column number when the error occurred + * errorAfter : prefix that was seen before this error occurred * curchar : the offending character * Note: You can customize the lexical error message by modifying this method. */ @@ -105,7 +109,7 @@ /** * You can also modify the body of this method to customize your error messages. * For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not - * of end-users concern, so you can return something like : + * of end-users concern, so you can return something like : * * "Internal Error : Please file a bug report .... " * @@ -119,15 +123,19 @@ * Constructors of various flavors follow. */ + /** No arg constructor. */ public TokenMgrError() { } + /** Constructor with message and reason. */ public TokenMgrError(String message, int reason) { super(message); errorCode = reason; } + /** Full Constructor. */ public TokenMgrError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar, int reason) { this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason); } } +/* JavaCC - OriginalChecksum=5ffb7e46d5ae93d8d59e6f4ae7eb36d1 (do not edit this line) */ Modified: lucene/java/trunk/src/java/org/apache/lucene/queryParser/CharStream.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/queryParser/CharStream.java?rev=826549&r1=826548&r2=826549&view=diff ============================================================================== --- lucene/java/trunk/src/java/org/apache/lucene/queryParser/CharStream.java (original) +++ lucene/java/trunk/src/java/org/apache/lucene/queryParser/CharStream.java Sun Oct 18 23:21:13 2009 @@ -109,4 +109,4 @@ void Done(); } -/* JavaCC - OriginalChecksum=a83909a2403f969f94d18375f9f143e4 (do not edit this line) */ +/* JavaCC - OriginalChecksum=32a89423891f765dde472f7ef0e3ef7b (do not edit this line) */ Modified: lucene/java/trunk/src/java/org/apache/lucene/queryParser/ParseException.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/queryParser/ParseException.java?rev=826549&r1=826548&r2=826549&view=diff ============================================================================== --- lucene/java/trunk/src/java/org/apache/lucene/queryParser/ParseException.java (original) +++ lucene/java/trunk/src/java/org/apache/lucene/queryParser/ParseException.java Sun Oct 18 23:21:13 2009 @@ -195,4 +195,4 @@ } } -/* JavaCC - OriginalChecksum=c63b396885c4ff44d7aa48d3feae60cd (do not edit this line) */ +/* JavaCC - OriginalChecksum=c7631a240f7446940695eac31d9483ca (do not edit this line) */ Modified: lucene/java/trunk/src/java/org/apache/lucene/queryParser/QueryParser.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/queryParser/QueryParser.java?rev=826549&r1=826548&r2=826549&view=diff ============================================================================== --- lucene/java/trunk/src/java/org/apache/lucene/queryParser/QueryParser.java (original) +++ lucene/java/trunk/src/java/org/apache/lucene/queryParser/QueryParser.java Sun Oct 18 23:21:13 2009 @@ -1754,7 +1754,7 @@ return (jj_ntk = jj_nt.kind); } - private java.util.List jj_expentries = new java.util.ArrayList(); + private java.util.List jj_expentries = new java.util.ArrayList(); private int[] jj_expentry; private int jj_kind = -1; private int[] jj_lasttokens = new int[100]; @@ -1817,7 +1817,7 @@ jj_add_error_token(0, 0); int[][] exptokseq = new int[jj_expentries.size()][]; for (int i = 0; i < jj_expentries.size(); i++) { - exptokseq[i] = (int[])jj_expentries.get(i); + exptokseq[i] = jj_expentries.get(i); } return new ParseException(token, exptokseq, tokenImage); } Modified: lucene/java/trunk/src/java/org/apache/lucene/queryParser/Token.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/queryParser/Token.java?rev=826549&r1=826548&r2=826549&view=diff ============================================================================== --- lucene/java/trunk/src/java/org/apache/lucene/queryParser/Token.java (original) +++ lucene/java/trunk/src/java/org/apache/lucene/queryParser/Token.java Sun Oct 18 23:21:13 2009 @@ -121,4 +121,4 @@ } } -/* JavaCC - OriginalChecksum=37b1923f964a5a434f5ea3d6952ff200 (do not edit this line) */ +/* JavaCC - OriginalChecksum=c147cc166a7cf8812c7c39bc8c5eb868 (do not edit this line) */ Modified: lucene/java/trunk/src/java/org/apache/lucene/queryParser/TokenMgrError.java URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/queryParser/TokenMgrError.java?rev=826549&r1=826548&r2=826549&view=diff ============================================================================== --- lucene/java/trunk/src/java/org/apache/lucene/queryParser/TokenMgrError.java (original) +++ lucene/java/trunk/src/java/org/apache/lucene/queryParser/TokenMgrError.java Sun Oct 18 23:21:13 2009 @@ -3,6 +3,7 @@ package org.apache.lucene.queryParser; /** Token Manager Error. */ +@SuppressWarnings("serial") public class TokenMgrError extends Error { @@ -137,4 +138,4 @@ this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason); } } -/* JavaCC - OriginalChecksum=55cddb2336a66b376c0bb59d916b326d (do not edit this line) */ +/* JavaCC - OriginalChecksum=1c94e13236c7e0121e49427992341ee3 (do not edit this line) */