Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 61B80200C20 for ; Sat, 4 Feb 2017 21:32:23 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 60476160B69; Sat, 4 Feb 2017 20:32:23 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 53926160B63 for ; Sat, 4 Feb 2017 21:32:21 +0100 (CET) Received: (qmail 96090 invoked by uid 500); 4 Feb 2017 20:32:20 -0000 Mailing-List: contact commits-help@lucenenet.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: lucene-net-dev@lucenenet.apache.org Delivered-To: mailing list commits@lucenenet.apache.org Received: (qmail 96040 invoked by uid 99); 4 Feb 2017 20:32:20 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 04 Feb 2017 20:32:20 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1D3ABDFC60; Sat, 4 Feb 2017 20:32:20 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: nightowl888@apache.org To: commits@lucenenet.apache.org Date: Sat, 04 Feb 2017 20:32:24 -0000 Message-Id: <5e567bc45d824d0dabceae35cacf7867@git.apache.org> In-Reply-To: <4b623b494bc34c2780b01bf40ded92c7@git.apache.org> References: <4b623b494bc34c2780b01bf40ded92c7@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [05/39] lucenenet git commit: Lucene.Net.Analysis.Standard refactor: member accessibility and documentation comments archived-at: Sat, 04 Feb 2017 20:32:23 -0000 http://git-wip-us.apache.org/repos/asf/lucenenet/blob/ab69b431/src/Lucene.Net.Analysis.Common/Analysis/Standard/Std34/StandardTokenizerImpl34.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Standard/Std34/StandardTokenizerImpl34.cs b/src/Lucene.Net.Analysis.Common/Analysis/Standard/Std34/StandardTokenizerImpl34.cs index fd446a3..855470d 100644 --- a/src/Lucene.Net.Analysis.Common/Analysis/Standard/Std34/StandardTokenizerImpl34.cs +++ b/src/Lucene.Net.Analysis.Common/Analysis/Standard/Std34/StandardTokenizerImpl34.cs @@ -29,26 +29,26 @@ namespace Lucene.Net.Analysis.Standard.Std34 [Obsolete("This class is only for exact backwards compatibility")] public sealed class StandardTokenizerImpl34 : IStandardTokenizerInterface { - /** This character denotes the end of file */ + /// This character denotes the end of file public static readonly int YYEOF = -1; - /** initial size of the lookahead buffer */ + /// initial size of the lookahead buffer private static readonly int ZZ_BUFFERSIZE = 4096; - /** lexical states */ + /// lexical states public const int YYINITIAL = 0; - /** - * ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l - * ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l - * at the beginning of a line - * l is of the form l = 2*k, k a non negative integer - */ + /// + /// ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l + /// ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l + /// at the beginning of a line + /// l is of the form l = 2*k, k a non negative integer + /// private static readonly int[] ZZ_LEXSTATE = { 0, 0 }; - /** - * Translates characters to character classes - */ + /// + /// Translates characters to character classes + /// private const string ZZ_CMAP_PACKED = "\x0027\x0000\x0001\x0060\x0004\x0000\x0001\x005F\x0001\x0000\x0001\x0060\x0001\x0000\x000A\x005C\x0001\x005E\x0001\x005F" + "\x0005\x0000\x001A\x005A\x0004\x0000\x0001\x0061\x0001\x0000\x001A\x005A\x002F\x0000\x0001\x005A\x0002\x0000\x0001\x005B" + @@ -184,14 +184,14 @@ namespace Lucene.Net.Analysis.Standard.Std34 "\x000B\x0000\x0038\x005D\x0002\x005B\x001F\x0066\x0003\x0000\x0006\x0066\x0002\x0000\x0006\x0066\x0002\x0000\x0006\x0066" + "\x0002\x0000\x0003\x0066\x001C\x0000\x0003\x005B\x0004\x0000"; - /** - * Translates characters to character classes - */ + /// + /// Translates characters to character classes + /// private static readonly char[] ZZ_CMAP = ZzUnpackCMap(ZZ_CMAP_PACKED); - /** - * Translates DFA states to action switch labels. - */ + /// + /// Translates DFA states to action switch labels. + /// private static readonly int[] ZZ_ACTION = ZzUnpackAction(); private const string ZZ_ACTION_PACKED_0 = @@ -222,9 +222,9 @@ namespace Lucene.Net.Analysis.Standard.Std34 } - /** - * Translates a state to a row index in the transition table - */ + /// + /// Translates a state to a row index in the transition table + /// private static readonly int[] ZZ_ROWMAP = ZzUnpackRowMap(); private const string ZZ_ROWMAP_PACKED_0 = @@ -266,9 +266,9 @@ namespace Lucene.Net.Analysis.Standard.Std34 return j; } - /** - * The transition table of the DFA - */ + /// + /// The transition table of the DFA + /// private static readonly int[] ZZ_TRANS = ZzUnpackTrans(); private const string ZZ_TRANS_PACKED_0 = @@ -631,9 +631,9 @@ namespace Lucene.Net.Analysis.Standard.Std34 "Error: pushback value was too large" }; - /** - * ZZ_ATTRIBUTE[aState] contains the attributes of state aState - */ + /// + /// ZZ_ATTRIBUTE[aState] contains the attributes of state aState + /// private static readonly int[] ZZ_ATTRIBUTE = ZzUnpackAttribute(); private const string ZZ_ATTRIBUTE_PACKED_0 = @@ -662,73 +662,78 @@ namespace Lucene.Net.Analysis.Standard.Std34 return j; } - /** the input device */ + /// the input device private TextReader zzReader; - /** the current state of the DFA */ + /// the current state of the DFA private int zzState; - /** the current lexical state */ + /// the current lexical state private int zzLexicalState = YYINITIAL; - /** this buffer contains the current text to be matched and is - the source of the YyText() string */ + /// + /// this buffer contains the current text to be matched and is + /// the source of the YyText string + /// private char[] zzBuffer = new char[ZZ_BUFFERSIZE]; - /** the textposition at the last accepting state */ + /// the textposition at the last accepting state private int zzMarkedPos; - /** the current text position in the buffer */ + /// the current text position in the buffer private int zzCurrentPos; - /** startRead marks the beginning of the YyText() string in the buffer */ + /// startRead marks the beginning of the YyText string in the buffer private int zzStartRead; - /** endRead marks the last character in the buffer, that has been read - from input */ + /// + /// endRead marks the last character in the buffer, that has been read + /// from input + /// private int zzEndRead; - /** number of newlines encountered up to the start of the matched text */ + /// number of newlines encountered up to the start of the matched text private int yyline; - /** the number of characters up to the start of the matched text */ + /// the number of characters up to the start of the matched text private int yyChar; #pragma warning disable 169, 414 - /** - * the number of characters from the last newline up to the start of the - * matched text - */ + + /// + /// the number of characters from the last newline up to the start of the + /// matched text + /// private int yycolumn; - /** - * zzAtBOL == true <=> the scanner is currently at the beginning of a line - */ + /// + /// zzAtBOL == true <=> the scanner is currently at the beginning of a line + /// private bool zzAtBOL = true; - /** zzAtEOF == true <=> the scanner is at the EOF */ + /// zzAtEOF == true <=> the scanner is at the EOF private bool zzAtEOF; - /** denotes if the user-EOF-code has already been executed */ + /// denotes if the user-EOF-code has already been executed private bool zzEOFDone; #pragma warning restore 169, 414 /* user code: */ - /** Alphanumeric sequences */ + /// Alphanumeric sequences public static readonly int WORD_TYPE = StandardTokenizer.ALPHANUM; - /** Numbers */ + /// Numbers public static readonly int NUMERIC_TYPE = StandardTokenizer.NUM; - /** - * Chars in class \p{Line_Break = Complex_Context} are from South East Asian - * scripts (Thai, Lao, Myanmar, Khmer, etc.). Sequences of these are kept - * together as as a single token rather than broken up, because the logic - * required to break them at word boundaries is too complex for UAX#29. - *

- * See Unicode Line Breaking Algorithm: http://www.unicode.org/reports/tr14/#SA - */ + ///

+ /// Chars in class \p{Line_Break = Complex_Context} are from South East Asian + /// scripts (Thai, Lao, Myanmar, Khmer, etc.). Sequences of these are kept + /// together as as a single token rather than broken up, because the logic + /// required to break them at word boundaries is too complex for UAX#29. + /// + /// See Unicode Line Breaking Algorithm: http://www.unicode.org/reports/tr14/#SA + /// public static readonly int SOUTH_EAST_ASIAN_TYPE = StandardTokenizer.SOUTHEAST_ASIAN; public static readonly int IDEOGRAPHIC_TYPE = StandardTokenizer.IDEOGRAPHIC; @@ -744,32 +749,28 @@ namespace Lucene.Net.Analysis.Standard.Std34 get { return yyChar; } } - /** - * Fills CharTermAttribute with the current token text. - */ + /// + /// Fills ICharTermAttribute with the current token text. + /// public void GetText(ICharTermAttribute t) { t.CopyBuffer(zzBuffer, zzStartRead, zzMarkedPos - zzStartRead); } - - /** - * Creates a new scanner - * - * @param in the TextReader to read input from. - */ + /// + /// Creates a new scanner + /// + /// the TextReader to read input from. public StandardTokenizerImpl34(TextReader @in) { this.zzReader = @in; } - - /** - * Unpacks the compressed character translation table. - * - * @param packed the packed character translation table - * @return the unpacked character translation table - */ + /// + /// Unpacks the compressed character translation table. + /// + /// the packed character translation table + /// the unpacked character translation table private static char[] ZzUnpackCMap(string packed) { char[] map = new char[0x10000]; @@ -784,14 +785,11 @@ namespace Lucene.Net.Analysis.Standard.Std34 return map; } - - /** - * Refills the input buffer. - * - * @return false, iff there was new input. - * - * @exception java.io.IOException if any I/O-Error occurs - */ + /// + /// Refills the input buffer. + /// + /// false, iff there was new input. + /// if any I/O-Error occurs private bool ZzRefill() { @@ -847,9 +845,9 @@ namespace Lucene.Net.Analysis.Standard.Std34 } - /** - * Closes the input stream. - */ + /// + /// Disposes the input stream. + /// public void YyClose() { zzAtEOF = true; /* indicate end of file */ @@ -859,19 +857,17 @@ namespace Lucene.Net.Analysis.Standard.Std34 zzReader.Dispose(); } - - /** - * Resets the scanner to read from a new input stream. - * Does not close the old reader. - * - * All internal variables are reset, the old input stream - * cannot be reused (internal buffer is discarded and lost). - * Lexical state is set to ZZ_INITIAL. - * - * Internal scan buffer is resized down to its initial length, if it has grown. - * - * @param reader the new input stream - */ + /// + /// Resets the scanner to read from a new input stream. + /// Does not close the old reader. + /// + /// All internal variables are reset, the old input stream + /// cannot be reused (internal buffer is discarded and lost). + /// Lexical state is set to . + /// + /// Internal scan buffer is resized down to its initial length, if it has grown. + /// + /// the new input stream public void YyReset(TextReader reader) { zzReader = reader; @@ -887,75 +883,67 @@ namespace Lucene.Net.Analysis.Standard.Std34 } - /** - * Returns the current lexical state. - */ + /// + /// Returns the current lexical state. + /// public int YyState { get { return zzLexicalState; } } - - /** - * Enters a new lexical state - * - * @param newState the new lexical state - */ + /// + /// Enters a new lexical state + /// + /// the new lexical state public void YyBegin(int newState) { zzLexicalState = newState; } - - /** - * Returns the text matched by the current regular expression. - */ + /// + /// Returns the text matched by the current regular expression. + /// public string YyText { get { return new string(zzBuffer, zzStartRead, zzMarkedPos - zzStartRead); } } - - /** - * Returns the character at position pos from the - * matched text. - * - * It is equivalent to YyText().charAt(pos), but faster - * - * @param pos the position of the character to fetch. - * A value from 0 to YyLength()-1. - * - * @return the character at position pos - */ + /// + /// Returns the character at position from the + /// matched text. + /// + /// It is equivalent to YyText[pos], but faster + /// + /// + /// the position of the character to fetch. + /// A value from 0 to YyLength-1. + /// + /// the character at position pos public char YyCharAt(int pos) { return zzBuffer[zzStartRead + pos]; } - /** - * Returns the length of the matched text region. - */ + /// + /// Returns the length of the matched text region. + /// public int YyLength { get { return zzMarkedPos - zzStartRead; } } - - /** - * Reports an error that occured while scanning. - * - * In a wellformed scanner (no or only correct usage of - * YyPushBack(int) and a match-all fallback rule) this method - * will only be called with things that "Can't Possibly Happen". - * If this method is called, something is seriously wrong - * (e.g. a JFlex bug producing a faulty scanner etc.). - * - * Usual syntax/scanner level error handling should be done - * in error fallback rules. - * - * @param errorCode the code of the errormessage to display - */ + /// + /// Reports an error that occured while scanning. + /// + /// In a wellformed scanner (no or only correct usage of + /// YyPushBack(int) and a match-all fallback rule) this method + /// will only be called with things that "Can't Possibly Happen". + /// If this method is called, something is seriously wrong + /// Usual syntax/scanner level error handling should be done + /// in error fallback rules. + /// + /// the code of the errormessage to display private void ZzScanError(int errorCode) { string message; @@ -971,15 +959,15 @@ namespace Lucene.Net.Analysis.Standard.Std34 throw new Exception(message); } - - /** - * Pushes the specified amount of characters back into the input stream. - * - * They will be read again by then next call of the scanning method - * - * @param number the number of characters to be read again. - * This number must not be greater than YyLength()! - */ + /// + /// Pushes the specified amount of characters back into the input stream. + /// + /// They will be read again by then next call of the scanning method + /// + /// + /// the number of characters to be read again. + /// This number must not be greater than YyLength! + /// public void YyPushBack(int number) { if (number > YyLength) @@ -988,14 +976,12 @@ namespace Lucene.Net.Analysis.Standard.Std34 zzMarkedPos -= number; } - - /** - * Resumes scanning until the next regular expression is matched, - * the end of input is encountered or an I/O-Error occurs. - * - * @return the next token - * @exception java.io.IOException if any I/O-Error occurs - */ + /// + /// Resumes scanning until the next regular expression is matched, + /// the end of input is encountered or an I/O-Error occurs. + /// + /// the next token + /// if any I/O-Error occurs public int GetNextToken() { int zzInput; http://git-wip-us.apache.org/repos/asf/lucenenet/blob/ab69b431/src/Lucene.Net.Analysis.Common/Analysis/Standard/Std34/UAX29URLEmailTokenizerImpl34.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Standard/Std34/UAX29URLEmailTokenizerImpl34.cs b/src/Lucene.Net.Analysis.Common/Analysis/Standard/Std34/UAX29URLEmailTokenizerImpl34.cs index c2066de..d774d6d 100644 --- a/src/Lucene.Net.Analysis.Common/Analysis/Standard/Std34/UAX29URLEmailTokenizerImpl34.cs +++ b/src/Lucene.Net.Analysis.Common/Analysis/Standard/Std34/UAX29URLEmailTokenizerImpl34.cs @@ -32,26 +32,26 @@ namespace Lucene.Net.Analysis.Standard.Std34 [Obsolete("This class is only for exact backwards compatibility")] public sealed class UAX29URLEmailTokenizerImpl34 : IStandardTokenizerInterface { - /** This character denotes the end of file */ + /// This character denotes the end of file public static readonly int YYEOF = -1; - /** initial size of the lookahead buffer */ + /// initial size of the lookahead buffer private static readonly int ZZ_BUFFERSIZE = 4096; - /** lexical states */ + /// lexical states public const int YYINITIAL = 0; - /** - * ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l - * ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l - * at the beginning of a line - * l is of the form l = 2*k, k a non negative integer - */ + /// + /// ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l + /// ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l + /// at the beginning of a line + /// l is of the form l = 2*k, k a non negative integer + /// private static readonly int[] ZZ_LEXSTATE = { 0, 0 }; - /** - * Translates characters to character classes - */ + /// + /// Translates characters to character classes + /// private static readonly string ZZ_CMAP_PACKED = "\x0001\x009F\x0008\x009D\x0002\x009F\x0002\x009D\x0001\x009F\x0013\x009D\x0001\x00A0\x0001\x009C\x0001\x0097\x0001\x00A0" + "\x0001\x0090\x0001\x008E\x0001\x0093\x0002\x0091\x0002\x00A0\x0001\x0092\x0001\x0082\x0001\x0067\x0001\x0096\x0001\x0083" + @@ -195,14 +195,14 @@ namespace Lucene.Net.Analysis.Standard.Std34 "\x001A\x005A\x000B\x0000\x0038\x005D\x0002\x005B\x001F\x0066\x0003\x0000\x0006\x0066\x0002\x0000\x0006\x0066\x0002\x0000" + "\x0006\x0066\x0002\x0000\x0003\x0066\x001C\x0000\x0003\x005B\x0004\x0000"; - /** - * Translates characters to character classes - */ + /// + /// Translates characters to character classes + /// private static readonly char[] ZZ_CMAP = ZzUnpackCMap(ZZ_CMAP_PACKED); - /** - * Translates DFA states to action switch labels. - */ + /// + /// Translates DFA states to action switch labels. + /// private static readonly int[] ZZ_ACTION = ZzUnpackAction(); private const string ZZ_ACTION_PACKED_0 = @@ -247,9 +247,9 @@ namespace Lucene.Net.Analysis.Standard.Std34 } - /** - * Translates a state to a row index in the transition table - */ + /// + /// Translates a state to a row index in the transition table + /// private static readonly int[] ZZ_ROWMAP = ZzUnpackRowMap(); private const string ZZ_ROWMAP_PACKED_0 = @@ -448,9 +448,9 @@ namespace Lucene.Net.Analysis.Standard.Std34 return j; } - /** - * The transition table of the DFA - */ + /// + /// The transition table of the DFA + /// private static readonly int[] ZZ_TRANS = ZzUnpackTrans(); private const string ZZ_TRANS_PACKED_0 = @@ -3261,9 +3261,9 @@ namespace Lucene.Net.Analysis.Standard.Std34 "Error: pushback value was too large" }; - /** - * ZZ_ATTRIBUTE[aState] contains the attributes of state aState - */ + /// + /// ZZ_ATTRIBUTE[aState] contains the attributes of state aState + /// private static readonly int[] ZZ_ATTRIBUTE = ZzUnpackAttribute(); private const string ZZ_ATTRIBUTE_PACKED_0 = @@ -3304,73 +3304,77 @@ namespace Lucene.Net.Analysis.Standard.Std34 return j; } - /** the input device */ + /// the input device private TextReader zzReader; - /** the current state of the DFA */ + /// the current state of the DFA private int zzState; - /** the current lexical state */ + /// the current lexical state private int zzLexicalState = YYINITIAL; - /** this buffer contains the current text to be matched and is - the source of the YyText() string */ + /// + /// this buffer contains the current text to be matched and is + /// the source of the YyText string + /// private char[] zzBuffer = new char[ZZ_BUFFERSIZE]; - /** the textposition at the last accepting state */ + /// the textposition at the last accepting state private int zzMarkedPos; - /** the current text position in the buffer */ + /// the current text position in the buffer private int zzCurrentPos; - /** startRead marks the beginning of the YyText() string in the buffer */ + /// startRead marks the beginning of the YyText string in the buffer private int zzStartRead; - /** endRead marks the last character in the buffer, that has been read - from input */ + /// + /// endRead marks the last character in the buffer, that has been read + /// from input + /// private int zzEndRead; - /** number of newlines encountered up to the start of the matched text */ + /// number of newlines encountered up to the start of the matched text private int yyline; - /** the number of characters up to the start of the matched text */ + /// the number of characters up to the start of the matched text private int yychar; #pragma warning disable 169, 414 - /** - * the number of characters from the last newline up to the start of the - * matched text - */ + /// + /// the number of characters from the last newline up to the start of the + /// matched text + /// private int yycolumn; - /** - * zzAtBOL == true <=> the scanner is currently at the beginning of a line - */ + /// + /// zzAtBOL == true <=> the scanner is currently at the beginning of a line + /// private bool zzAtBOL = true; - /** zzAtEOF == true <=> the scanner is at the EOF */ + /// zzAtEOF == true <=> the scanner is at the EOF private bool zzAtEOF; - /** denotes if the user-EOF-code has already been executed */ + /// denotes if the user-EOF-code has already been executed private bool zzEOFDone; #pragma warning restore 169, 414 /* user code: */ - /** Alphanumeric sequences */ + /// Alphanumeric sequences public static readonly int WORD_TYPE = UAX29URLEmailTokenizer.ALPHANUM; - /** Numbers */ + /// Numbers public static readonly int NUMERIC_TYPE = UAX29URLEmailTokenizer.NUM; - /** - * Chars in class \p{Line_Break = Complex_Context} are from South East Asian - * scripts (Thai, Lao, Myanmar, Khmer, etc.). Sequences of these are kept - * together as as a single token rather than broken up, because the logic - * required to break them at word boundaries is too complex for UAX#29. - *

- * See Unicode Line Breaking Algorithm: http://www.unicode.org/reports/tr14/#SA - */ + ///

+ /// Chars in class \p{Line_Break = Complex_Context} are from South East Asian + /// scripts (Thai, Lao, Myanmar, Khmer, etc.). Sequences of these are kept + /// together as as a single token rather than broken up, because the logic + /// required to break them at word boundaries is too complex for UAX#29. + /// + /// See Unicode Line Breaking Algorithm: http://www.unicode.org/reports/tr14/#SA + /// public static readonly int SOUTH_EAST_ASIAN_TYPE = UAX29URLEmailTokenizer.SOUTHEAST_ASIAN; public static readonly int IDEOGRAPHIC_TYPE = UAX29URLEmailTokenizer.IDEOGRAPHIC; @@ -3390,32 +3394,28 @@ namespace Lucene.Net.Analysis.Standard.Std34 get { return yychar; } } - /** - * Fills CharTermAttribute with the current token text. - */ + /// + /// Fills ICharTermAttribute with the current token text. + /// public void GetText(ICharTermAttribute t) { t.CopyBuffer(zzBuffer, zzStartRead, zzMarkedPos - zzStartRead); } - - /** - * Creates a new scanner - * - * @param in the TextReader to read input from. - */ + /// + /// Creates a new scanner + /// + /// the TextReader to read input from. public UAX29URLEmailTokenizerImpl34(TextReader @in) { this.zzReader = @in; } - - /** - * Unpacks the compressed character translation table. - * - * @param packed the packed character translation table - * @return the unpacked character translation table - */ + /// + /// Unpacks the compressed character translation table. + /// + /// the packed character translation table + /// the unpacked character translation table private static char[] ZzUnpackCMap(string packed) { char[] map = new char[0x10000]; @@ -3430,14 +3430,11 @@ namespace Lucene.Net.Analysis.Standard.Std34 return map; } - - /** - * Refills the input buffer. - * - * @return false, iff there was new input. - * - * @exception java.io.IOException if any I/O-Error occurs - */ + /// + /// Refills the input buffer. + /// + /// false, iff there was new input. + /// if any I/O-Error occurs private bool ZzRefill() { @@ -3493,9 +3490,9 @@ namespace Lucene.Net.Analysis.Standard.Std34 } - /** - * Closes the input stream. - */ + /// + /// Disposes the input stream. + /// public void YyClose() { zzAtEOF = true; /* indicate end of file */ @@ -3507,19 +3504,17 @@ namespace Lucene.Net.Analysis.Standard.Std34 } } - - /** - * Resets the scanner to read from a new input stream. - * Does not close the old reader. - * - * All internal variables are reset, the old input stream - * cannot be reused (internal buffer is discarded and lost). - * Lexical state is set to ZZ_INITIAL. - * - * Internal scan buffer is resized down to its initial length, if it has grown. - * - * @param reader the new input stream - */ + /// + /// Resets the scanner to read from a new input stream. + /// Does not close the old reader. + /// + /// All internal variables are reset, the old input stream + /// cannot be reused (internal buffer is discarded and lost). + /// Lexical state is set to . + /// + /// Internal scan buffer is resized down to its initial length, if it has grown. + /// + /// the new input stream public void YyReset(TextReader reader) { zzReader = reader; @@ -3535,75 +3530,69 @@ namespace Lucene.Net.Analysis.Standard.Std34 } - /** - * Returns the current lexical state. - */ + /// + /// Returns the current lexical state. + /// public int YyState { get { return zzLexicalState; } } - - /** - * Enters a new lexical state - * - * @param newState the new lexical state - */ + /// + /// Enters a new lexical state + /// + /// the new lexical state public void YyBegin(int newState) { zzLexicalState = newState; } - - /** - * Returns the text matched by the current regular expression. - */ + /// + /// Returns the text matched by the current regular expression. + /// public string YyText { get { return new string(zzBuffer, zzStartRead, zzMarkedPos - zzStartRead); } } - - /** - * Returns the character at position pos from the - * matched text. - * - * It is equivalent to YyText().charAt(pos), but faster - * - * @param pos the position of the character to fetch. - * A value from 0 to YyLength()-1. - * - * @return the character at position pos - */ + /// + /// Returns the character at position from the + /// matched text. + /// + /// It is equivalent to YyText[pos], but faster + /// + /// + /// the position of the character to fetch. + /// A value from 0 to YyLength-1. + /// + /// the character at position pos public char YyCharAt(int pos) { return zzBuffer[zzStartRead + pos]; } - /** - * Returns the length of the matched text region. - */ + /// + /// Returns the length of the matched text region. + /// public int YyLength { get { return zzMarkedPos - zzStartRead; } } - - /** - * Reports an error that occured while scanning. - * - * In a wellformed scanner (no or only correct usage of - * YyPushBack(int) and a match-all fallback rule) this method - * will only be called with things that "Can't Possibly Happen". - * If this method is called, something is seriously wrong - * (e.g. a JFlex bug producing a faulty scanner etc.). - * - * Usual syntax/scanner level error handling should be done - * in error fallback rules. - * - * @param errorCode the code of the errormessage to display - */ + /// + /// Reports an error that occured while scanning. + /// + /// In a wellformed scanner (no or only correct usage of + /// YyPushBack(int) and a match-all fallback rule) this method + /// will only be called with things that "Can't Possibly Happen". + /// If this method is called, something is seriously wrong + /// (e.g. a JFlex bug producing a faulty scanner etc.). + /// + /// Usual syntax/scanner level error handling should be done + /// in error fallback rules. + /// + /// the code of the errormessage to display private void ZzScanError(int errorCode) { string message; @@ -3619,15 +3608,15 @@ namespace Lucene.Net.Analysis.Standard.Std34 throw new Exception(message); } - - /** - * Pushes the specified amount of characters back into the input stream. - * - * They will be read again by then next call of the scanning method - * - * @param number the number of characters to be read again. - * This number must not be greater than YyLength()! - */ + /// + /// Pushes the specified amount of characters back into the input stream. + /// + /// They will be read again by then next call of the scanning method + /// + /// + /// the number of characters to be read again. + /// This number must not be greater than YyLength! + /// public void YyPushBack(int number) { if (number > YyLength) @@ -3636,14 +3625,12 @@ namespace Lucene.Net.Analysis.Standard.Std34 zzMarkedPos -= number; } - - /** - * Resumes scanning until the next regular expression is matched, - * the end of input is encountered or an I/O-Error occurs. - * - * @return the next token - * @exception java.io.IOException if any I/O-Error occurs - */ + /// + /// Resumes scanning until the next regular expression is matched, + /// the end of input is encountered or an I/O-Error occurs. + /// + /// the next token + /// if any I/O-Error occurs public int GetNextToken() { int zzInput; http://git-wip-us.apache.org/repos/asf/lucenenet/blob/ab69b431/src/Lucene.Net.Analysis.Common/Analysis/Standard/Std36/UAX29URLEmailTokenizerImpl36.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Standard/Std36/UAX29URLEmailTokenizerImpl36.cs b/src/Lucene.Net.Analysis.Common/Analysis/Standard/Std36/UAX29URLEmailTokenizerImpl36.cs index 9d8f68d..afa04ff 100644 --- a/src/Lucene.Net.Analysis.Common/Analysis/Standard/Std36/UAX29URLEmailTokenizerImpl36.cs +++ b/src/Lucene.Net.Analysis.Common/Analysis/Standard/Std36/UAX29URLEmailTokenizerImpl36.cs @@ -28,26 +28,26 @@ namespace Lucene.Net.Analysis.Standard.Std36 [Obsolete("This class is only for exact backwards compatibility")] public sealed class UAX29URLEmailTokenizerImpl36 : IStandardTokenizerInterface { - /** This character denotes the end of file */ + /// This character denotes the end of file public static readonly int YYEOF = -1; - /** initial size of the lookahead buffer */ + /// initial size of the lookahead buffer private static readonly int ZZ_BUFFERSIZE = 4096; - /** lexical states */ + /// lexical states public const int YYINITIAL = 0; - /** - * ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l - * ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l - * at the beginning of a line - * l is of the form l = 2*k, k a non negative integer - */ + /// + /// ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l + /// ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l + /// at the beginning of a line + /// l is of the form l = 2*k, k a non negative integer + /// private static readonly int[] ZZ_LEXSTATE = { 0, 0 }; - /** - * Translates characters to character classes - */ + /// + /// Translates characters to character classes + /// private const string ZZ_CMAP_PACKED = "\x0001\x009F\x0008\x009D\x0002\x009F\x0002\x009D\x0001\x009F\x0013\x009D\x0001\x00A0\x0001\x009C\x0001\x0097\x0001\x00A0" + "\x0001\x0090\x0001\x008E\x0001\x0093\x0002\x0091\x0002\x00A0\x0001\x0092\x0001\x0082\x0001\x0067\x0001\x0096\x0001\x0083" + @@ -191,14 +191,14 @@ namespace Lucene.Net.Analysis.Standard.Std36 "\x001A\x005A\x000B\x0000\x0038\x005D\x0002\x005B\x001F\x0066\x0003\x0000\x0006\x0066\x0002\x0000\x0006\x0066\x0002\x0000" + "\x0006\x0066\x0002\x0000\x0003\x0066\x001C\x0000\x0003\x005B\x0004\x0000"; - /** - * Translates characters to character classes - */ + /// + /// Translates characters to character classes + /// private static readonly char[] ZZ_CMAP = ZzUnpackCMap(ZZ_CMAP_PACKED); - /** - * Translates DFA states to action switch labels. - */ + /// + /// Translates DFA states to action switch labels. + /// private static readonly int[] ZZ_ACTION = ZzUnpackAction(); private const string ZZ_ACTION_PACKED_0 = @@ -245,9 +245,9 @@ namespace Lucene.Net.Analysis.Standard.Std36 } - /** - * Translates a state to a row index in the transition table - */ + /// + /// Translates a state to a row index in the transition table + /// private static readonly int[] ZZ_ROWMAP = ZzUnpackRowMap(); private const string ZZ_ROWMAP_PACKED_0 = @@ -487,9 +487,9 @@ namespace Lucene.Net.Analysis.Standard.Std36 return j; } - /** - * The transition table of the DFA - */ + /// + /// The transition table of the DFA + /// private static readonly int[] ZZ_TRANS = ZzUnpackTrans(); private const string ZZ_TRANS_PACKED_0 = @@ -3694,9 +3694,9 @@ namespace Lucene.Net.Analysis.Standard.Std36 "Error: pushback value was too large" }; - /** - * ZZ_ATTRIBUTE[aState] contains the attributes of state aState - */ + /// + /// ZZ_ATTRIBUTE[aState] contains the attributes of state aState + /// private static readonly int[] ZZ_ATTRIBUTE = ZzUnpackAttribute(); private const string ZZ_ATTRIBUTE_PACKED_0 = @@ -3739,73 +3739,77 @@ namespace Lucene.Net.Analysis.Standard.Std36 return j; } - /** the input device */ + /// the input device private TextReader zzReader; - /** the current state of the DFA */ + /// the current state of the DFA private int zzState; - /** the current lexical state */ + /// the current lexical state private int zzLexicalState = YYINITIAL; - /** this buffer contains the current text to be matched and is - the source of the YyText() string */ + /// + /// this buffer contains the current text to be matched and is + /// the source of the YyText string + /// private char[] zzBuffer = new char[ZZ_BUFFERSIZE]; - /** the textposition at the last accepting state */ + /// the textposition at the last accepting state private int zzMarkedPos; - /** the current text position in the buffer */ + /// the current text position in the buffer private int zzCurrentPos; - /** startRead marks the beginning of the YyText() string in the buffer */ + /// startRead marks the beginning of the YyText string in the buffer private int zzStartRead; - /** endRead marks the last character in the buffer, that has been read - from input */ + /// + /// endRead marks the last character in the buffer, that has been read + /// from input + /// private int zzEndRead; - /** number of newlines encountered up to the start of the matched text */ + /// number of newlines encountered up to the start of the matched text private int yyline; - /** the number of characters up to the start of the matched text */ + /// the number of characters up to the start of the matched text private int yychar; #pragma warning disable 169, 414 - /** - * the number of characters from the last newline up to the start of the - * matched text - */ + /// + /// the number of characters from the last newline up to the start of the + /// matched text + /// private int yycolumn; - /** - * zzAtBOL == true <=> the scanner is currently at the beginning of a line - */ + /// + /// zzAtBOL == true <=> the scanner is currently at the beginning of a line + /// private bool zzAtBOL = true; - /** zzAtEOF == true <=> the scanner is at the EOF */ + /// zzAtEOF == true <=> the scanner is at the EOF private bool zzAtEOF; - /** denotes if the user-EOF-code has already been executed */ + /// denotes if the user-EOF-code has already been executed private bool zzEOFDone; #pragma warning restore 169, 414 /* user code: */ - /** Alphanumeric sequences */ + /// Alphanumeric sequences public static readonly int WORD_TYPE = UAX29URLEmailTokenizer.ALPHANUM; - /** Numbers */ + /// Numbers public static readonly int NUMERIC_TYPE = UAX29URLEmailTokenizer.NUM; - /** - * Chars in class \p{Line_Break = Complex_Context} are from South East Asian - * scripts (Thai, Lao, Myanmar, Khmer, etc.). Sequences of these are kept - * together as as a single token rather than broken up, because the logic - * required to break them at word boundaries is too complex for UAX#29. - *

- * See Unicode Line Breaking Algorithm: http://www.unicode.org/reports/tr14/#SA - */ + ///

+ /// Chars in class \p{Line_Break = Complex_Context} are from South East Asian + /// scripts (Thai, Lao, Myanmar, Khmer, etc.). Sequences of these are kept + /// together as as a single token rather than broken up, because the logic + /// required to break them at word boundaries is too complex for UAX#29. + /// + /// See Unicode Line Breaking Algorithm: http://www.unicode.org/reports/tr14/#SA + /// public static readonly int SOUTH_EAST_ASIAN_TYPE = UAX29URLEmailTokenizer.SOUTHEAST_ASIAN; public static readonly int IDEOGRAPHIC_TYPE = UAX29URLEmailTokenizer.IDEOGRAPHIC; @@ -3825,32 +3829,28 @@ namespace Lucene.Net.Analysis.Standard.Std36 get { return yychar; } } - /** - * Fills CharTermAttribute with the current token text. - */ + /// + /// Fills ICharTermAttribute with the current token text. + /// public void GetText(ICharTermAttribute t) { t.CopyBuffer(zzBuffer, zzStartRead, zzMarkedPos - zzStartRead); } - - /** - * Creates a new scanner - * - * @param in the TextReader to read input from. - */ + /// + /// Creates a new scanner + /// + /// the TextReader to read input from. public UAX29URLEmailTokenizerImpl36(TextReader @in) { this.zzReader = @in; } - - /** - * Unpacks the compressed character translation table. - * - * @param packed the packed character translation table - * @return the unpacked character translation table - */ + /// + /// Unpacks the compressed character translation table. + /// + /// the packed character translation table + /// the unpacked character translation table private static char[] ZzUnpackCMap(string packed) { char[] map = new char[0x10000]; @@ -3865,14 +3865,11 @@ namespace Lucene.Net.Analysis.Standard.Std36 return map; } - - /** - * Refills the input buffer. - * - * @return false, iff there was new input. - * - * @exception java.io.IOException if any I/O-Error occurs - */ + /// + /// Refills the input buffer. + /// + /// false, iff there was new input. + /// if any I/O-Error occurs private bool ZzRefill() { @@ -3928,9 +3925,9 @@ namespace Lucene.Net.Analysis.Standard.Std36 } - /** - * Closes the input stream. - */ + /// + /// Disposes the input stream. + /// public void YyClose() { zzAtEOF = true; /* indicate end of file */ @@ -3942,19 +3939,17 @@ namespace Lucene.Net.Analysis.Standard.Std36 } } - - /** - * Resets the scanner to read from a new input stream. - * Does not close the old reader. - * - * All internal variables are reset, the old input stream - * cannot be reused (internal buffer is discarded and lost). - * Lexical state is set to ZZ_INITIAL. - * - * Internal scan buffer is resized down to its initial length, if it has grown. - * - * @param reader the new input stream - */ + /// + /// Resets the scanner to read from a new input stream. + /// Does not close the old reader. + /// + /// All internal variables are reset, the old input stream + /// cannot be reused (internal buffer is discarded and lost). + /// Lexical state is set to . + /// + /// Internal scan buffer is resized down to its initial length, if it has grown. + /// + /// the new input stream public void YyReset(TextReader reader) { zzReader = reader; @@ -3970,75 +3965,71 @@ namespace Lucene.Net.Analysis.Standard.Std36 } - /** - * Returns the current lexical state. - */ + /// + /// Returns the current lexical state. + /// public int YyState { get { return zzLexicalState; } } - - /** - * Enters a new lexical state - * - * @param newState the new lexical state - */ + /// + /// Enters a new lexical state + /// + /// the new lexical state public void YyBegin(int newState) { zzLexicalState = newState; } - - /** - * Returns the text matched by the current regular expression. - */ + /// + /// Returns the text matched by the current regular expression. + /// public string YyText { get { return new string(zzBuffer, zzStartRead, zzMarkedPos - zzStartRead); } } - /** - * Returns the character at position pos from the - * matched text. - * - * It is equivalent to YyText().charAt(pos), but faster - * - * @param pos the position of the character to fetch. - * A value from 0 to YyLength()-1. - * - * @return the character at position pos - */ + /// + /// Returns the character at position from the + /// matched text. + /// + /// It is equivalent to YyText[pos], but faster + /// + /// + /// the position of the character to fetch. + /// A value from 0 to YyLength-1. + /// + /// the character at position pos public char YyCharAt(int pos) { return zzBuffer[zzStartRead + pos]; } - /** - * Returns the length of the matched text region. - */ + /// + /// Returns the length of the matched text region. + /// public int YyLength { get { return zzMarkedPos - zzStartRead; } } - /** - * Reports an error that occured while scanning. - * - * In a wellformed scanner (no or only correct usage of - * YyPushBack(int) and a match-all fallback rule) this method - * will only be called with things that "Can't Possibly Happen". - * If this method is called, something is seriously wrong - * (e.g. a JFlex bug producing a faulty scanner etc.). - * - * Usual syntax/scanner level error handling should be done - * in error fallback rules. - * - * @param errorCode the code of the errormessage to display - */ + /// + /// Reports an error that occured while scanning. + /// + /// In a wellformed scanner (no or only correct usage of + /// YyPushBack(int) and a match-all fallback rule) this method + /// will only be called with things that "Can't Possibly Happen". + /// If this method is called, something is seriously wrong + /// (e.g. a JFlex bug producing a faulty scanner etc.). + /// + /// Usual syntax/scanner level error handling should be done + /// in error fallback rules. + /// + /// the code of the errormessage to display private void ZzScanError(int errorCode) { string message; @@ -4055,14 +4046,15 @@ namespace Lucene.Net.Analysis.Standard.Std36 } - /** - * Pushes the specified amount of characters back into the input stream. - * - * They will be read again by then next call of the scanning method - * - * @param number the number of characters to be read again. - * This number must not be greater than YyLength()! - */ + /// + /// Pushes the specified amount of characters back into the input stream. + /// + /// They will be read again by then next call of the scanning method + /// + /// + /// the number of characters to be read again. + /// This number must not be greater than YyLength! + /// public void YyPushBack(int number) { if (number > YyLength) @@ -4072,13 +4064,12 @@ namespace Lucene.Net.Analysis.Standard.Std36 } - /** - * Resumes scanning until the next regular expression is matched, - * the end of input is encountered or an I/O-Error occurs. - * - * @return the next token - * @exception java.io.IOException if any I/O-Error occurs - */ + /// + /// Resumes scanning until the next regular expression is matched, + /// the end of input is encountered or an I/O-Error occurs. + /// + /// the next token + /// if any I/O-Error occurs public int GetNextToken() { int zzInput; http://git-wip-us.apache.org/repos/asf/lucenenet/blob/ab69b431/src/Lucene.Net.Analysis.Common/Analysis/Standard/Std40/StandardTokenizerImpl40.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Standard/Std40/StandardTokenizerImpl40.cs b/src/Lucene.Net.Analysis.Common/Analysis/Standard/Std40/StandardTokenizerImpl40.cs index c0be18d..59bd49d 100644 --- a/src/Lucene.Net.Analysis.Common/Analysis/Standard/Std40/StandardTokenizerImpl40.cs +++ b/src/Lucene.Net.Analysis.Common/Analysis/Standard/Std40/StandardTokenizerImpl40.cs @@ -29,26 +29,26 @@ namespace Lucene.Net.Analysis.Standard.Std40 [Obsolete("This class is only for exact backwards compatibility")] public sealed class StandardTokenizerImpl40 : IStandardTokenizerInterface { - /** This character denotes the end of file */ + /// This character denotes the end of file public static readonly int YYEOF = -1; - /** initial size of the lookahead buffer */ + /// initial size of the lookahead buffer private static readonly int ZZ_BUFFERSIZE = 4096; - /** lexical states */ + /// lexical states public const int YYINITIAL = 0; - /** - * ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l - * ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l - * at the beginning of a line - * l is of the form l = 2*k, k a non negative integer - */ + /// + /// ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l + /// ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l + /// at the beginning of a line + /// l is of the form l = 2*k, k a non negative integer + /// private static readonly int[] ZZ_LEXSTATE = { 0, 0 }; - /** - * Translates characters to character classes - */ + /// + /// Translates characters to character classes + /// private const string ZZ_CMAP_PACKED = "\x0027\x0000\x0001\x0082\x0004\x0000\x0001\x0081\x0001\x0000\x0001\x0082\x0001\x0000\x000A\x007E\x0001\x0080\x0001\x0081" + "\x0005\x0000\x001A\x007C\x0004\x0000\x0001\x0083\x0001\x0000\x001A\x007C\x002F\x0000\x0001\x007C\x0002\x0000\x0001\x007D" + @@ -194,14 +194,14 @@ namespace Lucene.Net.Analysis.Standard.Std40 "\x0038\x007F\x0002\x007D\x001F\x0088\x0003\x0000\x0006\x0088\x0002\x0000\x0006\x0088\x0002\x0000\x0006\x0088\x0002\x0000" + "\x0003\x0088\x001C\x0000\x0003\x007D\x0004\x0000"; - /** - * Translates characters to character classes - */ + /// + /// Translates characters to character classes + /// private static readonly char[] ZZ_CMAP = ZzUnpackCMap(ZZ_CMAP_PACKED); - /** - * Translates DFA states to action switch labels. - */ + /// + /// Translates DFA states to action switch labels. + /// private static readonly int[] ZZ_ACTION = ZzUnpackAction(); private const string ZZ_ACTION_PACKED_0 = @@ -232,9 +232,9 @@ namespace Lucene.Net.Analysis.Standard.Std40 } - /** - * Translates a state to a row index in the transition table - */ + /// + /// Translates a state to a row index in the transition table + /// private static readonly int[] ZZ_ROWMAP = ZzUnpackRowMap(); private const string ZZ_ROWMAP_PACKED_0 = @@ -280,9 +280,9 @@ namespace Lucene.Net.Analysis.Standard.Std40 return j; } - /** - * The transition table of the DFA - */ + /// + /// The transition table of the DFA + /// private static readonly int[] ZZ_TRANS = ZzUnpackTrans(); private const string ZZ_TRANS_PACKED_0 = @@ -738,9 +738,9 @@ namespace Lucene.Net.Analysis.Standard.Std40 "Error: pushback value was too large" }; - /** - * ZZ_ATTRIBUTE[aState] contains the attributes of state aState - */ + /// + /// ZZ_ATTRIBUTE[aState] contains the attributes of state aState + /// private static readonly int[] ZZ_ATTRIBUTE = ZzUnpackAttribute(); private const string ZZ_ATTRIBUTE_PACKED_0 = @@ -769,73 +769,77 @@ namespace Lucene.Net.Analysis.Standard.Std40 return j; } - /** the input device */ + /// the input device private TextReader zzReader; - /** the current state of the DFA */ + /// the current state of the DFA private int zzState; - /** the current lexical state */ + /// the current lexical state private int zzLexicalState = YYINITIAL; - /** this buffer contains the current text to be matched and is - the source of the YyText() string */ + /// + /// this buffer contains the current text to be matched and is + /// the source of the YyText string + /// private char[] zzBuffer = new char[ZZ_BUFFERSIZE]; - /** the textposition at the last accepting state */ + /// the textposition at the last accepting state private int zzMarkedPos; - /** the current text position in the buffer */ + /// the current text position in the buffer private int zzCurrentPos; - /** startRead marks the beginning of the YyText() string in the buffer */ + /// startRead marks the beginning of the YyText string in the buffer private int zzStartRead; - /** endRead marks the last character in the buffer, that has been read - from input */ + /// + /// endRead marks the last character in the buffer, that has been read + /// from input + /// private int zzEndRead; - /** number of newlines encountered up to the start of the matched text */ + /// number of newlines encountered up to the start of the matched text private int yyline; - /** the number of characters up to the start of the matched text */ + /// the number of characters up to the start of the matched text private int yyChar; #pragma warning disable 169, 414 - /** - * the number of characters from the last newline up to the start of the - * matched text - */ + /// + /// the number of characters from the last newline up to the start of the + /// matched text + /// private int yycolumn; - /** - * zzAtBOL == true <=> the scanner is currently at the beginning of a line - */ + /// + /// zzAtBOL == true <=> the scanner is currently at the beginning of a line + /// private bool zzAtBOL = true; - /** zzAtEOF == true <=> the scanner is at the EOF */ + /// zzAtEOF == true <=> the scanner is at the EOF private bool zzAtEOF; - /** denotes if the user-EOF-code has already been executed */ + /// denotes if the user-EOF-code has already been executed private bool zzEOFDone; #pragma warning restore 169, 414 /* user code: */ - /** Alphanumeric sequences */ + /// Alphanumeric sequences public static readonly int WORD_TYPE = StandardTokenizer.ALPHANUM; - /** Numbers */ + /// Numbers public static readonly int NUMERIC_TYPE = StandardTokenizer.NUM; - /** - * Chars in class \p{Line_Break = Complex_Context} are from South East Asian - * scripts (Thai, Lao, Myanmar, Khmer, etc.). Sequences of these are kept - * together as as a single token rather than broken up, because the logic - * required to break them at word boundaries is too complex for UAX#29. - *

- * See Unicode Line Breaking Algorithm: http://www.unicode.org/reports/tr14/#SA - */ + ///

+ /// Chars in class \p{Line_Break = Complex_Context} are from South East Asian + /// scripts (Thai, Lao, Myanmar, Khmer, etc.). Sequences of these are kept + /// together as as a single token rather than broken up, because the logic + /// required to break them at word boundaries is too complex for UAX#29. + /// + /// See Unicode Line Breaking Algorithm: http://www.unicode.org/reports/tr14/#SA + /// public static readonly int SOUTH_EAST_ASIAN_TYPE = StandardTokenizer.SOUTHEAST_ASIAN; public static readonly int IDEOGRAPHIC_TYPE = StandardTokenizer.IDEOGRAPHIC; @@ -851,32 +855,30 @@ namespace Lucene.Net.Analysis.Standard.Std40 get { return yyChar; } } - /** - * Fills CharTermAttribute with the current token text. - */ + /// + /// Fills ICharTermAttribute with the current token text. + /// public void GetText(ICharTermAttribute t) { t.CopyBuffer(zzBuffer, zzStartRead, zzMarkedPos - zzStartRead); } - /** - * Creates a new scanner - * - * @param in the TextReader to read input from. - */ + /// + /// Creates a new scanner + /// + /// the TextReader to read input from. public StandardTokenizerImpl40(TextReader @in) { this.zzReader = @in; } - /** - * Unpacks the compressed character translation table. - * - * @param packed the packed character translation table - * @return the unpacked character translation table - */ + /// + /// Unpacks the compressed character translation table. + /// + /// the packed character translation table + /// the unpacked character translation table private static char[] ZzUnpackCMap(string packed) { char[] map = new char[0x10000]; @@ -892,13 +894,11 @@ namespace Lucene.Net.Analysis.Standard.Std40 } - /** - * Refills the input buffer. - * - * @return false, iff there was new input. - * - * @exception java.io.IOException if any I/O-Error occurs - */ + /// + /// Refills the input buffer. + /// + /// false, iff there was new input. + /// if any I/O-Error occurs private bool ZzRefill() { @@ -954,9 +954,9 @@ namespace Lucene.Net.Analysis.Standard.Std40 } - /** - * Closes the input stream. - */ + /// + /// Disposes the input stream. + /// public void YyClose() { zzAtEOF = true; /* indicate end of file */ @@ -967,18 +967,17 @@ namespace Lucene.Net.Analysis.Standard.Std40 } - /** - * Resets the scanner to read from a new input stream. - * Does not close the old reader. - * - * All internal variables are reset, the old input stream - * cannot be reused (internal buffer is discarded and lost). - * Lexical state is set to ZZ_INITIAL. - * - * Internal scan buffer is resized down to its initial length, if it has grown. - * - * @param reader the new input stream - */ + /// + /// Resets the scanner to read from a new input stream. + /// Does not close the old reader. + /// + /// All internal variables are reset, the old input stream + /// cannot be reused (internal buffer is discarded and lost). + /// Lexical state is set to . + /// + /// Internal scan buffer is resized down to its initial length, if it has grown. + /// + /// the new input stream public void YyReset(TextReader reader) { zzReader = reader; @@ -994,75 +993,73 @@ namespace Lucene.Net.Analysis.Standard.Std40 } - /** - * Returns the current lexical state. - */ + /// + /// Returns the current lexical state. + /// public int YyState { get { return zzLexicalState; } } - /** - * Enters a new lexical state - * - * @param newState the new lexical state - */ + /// + /// Enters a new lexical state + /// + /// the new lexical state public void YyBegin(int newState) { zzLexicalState = newState; } - /** - * Returns the text matched by the current regular expression. - */ + /// + /// Returns the text matched by the current regular expression. + /// public string YyText { get { return new string(zzBuffer, zzStartRead, zzMarkedPos - zzStartRead); } } - /** - * Returns the character at position pos from the - * matched text. - * - * It is equivalent to YyText().charAt(pos), but faster - * - * @param pos the position of the character to fetch. - * A value from 0 to YyLength()-1. - * - * @return the character at position pos - */ + /// + /// Returns the character at position from the + /// matched text. + /// + /// It is equivalent to YyText[pos], but faster + /// + /// + /// the position of the character to fetch. + /// A value from 0 to YyLength-1. + /// + /// the character at position pos public char YyCharAt(int pos) { return zzBuffer[zzStartRead + pos]; } - /** - * Returns the length of the matched text region. - */ + /// + /// Returns the length of the matched text region. + /// public int YyLength { get { return zzMarkedPos - zzStartRead; } } - /** - * Reports an error that occured while scanning. - * - * In a wellformed scanner (no or only correct usage of - * YyPushBack(int) and a match-all fallback rule) this method - * will only be called with things that "Can't Possibly Happen". - * If this method is called, something is seriously wrong - * (e.g. a JFlex bug producing a faulty scanner etc.). - * - * Usual syntax/scanner level error handling should be done - * in error fallback rules. - * - * @param errorCode the code of the errormessage to display - */ + /// + /// Reports an error that occured while scanning. + /// + /// In a wellformed scanner (no or only correct usage of + /// YyPushBack(int) and a match-all fallback rule) this method + /// will only be called with things that "Can't Possibly Happen". + /// If this method is called, something is seriously wrong + /// (e.g. a JFlex bug producing a faulty scanner etc.). + /// + /// Usual syntax/scanner level error handling should be done + /// in error fallback rules. + /// + /// the code of the errormessage to display private void ZzScanError(int errorCode) { string message; @@ -1079,14 +1076,15 @@ namespace Lucene.Net.Analysis.Standard.Std40 } - /** - * Pushes the specified amount of characters back into the input stream. - * - * They will be read again by then next call of the scanning method - * - * @param number the number of characters to be read again. - * This number must not be greater than YyLength()! - */ + /// + /// Pushes the specified amount of characters back into the input stream. + /// + /// They will be read again by then next call of the scanning method + /// + /// + /// the number of characters to be read again. + /// This number must not be greater than YyLength! + /// public void YyPushBack(int number) { if (number > YyLength) @@ -1096,13 +1094,12 @@ namespace Lucene.Net.Analysis.Standard.Std40 } - /** - * Resumes scanning until the next regular expression is matched, - * the end of input is encountered or an I/O-Error occurs. - * - * @return the next token - * @exception java.io.IOException if any I/O-Error occurs - */ + /// + /// Resumes scanning until the next regular expression is matched, + /// the end of input is encountered or an I/O-Error occurs. + /// + /// the next token + /// if any I/O-Error occurs public int GetNextToken() { int zzInput;