commons-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ebo...@apache.org
Subject svn commit: r1199872 - in /commons/sandbox/csv/trunk/src: main/java/org/apache/commons/csv/CSVParser.java test/java/org/apache/commons/csv/CSVParserTest.java
Date Wed, 09 Nov 2011 17:11:22 GMT
Author: ebourg
Date: Wed Nov  9 17:11:22 2011
New Revision: 1199872

URL: http://svn.apache.org/viewvc?rev=1199872&view=rev
Log:
Turned the token types into an Enum

Modified:
    commons/sandbox/csv/trunk/src/main/java/org/apache/commons/csv/CSVParser.java
    commons/sandbox/csv/trunk/src/test/java/org/apache/commons/csv/CSVParserTest.java

Modified: commons/sandbox/csv/trunk/src/main/java/org/apache/commons/csv/CSVParser.java
URL: http://svn.apache.org/viewvc/commons/sandbox/csv/trunk/src/main/java/org/apache/commons/csv/CSVParser.java?rev=1199872&r1=1199871&r2=1199872&view=diff
==============================================================================
--- commons/sandbox/csv/trunk/src/main/java/org/apache/commons/csv/CSVParser.java (original)
+++ commons/sandbox/csv/trunk/src/main/java/org/apache/commons/csv/CSVParser.java Wed Nov
 9 17:11:22 2011
@@ -22,6 +22,7 @@ import java.io.Reader;
 import java.util.ArrayList;
 import java.util.List;
 
+import static org.apache.commons.csv.CSVParser.Token.Type.*;
 
 /**
  * Parses CSV files according to the specified configuration.
@@ -54,19 +55,6 @@ public class CSVParser {
     /** length of the initial token (content-)buffer */
     private static final int INITIAL_TOKEN_LENGTH = 50;
 
-    // the token types
-    /** Token has no valid content, i.e. is in its initialized state. */
-    static final int TT_INVALID = -1;
-    
-    /** Token with content, at beginning or in the middle of a line. */
-    static final int TT_TOKEN = 0;
-    
-    /** Token (which can have content) when end of file is reached. */
-    static final int TT_EOF = 1;
-    
-    /** Token with content when end of a line is reached. */
-    static final int TT_EORECORD = 2;
-
     /** Immutable empty String array. */
     private static final String[] EMPTY_STRING_ARRAY = new String[0];
 
@@ -91,22 +79,33 @@ public class CSVParser {
      * It is used as contract between the lexer and the parser.
      */
     static class Token {
-        /**
-         * Token type, see TT_xxx constants.
-         */
-        int type = TT_INVALID;
-        /**
-         * The content buffer.
-         */
+
+        enum Type {
+            /** Token has no valid content, i.e. is in its initialized state. */
+            INVALID,
+            
+            /** Token with content, at beginning or in the middle of a line. */
+            TOKEN,
+            
+            /** Token (which can have content) when end of file is reached. */
+            EOF,
+            
+            /** Token with content when end of a line is reached. */
+            EORECORD
+        }
+        
+        /** Token type */
+        Type type = INVALID;
+        
+        /** The content buffer. */
         CharBuffer content = new CharBuffer(INITIAL_TOKEN_LENGTH);
-        /**
-         * Token ready flag: indicates a valid token with content (ready for the parser).
-         */
+        
+        /** Token ready flag: indicates a valid token with content (ready for the parser).
*/
         boolean isReady;
 
         Token reset() {
             content.clear();
-            type = TT_INVALID;
+            type = INVALID;
             isReady = false;
             return this;
         }
@@ -180,26 +179,26 @@ public class CSVParser {
             reusableToken.reset();
             nextToken(reusableToken);
             switch (reusableToken.type) {
-                case TT_TOKEN:
+                case TOKEN:
                     record.add(reusableToken.content.toString());
                     break;
-                case TT_EORECORD:
+                case EORECORD:
                     record.add(reusableToken.content.toString());
                     break;
-                case TT_EOF:
+                case EOF:
                     if (reusableToken.isReady) {
                         record.add(reusableToken.content.toString());
                     } else {
                         ret = null;
                     }
                     break;
-                case TT_INVALID:
+                case INVALID:
                 default:
                     // error: throw IOException
                     throw new IOException("(line " + getLineNumber() + ") invalid parse sequence");
                     // unreachable: break;
             }
-            if (reusableToken.type != TT_TOKEN) {
+            if (reusableToken.type != TOKEN) {
                 break;
             }
         }
@@ -272,19 +271,19 @@ public class CSVParser {
             c = in.readAgain();
             // reached end of file without any content (empty line at the end)
             if (isEndOfFile(c)) {
-                tkn.type = TT_EOF;
+                tkn.type = EOF;
                 return tkn;
             }
         }
 
-        // did we reach eof during the last iteration already ? TT_EOF
+        // did we reach eof during the last iteration already ? EOF
         if (isEndOfFile(lastChar) || (lastChar != format.getDelimiter() && isEndOfFile(c)))
{
-            tkn.type = TT_EOF;
+            tkn.type = EOF;
             return tkn;
         }
 
         //  important: make sure a new char gets consumed in each iteration
-        while (!tkn.isReady && tkn.type != TT_EOF) {
+        while (!tkn.isReady && tkn.type != EOF) {
             // ignore whitespaces at beginning of a token
             while (format.isLeadingSpacesIgnored() && isWhitespace(c) &&
!eol) {
                 wsBuf.append((char) c);
@@ -297,21 +296,21 @@ public class CSVParser {
                 in.readLine();
                 tkn = nextToken(tkn.reset());
             } else if (c == format.getDelimiter()) {
-                // empty token return TT_TOKEN("")
-                tkn.type = TT_TOKEN;
+                // empty token return TOKEN("")
+                tkn.type = TOKEN;
                 tkn.isReady = true;
             } else if (eol) {
-                // empty token return TT_EORECORD("")
+                // empty token return EORECORD("")
                 //noop: tkn.content.append("");
-                tkn.type = TT_EORECORD;
+                tkn.type = EORECORD;
                 tkn.isReady = true;
             } else if (c == format.getEncapsulator()) {
                 // consume encapsulated token
                 encapsulatedTokenLexer(tkn, c);
             } else if (isEndOfFile(c)) {
-                // end of file return TT_EOF()
+                // end of file return EOF()
                 //noop: tkn.content.append("");
-                tkn.type = TT_EOF;
+                tkn.type = EOF;
                 tkn.isReady = true;
             } else {
                 // next token must be a simple token
@@ -332,9 +331,9 @@ public class CSVParser {
      * A simple token might contain escaped delimiters (as \, or \;). The
      * token is finished when one of the following conditions become true:
      * <ul>
-     * <li>end of line has been reached (TT_EORECORD)</li>
-     * <li>end of stream has been reached (TT_EOF)</li>
-     * <li>an unescaped delimiter has been reached (TT_TOKEN)</li>
+     * <li>end of line has been reached (EORECORD)</li>
+     * <li>end of stream has been reached (EOF)</li>
+     * <li>an unescaped delimiter has been reached (TOKEN)</li>
      * </ul>
      *
      * @param tkn the current token
@@ -346,17 +345,17 @@ public class CSVParser {
         for (; ;) {
             if (isEndOfLine(c)) {
                 // end of record
-                tkn.type = TT_EORECORD;
+                tkn.type = EORECORD;
                 tkn.isReady = true;
                 break;
             } else if (isEndOfFile(c)) {
                 // end of file
-                tkn.type = TT_EOF;
+                tkn.type = EOF;
                 tkn.isReady = true;
                 break;
             } else if (c == format.getDelimiter()) {
                 // end of token
-                tkn.type = TT_TOKEN;
+                tkn.type = TOKEN;
                 tkn.isReady = true;
                 break;
             } else if (c == '\\' && format.isUnicodeEscapesInterpreted() &&
in.lookAhead() == 'u') {
@@ -414,16 +413,16 @@ public class CSVParser {
                     for (; ;) {
                         c = in.read();
                         if (c == format.getDelimiter()) {
-                            tkn.type = TT_TOKEN;
+                            tkn.type = TOKEN;
                             tkn.isReady = true;
                             return tkn;
                         } else if (isEndOfFile(c)) {
-                            tkn.type = TT_EOF;
+                            tkn.type = EOF;
                             tkn.isReady = true;
                             return tkn;
                         } else if (isEndOfLine(c)) {
                             // ok eo token reached
-                            tkn.type = TT_EORECORD;
+                            tkn.type = EORECORD;
                             tkn.isReady = true;
                             return tkn;
                         } else if (!isWhitespace(c)) {

Modified: commons/sandbox/csv/trunk/src/test/java/org/apache/commons/csv/CSVParserTest.java
URL: http://svn.apache.org/viewvc/commons/sandbox/csv/trunk/src/test/java/org/apache/commons/csv/CSVParserTest.java?rev=1199872&r1=1199871&r2=1199872&view=diff
==============================================================================
--- commons/sandbox/csv/trunk/src/test/java/org/apache/commons/csv/CSVParserTest.java (original)
+++ commons/sandbox/csv/trunk/src/test/java/org/apache/commons/csv/CSVParserTest.java Wed
Nov  9 17:11:22 2011
@@ -14,6 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.commons.csv;
 
 import java.io.IOException;
@@ -23,6 +24,8 @@ import java.util.Arrays;
 
 import junit.framework.TestCase;
 
+import static org.apache.commons.csv.CSVParser.Token.Type.*;
+
 /**
  * CSVParserTest
  *
@@ -60,7 +63,7 @@ public class CSVParserTest extends TestC
          */
         public String testNextToken() throws IOException {
             Token t = super.nextToken();
-            return Integer.toString(t.type) + ";" + t.content + ";";
+            return t.type.name() + ";" + t.content + ";";
         }
     }
 
@@ -72,16 +75,16 @@ public class CSVParserTest extends TestC
     public void testNextToken1() throws IOException {
         String code = "abc,def, hijk,  lmnop,   qrst,uv ,wxy   ,z , ,";
         TestCSVParser parser = new TestCSVParser(new StringReader(code));
-        assertEquals(CSVParser.TT_TOKEN + ";abc;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";def;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";hijk;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";lmnop;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";qrst;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";uv;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";wxy;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";z;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";;", parser.testNextToken());
-        assertEquals(CSVParser.TT_EOF + ";;", parser.testNextToken());
+        assertEquals(TOKEN + ";abc;", parser.testNextToken());
+        assertEquals(TOKEN + ";def;", parser.testNextToken());
+        assertEquals(TOKEN + ";hijk;", parser.testNextToken());
+        assertEquals(TOKEN + ";lmnop;", parser.testNextToken());
+        assertEquals(TOKEN + ";qrst;", parser.testNextToken());
+        assertEquals(TOKEN + ";uv;", parser.testNextToken());
+        assertEquals(TOKEN + ";wxy;", parser.testNextToken());
+        assertEquals(TOKEN + ";z;", parser.testNextToken());
+        assertEquals(TOKEN + ";;", parser.testNextToken());
+        assertEquals(EOF + ";;", parser.testNextToken());
     }
 
     // multiline including comments (and empty lines)
@@ -99,19 +102,19 @@ public class CSVParserTest extends TestC
         TestCSVParser parser = new TestCSVParser(new StringReader(code), format);
 
 
-        assertEquals(CSVParser.TT_TOKEN + ";1;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";2;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";3;", parser.testNextToken());
-        assertEquals(CSVParser.TT_EORECORD + ";;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";b x;", parser.testNextToken());
-        assertEquals(CSVParser.TT_EORECORD + ";c;", parser.testNextToken());
-        assertEquals(CSVParser.TT_EORECORD + ";;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";d;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";e;", parser.testNextToken());
-        assertEquals(CSVParser.TT_EORECORD + ";;", parser.testNextToken());
-        assertEquals(CSVParser.TT_EOF + ";;", parser.testNextToken());
-        assertEquals(CSVParser.TT_EOF + ";;", parser.testNextToken());
+        assertEquals(TOKEN + ";1;", parser.testNextToken());
+        assertEquals(TOKEN + ";2;", parser.testNextToken());
+        assertEquals(TOKEN + ";3;", parser.testNextToken());
+        assertEquals(EORECORD + ";;", parser.testNextToken());
+        assertEquals(TOKEN + ";a;", parser.testNextToken());
+        assertEquals(TOKEN + ";b x;", parser.testNextToken());
+        assertEquals(EORECORD + ";c;", parser.testNextToken());
+        assertEquals(EORECORD + ";;", parser.testNextToken());
+        assertEquals(TOKEN + ";d;", parser.testNextToken());
+        assertEquals(TOKEN + ";e;", parser.testNextToken());
+        assertEquals(EORECORD + ";;", parser.testNextToken());
+        assertEquals(EOF + ";;", parser.testNextToken());
+        assertEquals(EOF + ";;", parser.testNextToken());
 
     }
 
@@ -124,15 +127,15 @@ public class CSVParserTest extends TestC
         CSVFormat format = CSVFormat.DEFAULT.withCommentStart('#');
         TestCSVParser parser = new TestCSVParser(new StringReader(code), format);
 
-        assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
+        assertEquals(TOKEN + ";a;", parser.testNextToken());
         // an unquoted single backslash is not an escape char
-        assertEquals(CSVParser.TT_TOKEN + ";\\;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";;", parser.testNextToken());
-        assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
+        assertEquals(TOKEN + ";\\;", parser.testNextToken());
+        assertEquals(TOKEN + ";;", parser.testNextToken());
+        assertEquals(EORECORD + ";b;", parser.testNextToken());
         // an unquoted single backslash is not an escape char
-        assertEquals(CSVParser.TT_TOKEN + ";\\;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";;", parser.testNextToken());
-        assertEquals(CSVParser.TT_EOF + ";;", parser.testNextToken());
+        assertEquals(TOKEN + ";\\;", parser.testNextToken());
+        assertEquals(TOKEN + ";;", parser.testNextToken());
+        assertEquals(EOF + ";;", parser.testNextToken());
     }
 
     // encapsulator tokenizer (sinle line)
@@ -145,19 +148,19 @@ public class CSVParserTest extends TestC
         String code =
                 "a,\"foo\",b\na,   \" foo\",b\na,\"foo \"  ,b\na,  \" foo \"  ,b";
         TestCSVParser parser = new TestCSVParser(new StringReader(code));
-        assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";foo;", parser.testNextToken());
-        assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + "; foo;", parser.testNextToken());
-        assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";foo ;", parser.testNextToken());
-        assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + "; foo ;", parser.testNextToken());
-//     assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
-        assertEquals(CSVParser.TT_EOF + ";b;", parser.testNextToken());
+        assertEquals(TOKEN + ";a;", parser.testNextToken());
+        assertEquals(TOKEN + ";foo;", parser.testNextToken());
+        assertEquals(EORECORD + ";b;", parser.testNextToken());
+        assertEquals(TOKEN + ";a;", parser.testNextToken());
+        assertEquals(TOKEN + "; foo;", parser.testNextToken());
+        assertEquals(EORECORD + ";b;", parser.testNextToken());
+        assertEquals(TOKEN + ";a;", parser.testNextToken());
+        assertEquals(TOKEN + ";foo ;", parser.testNextToken());
+        assertEquals(EORECORD + ";b;", parser.testNextToken());
+        assertEquals(TOKEN + ";a;", parser.testNextToken());
+        assertEquals(TOKEN + "; foo ;", parser.testNextToken());
+//     assertEquals(EORECORD + ";b;", parser.testNextToken());
+        assertEquals(EOF + ";b;", parser.testNextToken());
     }
 
     // encapsulator tokenizer (multi line, delimiter in string)
@@ -165,12 +168,12 @@ public class CSVParserTest extends TestC
         String code =
                 "a,\"foo\n\",b\n\"foo\n  baar ,,,\"\n\"\n\t \n\"";
         TestCSVParser parser = new TestCSVParser(new StringReader(code));
-        assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";foo\n;", parser.testNextToken());
-        assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
-        assertEquals(CSVParser.TT_EORECORD + ";foo\n  baar ,,,;",
+        assertEquals(TOKEN + ";a;", parser.testNextToken());
+        assertEquals(TOKEN + ";foo\n;", parser.testNextToken());
+        assertEquals(EORECORD + ";b;", parser.testNextToken());
+        assertEquals(EORECORD + ";foo\n  baar ,,,;",
                 parser.testNextToken());
-        assertEquals(CSVParser.TT_EOF + ";\n\t \n;", parser.testNextToken());
+        assertEquals(EOF + ";\n\t \n;", parser.testNextToken());
 
     }
 
@@ -183,9 +186,9 @@ public class CSVParserTest extends TestC
         */
         String code = "a;'b and '' more\n'\n!comment;;;;\n;;";
         TestCSVParser parser = new TestCSVParser(new StringReader(code), new CSVFormat(';',
'\'', '!'));
-        assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
+        assertEquals(TOKEN + ";a;", parser.testNextToken());
         assertEquals(
-                CSVParser.TT_EORECORD + ";b and ' more\n;",
+                EORECORD + ";b and ' more\n;",
                 parser.testNextToken());
     }
 
@@ -209,13 +212,11 @@ public class CSVParserTest extends TestC
 
     public void testGetLine() throws IOException {
         CSVParser parser = new CSVParser(new StringReader(code));
-        String[] tmp = null;
-        for (int i = 0; i < res.length; i++) {
-            tmp = parser.getLine();
-            assertTrue(Arrays.equals(res[i], tmp));
+        for (String[] re : res) {
+            assertTrue(Arrays.equals(re, parser.getLine()));
         }
-        tmp = parser.getLine();
-        assertTrue(tmp == null);
+        
+        assertTrue(parser.getLine() == null);
     }
 
     public void testGetAllValues() throws IOException {
@@ -282,9 +283,8 @@ public class CSVParserTest extends TestC
                 {""},  // Excel format does not ignore empty lines
                 {"world", ""}
         };
-        String code;
-        for (int codeIndex = 0; codeIndex < codes.length; codeIndex++) {
-            code = codes[codeIndex];
+        
+        for (String code : codes) {
             CSVParser parser = new CSVParser(new StringReader(code), CSVFormat.EXCEL);
             String[][] tmp = parser.getAllValues();
             assertEquals(res.length, tmp.length);
@@ -558,11 +558,11 @@ public class CSVParserTest extends TestC
     public void testDelimiterIsWhitespace() throws IOException {
         String code = "one\ttwo\t\tfour \t five\t six";
         TestCSVParser parser = new TestCSVParser(new StringReader(code), CSVFormat.TDF);
-        assertEquals(CSVParser.TT_TOKEN + ";one;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";two;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";four;", parser.testNextToken());
-        assertEquals(CSVParser.TT_TOKEN + ";five;", parser.testNextToken());
-        assertEquals(CSVParser.TT_EOF + ";six;", parser.testNextToken());
+        assertEquals(TOKEN + ";one;", parser.testNextToken());
+        assertEquals(TOKEN + ";two;", parser.testNextToken());
+        assertEquals(TOKEN + ";;", parser.testNextToken());
+        assertEquals(TOKEN + ";four;", parser.testNextToken());
+        assertEquals(TOKEN + ";five;", parser.testNextToken());
+        assertEquals(EOF + ";six;", parser.testNextToken());
     }
 }



Mime
View raw message