commons-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s...@apache.org
Subject svn commit: r1306890 - in /commons/proper/csv/trunk/src: main/java/org/apache/commons/csv/Lexer.java test/java/org/apache/commons/csv/CSVLexerTest.java
Date Thu, 29 Mar 2012 15:01:05 GMT
Author: sebb
Date: Thu Mar 29 15:01:04 2012
New Revision: 1306890

URL: http://svn.apache.org/viewvc?rev=1306890&view=rev
Log:
Improve escape tests
Fix bug in readEscape() - was not handling EOF

Modified:
    commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/Lexer.java
    commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/CSVLexerTest.java

Modified: commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/Lexer.java
URL: http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/Lexer.java?rev=1306890&r1=1306889&r2=1306890&view=diff
==============================================================================
--- commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/Lexer.java (original)
+++ commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/Lexer.java Thu Mar 29 15:01:04
2012
@@ -60,6 +60,7 @@ abstract class Lexer {
         return in.getLineNumber();
     }
 
+    // TODO escape handling needs more work
     int readEscape() throws IOException {
         // assume c is the escape char (normally a backslash)
         int c = in.read();
@@ -74,6 +75,8 @@ abstract class Lexer {
                 return '\b';
             case 'f':
                 return '\f';
+            case ExtendedBufferedReader.END_OF_STREAM:
+                throw new IOException("EOF whilst processing escape sequence");
             default:
                 return c;
         }

Modified: commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/CSVLexerTest.java
URL: http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/CSVLexerTest.java?rev=1306890&r1=1306889&r2=1306890&view=diff
==============================================================================
--- commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/CSVLexerTest.java (original)
+++ commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/CSVLexerTest.java Thu Mar
29 15:01:04 2012
@@ -147,28 +147,66 @@ public class CSVLexerTest {
 
     }
 
-    // simple token with escaping
+    // simple token with escaping not enabled
     @Test
     public void testNextToken3() throws IOException {
         /* file: a,\,,b
         *       \,,
         */
-        String code = "a,\\,,b\n\\,,";
-        CSVFormat format = CSVFormat.DEFAULT.withCommentStart('#');
+        String code = "a,\\,,b\\\n\\,,";
+        CSVFormat format = CSVFormat.DEFAULT;
+        assertFalse(format.isEscaping());
         Lexer parser = getLexer(code, format);
 
         assertTokenEquals(TOKEN, "a", parser.nextToken(new Token()));
         // an unquoted single backslash is not an escape char
         assertTokenEquals(TOKEN, "\\", parser.nextToken(new Token()));
         assertTokenEquals(TOKEN, "", parser.nextToken(new Token()));
-        assertTokenEquals(EORECORD, "b", parser.nextToken(new Token()));
+        assertTokenEquals(EORECORD, "b\\", parser.nextToken(new Token()));
         // an unquoted single backslash is not an escape char
         assertTokenEquals(TOKEN, "\\", parser.nextToken(new Token()));
         assertTokenEquals(TOKEN, "", parser.nextToken(new Token()));
         assertTokenEquals(EOF, "", parser.nextToken(new Token()));
     }
 
-    // encapsulator tokenizer (sinle line)
+    // simple token with escaping enabled
+    @Test
+    public void testNextToken3Escaping() throws IOException {
+        /* file: a,\,,b
+        *       \,,
+        */
+        String code = "a,\\,,b\\\\\n\\,,\\\nc,d\\\n";
+        CSVFormat format = CSVFormat.DEFAULT.withEscape('\\');
+        assertTrue(format.isEscaping());
+        Lexer parser = getLexer(code, format);
+
+        assertTokenEquals(TOKEN, "a", parser.nextToken(new Token()));
+        assertTokenEquals(TOKEN, ",", parser.nextToken(new Token()));
+        assertTokenEquals(EORECORD, "b\\", parser.nextToken(new Token()));
+        assertTokenEquals(TOKEN, ",", parser.nextToken(new Token()));
+        assertTokenEquals(TOKEN, "\nc", parser.nextToken(new Token()));
+        assertTokenEquals(EOF, "d\n", parser.nextToken(new Token()));
+        assertTokenEquals(EOF, "", parser.nextToken(new Token()));
+    }
+
+    // simple token with escaping enabled
+    @Test
+    public void testNextToken3BadEscaping() throws IOException {
+        String code = "a,b,c\\";
+        CSVFormat format = CSVFormat.DEFAULT.withEscape('\\');
+        assertTrue(format.isEscaping());
+        Lexer parser = getLexer(code, format);
+
+        assertTokenEquals(TOKEN, "a", parser.nextToken(new Token()));
+        assertTokenEquals(TOKEN, "b", parser.nextToken(new Token()));
+        try {
+            Token tkn = parser.nextToken(new Token());
+            fail("Expected IOE, found "+tkn);
+        } catch (IOException e) {
+        }
+    }
+
+    // encapsulator tokenizer (single line)
     @Test
     public void testNextToken4() throws IOException {
         /* file:  a,"foo",b



Mime
View raw message