commons-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yo...@apache.org
Subject svn commit: r609155 - in /commons/sandbox/csv/trunk/src: java/org/apache/commons/csv/CSVParser.java java/org/apache/commons/csv/CSVStrategy.java test/org/apache/commons/csv/CSVParserTest.java test/org/apache/commons/csv/CSVStrategyTest.java
Date Sat, 05 Jan 2008 15:39:06 GMT
Author: yonik
Date: Sat Jan  5 07:37:26 2008
New Revision: 609155

URL: http://svn.apache.org/viewvc?rev=609155&view=rev
Log:
SANDBOX-206: add escape to strategy, turn off backslash-style escaping by default

Modified:
    commons/sandbox/csv/trunk/src/java/org/apache/commons/csv/CSVParser.java
    commons/sandbox/csv/trunk/src/java/org/apache/commons/csv/CSVStrategy.java
    commons/sandbox/csv/trunk/src/test/org/apache/commons/csv/CSVParserTest.java
    commons/sandbox/csv/trunk/src/test/org/apache/commons/csv/CSVStrategyTest.java

Modified: commons/sandbox/csv/trunk/src/java/org/apache/commons/csv/CSVParser.java
URL: http://svn.apache.org/viewvc/commons/sandbox/csv/trunk/src/java/org/apache/commons/csv/CSVParser.java?rev=609155&r1=609154&r2=609155&view=diff
==============================================================================
--- commons/sandbox/csv/trunk/src/java/org/apache/commons/csv/CSVParser.java (original)
+++ commons/sandbox/csv/trunk/src/java/org/apache/commons/csv/CSVParser.java Sat Jan  5 07:37:26
2008
@@ -134,7 +134,7 @@
    * @deprecated use {@link #CSVParser(Reader,CSVStrategy)}.
    */
   public CSVParser(Reader input, char delimiter) {
-    this(input, delimiter, '"', (char) 0);
+    this(input, delimiter, '"', CSVStrategy.COMMENTS_DISABLED);
   }
   
   /**
@@ -347,7 +347,7 @@
         eol = isEndOfLine(c);
       }
       // ok, start of token reached: comment, encapsulated, or token
-      if (!strategy.isCommentingDisabled() && c == strategy.getCommentStart()) {
+      if (c == strategy.getCommentStart()) {
         // ignore everything till end of line and continue (incr linecount)
         in.readLine();
         tkn = nextToken(tkn.reset());
@@ -400,19 +400,22 @@
    */
   private Token simpleTokenLexer(Token tkn, int c) throws IOException {
     wsBuf.clear();
-    while (!tkn.isReady) {
+    for (;;) {
       if (isEndOfLine(c)) {
         // end of record
         tkn.type = TT_EORECORD;
         tkn.isReady = true;
+        return tkn;
       } else if (isEndOfFile(c)) {
         // end of file
         tkn.type = TT_EOF;
         tkn.isReady = true;
+        return tkn;
       } else if (c == strategy.getDelimiter()) {
         // end of token
         tkn.type = TT_TOKEN;
         tkn.isReady = true;
+        return tkn;
       } else if (c == '\\' && strategy.getUnicodeEscapeInterpretation() &&
in.lookAhead() == 'u') {
         // interpret unicode escaped chars (like \u0070 -> p)
         tkn.content.append((char) unicodeEscapeLexer(c));
@@ -422,6 +425,8 @@
         if (tkn.content.length() > 0) {
           wsBuf.append((char) c);
         }
+      } else if (c == strategy.getEscape()) {
+        tkn.content.append((char)readEscape(c));
       } else {
         // prepend whitespaces (if we have)
         if (wsBuf.length() > 0) {
@@ -435,7 +440,6 @@
         c = in.read();
       }
     }
-    return tkn;
   }
   
   
@@ -457,70 +461,55 @@
     int startLineNumber = getLineNumber();
     // ignore the given delimiter
     // assert c == delimiter;
-    c = in.read();
-    while (!tkn.isReady) {
-      boolean skipRead = false;
-      if (c == strategy.getEncapsulator() || c == '\\') {
-        // check lookahead
+    for (;;) {
+      c = in.read();
+
+      if (c == '\\' && strategy.getUnicodeEscapeInterpretation() && in.lookAhead()=='u')
{
+        tkn.content.append((char) unicodeEscapeLexer(c));
+      } else if (c == strategy.getEscape()) {
+        tkn.content.append((char)readEscape(c));
+      } else if (c == strategy.getEncapsulator()) {
         if (in.lookAhead() == strategy.getEncapsulator()) {
           // double or escaped encapsulator -> add single encapsulator to token
           c = in.read();
           tkn.content.append((char) c);
-        } else if (c == '\\' && in.lookAhead() == '\\') {
-          // doubled escape char, it does not escape itself, only encapsulator 
-          // -> add both escape chars to stream
-          tkn.content.append((char) c);
-          c = in.read();
-          tkn.content.append((char) c);
-        } else if (
-          strategy.getUnicodeEscapeInterpretation()
-          && c == '\\' 
-          && in.lookAhead() == 'u') {
-          // interpret unicode escaped chars (like \u0070 -> p)
-          tkn.content.append((char) unicodeEscapeLexer(c));
-        } else if (c == '\\') {
-          // use a single escape character -> add it to stream
-          tkn.content.append((char) c);
         } else {
           // token finish mark (encapsulator) reached: ignore whitespace till delimiter
-          while (!tkn.isReady) {
+          for (;;) {
             c = in.read();
             if (c == strategy.getDelimiter()) {
               tkn.type = TT_TOKEN;
               tkn.isReady = true;
+              return tkn;
             } else if (isEndOfFile(c)) {
               tkn.type = TT_EOF;
               tkn.isReady = true;
+              return tkn;
             } else if (isEndOfLine(c)) {
               // ok eo token reached
               tkn.type = TT_EORECORD;
               tkn.isReady = true;
+              return tkn;
             } else if (!isWhitespace(c)) {
-                // error invalid char between token and next delimiter
-                throw new IOException(
-                  "(line " + getLineNumber() 
-                  + ") invalid char between encapsulated token end delimiter"
-                );
-              }
+              // error invalid char between token and next delimiter
+              throw new IOException(
+                      "(line " + getLineNumber()
+                              + ") invalid char between encapsulated token end delimiter"
+              );
+            }
           }
-          skipRead = true;
         }
       } else if (isEndOfFile(c)) {
         // error condition (end of file before end of token)
         throw new IOException(
-          "(startline " + startLineNumber + ")"
-          + "eof reached before encapsulated token finished"
-          );
+                "(startline " + startLineNumber + ")"
+                        + "eof reached before encapsulated token finished"
+        );
       } else {
         // consume character
         tkn.content.append((char) c);
       }
-      // get the next char
-      if (!tkn.isReady && !skipRead) {
-        c = in.read();
-      }
     }
-    return tkn;
   }
   
   
@@ -553,6 +542,21 @@
         + code.toString() + "'" + e.toString());
     }
     return ret;
+  }
+
+  private int readEscape(int c) throws IOException {
+    // assume c is the escape char (normally a backslash)
+    c = in.read();
+    int out;
+    switch (c) {
+      case 'r': out='\r'; break;
+      case 'n': out='\n'; break;
+      case 't': out='\t'; break;
+      case 'b': out='\b'; break;
+      case 'f': out='\f'; break;
+      default : out=c;
+    }
+    return out;
   }
   
   // ======================================================

Modified: commons/sandbox/csv/trunk/src/java/org/apache/commons/csv/CSVStrategy.java
URL: http://svn.apache.org/viewvc/commons/sandbox/csv/trunk/src/java/org/apache/commons/csv/CSVStrategy.java?rev=609155&r1=609154&r2=609155&view=diff
==============================================================================
--- commons/sandbox/csv/trunk/src/java/org/apache/commons/csv/CSVStrategy.java (original)
+++ commons/sandbox/csv/trunk/src/java/org/apache/commons/csv/CSVStrategy.java Sat Jan  5
07:37:26 2008
@@ -28,15 +28,21 @@
     private char delimiter;
     private char encapsulator;
     private char commentStart;
+    private char escape;
     private boolean ignoreLeadingWhitespaces;
     private boolean interpretUnicodeEscapes;
     private boolean ignoreEmptyLines;
 
-    public static char COMMENTS_DISABLED       = (char) 0;
-
-    public static CSVStrategy DEFAULT_STRATEGY = new CSVStrategy(',', '"', COMMENTS_DISABLED,
true,  false, true);
-    public static CSVStrategy EXCEL_STRATEGY   = new CSVStrategy(',', '"', COMMENTS_DISABLED,
false, false, false);
-    public static CSVStrategy TDF_STRATEGY     = new CSVStrategy('	', '"', COMMENTS_DISABLED,
true,  false, true);
+    // -2 is used to signal disabled, because it won't be confused with
+    // an EOF signal (-1), and because \ufffe in UTF-16 would be
+    // encoded as two chars (using surrogates) and thus there should never
+    // be a collision with a real text char.
+    public static char COMMENTS_DISABLED       = (char)-2;
+    public static char ESCAPE_DISABLED         = (char)-2;
+
+    public static CSVStrategy DEFAULT_STRATEGY = new CSVStrategy(',', '"', COMMENTS_DISABLED,
ESCAPE_DISABLED, true,  false, true);
+    public static CSVStrategy EXCEL_STRATEGY   = new CSVStrategy(',', '"', COMMENTS_DISABLED,
ESCAPE_DISABLED, false, false, false);
+    public static CSVStrategy TDF_STRATEGY     = new CSVStrategy('	', '"', COMMENTS_DISABLED,
ESCAPE_DISABLED, true,  false, true);
 
 
     public CSVStrategy(char delimiter, char encapsulator, char commentStart) {
@@ -58,7 +64,8 @@
     public CSVStrategy(
         char delimiter, 
         char encapsulator, 
-        char commentStart, 
+        char commentStart,
+        char escape,
         boolean ignoreLeadingWhitespace, 
         boolean interpretUnicodeEscapes,
         boolean ignoreEmptyLines) 
@@ -66,11 +73,25 @@
         setDelimiter(delimiter);
         setEncapsulator(encapsulator);
         setCommentStart(commentStart);
+        setEscape(escape);
         setIgnoreLeadingWhitespaces(ignoreLeadingWhitespace);
         setUnicodeEscapeInterpretation(interpretUnicodeEscapes);
         setIgnoreEmptyLines(ignoreEmptyLines);
     }
 
+    /** @deprecated */
+    public CSVStrategy(
+        char delimiter,
+        char encapsulator,
+        char commentStart,
+        boolean ignoreLeadingWhitespace,
+        boolean interpretUnicodeEscapes,
+        boolean ignoreEmptyLines)
+    {
+        this(delimiter,encapsulator,commentStart,CSVStrategy.ESCAPE_DISABLED,ignoreLeadingWhitespace,interpretUnicodeEscapes,ignoreEmptyLines);
+    }
+
+
     public void setDelimiter(char delimiter) { this.delimiter = delimiter; }
     public char getDelimiter() { return this.delimiter; }
 
@@ -80,6 +101,9 @@
     public void setCommentStart(char commentStart) { this.commentStart = commentStart; }
     public char getCommentStart() { return this.commentStart; }
     public boolean isCommentingDisabled() { return this.commentStart == COMMENTS_DISABLED;
}
+
+    public void setEscape(char escape) { this.escape = escape; }
+    public char getEscape() { return this.escape; }
 
     public void setIgnoreLeadingWhitespaces(boolean ignoreLeadingWhitespaces) { this.ignoreLeadingWhitespaces
= ignoreLeadingWhitespaces; }
     public boolean getIgnoreLeadingWhitespaces() { return this.ignoreLeadingWhitespaces;
}

Modified: commons/sandbox/csv/trunk/src/test/org/apache/commons/csv/CSVParserTest.java
URL: http://svn.apache.org/viewvc/commons/sandbox/csv/trunk/src/test/org/apache/commons/csv/CSVParserTest.java?rev=609155&r1=609154&r2=609155&view=diff
==============================================================================
--- commons/sandbox/csv/trunk/src/test/org/apache/commons/csv/CSVParserTest.java (original)
+++ commons/sandbox/csv/trunk/src/test/org/apache/commons/csv/CSVParserTest.java Sat Jan 
5 07:37:26 2008
@@ -182,9 +182,7 @@
   // encapsulator tokenizer (multi line, delimiter in string)
   public void testNextToken5() throws IOException {   
     String code = 
-      "a,\"foo\n\",b\n\"foo\n  baar ,,,\"\n\"\n\t \n\",\"\\\"\""
-      + ",\"\\,\"" 
-      + ",\"\"\"\"";
+      "a,\"foo\n\",b\n\"foo\n  baar ,,,\"\n\"\n\t \n\"";
     TestCSVParser parser = new TestCSVParser(new StringReader(code));
     parser.setStrategy(CSVStrategy.DEFAULT_STRATEGY);
     System.out.println("---------\n" + code + "\n-------------");
@@ -193,11 +191,8 @@
     assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
     assertEquals(CSVParser.TT_EORECORD + ";foo\n  baar ,,,;",
         parser.testNextToken());
-    assertEquals(CSVParser.TT_TOKEN + ";\n\t \n;", parser.testNextToken());
-    assertEquals(CSVParser.TT_TOKEN + ";\";", parser.testNextToken());
-    // escape char in quoted input only escapes delimiter
-    assertEquals(CSVParser.TT_TOKEN + ";\\,;", parser.testNextToken());
-    assertEquals(CSVParser.TT_EOF + ";\";", parser.testNextToken());
+    assertEquals(CSVParser.TT_EOF + ";\n\t \n;", parser.testNextToken());
+
   }
   
   // change delimiters, comment, encapsulater
@@ -207,7 +202,7 @@
      *       !comment;;;;
      *       ;;
      */
-    String code = "a;'b and \\' more\n'\n!comment;;;;\n;;";
+    String code = "a;'b and '' more\n'\n!comment;;;;\n;;";
     TestCSVParser parser = new TestCSVParser(new StringReader(code));
     parser.setStrategy( new CSVStrategy(';', '\'', '!') );
     System.out.println("---------\n" + code + "\n-------------");
@@ -226,8 +221,9 @@
     "a,b,c,d\n"
     + " a , b , 1 2 \n"
     + "\"foo baar\", b,\n"
-    + "   \"foo\n,,\n\"\",,\n\\\"\",d,e\n";
-  String[][] res = { 
+   // + "   \"foo\n,,\n\"\",,\n\\\"\",d,e\n";
+      + "   \"foo\n,,\n\"\",,\n\"\"\",d,e\n";   // changed to use standard CSV escaping
+  String[][] res = {
     {"a", "b", "c", "d"},
     {"a", "b", "1 2"}, 
     {"foo baar", "b", ""}, 
@@ -439,7 +435,7 @@
     }
   }
   
-  public void testBackslashEscaping() throws IOException {
+  public void OLDtestBackslashEscaping() throws IOException {
     String code =
       "one,two,three\n"
       + "on\\\"e,two\n"
@@ -474,6 +470,49 @@
     }
   }
   
+  public void testBackslashEscaping() throws IOException {
+
+    // To avoid confusion over the need for escaping chars in java code,
+    // We will test with a forward slash as the escape char, and a single
+    // quote as the encapsulator.
+
+    String code =
+      "one,two,three\n" // 0
+      + "'',''\n"       // 1) empty encapsulators
+      + "/',/'\n"       // 2) single encapsulators
+      + "'/'','/''\n"   // 3) single encapsulators encapsulated via escape
+      + "'''',''''\n"   // 4) single encapsulators encapsulated via doubling
+      + "/,,/,\n"       // 5) separator escaped
+      + "//,//\n"       // 6) escape escaped
+      + "'//','//'\n"   // 7) escape escaped in encapsulation
+      + "";
+    String[][] res = {
+        { "one", "two", "three" }, // 0
+        { "", "" },                // 1
+        { "'", "'" },              // 2
+        { "'", "'" },              // 3
+        { "'", "'" },              // 4
+        { ",", "," },              // 5
+        { "/", "/" },              // 6
+        { "/", "/" },              // 7
+      };
+
+
+    CSVStrategy strategy = new CSVStrategy(',','\'',CSVStrategy.COMMENTS_DISABLED,'/',true,true,true);
+
+    CSVParser parser = new CSVParser(new StringReader(code), strategy);
+    System.out.println("---------\n" + code + "\n-------------");
+    String[][] tmp = parser.getAllValues();
+    assertTrue(tmp.length > 0);
+    for (int i = 0; i < res.length; i++) {
+      for (int j = 0; j < tmp[i].length; j++) {
+        System.out.println("'" + tmp[i][j] + "'  should be '" + res[i][j] + "'");
+      }
+      assertTrue(Arrays.equals(res[i], tmp[i]));
+    }
+  }
+
+
     public void testUnicodeEscape() throws IOException {
       String code = "abc,\\u0070\\u0075\\u0062\\u006C\\u0069\\u0063";
       CSVParser parser = new CSVParser(new StringReader(code));

Modified: commons/sandbox/csv/trunk/src/test/org/apache/commons/csv/CSVStrategyTest.java
URL: http://svn.apache.org/viewvc/commons/sandbox/csv/trunk/src/test/org/apache/commons/csv/CSVStrategyTest.java?rev=609155&r1=609154&r2=609155&view=diff
==============================================================================
--- commons/sandbox/csv/trunk/src/test/org/apache/commons/csv/CSVStrategyTest.java (original)
+++ commons/sandbox/csv/trunk/src/test/org/apache/commons/csv/CSVStrategyTest.java Sat Jan
 5 07:37:26 2008
@@ -91,7 +91,7 @@
     // default settings
     assertEquals(strategy.getDelimiter(), ',');
     assertEquals(strategy.getEncapsulator(), '"');
-    assertEquals(strategy.getCommentStart(), '\0');
+    assertEquals(strategy.getCommentStart(), CSVStrategy.COMMENTS_DISABLED);
     assertEquals(true,  strategy.getIgnoreLeadingWhitespaces());
     assertEquals(false, strategy.getUnicodeEscapeInterpretation());
     assertEquals(true,  strategy.getIgnoreEmptyLines());
@@ -99,7 +99,7 @@
     parser.setStrategy(CSVStrategy.DEFAULT_STRATEGY);
     assertEquals(strategy.getDelimiter(), ',');
     assertEquals(strategy.getEncapsulator(), '"');
-    assertEquals(strategy.getCommentStart(), '\0');
+    assertEquals(strategy.getCommentStart(), CSVStrategy.COMMENTS_DISABLED);
     assertEquals(true,  strategy.getIgnoreLeadingWhitespaces());
     assertEquals(false, strategy.getUnicodeEscapeInterpretation());
     assertEquals(true,  strategy.getIgnoreEmptyLines());
@@ -109,7 +109,7 @@
     CSVStrategy strategy = CSVStrategy.EXCEL_STRATEGY;
     assertEquals(strategy.getDelimiter(), ',');
     assertEquals(strategy.getEncapsulator(), '"');
-    assertEquals(strategy.getCommentStart(), '\0');
+    assertEquals(strategy.getCommentStart(), CSVStrategy.COMMENTS_DISABLED);
     assertEquals(false,  strategy.getIgnoreLeadingWhitespaces());
     assertEquals(false, strategy.getUnicodeEscapeInterpretation());
     assertEquals(false, strategy.getIgnoreEmptyLines());



Mime
View raw message