lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Chris Male <gento...@gmail.com>
Subject Re: svn commit: r1308225 - in /lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal: ./ csv/ csv/writer/
Date Mon, 02 Apr 2012 11:41:05 GMT
Alrighty.

On Mon, Apr 2, 2012 at 11:29 PM, Robert Muir <rcmuir@gmail.com> wrote:

> Since i've already svn merge --reintegrated this branch, I don't think
> we can safely merge any further changes... I think any future
> development should be against trunk?
>
> "In Subversion 1.5, once a --reintegrate merge is done from branch to
> trunk, the branch is no longer usable for further work. It's not able
> to correctly absorb new trunk changes, nor can it be properly
> reintegrated to trunk again."
>
> http://svnbook.red-bean.com/en/1.5/svn.branchmerge.basicmerging.html
>
> On Sun, Apr 1, 2012 at 9:29 PM,  <chrism@apache.org> wrote:
> > Author: chrism
> > Date: Mon Apr  2 01:29:20 2012
> > New Revision: 1308225
> >
> > URL: http://svn.apache.org/viewvc?rev=1308225&view=rev
> > Log:
> > LUCENE-3930: Imported commons-csv tests
> >
> > Added:
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVParserTest.java
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVPrinterTest.java
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVStrategyTest.java
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVUtilsTest.java
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CharBufferTest.java
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/ExtendedBufferedReaderTest.java
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigGuesserTest.java
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigTest.java
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVFieldTest.java
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVWriterTest.java
> >
> > Added:
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVParserTest.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVParserTest.java?rev=1308225&view=auto
> >
> ==============================================================================
> > ---
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVParserTest.java
> (added)
> > +++
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVParserTest.java
> Mon Apr  2 01:29:20 2012
> > @@ -0,0 +1,581 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one or more
> > + * contributor license agreements.  See the NOTICE file distributed with
> > + * this work for additional information regarding copyright ownership.
> > + * The ASF licenses this file to You under the Apache License, Version
> 2.0
> > + * (the "License"); you may not use this file except in compliance with
> > + * the License.  You may obtain a copy of the License at
> > + *
> > + *      http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing, software
> > + * distributed under the License is distributed on an "AS IS" BASIS,
> > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> > + * See the License for the specific language governing permissions and
> > + * limitations under the License.
> > + */
> > +package org.apache.solr.internal.csv;
> > +
> > +import java.io.IOException;
> > +import java.io.Reader;
> > +import java.io.StringReader;
> > +import java.util.Arrays;
> > +
> > +import junit.framework.TestCase;
> > +
> > +/**
> > + * CSVParserTest
> > + *
> > + * The test are organized in three different sections:
> > + * The 'setter/getter' section, the lexer section and finally the parser
> > + * section. In case a test fails, you should follow a top-down approach
> for
> > + * fixing a potential bug (its likely that the parser itself fails if
> the lexer
> > + * has problems...).
> > + */
> > +public class CSVParserTest extends TestCase {
> > +
> > +  /**
> > +   * TestCSVParser.
> > +   */
> > +  class TestCSVParser extends CSVParser {
> > +    /**
> > +     * Test parser to investigate the type of the internal Token.
> > +     * @param in a Reader
> > +     */
> > +    TestCSVParser(Reader in) {
> > +      super(in);
> > +    }
> > +
> > +    TestCSVParser(Reader in, CSVStrategy strategy) {
> > +      super(in, strategy);
> > +    }
> > +    /**
> > +     * Calls super.nextToken() and prints out a String representation
> of token
> > +     * type and content.
> > +     * @return String representation of token type and content
> > +     * @throws IOException like {@link CSVParser#nextToken()}
> > +     */
> > +    public String testNextToken() throws IOException {
> > +      Token t = super.nextToken();
> > +      return Integer.toString(t.type) + ";" + t.content + ";";
> > +    }
> > +  }
> > +
> > +  // ======================================================
> > +  //   lexer tests
> > +  // ======================================================
> > +
> > +  // Single line (without comment)
> > +  public void testNextToken1() throws IOException {
> > +    String code = "abc,def, hijk,  lmnop,   qrst,uv ,wxy   ,z , ,";
> > +    TestCSVParser parser = new TestCSVParser(new StringReader(code));
> > +    assertEquals(CSVParser.TT_TOKEN + ";abc;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";def;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";hijk;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";lmnop;",
> parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";qrst;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";uv;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";wxy;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";z;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EOF + ";;", parser.testNextToken());
> > +  }
> > +
> > +  // multiline including comments (and empty lines)
> > +  public void testNextToken2() throws IOException {
> > +    /*   file:   1,2,3,
> > +     *           a,b x,c
> > +     *
> > +     *           # this is a comment
> > +     *           d,e,
> > +     *
> > +     */
> > +    String code = "1,2,3,\na,b x,c\n#foo\n\nd,e,\n\n";
> > +    CSVStrategy strategy =
> (CSVStrategy)CSVStrategy.DEFAULT_STRATEGY.clone();
> > +    // strategy.setIgnoreEmptyLines(false);
> > +    strategy.setCommentStart('#');
> > +
> > +    TestCSVParser parser = new TestCSVParser(new StringReader(code),
> strategy);
> > +
> > +
> > +    assertEquals(CSVParser.TT_TOKEN + ";1;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";2;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";3;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EORECORD + ";;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";b x;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EORECORD + ";c;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EORECORD + ";;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";d;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";e;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EORECORD + ";;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EOF + ";;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EOF + ";;", parser.testNextToken());
> > +
> > +  }
> > +
> > +  // simple token with escaping
> > +  public void testNextToken3() throws IOException {
> > +    /* file: a,\,,b
> > +     *       \,,
> > +     */
> > +    String code = "a,\\,,b\n\\,,";
> > +    CSVStrategy strategy =
> (CSVStrategy)CSVStrategy.DEFAULT_STRATEGY.clone();
> > +    strategy.setCommentStart('#');
> > +    TestCSVParser parser = new TestCSVParser(new StringReader(code),
> strategy);
> > +
> > +    assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> > +    // an unquoted single backslash is not an escape char
> > +    assertEquals(CSVParser.TT_TOKEN + ";\\;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
> > +    // an unquoted single backslash is not an escape char
> > +    assertEquals(CSVParser.TT_TOKEN + ";\\;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EOF + ";;", parser.testNextToken());
> > +  }
> > +
> > +  // encapsulator tokenizer (sinle line)
> > +  public void testNextToken4() throws IOException {
> > +    /* file:  a,"foo",b
> > +     *        a,   " foo",b
> > +     *        a,"foo "   ,b     // whitespace after closing encapsulator
> > +     *        a,  " foo " ,b
> > +     */
> > +     String code =
> > +      "a,\"foo\",b\na,   \" foo\",b\na,\"foo \"  ,b\na,  \" foo \"  ,b";
> > +     TestCSVParser parser = new TestCSVParser(new StringReader(code));
> > +     assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> > +     assertEquals(CSVParser.TT_TOKEN + ";foo;", parser.testNextToken());
> > +     assertEquals(CSVParser.TT_EORECORD + ";b;",
> parser.testNextToken());
> > +     assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> > +     assertEquals(CSVParser.TT_TOKEN + "; foo;",
> parser.testNextToken());
> > +     assertEquals(CSVParser.TT_EORECORD + ";b;",
> parser.testNextToken());
> > +     assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> > +     assertEquals(CSVParser.TT_TOKEN + ";foo ;",
> parser.testNextToken());
> > +     assertEquals(CSVParser.TT_EORECORD + ";b;",
> parser.testNextToken());
> > +     assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> > +     assertEquals(CSVParser.TT_TOKEN + "; foo ;",
> parser.testNextToken());
> > +//     assertEquals(CSVParser.TT_EORECORD + ";b;",
> parser.testNextToken());
> > +     assertEquals(CSVParser.TT_EOF + ";b;", parser.testNextToken());
> > +  }
> > +
> > +  // encapsulator tokenizer (multi line, delimiter in string)
> > +  public void testNextToken5() throws IOException {
> > +    String code =
> > +      "a,\"foo\n\",b\n\"foo\n  baar ,,,\"\n\"\n\t \n\"";
> > +    TestCSVParser parser = new TestCSVParser(new StringReader(code));
> > +    assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";foo\n;",
> parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EORECORD + ";foo\n  baar ,,,;",
> > +        parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EOF + ";\n\t \n;",
> parser.testNextToken());
> > +
> > +  }
> > +
> > +  // change delimiters, comment, encapsulater
> > +  public void testNextToken6() throws IOException {
> > +    /* file: a;'b and \' more
> > +     *       '
> > +     *       !comment;;;;
> > +     *       ;;
> > +     */
> > +    String code = "a;'b and '' more\n'\n!comment;;;;\n;;";
> > +    TestCSVParser parser = new TestCSVParser(new StringReader(code),
> new CSVStrategy(';', '\'', '!'));
> > +    assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> > +    assertEquals(
> > +      CSVParser.TT_EORECORD + ";b and ' more\n;",
> > +      parser.testNextToken());
> > +  }
> > +
> > +
> > +  // ======================================================
> > +  //   parser tests
> > +  // ======================================================
> > +
> > +  String code =
> > +    "a,b,c,d\n"
> > +    + " a , b , 1 2 \n"
> > +    + "\"foo baar\", b,\n"
> > +   // + "   \"foo\n,,\n\"\",,\n\\\"\",d,e\n";
> > +      + "   \"foo\n,,\n\"\",,\n\"\"\",d,e\n";   // changed to use
> standard CSV escaping
> > +  String[][] res = {
> > +    {"a", "b", "c", "d"},
> > +    {"a", "b", "1 2"},
> > +    {"foo baar", "b", ""},
> > +    {"foo\n,,\n\",,\n\"", "d", "e"}
> > +  };
> > +  public void testGetLine() throws IOException {
> > +    CSVParser parser = new CSVParser(new StringReader(code));
> > +    String[] tmp = null;
> > +    for (int i = 0; i < res.length; i++) {
> > +      tmp = parser.getLine();
> > +      assertTrue(Arrays.equals(res[i], tmp));
> > +    }
> > +    tmp = parser.getLine();
> > +    assertTrue(tmp == null);
> > +  }
> > +
> > +  public void testNextValue() throws IOException {
> > +    CSVParser parser = new CSVParser(new StringReader(code));
> > +    String tmp = null;
> > +    for (int i = 0; i < res.length; i++) {
> > +      for (int j = 0; j < res[i].length; j++) {
> > +        tmp = parser.nextValue();
> > +        assertEquals(res[i][j], tmp);
> > +      }
> > +    }
> > +    tmp = parser.nextValue();
> > +    assertTrue(tmp == null);
> > +  }
> > +
> > +  public void testGetAllValues() throws IOException {
> > +    CSVParser parser = new CSVParser(new StringReader(code));
> > +    String[][] tmp = parser.getAllValues();
> > +    assertEquals(res.length, tmp.length);
> > +    assertTrue(tmp.length > 0);
> > +    for (int i = 0; i < res.length; i++) {
> > +      assertTrue(Arrays.equals(res[i], tmp[i]));
> > +    }
> > +  }
> > +
> > +  public void testExcelStrategy1() throws IOException {
> > +    String code =
> > +      "value1,value2,value3,value4\r\na,b,c,d\r\n  x,,,"
> > +      + "\r\n\r\n\"\"\"hello\"\"\",\"
>  \"\"world\"\"\",\"abc\ndef\",\r\n";
> > +    String[][] res = {
> > +      {"value1", "value2", "value3", "value4"},
> > +      {"a", "b", "c", "d"},
> > +      {"  x", "", "", ""},
> > +      {""},
> > +      {"\"hello\"", "  \"world\"", "abc\ndef", ""}
> > +    };
> > +    CSVParser parser = new CSVParser(new StringReader(code),
> CSVStrategy.EXCEL_STRATEGY);
> > +    String[][] tmp = parser.getAllValues();
> > +    assertEquals(res.length, tmp.length);
> > +    assertTrue(tmp.length > 0);
> > +    for (int i = 0; i < res.length; i++) {
> > +      assertTrue(Arrays.equals(res[i], tmp[i]));
> > +    }
> > +  }
> > +
> > +  public void testExcelStrategy2() throws Exception {
> > +    String code = "foo,baar\r\n\r\nhello,\r\n\r\nworld,\r\n";
> > +    String[][] res = {
> > +      {"foo", "baar"},
> > +      {""},
> > +      {"hello", ""},
> > +      {""},
> > +      {"world", ""}
> > +    };
> > +    CSVParser parser = new CSVParser(new StringReader(code),
> CSVStrategy.EXCEL_STRATEGY);
> > +    String[][] tmp = parser.getAllValues();
> > +    assertEquals(res.length, tmp.length);
> > +    assertTrue(tmp.length > 0);
> > +    for (int i = 0; i < res.length; i++) {
> > +      assertTrue(Arrays.equals(res[i], tmp[i]));
> > +    }
> > +  }
> > +
> > +  public void testEndOfFileBehaviourExcel() throws Exception {
> > +    String[] codes = {
> > +        "hello,\r\n\r\nworld,\r\n",
> > +        "hello,\r\n\r\nworld,",
> > +        "hello,\r\n\r\nworld,\"\"\r\n",
> > +        "hello,\r\n\r\nworld,\"\"",
> > +        "hello,\r\n\r\nworld,\n",
> > +        "hello,\r\n\r\nworld,",
> > +        "hello,\r\n\r\nworld,\"\"\n",
> > +        "hello,\r\n\r\nworld,\"\""
> > +        };
> > +    String[][] res = {
> > +      {"hello", ""},
> > +      {""},  // ExcelStrategy does not ignore empty lines
> > +      {"world", ""}
> > +    };
> > +    String code;
> > +    for (int codeIndex = 0; codeIndex < codes.length; codeIndex++) {
> > +      code = codes[codeIndex];
> > +      CSVParser parser = new CSVParser(new StringReader(code),
> CSVStrategy.EXCEL_STRATEGY);
> > +      String[][] tmp = parser.getAllValues();
> > +      assertEquals(res.length, tmp.length);
> > +      assertTrue(tmp.length > 0);
> > +      for (int i = 0; i < res.length; i++) {
> > +        assertTrue(Arrays.equals(res[i], tmp[i]));
> > +      }
> > +    }
> > +  }
> > +
> > +  public void testEndOfFileBehaviorCSV() throws Exception {
> > +    String[] codes = {
> > +        "hello,\r\n\r\nworld,\r\n",
> > +        "hello,\r\n\r\nworld,",
> > +        "hello,\r\n\r\nworld,\"\"\r\n",
> > +        "hello,\r\n\r\nworld,\"\"",
> > +        "hello,\r\n\r\nworld,\n",
> > +        "hello,\r\n\r\nworld,",
> > +        "hello,\r\n\r\nworld,\"\"\n",
> > +        "hello,\r\n\r\nworld,\"\""
> > +        };
> > +    String[][] res = {
> > +      {"hello", ""},  // CSV Strategy ignores empty lines
> > +      {"world", ""}
> > +    };
> > +    String code;
> > +    for (int codeIndex = 0; codeIndex < codes.length; codeIndex++) {
> > +      code = codes[codeIndex];
> > +      CSVParser parser = new CSVParser(new StringReader(code));
> > +      String[][] tmp = parser.getAllValues();
> > +      assertEquals(res.length, tmp.length);
> > +      assertTrue(tmp.length > 0);
> > +      for (int i = 0; i < res.length; i++) {
> > +        assertTrue(Arrays.equals(res[i], tmp[i]));
> > +      }
> > +    }
> > +  }
> > +
> > +  public void testEmptyLineBehaviourExcel() throws Exception {
> > +    String[] codes = {
> > +        "hello,\r\n\r\n\r\n",
> > +        "hello,\n\n\n",
> > +        "hello,\"\"\r\n\r\n\r\n",
> > +        "hello,\"\"\n\n\n"
> > +        };
> > +    String[][] res = {
> > +      {"hello", ""},
> > +      {""},  // ExcelStrategy does not ignore empty lines
> > +      {""}
> > +    };
> > +    String code;
> > +    for (int codeIndex = 0; codeIndex < codes.length; codeIndex++) {
> > +      code = codes[codeIndex];
> > +      CSVParser parser = new CSVParser(new StringReader(code),
> CSVStrategy.EXCEL_STRATEGY);
> > +      String[][] tmp = parser.getAllValues();
> > +      assertEquals(res.length, tmp.length);
> > +      assertTrue(tmp.length > 0);
> > +      for (int i = 0; i < res.length; i++) {
> > +        assertTrue(Arrays.equals(res[i], tmp[i]));
> > +      }
> > +    }
> > +  }
> > +
> > +  public void testEmptyLineBehaviourCSV() throws Exception {
> > +    String[] codes = {
> > +        "hello,\r\n\r\n\r\n",
> > +        "hello,\n\n\n",
> > +        "hello,\"\"\r\n\r\n\r\n",
> > +        "hello,\"\"\n\n\n"
> > +        };
> > +    String[][] res = {
> > +      {"hello", ""}  // CSV Strategy ignores empty lines
> > +    };
> > +    String code;
> > +    for (int codeIndex = 0; codeIndex < codes.length; codeIndex++) {
> > +      code = codes[codeIndex];
> > +      CSVParser parser = new CSVParser(new StringReader(code));
> > +      String[][] tmp = parser.getAllValues();
> > +      assertEquals(res.length, tmp.length);
> > +      assertTrue(tmp.length > 0);
> > +      for (int i = 0; i < res.length; i++) {
> > +        assertTrue(Arrays.equals(res[i], tmp[i]));
> > +      }
> > +    }
> > +  }
> > +
> > +  public void OLDtestBackslashEscaping() throws IOException {
> > +    String code =
> > +      "one,two,three\n"
> > +      + "on\\\"e,two\n"
> > +      + "on\"e,two\n"
> > +      + "one,\"tw\\\"o\"\n"
> > +      + "one,\"t\\,wo\"\n"
> > +      + "one,two,\"th,ree\"\n"
> > +      + "\"a\\\\\"\n"
> > +      + "a\\,b\n"
> > +      + "\"a\\\\,b\"";
> > +    String[][] res = {
> > +        { "one", "two", "three" },
> > +        { "on\\\"e", "two" },
> > +        { "on\"e", "two" },
> > +        { "one", "tw\"o" },
> > +        { "one", "t\\,wo" },  // backslash in quotes only escapes a
> delimiter (",")
> > +        { "one", "two", "th,ree" },
> > +        { "a\\\\" },     // backslash in quotes only escapes a
> delimiter (",")
> > +        { "a\\", "b" },  // a backslash must be returnd
> > +        { "a\\\\,b" }    // backslash in quotes only escapes a
> delimiter (",")
> > +      };
> > +    CSVParser parser = new CSVParser(new StringReader(code));
> > +    String[][] tmp = parser.getAllValues();
> > +    assertEquals(res.length, tmp.length);
> > +    assertTrue(tmp.length > 0);
> > +    for (int i = 0; i < res.length; i++) {
> > +      assertTrue(Arrays.equals(res[i], tmp[i]));
> > +    }
> > +  }
> > +
> > +  public void testBackslashEscaping() throws IOException {
> > +
> > +    // To avoid confusion over the need for escaping chars in java code,
> > +    // We will test with a forward slash as the escape char, and a
> single
> > +    // quote as the encapsulator.
> > +
> > +    String code =
> > +      "one,two,three\n" // 0
> > +      + "'',''\n"       // 1) empty encapsulators
> > +      + "/',/'\n"       // 2) single encapsulators
> > +      + "'/'','/''\n"   // 3) single encapsulators encapsulated via
> escape
> > +      + "'''',''''\n"   // 4) single encapsulators encapsulated via
> doubling
> > +      + "/,,/,\n"       // 5) separator escaped
> > +      + "//,//\n"       // 6) escape escaped
> > +      + "'//','//'\n"   // 7) escape escaped in encapsulation
> > +      + "   8   ,   \"quoted \"\" /\" // string\"   \n"     // don't
> eat spaces
> > +      + "9,   /\n   \n"  // escaped newline
> > +      + "";
> > +    String[][] res = {
> > +        { "one", "two", "three" }, // 0
> > +        { "", "" },                // 1
> > +        { "'", "'" },              // 2
> > +        { "'", "'" },              // 3
> > +        { "'", "'" },              // 4
> > +        { ",", "," },              // 5
> > +        { "/", "/" },              // 6
> > +        { "/", "/" },              // 7
> > +        { "   8   ", "   \"quoted \"\" \" / string\"   " },
> > +        { "9", "   \n   " },
> > +      };
> > +
> > +
> > +    CSVStrategy strategy = new
> CSVStrategy(',','\'',CSVStrategy.COMMENTS_DISABLED,'/',false,false,true,true);
> > +
> > +    CSVParser parser = new CSVParser(new StringReader(code), strategy);
> > +    String[][] tmp = parser.getAllValues();
> > +    assertTrue(tmp.length > 0);
> > +    for (int i = 0; i < res.length; i++) {
> > +      assertTrue(Arrays.equals(res[i], tmp[i]));
> > +    }
> > +  }
> > +
> > +  public void testBackslashEscaping2() throws IOException {
> > +
> > +    // To avoid confusion over the need for escaping chars in java code,
> > +    // We will test with a forward slash as the escape char, and a
> single
> > +    // quote as the encapsulator.
> > +
> > +    String code = ""
> > +      + " , , \n"           // 1)
> > +      + " \t ,  , \n"       // 2)
> > +      + " // , /, , /,\n"   // 3)
> > +      + "";
> > +    String[][] res = {
> > +        { " ", " ", " " },         // 1
> > +        { " \t ", "  ", " " },         // 2
> > +        { " / ", " , ", " ," },         //3
> > +      };
> > +
> > +
> > +    CSVStrategy strategy = new
> CSVStrategy(',',CSVStrategy.ENCAPSULATOR_DISABLED,CSVStrategy.COMMENTS_DISABLED,'/',false,false,true,true);
> > +
> > +    CSVParser parser = new CSVParser(new StringReader(code), strategy);
> > +    String[][] tmp = parser.getAllValues();
> > +    assertTrue(tmp.length > 0);
> > +
> > +    if (!CSVPrinterTest.equals(res, tmp)) {
> > +      assertTrue(false);
> > +    }
> > +
> > +  }
> > +
> > +
> > +  public void testDefaultStrategy() throws IOException {
> > +
> > +    String code = ""
> > +        + "a,b\n"            // 1)
> > +        + "\"\n\",\" \"\n"   // 2)
> > +        + "\"\",#\n"   // 2)
> > +        ;
> > +    String[][] res = {
> > +        { "a", "b" },
> > +        { "\n", " " },
> > +        { "", "#" },
> > +    };
> > +
> > +    CSVStrategy strategy = CSVStrategy.DEFAULT_STRATEGY;
> > +    assertEquals(CSVStrategy.COMMENTS_DISABLED,
> strategy.getCommentStart());
> > +
> > +    CSVParser parser = new CSVParser(new StringReader(code), strategy);
> > +    String[][] tmp = parser.getAllValues();
> > +    assertTrue(tmp.length > 0);
> > +
> > +    if (!CSVPrinterTest.equals(res, tmp)) {
> > +      assertTrue(false);
> > +    }
> > +
> > +    String[][] res_comments = {
> > +        { "a", "b" },
> > +        { "\n", " " },
> > +        { ""},
> > +    };
> > +
> > +    strategy = new CSVStrategy(',','"','#');
> > +    parser = new CSVParser(new StringReader(code), strategy);
> > +    tmp = parser.getAllValues();
> > +
> > +    if (!CSVPrinterTest.equals(res_comments, tmp)) {
> > +      assertTrue(false);
> > +    }
> > +  }
> > +
> > +
> > +    public void testUnicodeEscape() throws IOException {
> > +      String code = "abc,\\u0070\\u0075\\u0062\\u006C\\u0069\\u0063";
> > +      CSVParser parser = new CSVParser(new StringReader(code));
> > +      parser.getStrategy().setUnicodeEscapeInterpretation(true);
> > +      String[] data = parser.getLine();
> > +      assertEquals(2, data.length);
> > +      assertEquals("abc", data[0]);
> > +      assertEquals("public", data[1]);
> > +    }
> > +
> > +    public void testCarriageReturnLineFeedEndings() throws IOException {
> > +     String code = "foo\r\nbaar,\r\nhello,world\r\n,kanu";
> > +     CSVParser parser = new CSVParser(new StringReader(code));
> > +     String[][] data = parser.getAllValues();
> > +     assertEquals(4, data.length);
> > +    }
> > +
> > +    public void testIgnoreEmptyLines() throws IOException {
> > +      String code = "\nfoo,baar\n\r\n,\n\n,world\r\n\n";
> > +      //String code = "world\r\n\n";
> > +      //String code = "foo;baar\r\n\r\nhello;\r\n\r\nworld;\r\n";
> > +      CSVParser parser = new CSVParser(new StringReader(code));
> > +      String[][] data = parser.getAllValues();
> > +      assertEquals(3, data.length);
> > +    }
> > +
> > +    public void testLineTokenConsistency() throws IOException {
> > +      String code = "\nfoo,baar\n\r\n,\n\n,world\r\n\n";
> > +      CSVParser parser = new CSVParser(new StringReader(code));
> > +      String[][] data = parser.getAllValues();
> > +      parser = new CSVParser(new StringReader(code));
> > +      CSVParser parser1 = new CSVParser(new StringReader(code));
> > +      for (int i = 0; i < data.length; i++) {
> > +        assertTrue(Arrays.equals(parser1.getLine(), data[i]));
> > +        for (int j = 0; j < data[i].length; j++) {
> > +          assertEquals(parser.nextValue(), data[i][j]);
> > +        }
> > +      }
> > +    }
> > +
> > +    // From SANDBOX-153
> > +     public void testDelimiterIsWhitespace() throws IOException {
> > +         String code = "one\ttwo\t\tfour \t five\t six";
> > +         TestCSVParser parser = new TestCSVParser(new
> StringReader(code), CSVStrategy.TDF_STRATEGY);
> > +         assertEquals(CSVParser.TT_TOKEN + ";one;",
> parser.testNextToken());
> > +         assertEquals(CSVParser.TT_TOKEN + ";two;",
> parser.testNextToken());
> > +         assertEquals(CSVParser.TT_TOKEN + ";;",
> parser.testNextToken());
> > +         assertEquals(CSVParser.TT_TOKEN + ";four;",
> parser.testNextToken());
> > +         assertEquals(CSVParser.TT_TOKEN + ";five;",
> parser.testNextToken());
> > +         assertEquals(CSVParser.TT_EOF + ";six;",
> parser.testNextToken());
> > +     }
> > +}
> >
> > Added:
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVPrinterTest.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVPrinterTest.java?rev=1308225&view=auto
> >
> ==============================================================================
> > ---
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVPrinterTest.java
> (added)
> > +++
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVPrinterTest.java
> Mon Apr  2 01:29:20 2012
> > @@ -0,0 +1,197 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one or more
> > + * contributor license agreements.  See the NOTICE file distributed with
> > + * this work for additional information regarding copyright ownership.
> > + * The ASF licenses this file to You under the Apache License, Version
> 2.0
> > + * (the "License"); you may not use this file except in compliance with
> > + * the License.  You may obtain a copy of the License at
> > + *
> > + *      http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing, software
> > + * distributed under the License is distributed on an "AS IS" BASIS,
> > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> > + * See the License for the specific language governing permissions and
> > + * limitations under the License.
> > + */
> > +package org.apache.solr.internal.csv;
> > +
> > +import java.io.IOException;
> > +import java.io.StringReader;
> > +import java.io.StringWriter;
> > +import java.util.Arrays;
> > +import java.util.Random;
> > +
> > +import junit.framework.Test;
> > +import junit.framework.TestCase;
> > +import junit.framework.TestSuite;
> > +
> > +/**
> > + * CSVPrinterTest
> > + */
> > +public class CSVPrinterTest extends TestCase {
> > +
> > +  String lineSeparator = "\n";
> > +
> > +  public void testPrinter1() throws IOException {
> > +    StringWriter sw = new StringWriter();
> > +    CSVPrinter printer = new CSVPrinter(sw,
> CSVStrategy.DEFAULT_STRATEGY);
> > +    String[] line1 = {"a", "b"};
> > +    printer.println(line1);
> > +    assertEquals("a,b" + lineSeparator, sw.toString());
> > +  }
> > +
> > +  public void testPrinter2() throws IOException {
> > +    StringWriter sw = new StringWriter();
> > +    CSVPrinter printer = new CSVPrinter(sw,
> CSVStrategy.DEFAULT_STRATEGY);
> > +    String[] line1 = {"a,b", "b"};
> > +    printer.println(line1);
> > +    assertEquals("\"a,b\",b" + lineSeparator, sw.toString());
> > +  }
> > +
> > +  public void testPrinter3() throws IOException {
> > +    StringWriter sw = new StringWriter();
> > +    CSVPrinter printer = new CSVPrinter(sw,
> CSVStrategy.DEFAULT_STRATEGY);
> > +    String[] line1 = {"a, b", "b "};
> > +    printer.println(line1);
> > +    assertEquals("\"a, b\",\"b \"" + lineSeparator, sw.toString());
> > +  }
> > +
> > +  public void testExcelPrinter1() throws IOException {
> > +    StringWriter sw = new StringWriter();
> > +    CSVPrinter printer = new CSVPrinter(sw, CSVStrategy.EXCEL_STRATEGY);
> > +    String[] line1 = {"a", "b"};
> > +    printer.println(line1);
> > +    assertEquals("a,b" + lineSeparator, sw.toString());
> > +  }
> > +
> > +  public void testExcelPrinter2() throws IOException {
> > +    StringWriter sw = new StringWriter();
> > +    CSVPrinter printer = new CSVPrinter(sw, CSVStrategy.EXCEL_STRATEGY);
> > +    String[] line1 = {"a,b", "b"};
> > +    printer.println(line1);
> > +    assertEquals("\"a,b\",b" + lineSeparator, sw.toString());
> > +  }
> > +
> > +
> > +
> > +  public void testRandom() throws Exception {
> > +    int iter=10000;
> > +    strategy = CSVStrategy.DEFAULT_STRATEGY;
> > +    doRandom(iter);
> > +    strategy = CSVStrategy.EXCEL_STRATEGY;
> > +    doRandom(iter);
> > +
> > +    // Strategy for MySQL
> > +    strategy = new CSVStrategy('\t', CSVStrategy.ENCAPSULATOR_DISABLED,
> CSVStrategy.COMMENTS_DISABLED,'\\',false, false, false, false);
> > +    doRandom(iter);
> > +  }
> > +
> > +  Random r = new Random();
> > +  CSVStrategy strategy;
> > +
> > +  public void doRandom(int iter) throws Exception {
> > +    for (int i=0; i<iter; i++) {
> > +      doOneRandom();
> > +    }
> > +  }
> > +
> > +  public void doOneRandom() throws Exception {
> > +    int nLines = r.nextInt(4)+1;
> > +    int nCol = r.nextInt(3)+1;
> > +    // nLines=1;nCol=2;
> > +    String[][] lines = new String[nLines][];
> > +    for (int i=0; i<nLines; i++) {
> > +      String[] line = new String[nCol];
> > +      lines[i] = line;
> > +      for (int j=0; j<nCol; j++) {
> > +        line[j] = randStr();
> > +      }
> > +    }
> > +
> > +    StringWriter sw = new StringWriter();
> > +    CSVPrinter printer = new CSVPrinter(sw, strategy);
> > +
> > +    for (int i=0; i<nLines; i++) {
> > +      // for (int j=0; j<lines[i].length; j++) System.out.println("###
> VALUE=:" + printable(lines[i][j]));
> > +      printer.println(lines[i]);
> > +    }
> > +
> > +    printer.flush();
> > +    String result = sw.toString();
> > +    // System.out.println("### :" + printable(result));
> > +
> > +    StringReader reader = new StringReader(result);
> > +
> > +    CSVParser parser = new CSVParser(reader, strategy);
> > +    String[][] parseResult = parser.getAllValues();
> > +
> > +    if (!equals(lines, parseResult)) {
> > +      System.out.println("Printer output :" + printable(result));
> > +      assertTrue(false);
> > +    }
> > +  }
> > +
> > +  public static boolean equals(String[][] a, String[][] b) {
> > +    if (a.length != b.length) {
> > +      return false;
> > +    }
> > +    for (int i=0; i<a.length; i++) {
> > +      String[] linea = a[i];
> > +      String[] lineb = b[i];
> > +      if (linea.length != lineb.length) {
> > +        return false;
> > +      }
> > +      for (int j=0; j<linea.length; j++) {
> > +        String aval = linea[j];
> > +        String bval = lineb[j];
> > +        if (!aval.equals(bval)) {
> > +          System.out.println("expected  :" + printable(aval));
> > +          System.out.println("got       :" + printable(bval));
> > +          return false;
> > +        }
> > +      }
> > +    }
> > +    return true;
> > +  }
> > +
> > +  public static String printable(String s) {
> > +    StringBuffer sb = new StringBuffer();
> > +    for (int i=0; i<s.length(); i++) {
> > +      char ch = s.charAt(i);
> > +      if (ch<=' ' || ch>=128) {
> > +        sb.append("(" + (int)ch + ")");
> > +      } else {
> > +        sb.append(ch);
> > +      }
> > +    }
> > +    return sb.toString();
> > +  }
> > +
> > +  public String randStr() {
> > +    int sz = r.nextInt(20);
> > +    // sz = r.nextInt(3);
> > +    char[] buf = new char[sz];
> > +    for (int i=0; i<sz; i++) {
> > +      // stick in special chars with greater frequency
> > +      char ch;
> > +      int what = r.nextInt(20);
> > +      switch (what) {
> > +        case 0: ch = '\r'; break;
> > +        case 1: ch = '\n'; break;
> > +        case 2: ch = '\t'; break;
> > +        case 3: ch = '\f'; break;
> > +        case 4: ch = ' ';  break;
> > +        case 5: ch = ',';  break;
> > +        case 6: ch = '"';  break;
> > +        case 7: ch = '\''; break;
> > +        case 8: ch = '\\'; break;
> > +        default: ch = (char)r.nextInt(300); break;
> > +        // default: ch = 'a'; break;
> > +      }
> > +      buf[i] = ch;
> > +    }
> > +    return new String(buf);
> > +  }
> > +
> > +}
> >
> > Added:
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVStrategyTest.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVStrategyTest.java?rev=1308225&view=auto
> >
> ==============================================================================
> > ---
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVStrategyTest.java
> (added)
> > +++
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVStrategyTest.java
> Mon Apr  2 01:29:20 2012
> > @@ -0,0 +1,91 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one or more
> > + * contributor license agreements.  See the NOTICE file distributed with
> > + * this work for additional information regarding copyright ownership.
> > + * The ASF licenses this file to You under the Apache License, Version
> 2.0
> > + * (the "License"); you may not use this file except in compliance with
> > + * the License.  You may obtain a copy of the License at
> > + *
> > + *      http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing, software
> > + * distributed under the License is distributed on an "AS IS" BASIS,
> > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> > + * See the License for the specific language governing permissions and
> > + * limitations under the License.
> > + */
> > +package org.apache.solr.internal.csv;
> > +
> > +import java.io.StringReader;
> > +
> > +import junit.framework.TestCase;
> > +
> > +/**
> > + * CSVStrategyTest
> > + *
> > + * The test are organized in three different sections:
> > + * The 'setter/getter' section, the lexer section and finally the
> strategy
> > + * section. In case a test fails, you should follow a top-down approach
> for
> > + * fixing a potential bug (its likely that the strategy itself fails if
> the lexer
> > + * has problems...).
> > + */
> > +public class CSVStrategyTest extends TestCase {
> > +
> > +  // ======================================================
> > +  //   getters / setters
> > +  // ======================================================
> > +  public void testGetSetCommentStart() {
> > +    CSVStrategy strategy =
> (CSVStrategy)CSVStrategy.DEFAULT_STRATEGY.clone();
> > +    strategy.setCommentStart('#');
> > +    assertEquals(strategy.getCommentStart(), '#');
> > +    strategy.setCommentStart('!');
> > +    assertEquals(strategy.getCommentStart(), '!');
> > +  }
> > +
> > +  public void testGetSetEncapsulator() {
> > +    CSVStrategy strategy =
> (CSVStrategy)CSVStrategy.DEFAULT_STRATEGY.clone();
> > +    strategy.setEncapsulator('"');
> > +    assertEquals(strategy.getEncapsulator(), '"');
> > +    strategy.setEncapsulator('\'');
> > +    assertEquals(strategy.getEncapsulator(), '\'');
> > +  }
> > +
> > +  public void testGetSetDelimiter() {
> > +    CSVStrategy strategy =
> (CSVStrategy)CSVStrategy.DEFAULT_STRATEGY.clone();
> > +    strategy.setDelimiter(';');
> > +    assertEquals(strategy.getDelimiter(), ';');
> > +    strategy.setDelimiter(',');
> > +    assertEquals(strategy.getDelimiter(), ',');
> > +    strategy.setDelimiter('\t');
> > +    assertEquals(strategy.getDelimiter(), '\t');
> > +  }
> > +
> > +  public void testSetCSVStrategy() {
> > +    CSVStrategy strategy = CSVStrategy.DEFAULT_STRATEGY;
> > +    // default settings
> > +    assertEquals(strategy.getDelimiter(), ',');
> > +    assertEquals(strategy.getEncapsulator(), '"');
> > +    assertEquals(strategy.getCommentStart(),
> CSVStrategy.COMMENTS_DISABLED);
> > +    assertEquals(true,  strategy.getIgnoreLeadingWhitespaces());
> > +    assertEquals(false, strategy.getUnicodeEscapeInterpretation());
> > +    assertEquals(true,  strategy.getIgnoreEmptyLines());
> > +    // explicit csv settings
> > +    assertEquals(strategy.getDelimiter(), ',');
> > +    assertEquals(strategy.getEncapsulator(), '"');
> > +    assertEquals(strategy.getCommentStart(),
> CSVStrategy.COMMENTS_DISABLED);
> > +    assertEquals(true,  strategy.getIgnoreLeadingWhitespaces());
> > +    assertEquals(false, strategy.getUnicodeEscapeInterpretation());
> > +    assertEquals(true,  strategy.getIgnoreEmptyLines());
> > +  }
> > +
> > +  public void testSetExcelStrategy() {
> > +    CSVStrategy strategy = CSVStrategy.EXCEL_STRATEGY;
> > +    assertEquals(strategy.getDelimiter(), ',');
> > +    assertEquals(strategy.getEncapsulator(), '"');
> > +    assertEquals(strategy.getCommentStart(),
> CSVStrategy.COMMENTS_DISABLED);
> > +    assertEquals(false,  strategy.getIgnoreLeadingWhitespaces());
> > +    assertEquals(false, strategy.getUnicodeEscapeInterpretation());
> > +    assertEquals(false, strategy.getIgnoreEmptyLines());
> > +  }
> > +
> > +}
> >
> > Added:
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVUtilsTest.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVUtilsTest.java?rev=1308225&view=auto
> >
> ==============================================================================
> > ---
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVUtilsTest.java
> (added)
> > +++
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVUtilsTest.java
> Mon Apr  2 01:29:20 2012
> > @@ -0,0 +1,150 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one or more
> > + * contributor license agreements.  See the NOTICE file distributed with
> > + * this work for additional information regarding copyright ownership.
> > + * The ASF licenses this file to You under the Apache License, Version
> 2.0
> > + * (the "License"); you may not use this file except in compliance with
> > + * the License.  You may obtain a copy of the License at
> > + *
> > + *      http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing, software
> > + * distributed under the License is distributed on an "AS IS" BASIS,
> > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> > + * See the License for the specific language governing permissions and
> > + * limitations under the License.
> > + */
> > +package org.apache.solr.internal.csv;
> > +
> > +import java.io.IOException;
> > +
> > +import junit.framework.TestCase;
> > +
> > +/**
> > + * CSVUtilsTest
> > + */
> > +public class CSVUtilsTest extends TestCase {
> > +
> > +  // ======================================================
> > +  //   static parser tests
> > +  // ======================================================
> > +  public void testParse1() throws IOException {
> > +      String[][] data = CSVUtils.parse("abc\ndef");
> > +      assertEquals(2, data.length);
> > +      assertEquals(1, data[0].length);
> > +      assertEquals(1, data[1].length);
> > +      assertEquals("abc", data[0][0]);
> > +      assertEquals("def", data[1][0]);
> > +    }
> > +
> > +    public void testParse2() throws IOException {
> > +      String[][] data = CSVUtils.parse("abc,def,\"ghi,jkl\"\ndef");
> > +      assertEquals(2, data.length);
> > +      assertEquals(3, data[0].length);
> > +      assertEquals(1, data[1].length);
> > +      assertEquals("abc", data[0][0]);
> > +      assertEquals("def", data[0][1]);
> > +      assertEquals("ghi,jkl", data[0][2]);
> > +      assertEquals("def", data[1][0]);
> > +    }
> > +
> > +    public void testParse3() throws IOException {
> > +      String[][] data = CSVUtils.parse("abc,\"def\nghi\"\njkl");
> > +      assertEquals(2, data.length);
> > +      assertEquals(2, data[0].length);
> > +      assertEquals(1, data[1].length);
> > +      assertEquals("abc", data[0][0]);
> > +      assertEquals("def\nghi", data[0][1]);
> > +      assertEquals("jkl", data[1][0]);
> > +    }
> > +
> > +    public void testParse4() throws IOException {
> > +      String[][] data = CSVUtils.parse("abc,\"def\\\\nghi\"\njkl");
> > +      assertEquals(2, data.length);
> > +      assertEquals(2, data[0].length);
> > +      assertEquals(1, data[1].length);
> > +      assertEquals("abc", data[0][0]);
> > +      // an escape char in quotes only escapes a delimiter, not itself
> > +      assertEquals("def\\\\nghi", data[0][1]);
> > +      assertEquals("jkl", data[1][0]);
> > +    }
> > +
> > +    public void testParse5() throws IOException {
> > +      String[][] data = CSVUtils.parse("abc,def\\nghi\njkl");
> > +      assertEquals(2, data.length);
> > +      assertEquals(2, data[0].length);
> > +      assertEquals(1, data[1].length);
> > +      assertEquals("abc", data[0][0]);
> > +      assertEquals("def\\nghi", data[0][1]);
> > +      assertEquals("jkl", data[1][0]);
> > +    }
> > +
> > +    public void testParse6() throws IOException {
> > +      String[][] data = CSVUtils.parse("");
> > +      // default strategy is CSV, which ignores empty lines
> > +      assertEquals(0, data.length);
> > +    }
> > +
> > +    public void testParse7() throws IOException {
> > +      boolean io = false;
> > +      try {
> > +        CSVUtils.parse(null);
> > +      } catch (IllegalArgumentException e) {
> > +        io = true;
> > +      }
> > +      assertTrue(io);
> > +    }
> > +
> > +    public void testParseLine1() throws IOException {
> > +      String[] data = CSVUtils.parseLine("abc,def,ghi");
> > +      assertEquals(3, data.length);
> > +      assertEquals("abc", data[0]);
> > +      assertEquals("def", data[1]);
> > +      assertEquals("ghi", data[2]);
> > +    }
> > +
> > +    public void testParseLine2() throws IOException {
> > +      String[] data = CSVUtils.parseLine("abc,def,ghi\n");
> > +      assertEquals(3, data.length);
> > +      assertEquals("abc", data[0]);
> > +      assertEquals("def", data[1]);
> > +      assertEquals("ghi", data[2]);
> > +    }
> > +
> > +    public void testParseLine3() throws IOException {
> > +      String[] data = CSVUtils.parseLine("abc,\"def,ghi\"");
> > +      assertEquals(2, data.length);
> > +      assertEquals("abc", data[0]);
> > +      assertEquals("def,ghi", data[1]);
> > +    }
> > +
> > +    public void testParseLine4() throws IOException {
> > +      String[] data = CSVUtils.parseLine("abc,\"def\nghi\"");
> > +      assertEquals(2, data.length);
> > +      assertEquals("abc", data[0]);
> > +      assertEquals("def\nghi", data[1]);
> > +    }
> > +
> > +    public void testParseLine5() throws IOException {
> > +      String[] data = CSVUtils.parseLine("");
> > +      assertEquals(0, data.length);
> > +      // assertEquals("", data[0]);
> > +    }
> > +
> > +    public void testParseLine6() throws IOException {
> > +      boolean io = false;
> > +      try {
> > +        CSVUtils.parseLine(null);
> > +      } catch (IllegalArgumentException e) {
> > +        io = true;
> > +      }
> > +      assertTrue(io);
> > +    }
> > +
> > +    public void testParseLine7() throws IOException {
> > +      String[] res = CSVUtils.parseLine("");
> > +      assertNotNull(res);
> > +      assertEquals(0, res.length);
> > +    }
> > +
> > +}
> >
> > Added:
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CharBufferTest.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CharBufferTest.java?rev=1308225&view=auto
> >
> ==============================================================================
> > ---
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CharBufferTest.java
> (added)
> > +++
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CharBufferTest.java
> Mon Apr  2 01:29:20 2012
> > @@ -0,0 +1,192 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one
> > + * or more contributor license agreements.  See the NOTICE file
> > + * distributed with this work for additional information
> > + * regarding copyright ownership.  The ASF licenses this file
> > + * to you under the Apache License, Version 2.0 (the
> > + * "License"); you may not use this file except in compliance
> > + * with the License.  You may obtain a copy of the License at
> > + *
> > + * http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing,
> > + * software distributed under the License is distributed on an
> > + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> > + * KIND, either express or implied.  See the License for the
> > + * specific language governing permissions and limitations
> > + * under the License.
> > + */
> > +package org.apache.solr.internal.csv;
> > +
> > +import junit.framework.TestCase;
> > +
> > +public class CharBufferTest extends TestCase {
> > +    public void testCreate() {
> > +        CharBuffer cb = new CharBuffer();
> > +        assertEquals(0, cb.length());
> > +        try {
> > +            cb = new CharBuffer(0);
> > +            fail("Should not be possible");
> > +        } catch(IllegalArgumentException e) {
> > +            // expected
> > +        }
> > +
> > +        cb = new CharBuffer(128);
> > +        assertEquals(0, cb.length());
> > +    }
> > +
> > +    public void testAppendChar() {
> > +        CharBuffer cb = new CharBuffer(1);
> > +        String expected = "";
> > +        for (char c = 'a'; c < 'z'; c++) {
> > +            cb.append(c);
> > +            expected += c;
> > +            assertEquals(expected, cb.toString());
> > +            assertEquals(expected.length(), cb.length());
> > +        }
> > +    }
> > +
> > +    public void testAppendCharArray() {
> > +        CharBuffer cb = new CharBuffer(1);
> > +        char[] abcd = "abcd".toCharArray();
> > +        String expected = "";
> > +        for (int i=0; i<10; i++) {
> > +            cb.append(abcd);
> > +            expected += "abcd";
> > +            assertEquals(expected, cb.toString());
> > +            assertEquals(4*(i+1), cb.length());
> > +        }
> > +    }
> > +
> > +    public void testAppendString() {
> > +        CharBuffer cb = new CharBuffer(1);
> > +        String abcd = "abcd";
> > +        String expected = "";
> > +        for (int i=0; i<10; i++) {
> > +            cb.append(abcd);
> > +            expected += abcd;
> > +            assertEquals(expected, cb.toString());
> > +            assertEquals(4*(i+1), cb.length());
> > +        }
> > +    }
> > +
> > +    public void testAppendStringBuffer() {
> > +        CharBuffer cb = new CharBuffer(1);
> > +        StringBuffer abcd = new StringBuffer("abcd");
> > +        String expected = "";
> > +        for (int i=0; i<10; i++) {
> > +            cb.append(abcd);
> > +            expected += "abcd";
> > +            assertEquals(expected, cb.toString());
> > +            assertEquals(4*(i+1), cb.length());
> > +        }
> > +    }
> > +
> > +    public void testAppendCharBuffer() {
> > +        CharBuffer cb = new CharBuffer(1);
> > +        CharBuffer abcd = new CharBuffer(17);
> > +        abcd.append("abcd");
> > +        String expected = "";
> > +        for (int i=0; i<10; i++) {
> > +            cb.append(abcd);
> > +            expected += "abcd";
> > +            assertEquals(expected, cb.toString());
> > +            assertEquals(4*(i+1), cb.length());
> > +        }
> > +    }
> > +
> > +    public void testShrink() {
> > +        String data = "123456789012345678901234567890";
> > +
> > +        CharBuffer cb = new CharBuffer(data.length() + 100);
> > +        assertEquals(data.length() + 100, cb.capacity());
> > +        cb.append(data);
> > +        assertEquals(data.length() + 100, cb.capacity());
> > +        assertEquals(data.length(), cb.length());
> > +        cb.shrink();
> > +        assertEquals(data.length(), cb.capacity());
> > +        assertEquals(data.length(), cb.length());
> > +        assertEquals(data, cb.toString());
> > +    }
> > +
> > +    //-- the following test cases have been adapted from the
> HttpComponents project
> > +    //-- written by Oleg Kalnichevski
> > +
> > +    public void testSimpleAppend() throws Exception {
> > +        CharBuffer buffer = new CharBuffer(16);
> > +        assertEquals(16, buffer.capacity());
> > +        assertEquals(0, buffer.length());
> > +        char[] b1 = buffer.getCharacters();
> > +        assertNotNull(b1);
> > +        assertEquals(0, b1.length);
> > +        assertEquals(0, buffer.length());
> > +
> > +        char[] tmp = new char[] { '1', '2', '3', '4'};
> > +        buffer.append(tmp);
> > +        assertEquals(16, buffer.capacity());
> > +        assertEquals(4, buffer.length());
> > +
> > +        char[] b2 = buffer.getCharacters();
> > +        assertNotNull(b2);
> > +        assertEquals(4, b2.length);
> > +        for (int i = 0; i < tmp.length; i++) {
> > +            assertEquals(tmp[i], b2[i]);
> > +        }
> > +        assertEquals("1234", buffer.toString());
> > +
> > +        buffer.clear();
> > +        assertEquals(16, buffer.capacity());
> > +        assertEquals(0, buffer.length());
> > +    }
> > +
> > +    public void testAppendString2() throws Exception {
> > +        CharBuffer buffer = new CharBuffer(8);
> > +        buffer.append("stuff");
> > +        buffer.append(" and more stuff");
> > +        assertEquals("stuff and more stuff", buffer.toString());
> > +    }
> > +
> > +    public void testAppendNull() throws Exception {
> > +        CharBuffer buffer = new CharBuffer(8);
> > +
> > +        buffer.append((StringBuffer)null);
> > +        assertEquals("", buffer.toString());
> > +
> > +        buffer.append((String)null);
> > +        assertEquals("", buffer.toString());
> > +
> > +        buffer.append((CharBuffer)null);
> > +        assertEquals("", buffer.toString());
> > +
> > +        buffer.append((char[])null);
> > +        assertEquals("", buffer.toString());
> > +    }
> > +
> > +    public void testAppendCharArrayBuffer() throws Exception {
> > +        CharBuffer buffer1 = new CharBuffer(8);
> > +        buffer1.append(" and more stuff");
> > +        CharBuffer buffer2 = new CharBuffer(8);
> > +        buffer2.append("stuff");
> > +        buffer2.append(buffer1);
> > +        assertEquals("stuff and more stuff", buffer2.toString());
> > +    }
> > +
> > +    public void testAppendSingleChar() throws Exception {
> > +        CharBuffer buffer = new CharBuffer(4);
> > +        buffer.append('1');
> > +        buffer.append('2');
> > +        buffer.append('3');
> > +        buffer.append('4');
> > +        buffer.append('5');
> > +        buffer.append('6');
> > +        assertEquals("123456", buffer.toString());
> > +    }
> > +
> > +    public void testProvideCapacity() throws Exception {
> > +        CharBuffer buffer = new CharBuffer(4);
> > +        buffer.provideCapacity(2);
> > +        assertEquals(4, buffer.capacity());
> > +        buffer.provideCapacity(8);
> > +        assertTrue(buffer.capacity() >= 8);
> > +    }
> > +}
> >
> > Added:
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/ExtendedBufferedReaderTest.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/ExtendedBufferedReaderTest.java?rev=1308225&view=auto
> >
> ==============================================================================
> > ---
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/ExtendedBufferedReaderTest.java
> (added)
> > +++
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/ExtendedBufferedReaderTest.java
> Mon Apr  2 01:29:20 2012
> > @@ -0,0 +1,222 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one or more
> > + * contributor license agreements.  See the NOTICE file distributed with
> > + * this work for additional information regarding copyright ownership.
> > + * The ASF licenses this file to You under the Apache License, Version
> 2.0
> > + * (the "License"); you may not use this file except in compliance with
> > + * the License.  You may obtain a copy of the License at
> > + *
> > + *      http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing, software
> > + * distributed under the License is distributed on an "AS IS" BASIS,
> > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> > + * See the License for the specific language governing permissions and
> > + * limitations under the License.
> > + */
> > +package org.apache.solr.internal.csv;
> > +
> > +import java.io.StringReader;
> > +import java.util.Arrays;
> > +
> > +import junit.framework.Test;
> > +import junit.framework.TestCase;
> > +import junit.framework.TestSuite;
> > +
> > +/**
> > + * ExtendedBufferedReaderTest
> > + *
> > + */
> > +public class ExtendedBufferedReaderTest extends TestCase {
> > +
> > +  // ======================================================
> > +  //   the test cases
> > +  // ======================================================
> > +
> > +  public void testConstructors() {
> > +    ExtendedBufferedReader br = new ExtendedBufferedReader(new
> StringReader(""));
> > +    br = new ExtendedBufferedReader(new StringReader(""), 10);
> > +  }
> > +
> > +  public void testReadLookahead1() throws Exception {
> > +
> > +    assertEquals(ExtendedBufferedReader.END_OF_STREAM,
> getEBR("").read());
> > +    ExtendedBufferedReader br = getEBR("1\n2\r3\n");
> > +    assertEquals('1', br.lookAhead());
> > +    assertEquals(ExtendedBufferedReader.UNDEFINED, br.readAgain());
> > +    assertEquals('1', br.read());
> > +    assertEquals('1', br.readAgain());
> > +
> > +    assertEquals(0, br.getLineNumber());
> > +    assertEquals('\n', br.lookAhead());
> > +    assertEquals(0, br.getLineNumber());
> > +    assertEquals('1', br.readAgain());
> > +    assertEquals('\n', br.read());
> > +    assertEquals(1, br.getLineNumber());
> > +    assertEquals('\n', br.readAgain());
> > +    assertEquals(1, br.getLineNumber());
> > +
> > +    assertEquals('2', br.lookAhead());
> > +    assertEquals(1, br.getLineNumber());
> > +    assertEquals('\n', br.readAgain());
> > +    assertEquals(1, br.getLineNumber());
> > +    assertEquals('2', br.read());
> > +    assertEquals('2', br.readAgain());
> > +
> > +    assertEquals('\r', br.lookAhead());
> > +    assertEquals('2', br.readAgain());
> > +    assertEquals('\r', br.read());
> > +    assertEquals('\r', br.readAgain());
> > +
> > +    assertEquals('3', br.lookAhead());
> > +    assertEquals('\r', br.readAgain());
> > +    assertEquals('3', br.read());
> > +    assertEquals('3', br.readAgain());
> > +
> > +    assertEquals('\n', br.lookAhead());
> > +    assertEquals(1, br.getLineNumber());
> > +    assertEquals('3', br.readAgain());
> > +    assertEquals('\n', br.read());
> > +    assertEquals(2, br.getLineNumber());
> > +    assertEquals('\n', br.readAgain());
> > +    assertEquals(2, br.getLineNumber());
> > +
> > +    assertEquals(ExtendedBufferedReader.END_OF_STREAM, br.lookAhead());
> > +    assertEquals('\n', br.readAgain());
> > +    assertEquals(ExtendedBufferedReader.END_OF_STREAM, br.read());
> > +    assertEquals(ExtendedBufferedReader.END_OF_STREAM, br.readAgain());
> > +    assertEquals(ExtendedBufferedReader.END_OF_STREAM, br.read());
> > +    assertEquals(ExtendedBufferedReader.END_OF_STREAM, br.lookAhead());
> > +
> > +  }
> > +
> > +
> > +  public void testReadLookahead2() throws Exception {
> > +    char[] ref = new char[5];
> > +    char[] res = new char[5];
> > +
> > +    ExtendedBufferedReader br = getEBR("");
> > +    assertEquals(0, br.read(res, 0, 0));
> > +    assertTrue(Arrays.equals(res, ref));
> > +
> > +    br = getEBR("abcdefg");
> > +    ref[0] = 'a';
> > +    ref[1] = 'b';
> > +    ref[2] = 'c';
> > +    assertEquals(3, br.read(res, 0, 3));
> > +    assertTrue(Arrays.equals(res, ref));
> > +    assertEquals('c', br.readAgain());
> > +
> > +    assertEquals('d', br.lookAhead());
> > +    ref[4] = 'd';
> > +    assertEquals(1, br.read(res, 4, 1));
> > +    assertTrue(Arrays.equals(res, ref));
> > +    assertEquals('d', br.readAgain());
> > +
> > +  }
> > +
> > +  public void testMarkSupported() {
> > +    assertFalse(getEBR("foo").markSupported());
> > +  }
> > +
> > +  public void testReadLine() throws Exception {
> > +    ExtendedBufferedReader br = getEBR("");
> > +    assertTrue(br.readLine() == null);
> > +
> > +    br = getEBR("\n");
> > +    assertTrue(br.readLine().equals(""));
> > +    assertTrue(br.readLine() == null);
> > +
> > +    br = getEBR("foo\n\nhello");
> > +    assertEquals(0, br.getLineNumber());
> > +    assertTrue(br.readLine().equals("foo"));
> > +    assertEquals(1, br.getLineNumber());
> > +    assertTrue(br.readLine().equals(""));
> > +    assertEquals(2, br.getLineNumber());
> > +    assertTrue(br.readLine().equals("hello"));
> > +    assertEquals(3, br.getLineNumber());
> > +    assertTrue(br.readLine() == null);
> > +    assertEquals(3, br.getLineNumber());
> > +
> > +    br = getEBR("foo\n\nhello");
> > +    assertEquals('f', br.read());
> > +    assertEquals('o', br.lookAhead());
> > +    assertTrue(br.readLine().equals("oo"));
> > +    assertEquals(1, br.getLineNumber());
> > +    assertEquals('\n', br.lookAhead());
> > +    assertTrue(br.readLine().equals(""));
> > +    assertEquals(2, br.getLineNumber());
> > +    assertEquals('h', br.lookAhead());
> > +    assertTrue(br.readLine().equals("hello"));
> > +    assertTrue(br.readLine() == null);
> > +    assertEquals(3, br.getLineNumber());
> > +
> > +
> > +    br = getEBR("foo\rbaar\r\nfoo");
> > +    assertTrue(br.readLine().equals("foo"));
> > +    assertEquals('b', br.lookAhead());
> > +    assertTrue(br.readLine().equals("baar"));
> > +    assertEquals('f', br.lookAhead());
> > +    assertTrue(br.readLine().equals("foo"));
> > +    assertTrue(br.readLine() == null);
> > +  }
> > +
> > +  public void testSkip0() throws Exception {
> > +
> > +    ExtendedBufferedReader br = getEBR("");
> > +    assertEquals(0, br.skip(0));
> > +    assertEquals(0, br.skip(1));
> > +
> > +    br = getEBR("");
> > +    assertEquals(0, br.skip(1));
> > +
> > +    br = getEBR("abcdefg");
> > +    assertEquals(0, br.skip(0));
> > +    assertEquals('a', br.lookAhead());
> > +    assertEquals(0, br.skip(0));
> > +    assertEquals('a', br.lookAhead());
> > +    assertEquals(1, br.skip(1));
> > +    assertEquals('b', br.lookAhead());
> > +    assertEquals('b', br.read());
> > +    assertEquals(3, br.skip(3));
> > +    assertEquals('f', br.lookAhead());
> > +    assertEquals(2, br.skip(5));
> > +    assertTrue(br.readLine() == null);
> > +
> > +    br = getEBR("12345");
> > +    assertEquals(5, br.skip(5));
> > +    assertTrue (br.lookAhead() == ExtendedBufferedReader.END_OF_STREAM);
> > +  }
> > +
> > +  public void testSkipUntil() throws Exception {
> > +    ExtendedBufferedReader br = getEBR("");
> > +    assertEquals(0, br.skipUntil(';'));
> > +    br = getEBR("ABCDEF,GHL,,MN");
> > +    assertEquals(6, br.skipUntil(','));
> > +    assertEquals(0, br.skipUntil(','));
> > +    br.skip(1);
> > +    assertEquals(3, br.skipUntil(','));
> > +    br.skip(1);
> > +    assertEquals(0, br.skipUntil(','));
> > +    br.skip(1);
> > +    assertEquals(2, br.skipUntil(','));
> > +  }
> > +
> > +  public void testReadUntil() throws Exception {
> > +    ExtendedBufferedReader br = getEBR("");
> > +    assertTrue(br.readUntil(';').equals(""));
> > +    br = getEBR("ABCDEF;GHL;;MN");
> > +    assertTrue(br.readUntil(';').equals("ABCDEF"));
> > +    assertTrue(br.readUntil(';').length() == 0);
> > +    br.skip(1);
> > +    assertTrue(br.readUntil(';').equals("GHL"));
> > +    br.skip(1);
> > +    assertTrue(br.readUntil(';').equals(""));
> > +    br.skip(1);
> > +    assertTrue(br.readUntil(',').equals("MN"));
> > +  }
> > +
> > +  private ExtendedBufferedReader getEBR(String s) {
> > +    return new ExtendedBufferedReader(new StringReader(s));
> > +  }
> > +}
> >
> > Added:
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigGuesserTest.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigGuesserTest.java?rev=1308225&view=auto
> >
> ==============================================================================
> > ---
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigGuesserTest.java
> (added)
> > +++
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigGuesserTest.java
> Mon Apr  2 01:29:20 2012
> > @@ -0,0 +1,88 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one
> > + * or more contributor license agreements.  See the NOTICE file
> > + * distributed with this work for additional information
> > + * regarding copyright ownership.  The ASF licenses this file
> > + * to you under the Apache License, Version 2.0 (the
> > + * "License"); you may not use this file except in compliance
> > + * with the License.  You may obtain a copy of the License at
> > + *
> > + * http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing,
> > + * software distributed under the License is distributed on an
> > + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> > + * KIND, either express or implied.  See the License for the
> > + * specific language governing permissions and limitations
> > + * under the License.
> > + */
> > +package org.apache.solr.internal.csv.writer;
> > +
> > +import java.io.ByteArrayInputStream;
> > +
> > +import junit.framework.TestCase;
> > +
> > +/**
> > + * Tests for the config guesser.
> > + */
> > +public class CSVConfigGuesserTest extends TestCase {
> > +
> > +    public void testSetters() throws Exception {
> > +        CSVConfigGuesser guesser = new CSVConfigGuesser();
> > +        ByteArrayInputStream in = new ByteArrayInputStream(new byte[0]);
> > +        guesser.setInputStream(in);
> > +        assertEquals(in, guesser.getInputStream());
> > +        guesser = new CSVConfigGuesser(in);
> > +        assertEquals(in, guesser.getInputStream());
> > +        assertEquals(false, guesser.hasFieldHeader());
> > +        guesser.setHasFieldHeader(true);
> > +        assertEquals(true, guesser.hasFieldHeader());
> > +    }
> > +    /**
> > +     * Test a format like
> > +     *  1234 ; abcd ; 1234 ;
> > +     *
> > +     */
> > +    public void testConfigGuess1() {
> > +        CSVConfig expected = new CSVConfig();
> > +        expected.setDelimiter(';');
> > +        expected.setValueDelimiter(' ');
> > +        expected.setFill(CSVConfig.FILLRIGHT);
> > +        expected.setIgnoreValueDelimiter(false);
> > +        expected.setFixedWidth(true);
> > +        CSVField field = new CSVField();
> > +        field.setSize(4);
> > +        expected.addField(field);
> > +        expected.addField(field);
> > +        StringBuffer sb = new StringBuffer();
> > +        sb.append("1234;abcd;1234\n");
> > +        sb.append("abcd;1234;abcd");
> > +        ByteArrayInputStream in = new
> ByteArrayInputStream(sb.toString().getBytes());
> > +        CSVConfigGuesser guesser = new CSVConfigGuesser(in);
> > +        CSVConfig guessed = guesser.guess();
> > +        assertEquals(expected.isFixedWidth(), guessed.isFixedWidth());
> > +        assertEquals(expected.getFields().length,
> guessed.getFields().length);
> > +        assertEquals(expected.getFields()[0].getSize(),
> guessed.getFields()[0].getSize());
> > +    }
> > +    /**
> > +     * Test a format like
> > +     *  1234,123123,12312312,213123
> > +     *  1,2,3,4
> > +     *
> > +     */
> > +    public void testConfigGuess2() {
> > +        CSVConfig expected = new CSVConfig();
> > +        expected.setDelimiter(';');
> > +        expected.setValueDelimiter(' ');
> > +        expected.setFill(CSVConfig.FILLRIGHT);
> > +        expected.setIgnoreValueDelimiter(false);
> > +//        expected.setFixedWidth(false);
> > +        StringBuffer sb = new StringBuffer();
> > +        sb.append("1,2,3,4\n");
> > +        sb.append("abcd,1234,abcd,1234");
> > +        ByteArrayInputStream in = new
> ByteArrayInputStream(sb.toString().getBytes());
> > +        CSVConfigGuesser guesser = new CSVConfigGuesser(in);
> > +        CSVConfig guessed = guesser.guess();
> > +        assertEquals(expected.isFixedWidth(), guessed.isFixedWidth());
> > +    }
> > +}
> >
> > Added:
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigTest.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigTest.java?rev=1308225&view=auto
> >
> ==============================================================================
> > ---
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigTest.java
> (added)
> > +++
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigTest.java
> Mon Apr  2 01:29:20 2012
> > @@ -0,0 +1,100 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one
> > + * or more contributor license agreements.  See the NOTICE file
> > + * distributed with this work for additional information
> > + * regarding copyright ownership.  The ASF licenses this file
> > + * to you under the Apache License, Version 2.0 (the
> > + * "License"); you may not use this file except in compliance
> > + * with the License.  You may obtain a copy of the License at
> > + *
> > + * http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing,
> > + * software distributed under the License is distributed on an
> > + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> > + * KIND, either express or implied.  See the License for the
> > + * specific language governing permissions and limitations
> > + * under the License.
> > + */
> > +package org.apache.solr.internal.csv.writer;
> > +
> > +import java.util.Collection;
> > +
> > +import junit.framework.TestCase;
> > +
> > +/**
> > + * Testcase for the CSVConfig
> > + */
> > +public class CSVConfigTest extends TestCase {
> > +
> > +
> > +    public void testFixedWith() {
> > +        CSVConfig config = new CSVConfig();
> > +        assertEquals(false, config.isFixedWidth());
> > +        config.setFixedWidth(true);
> > +        assertEquals(true, config.isFixedWidth());
> > +    }
> > +
> > +    public void testFields() {
> > +        CSVConfig config = new CSVConfig();
> > +        assertEquals(0, config.getFields().length);
> > +        config.setFields((CSVField[])null);
> > +        assertEquals(0, config.getFields().length);
> > +        config.setFields((Collection)null);
> > +        assertEquals(0, config.getFields().length);
> > +        CSVField field = new CSVField();
> > +        field.setName("field1");
> > +        config.addField(field);
> > +        assertEquals(field, config.getFields()[0]);
> > +        assertEquals(null, config.getField(null));
> > +        assertEquals(null, config.getField("field11"));
> > +        assertEquals(field, config.getField("field1"));
> > +    }
> > +
> > +    public void testFill() {
> > +        CSVConfig config = new CSVConfig();
> > +        assertEquals(CSVConfig.FILLNONE, config.getFill());
> > +        config.setFill(CSVConfig.FILLLEFT);
> > +        assertEquals(CSVConfig.FILLLEFT, config.getFill());
> > +        config.setFill(CSVConfig.FILLRIGHT);
> > +        assertEquals(CSVConfig.FILLRIGHT, config.getFill());
> > +        assertEquals(' ', config.getFillChar());
> > +        config.setFillChar('m');
> > +        assertEquals('m', config.getFillChar());
> > +    }
> > +
> > +    public void testDelimiter() {
> > +        CSVConfig config = new CSVConfig();
> > +        assertEquals(',', config.getDelimiter());
> > +        config.setDelimiter(';');
> > +        assertEquals(';', config.getDelimiter());
> > +        assertEquals(false, config.isDelimiterIgnored());
> > +        config.setIgnoreDelimiter(true);
> > +        assertEquals(true, config.isDelimiterIgnored());
> > +    }
> > +
> > +    public void testValueDelimiter() {
> > +        CSVConfig config = new CSVConfig();
> > +        assertEquals('"', config.getValueDelimiter());
> > +        config.setValueDelimiter('m');
> > +        assertEquals('m', config.getValueDelimiter());
> > +        assertEquals(true, config.isValueDelimiterIgnored());
> > +        config.setIgnoreValueDelimiter(false);
> > +        assertEquals(false, config.isValueDelimiterIgnored());
> > +    }
> > +
> > +    public void testFieldHeader() {
> > +        CSVConfig config = new CSVConfig();
> > +        assertEquals(false, config.isFieldHeader());
> > +        config.setFieldHeader(true);
> > +        assertEquals(true, config.isFieldHeader());
> > +    }
> > +
> > +    public void testTrimEnd() {
> > +        CSVConfig config = new CSVConfig();
> > +        assertEquals(false, config.isEndTrimmed());
> > +        config.setEndTrimmed(true);
> > +        assertEquals(true, config.isEndTrimmed());
> > +    }
> > +
> > +}
> >
> > Added:
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVFieldTest.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVFieldTest.java?rev=1308225&view=auto
> >
> ==============================================================================
> > ---
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVFieldTest.java
> (added)
> > +++
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVFieldTest.java
> Mon Apr  2 01:29:20 2012
> > @@ -0,0 +1,48 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one
> > + * or more contributor license agreements.  See the NOTICE file
> > + * distributed with this work for additional information
> > + * regarding copyright ownership.  The ASF licenses this file
> > + * to you under the Apache License, Version 2.0 (the
> > + * "License"); you may not use this file except in compliance
> > + * with the License.  You may obtain a copy of the License at
> > + *
> > + * http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing,
> > + * software distributed under the License is distributed on an
> > + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> > + * KIND, either express or implied.  See the License for the
> > + * specific language governing permissions and limitations
> > + * under the License.
> > + */
> > +package org.apache.solr.internal.csv.writer;
> > +
> > +import junit.framework.TestCase;
> > +
> > +public class CSVFieldTest extends TestCase {
> > +
> > +    public void testCSVField() {
> > +        CSVField field = new CSVField();
> > +        assertEquals(null, field.getName());
> > +        field.setName("id");
> > +        assertEquals("id", field.getName());
> > +        assertEquals(0, field.getSize());
> > +        field.setSize(10);
> > +        assertEquals(10, field.getSize());
> > +        field = new CSVField("name");
> > +        assertEquals("name", field.getName());
> > +        field = new CSVField("name", 10);
> > +        assertEquals("name", field.getName());
> > +        assertEquals(10, field.getSize());
> > +    }
> > +
> > +    public void testFill() {
> > +        CSVField field = new CSVField();
> > +        assertEquals(CSVConfig.FILLNONE, field.getFill());
> > +        assertEquals(false, field.overrideFill());
> > +        field.setFill(CSVConfig.FILLLEFT);
> > +        assertEquals(true, field.overrideFill());
> > +        assertEquals(CSVConfig.FILLLEFT, field.getFill());
> > +    }
> > +}
> >
> > Added:
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVWriterTest.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVWriterTest.java?rev=1308225&view=auto
> >
> ==============================================================================
> > ---
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVWriterTest.java
> (added)
> > +++
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVWriterTest.java
> Mon Apr  2 01:29:20 2012
> > @@ -0,0 +1,56 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one
> > + * or more contributor license agreements.  See the NOTICE file
> > + * distributed with this work for additional information
> > + * regarding copyright ownership.  The ASF licenses this file
> > + * to you under the Apache License, Version 2.0 (the
> > + * "License"); you may not use this file except in compliance
> > + * with the License.  You may obtain a copy of the License at
> > + *
> > + * http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing,
> > + * software distributed under the License is distributed on an
> > + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> > + * KIND, either express or implied.  See the License for the
> > + * specific language governing permissions and limitations
> > + * under the License.
> > + */
> > +package org.apache.solr.internal.csv.writer;
> > +
> > +import java.io.StringWriter;
> > +import java.util.HashMap;
> > +import java.util.Map;
> > +
> > +import junit.framework.TestCase;
> > +
> > +/**
> > + * The testcase for the csv writer.
> > + */
> > +public class CSVWriterTest extends TestCase {
> > +
> > +    public void testCSVConfig() {
> > +        CSVWriter writer = new CSVWriter();
> > +        assertEquals(null, writer.getConfig());
> > +        CSVConfig config = new CSVConfig();
> > +        writer.setConfig(config);
> > +        assertEquals(config, writer.getConfig());
> > +        writer = new CSVWriter(config);
> > +        assertEquals(config, writer.getConfig());
> > +    }
> > +
> > +    public void testWriter() {
> > +        CSVWriter writer = new CSVWriter();
> > +        CSVConfig config = new CSVConfig();
> > +        config.addField(new CSVField("field1", 5));
> > +        config.addField(new CSVField("field2", 4));
> > +        writer.setConfig(config);
> > +        StringWriter sw = new StringWriter();
> > +        writer.setWriter(sw);
> > +        Map map = new HashMap();
> > +        map.put("field1", "12345");
> > +        map.put("field2", "1234");
> > +        writer.writeRecord(map);
> > +        assertEquals("12345,1234\n",sw.toString());
> > +    }
> > +}
> >
> >
>
>
>
> --
> lucidimagination.com
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>
>


-- 
Chris Male | Software Developer | DutchWorks | www.dutchworks.nl

Mime
View raw message