camel-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (CAMEL-10053) Bindy does not seem to support Cp922 character encoding
Date Wed, 15 Jun 2016 20:45:10 GMT

    [ https://issues.apache.org/jira/browse/CAMEL-10053?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15332518#comment-15332518
] 

ASF GitHub Bot commented on CAMEL-10053:
----------------------------------------

GitHub user anoordover opened a pull request:

    https://github.com/apache/camel/pull/1038

    CAMEL-10053: added note to documentation about characterset-conversio…

    Tried to document the behaviour seen with CAMEL-10053.
    I will close the issue.

You can merge this pull request into a Git repository by running:

    $ git pull https://github.com/anoordover/camel CAMEL-10053

Alternatively you can review and apply these changes as the patch at:

    https://github.com/apache/camel/pull/1038.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

    This closes #1038
    
----
commit b21051909a8a3417279421f4576869e326c5a827
Author: Arno Noordover <anoordover@users.noreply.github.com>
Date:   2016-06-15T20:36:36Z

    CAMEL-10053: added note to documentation about characterset-conversion and removal of
header/property

----


> Bindy does not seem to support Cp922 character encoding
> -------------------------------------------------------
>
>                 Key: CAMEL-10053
>                 URL: https://issues.apache.org/jira/browse/CAMEL-10053
>             Project: Camel
>          Issue Type: Bug
>          Components: camel-bindy
>    Affects Versions: 2.16.2
>            Reporter: David Cornforth
>
> If I use the {{charset}} option when consuming from a non UTF8 encoded file and then
unmarshal the result using a bindy object the result is either an exception or incorrect data.
 Below are some unit tests that illustrate the issue.  The UTF8 ones work, but all of the
Cp922 ones fail for a variety of reasons, both using {{BindyFixedLengthDataFormat}} and {{BindyCsvDataFormat}}.
 I have only tested this with Cp922, but it may be the case that other non-standard encodings
also exhibit the same behaviour.
> {code}
> import org.apache.camel.EndpointInject;
> import org.apache.camel.builder.RouteBuilder;
> import org.apache.camel.component.mock.MockEndpoint;
> import org.apache.camel.dataformat.bindy.annotation.CsvRecord;
> import org.apache.camel.dataformat.bindy.annotation.DataField;
> import org.apache.camel.dataformat.bindy.annotation.FixedLengthRecord;
> import org.apache.camel.dataformat.bindy.csv.BindyCsvDataFormat;
> import org.apache.camel.dataformat.bindy.fixed.BindyFixedLengthDataFormat;
> import org.apache.camel.test.junit4.CamelTestSupport;
> import org.junit.After;
> import org.junit.Test;
> import java.io.IOException;
> import java.nio.file.Files;
> import java.nio.file.Paths;
> import java.util.List;
> public class BindyEncodingTest extends CamelTestSupport {
>     private static final String RESULT_FIXED_LENGTH_UTF8 = "mock:result_fixed_length_utf8";
>     private static final String RESULT_FIXED_LENGTH_CP922 = "mock:result_fixed_length_cp922";
>     private static final String RESULT_CSV_UTF8 = "mock:result_csv_utf8";
>     private static final String RESULT_CSV_CP922 = "mock:result_csv_cp922";
>     private static final String TMP = "C:/tmp";
>     private static final String FIXED_LENGTH_UTF8 = "BindyEncodingTest.FixedLength.utf8.txt";
>     private static final String FIXED_LENGTH_CP922 = "BindyEncodingTest.FixedLength.cp922.txt";
>     private static final String CSV_UTF8 = "BindyEncodingTest.csv.utf8.txt";
>     private static final String CSV_CP922 = "BindyEncodingTest.csv.cp922.txt";
>     @EndpointInject(uri = RESULT_FIXED_LENGTH_UTF8)
>     protected MockEndpoint resultEndpointFixedLengthUtf8;
>     @EndpointInject(uri = RESULT_FIXED_LENGTH_CP922)
>     protected MockEndpoint resultEndpointFixedLengthCp922;
>     @EndpointInject(uri = RESULT_CSV_UTF8)
>     protected MockEndpoint resultEndpointCsvUtf8;
>     @EndpointInject(uri = RESULT_CSV_CP922)
>     protected MockEndpoint resultEndpointCsvCp922;
>     @Override
>     protected RouteBuilder createRouteBuilder() throws Exception {
>         return new RouteBuilder() {
>             @Override
>             public void configure() throws Exception {
>                 from("file://"+TMP+"?fileName="+ FIXED_LENGTH_UTF8 +"&charset=UTF8")
>                         .unmarshal(new BindyFixedLengthDataFormat(FixedLengthBindy.class))
>                         .to(RESULT_FIXED_LENGTH_UTF8);
>                 from("file://"+TMP+"?fileName="+ FIXED_LENGTH_CP922 +"&charset=Cp922")
>                         .unmarshal(new BindyFixedLengthDataFormat(FixedLengthBindy.class))
>                         .to(RESULT_FIXED_LENGTH_CP922);
>                 from("file://"+TMP+"?fileName="+ CSV_UTF8 +"&charset=UTF8")
>                         .unmarshal(new BindyCsvDataFormat(CsvBindy.class))
>                         .to(RESULT_CSV_UTF8);
>                 from("file://"+TMP+"?fileName="+ CSV_CP922 +"&charset=Cp922")
>                         .unmarshal(new BindyCsvDataFormat(CsvBindy.class))
>                         .to(RESULT_CSV_CP922);
>             }
>         };
>     }
>     @After
>     public void after() throws InterruptedException {
>         assertMockEndpointsSatisfied();
>     }
>     /* works */
>     @Test
>     public void fixedLengthUtf8() throws IOException, InterruptedException {
>         byte[] input = new byte[]{
>                 (byte)0xC3,(byte)0x85, // Å
>                 (byte)0xC3,(byte)0x98, // Ø
>                 (byte)0xC3,(byte)0x86, // Æ
>                 (byte)0x33,(byte)0x34,(byte)0x35,(byte)0x36,(byte)0x37,(byte)0x38,(byte)0x39
// 3456789
>         };
>         writeFile(TMP+"/"+ FIXED_LENGTH_UTF8, input);
>         resultEndpointFixedLengthUtf8.expectedMessageCount(1);
>         Thread.sleep(5000);
>         Object body = resultEndpointFixedLengthUtf8.getExchanges().get(0).getIn().getBody();
>         assertTrue("The message body is null", body != null);
>         assertTrue(body instanceof FixedLengthBindy);
>         FixedLengthBindy fixedLengthBindy = (FixedLengthBindy) body;
>         assertEquals("First is incorrect","Å",fixedLengthBindy.getFirst());
>         assertEquals("Second is incorrect","ØÆ",fixedLengthBindy.getSecond());
>         assertEquals("Third is incorrect","345678",fixedLengthBindy.getThird());
>         assertEquals("Fourth is incorrect","9",fixedLengthBindy.getFourth());
>         System.out.println(body);
>     }
>     /* fails with java.lang.StringIndexOutOfBoundsException */
>     @Test
>     public void fixedLengthCp922() throws IOException, InterruptedException {
>         // ftp://ftp.software.ibm.com/software/globalization/gcoc/attachments/CP00922.pdf
>         byte[] input = new byte[]{
>                 (byte)0xC5, // Å
>                 (byte)0xD8, // Ø
>                 (byte)0xC6, // Æ
>                 (byte)0x33,(byte)0x34,(byte)0x35,(byte)0x36,(byte)0x37,(byte)0x38,(byte)0x39
// 3456789
>         };
>         writeFile(TMP + "/" + FIXED_LENGTH_CP922, input);
>         resultEndpointFixedLengthCp922.expectedMessageCount(1);
>         Thread.sleep(5000);
>         Object body = resultEndpointFixedLengthCp922.getExchanges().get(0).getIn().getBody();
>         assertTrue("The message body is null", body != null);
>         assertTrue(body instanceof FixedLengthBindy);
>         FixedLengthBindy fixedLengthBindy = (FixedLengthBindy) body;
>         assertEquals("First is incorrect","Å",fixedLengthBindy.getFirst());
>         assertEquals("Second is incorrect","ØÆ",fixedLengthBindy.getSecond());
>         assertEquals("Third is incorrect","345678",fixedLengthBindy.getThird());
>         assertEquals("Fourth is incorrect","9",fixedLengthBindy.getFourth());
>         System.out.println(body);
>     }
>     /* fails as Third is incorrect */
>     @Test
>     public void fixedLengthCp922a() throws IOException, InterruptedException {
>         // ftp://ftp.software.ibm.com/software/globalization/gcoc/attachments/CP00922.pdf
>         byte[] input = new byte[]{
>                 (byte)0x31,(byte)0x32,(byte)0x33,(byte)0x34,(byte)0x35,(byte)0x36,(byte)0x37,
// 1234567
>                 (byte)0xD8, // Ø
>                 (byte)0xC6, // Æ
>                 (byte)0xC5 // Å
>         };
>         writeFile(TMP + "/" + FIXED_LENGTH_CP922, input);
>         resultEndpointFixedLengthCp922.expectedMessageCount(1);
>         Thread.sleep(5000);
>         Object body = resultEndpointFixedLengthCp922.getExchanges().get(0).getIn().getBody();
>         assertTrue("The message body is null", body != null);
>         assertTrue(body instanceof FixedLengthBindy);
>         FixedLengthBindy fixedLengthBindy = (FixedLengthBindy) body;
>         assertEquals("First is incorrect","1",fixedLengthBindy.getFirst());
>         assertEquals("Second is incorrect","23",fixedLengthBindy.getSecond());
>         assertEquals("Third is incorrect","4567ØÆ",fixedLengthBindy.getThird());
>         assertEquals("Fourth is incorrect","Å",fixedLengthBindy.getFourth());
>         System.out.println(body);
>     }
>     /* works */
>     @Test
>     public void csvUtf8() throws IOException, InterruptedException {
>         byte[] input = new byte[]{
>                 (byte)0xC3,(byte)0x85, // Å
>                 (byte)0x2C, // ,
>                 (byte)0xC3,(byte)0x98, // Ø
>                 (byte)0xC3,(byte)0x86, // Æ
>                 (byte)0x2C, // ,
>                 (byte)0x33,(byte)0x34,(byte)0x35,(byte)0x36,(byte)0x37,(byte)0x38,
>                 (byte)0x2C, // ,
>                 (byte)0x39 // 3456789
>         };
>         writeFile(TMP+"/"+ CSV_UTF8, input);
>         resultEndpointCsvUtf8.expectedMessageCount(1);
>         Thread.sleep(5000);
>         Object body = resultEndpointCsvUtf8.getExchanges().get(0).getIn().getBody();
>         assertTrue("The message body is null", body != null);
>         assertTrue(body instanceof CsvBindy);
>         CsvBindy csvBindy = (CsvBindy) body;
>         assertEquals("First is incorrect","Å",csvBindy.getFirst());
>         assertEquals("First is incorrect","ØÆ",csvBindy.getSecond());
>         assertEquals("First is incorrect","345678",csvBindy.getThird());
>         assertEquals("First is incorrect","9",csvBindy.getFourth());
>         System.out.println(body);
>     }
>     /* fails as First is incorrect */
>     @Test
>     public void csvCp922() throws IOException, InterruptedException {
>         // ftp://ftp.software.ibm.com/software/globalization/gcoc/attachments/CP00922.pdf
>         byte[] input = new byte[]{
>                 (byte)0xC5, // Å
>                 (byte)0x2C, // ,
>                 (byte)0xD8, // Ø
>                 (byte)0xC6, // Æ
>                 (byte)0x2C, // ,
>                 (byte)0x33,(byte)0x34,(byte)0x35,(byte)0x36,(byte)0x37,(byte)0x38, //
345678
>                 (byte)0x2C, // ,
>                 (byte)0x39  // 9
>         };
>         writeFile(TMP+"/"+ CSV_CP922, input);
>         resultEndpointCsvCp922.expectedMessageCount(1);
>         Thread.sleep(5000);
>         Object body = resultEndpointCsvCp922.getExchanges().get(0).getIn().getBody();
>         assertTrue("The message body is null", body != null);
>         assertTrue("body is not List it is "+body.getClass(), body instanceof List);
>         List<?> list = (List<?>)body;
>         assertEquals("incorrect list size", 2, list.size());
>         CsvBindy csvBindy = (CsvBindy) list.get(0);
>         assertEquals("First is incorrect","Å",csvBindy.getFirst());
>         assertEquals("First is incorrect","ØÆ",csvBindy.getSecond());
>         assertEquals("First is incorrect","345678",csvBindy.getThird());
>         assertEquals("First is incorrect","9",csvBindy.getFourth());
>         System.out.println(body);
>     }
>     private void writeFile(String path, byte[] content) throws IOException {
>         Files.write(Paths.get(path),content);
>     }
>     @FixedLengthRecord(ignoreTrailingChars = true)
>     public static class FixedLengthBindy {
>         @DataField(pos = 1, length = 1) // 1
>         private String first;
>         @DataField(pos = 2, length = 2) // 2 - 3
>         private String second;
>         @DataField(pos = 3, length = 6) // 4 - 9
>         private String third;
>         @DataField(pos = 4, length = 1) // 10
>         private String fourth;
>         public String getFirst() {
>             return first;
>         }
>         public void setFirst(String first) {
>             this.first = first;
>         }
>         public String getSecond() {
>             return second;
>         }
>         public void setSecond(String second) {
>             this.second = second;
>         }
>         public String getThird() {
>             return third;
>         }
>         public void setThird(String third) {
>             this.third = third;
>         }
>         public String getFourth() {
>             return fourth;
>         }
>         public void setFourth(String fourth) {
>             this.fourth = fourth;
>         }
>         @Override
>         public String toString() {
>             return "FixedLengthBindy{" +
>                     "first='" + first + '\'' +
>                     ", second='" + second + '\'' +
>                     ", third='" + third + '\'' +
>                     ", fourth='" + fourth + '\'' +
>                     '}';
>         }
>     }
>     @CsvRecord(separator = ",", skipFirstLine = false)
>     public static class CsvBindy{
>         @DataField(pos = 1)
>         private String first;
>         @DataField(pos = 2)
>         private String second;
>         @DataField(pos = 3)
>         private String third;
>         @DataField(pos = 4)
>         private String fourth;
>         public String getFirst() {
>             return first;
>         }
>         public void setFirst(String first) {
>             this.first = first;
>         }
>         public String getSecond() {
>             return second;
>         }
>         public void setSecond(String second) {
>             this.second = second;
>         }
>         public String getThird() {
>             return third;
>         }
>         public void setThird(String third) {
>             this.third = third;
>         }
>         public String getFourth() {
>             return fourth;
>         }
>         public void setFourth(String fourth) {
>             this.fourth = fourth;
>         }
>         @Override
>         public String toString() {
>             return "CsvBindy{" +
>                     "first='" + first + '\'' +
>                     ", second='" + second + '\'' +
>                     ", third='" + third + '\'' +
>                     ", fourth='" + fourth + '\'' +
>                     '}';
>         }
>     }
> }
> {code}
> The relevant parts of the pom:
> {code:xml}
>     <properties>
>         <camel.version>2.16.2</camel.version>
>     </properties>
>     ...
>     <build>
>         <plugins>
>             <plugin>
>                 <groupId>org.apache.maven.plugins</groupId>
>                 <artifactId>maven-compiler-plugin</artifactId>
>                 <version>2.5.1</version>
>                 <configuration>
>                     <source>1.7</source>
>                     <target>1.7</target>
>                 </configuration>
>             </plugin>
>         </plugins>
>     </build>
>     <dependencies>
>         <dependency>
>             <groupId>org.apache.camel</groupId>
>             <artifactId>camel-core</artifactId>
>             <version>${camel.version}</version>
>         </dependency>
>         <dependency>
>             <groupId>org.apache.camel</groupId>
>             <artifactId>camel-bindy</artifactId>
>             <version>${camel.version}</version>
>         </dependency>
>         <dependency>
>             <groupId>org.apache.camel</groupId>
>             <artifactId>camel-jackson</artifactId>
>             <version>${camel.version}</version>
>         </dependency>
>         <dependency>
>             <groupId>org.apache.camel</groupId>
>             <artifactId>camel-test</artifactId>
>             <version>${camel.version}</version>
>             <scope>test</scope>
>         </dependency>
>         <dependency>
>             <groupId>commons-lang</groupId>
>             <artifactId>commons-lang</artifactId>
>             <version>2.6</version>
>         </dependency>
>         <dependency>
>             <groupId>commons-io</groupId>
>             <artifactId>commons-io</artifactId>
>             <version>2.4</version>
>         </dependency>
>         <dependency>
>             <groupId>org.slf4j</groupId>
>             <artifactId>slf4j-jdk14</artifactId>
>             <version>1.7.21</version>
>         </dependency>
>     </dependencies>
> {code}
> Note that the tests write to the filesystem, defaulting to {{C:/tmp}}.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Mime
View raw message