From issues-return-49001-archive-asf-public=cust-asf.ponee.io@camel.apache.org Thu Feb 1 09:38:09 2018 Return-Path: X-Original-To: archive-asf-public@eu.ponee.io Delivered-To: archive-asf-public@eu.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by mx-eu-01.ponee.io (Postfix) with ESMTP id C90C2180652 for ; Thu, 1 Feb 2018 09:38:09 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id B9422160C26; Thu, 1 Feb 2018 08:38:09 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 8E179160C56 for ; Thu, 1 Feb 2018 09:38:08 +0100 (CET) Received: (qmail 13676 invoked by uid 500); 1 Feb 2018 08:38:07 -0000 Mailing-List: contact issues-help@camel.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@camel.apache.org Delivered-To: mailing list issues@camel.apache.org Received: (qmail 13666 invoked by uid 99); 1 Feb 2018 08:38:07 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd3-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 01 Feb 2018 08:38:07 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd3-us-west.apache.org (ASF Mail Server at spamd3-us-west.apache.org) with ESMTP id 4CD6E199704 for ; Thu, 1 Feb 2018 08:38:07 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd3-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -109.51 X-Spam-Level: X-Spam-Status: No, score=-109.51 tagged_above=-999 required=6.31 tests=[ENV_AND_HDR_SPF_MATCH=-0.5, KAM_ASCII_DIVIDERS=0.8, KAM_SHORT=0.001, RCVD_IN_DNSWL_MED=-2.3, SPF_PASS=-0.001, T_RP_MATCHES_RCVD=-0.01, USER_IN_DEF_SPF_WL=-7.5, USER_IN_WHITELIST=-100] autolearn=disabled Received: from mx1-lw-us.apache.org ([10.40.0.8]) by localhost (spamd3-us-west.apache.org [10.40.0.10]) (amavisd-new, port 10024) with ESMTP id P4DmDOLVahGF for ; Thu, 1 Feb 2018 08:38:03 +0000 (UTC) Received: from mailrelay1-us-west.apache.org (mailrelay1-us-west.apache.org [209.188.14.139]) by mx1-lw-us.apache.org (ASF Mail Server at mx1-lw-us.apache.org) with ESMTP id 629EB5F3B8 for ; Thu, 1 Feb 2018 08:38:03 +0000 (UTC) Received: from jira-lw-us.apache.org (unknown [207.244.88.139]) by mailrelay1-us-west.apache.org (ASF Mail Server at mailrelay1-us-west.apache.org) with ESMTP id EEE61E0309 for ; Thu, 1 Feb 2018 08:38:01 +0000 (UTC) Received: from jira-lw-us.apache.org (localhost [127.0.0.1]) by jira-lw-us.apache.org (ASF Mail Server at jira-lw-us.apache.org) with ESMTP id A110622D07 for ; Thu, 1 Feb 2018 08:38:00 +0000 (UTC) Date: Thu, 1 Feb 2018 08:38:00 +0000 (UTC) From: "ASF GitHub Bot (JIRA)" To: issues@camel.apache.org Message-ID: In-Reply-To: References: Subject: [jira] [Commented] (CAMEL-12192) BindyCsvDataFormat does not support skip fields MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-JIRA-FingerPrint: 30527f35849b9dde25b450d4833f0394 [ https://issues.apache.org/jira/browse/CAMEL-12192?page=3Dcom.atlassia= n.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=3D163= 48201#comment-16348201 ]=20 ASF GitHub Bot commented on CAMEL-12192: ---------------------------------------- davsclaus closed pull request #2200: CAMEL-12192 support csv bindy skip fi= elds URL: https://github.com/apache/camel/pull/2200 =20 =20 =20 This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/components/camel-bindy/src/main/java/org/apache/camel/dataform= at/bindy/BindyCsvFactory.java b/components/camel-bindy/src/main/java/org/ap= ache/camel/dataformat/bindy/BindyCsvFactory.java index 9906402ef6b..e2046c91f50 100755 --- a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bind= y/BindyCsvFactory.java +++ b/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bind= y/BindyCsvFactory.java @@ -37,8 +37,6 @@ import org.apache.camel.dataformat.bindy.annotation.Section; import org.apache.camel.dataformat.bindy.format.FormatException; import org.apache.camel.dataformat.bindy.util.ConverterUtils; -import org.apache.camel.impl.DefaultClassResolver; -import org.apache.camel.spi.ClassResolver; import org.apache.camel.util.ObjectHelper; import org.apache.camel.util.ReflectionHelper; import org.slf4j.Logger; @@ -67,6 +65,7 @@ =20 private String separator; private boolean skipFirstLine; + private boolean skipField; private boolean generateHeaderColumnNames; private boolean messageOrdered; private String quote; @@ -174,105 +173,118 @@ public void bind(CamelContext camelContext, List tokens, Map model, int line, int pos, int counterMandatoryFields, String data, D= ataField dataField) throws Exception { + ObjectHelper.notNull(dataField, "No position " + pos + " defined f= or the field: " + data + ", line: " + line); + + if (dataField.trim()) { + data =3D data.trim(); + } + + if (dataField.required()) { + // Increment counter of mandatory fields + ++counterMandatoryFields; + + // Check if content of the field is empty + // This is not possible for mandatory fields + if (data.equals("")) { + throw new IllegalArgumentException("The mandatory field de= fined at the position " + pos + " is empty for the line: " + line); } + } =20 - // Create format object to format the field - FormattingOptions formattingOptions =3D ConverterUtils.convert= (dataField, - field.getType(), - field.getAnnotation(BindyConverter.class), - getLocale()); - Format format =3D formatFactory.getFormat(formattingOptions= ); + // Get Field to be setted + Field field =3D annotatedFields.get(pos); + field.setAccessible(true); =20 - // field object to be set - Object modelField =3D model.get(field.getDeclaringClass().getN= ame()); + if (LOG.isDebugEnabled()) { + LOG.debug("Pos: {}, Data: {}, Field type: {}", new Object[]{po= s, data, field.getType()}); + } =20 - // format the data received - Object value =3D null; + // Create format object to format the field + FormattingOptions formattingOptions =3D ConverterUtils.convert(dat= aField, + field.getType(), + field.getAnnotation(BindyConverter.class), + getLocale()); + Format format =3D formatFactory.getFormat(formattingOptions); =20 - if (!data.equals("")) { - try { - if (quoting && quote !=3D null && (data.contains("\\" = + quote) || data.contains(quote)) && quotingEscaped) { - value =3D format.parse(data.replaceAll("\\\\" + qu= ote, "\\" + quote)); - } else { - value =3D format.parse(data); - } - } catch (FormatException ie) { - throw new IllegalArgumentException(ie.getMessage() + "= , position: " + pos + ", line: " + line, ie); - } catch (Exception e) { - throw new IllegalArgumentException("Parsing error dete= cted for field defined at the position: " + pos + ", line: " + line, e); - } - } else { - if (!dataField.defaultValue().isEmpty()) { - value =3D format.parse(dataField.defaultValue()); + // field object to be set + Object modelField =3D model.get(field.getDeclaringClass().getName(= )); + + // format the data received + Object value =3D null; + + if (!data.equals("")) { + try { + if (quoting && quote !=3D null && (data.contains("\\" + qu= ote) || data.contains(quote)) && quotingEscaped) { + value =3D format.parse(data.replaceAll("\\\\" + quote,= "\\" + quote)); } else { - value =3D getDefaultValueForPrimitive(field.getType())= ; + value =3D format.parse(data); } + } catch (FormatException ie) { + throw new IllegalArgumentException(ie.getMessage() + ", po= sition: " + pos + ", line: " + line, ie); + } catch (Exception e) { + throw new IllegalArgumentException("Parsing error detected= for field defined at the position: " + pos + ", line: " + line, e); } - =20 - if (value !=3D null && !dataField.method().isEmpty()) { - Class clazz; - if (dataField.method().contains(".")) { - clazz =3D camelContext.getClassResolver().resolveManda= toryClass(dataField.method().substring(0, dataField.method().lastIndexOf(".= "))); - } else { - clazz =3D field.getType(); - } - =20 - String methodName =3D dataField.method().substring(dataFie= ld.method().lastIndexOf(".") + 1, - dataFie= ld.method().length()); - =20 - Method m =3D ReflectionHelper.findMethod(clazz, methodName= , field.getType()); - if (m !=3D null) { - // this method must be static and return type - // must be the same as the datafield and=20 - // must receive only the datafield value=20 - // as the method argument - value =3D ObjectHelper.invokeMethod(m, null, value); - } else { - // fallback to method without parameter, that is on th= e value itself - m =3D ReflectionHelper.findMethod(clazz, methodName); - value =3D ObjectHelper.invokeMethod(m, value); - } + } else { + if (!dataField.defaultValue().isEmpty()) { + value =3D format.parse(dataField.defaultValue()); + } else { + value =3D getDefaultValueForPrimitive(field.getType()); } - - field.set(modelField, value); - - ++pos; - } =20 - LOG.debug("Counter mandatory fields: {}", counterMandatoryFields); + if (value !=3D null && !dataField.method().isEmpty()) { + Class clazz; + if (dataField.method().contains(".")) { + clazz =3D camelContext.getClassResolver().resolveMandatory= Class(dataField.method().substring(0, dataField.method().lastIndexOf(".")))= ; + } else { + clazz =3D field.getType(); + } =20 - if (counterMandatoryFields < numberMandatoryFields) { - throw new IllegalArgumentException("Some mandatory fields are = missing, line: " + line); - } + String methodName =3D dataField.method().substring(dataField.m= ethod().lastIndexOf(".") + 1, + dataField.method().length()); =20 - if (pos < totalFields) { - setDefaultValuesForFields(model); + Method m =3D ReflectionHelper.findMethod(clazz, methodName, fi= eld.getType()); + if (m !=3D null) { + // this method must be static and return type + // must be the same as the datafield and=20 + // must receive only the datafield value=20 + // as the method argument + value =3D ObjectHelper.invokeMethod(m, null, value); + } else { + // fallback to method without parameter, that is on the va= lue itself + m =3D ReflectionHelper.findMethod(clazz, methodName); + value =3D ObjectHelper.invokeMethod(m, value); + } } =20 + field.set(modelField, value); + return counterMandatoryFields; } =20 @Override @@ -590,6 +602,10 @@ private void initCsvRecordParameters() { skipFirstLine =3D record.skipFirstLine(); LOG.debug("Skip First Line parameter of the CSV: {}" += skipFirstLine); =20 + // Get skipFirstLine parameter + skipField =3D record.skipField(); + LOG.debug("Skip Field parameter of the CSV: {}" + skip= Field); + // Get generateHeaderColumnNames parameter generateHeaderColumnNames =3D record.generateHeaderCol= umns(); LOG.debug("Generate header column names parameter of t= he CSV: {}", generateHeaderColumnNames); @@ -688,6 +704,15 @@ public boolean getSkipFirstLine() { return skipFirstLine; } =20 + /** + * Indicate if can skip fields + *=20 + * @return boolean + */ + public boolean isSkipField() { + return this.skipField; + } + /** * If last record is to span the rest of the line */ @@ -719,4 +744,5 @@ public boolean isAllowEmptyStream() { public boolean isEndWithLineBreak() { return endWithLineBreak; } + =20 } diff --git a/components/camel-bindy/src/main/java/org/apache/camel/dataform= at/bindy/annotation/CsvRecord.java b/components/camel-bindy/src/main/java/o= rg/apache/camel/dataformat/bindy/annotation/CsvRecord.java index dd7bd186304..6175e5c22c3 100755 --- a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bind= y/annotation/CsvRecord.java +++ b/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bind= y/annotation/CsvRecord.java @@ -53,6 +53,12 @@ */ boolean skipFirstLine() default false; =20 + /** + * The skipField parameter will allow to skip fields of a CSV file.=20 + * If some fields are not necessary, they can be skipped. + */ + boolean skipField() default false; + /** * Character to be used to add a carriage return after each record * (optional) Three values can be used : WINDOWS, UNIX or MAC. diff --git a/components/camel-bindy/src/test/java/org/apache/camel/dataform= at/bindy/csv/BindyCsvSkipFieldTest.java b/components/camel-bindy/src/test/j= ava/org/apache/camel/dataformat/bindy/csv/BindyCsvSkipFieldTest.java new file mode 100644 index 00000000000..7f4df2fe4b2 --- /dev/null +++ b/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bind= y/csv/BindyCsvSkipFieldTest.java @@ -0,0 +1,170 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied= . + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dataformat.bindy.csv; + +import org.apache.camel.EndpointInject; +import org.apache.camel.Exchange; +import org.apache.camel.Processor; +import org.apache.camel.Produce; +import org.apache.camel.ProducerTemplate; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.mock.MockEndpoint; +import org.apache.camel.dataformat.bindy.annotation.CsvRecord; +import org.apache.camel.dataformat.bindy.annotation.DataField; +import org.junit.Test; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.AbstractJUnit4SpringContext= Tests; + + +@ContextConfiguration +public class BindyCsvSkipFieldTest extends AbstractJUnit4SpringContextTes= ts { + + private static final String URI_MOCK_RESULT =3D "mock:result"; + private static final String URI_DIRECT_START =3D "direct:start"; + =20 + private static String input =3D "VOA,12 abc street,Skip Street,Melbour= ne,VIC,3000,Australia,Skip dummy1,end of record"; + + @Produce(uri =3D URI_DIRECT_START) + private ProducerTemplate template; + + @EndpointInject(uri =3D URI_MOCK_RESULT) + private MockEndpoint result; + + private String expected; + =20 + @Test + @DirtiesContext + public void testUnMarshalAndMarshal() throws Exception { + =20 + template.sendBody(input); + result.expectedMessageCount(1); + result.assertIsSatisfied(); + } + + public static class ContextConfig extends RouteBuilder { + BindyCsvDataFormat camelDataFormat =3D new BindyCsvDataFormat(CsvS= kipField.class); + + public void configure() { + from(URI_DIRECT_START).unmarshal(camelDataFormat) + .process(new Processor() { + @Override + public void process(Exchange exchange) throws Exce= ption { + CsvSkipField csvSkipField =3D (CsvSkipField) e= xchange.getIn().getBody(); + assert csvSkipField.getAttention().equals("VO= A"); + assert csvSkipField.getAddressLine1().equals(= "12 abc street"); + assert csvSkipField.getCity().equals("Melbour= ne"); + assert csvSkipField.getState().equals("VIC"); + assert csvSkipField.getZip().equals("3000"); + assert csvSkipField.getCountry() .equals("Aus= tralia"); + assert csvSkipField.getDummy2().equals("end o= f record"); + } + }) + =20 + .marshal(camelDataFormat) + .convertBodyTo(String.class) + .to(URI_MOCK_RESULT); + } + + } + =20 + @CsvRecord(separator =3D ",", skipField =3D true) + public static class CsvSkipField { + @DataField(pos =3D 1) + private String attention; + =20 + @DataField(pos =3D 2) + private String addressLine1; + =20 + @DataField(pos =3D 4) + private String city; + =20 + @DataField(pos =3D 5) + private String state; + =20 + @DataField(pos =3D 6) + private String zip; + =20 + @DataField(pos =3D 7) + private String country; + =20 + @DataField(pos =3D 9) + private String dummy2; + + public String getAttention() { + return attention; + } + + public void setAttention(String attention) { + this.attention =3D attention; + } + + public String getAddressLine1() { + return addressLine1; + } + + public void setAddressLine1(String addressLine1) { + this.addressLine1 =3D addressLine1; + } + + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city =3D city; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state =3D state; + } + + public String getZip() { + return zip; + } + + public void setZip(String zip) { + this.zip =3D zip; + } + + public String getCountry() { + return country; + } + + public void setCountry(String country) { + this.country =3D country; + } + + public String getDummy2() { + return dummy2; + } + + public void setDummy2(String dummy2) { + this.dummy2 =3D dummy2; + } + + @Override + public String toString() { + return "Record [attention=3D" + getAttention() + ", addressLin= e1=3D" + getAddressLine1() + ", " + "city=3D" + getCity() + ", state=3D" + = getState() + ", zip=3D" + getZip() + ", country=3D" + + getCountry() + ", dummy2=3D" + getDummy2() + "]"; + } + } +} diff --git a/components/camel-bindy/src/test/resources/org/apache/camel/dat= aformat/bindy/csv/BindyCsvSkipFieldTest-context.xml b/components/camel-bind= y/src/test/resources/org/apache/camel/dataformat/bindy/csv/BindyCsvSkipFiel= dTest-context.xml new file mode 100644 index 00000000000..c1c6ebd1380 --- /dev/null +++ b/components/camel-bindy/src/test/resources/org/apache/camel/dataformat= /bindy/csv/BindyCsvSkipFieldTest-context.xml @@ -0,0 +1,34 @@ + + + + =20 +=09 +=09=09=20 +=09 +=09 +=09 +=09 + \ No newline at end of file =20 ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. =20 For queries about this service, please contact Infrastructure at: users@infra.apache.org > BindyCsvDataFormat does not support skip fields=20 > ------------------------------------------------ > > Key: CAMEL-12192 > URL: https://issues.apache.org/jira/browse/CAMEL-12192 > Project: Camel > Issue Type: Improvement > Components: camel-bindy > Affects Versions: 2.15.1, 2.20.1 > Reporter: Neo Xu > Assignee: Claus Ibsen > Priority: Major > Fix For: 2.21.0 > > Original Estimate: 48h > Remaining Estimate: 48h > > Currently:=C2=A0BindyCsvDataFormat has to map all the fields in a CSV fil= e. > =C2=A0If there are 200 fields in a CSV file, it has to create 200 fields = in a class. But sometimes, it is only interested in 20 fields. So it needs = to skip some positions if they are not required. But=C2=A0BindyCsvDataForma= t will throw an exception in this case.=C2=A0 > =C2=A0 -- This message was sent by Atlassian JIRA (v7.6.3#76005)