Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 78D58200D29 for ; Thu, 26 Oct 2017 20:42:59 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 3EE0E1609E5; Thu, 26 Oct 2017 18:42:54 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 3C3BF1609E8 for ; Thu, 26 Oct 2017 20:42:53 +0200 (CEST) Received: (qmail 93356 invoked by uid 500); 26 Oct 2017 18:42:52 -0000 Mailing-List: contact commits-help@trafodion.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: commits@trafodion.apache.org Delivered-To: mailing list commits@trafodion.apache.org Received: (qmail 93347 invoked by uid 99); 26 Oct 2017 18:42:52 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd2-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 26 Oct 2017 18:42:52 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd2-us-west.apache.org (ASF Mail Server at spamd2-us-west.apache.org) with ESMTP id 394571A07A9 for ; Thu, 26 Oct 2017 18:42:51 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd2-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -3.721 X-Spam-Level: X-Spam-Status: No, score=-3.721 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_NUMSUBJECT=0.5, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-0.001, SPF_PASS=-0.001, URIBL_BLOCKED=0.001] autolearn=disabled Received: from mx1-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd2-us-west.apache.org [10.40.0.9]) (amavisd-new, port 10024) with ESMTP id SKk-_wGYiXEV for ; Thu, 26 Oct 2017 18:42:49 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-eu.apache.org (ASF Mail Server at mx1-lw-eu.apache.org) with SMTP id E98895F5B8 for ; Thu, 26 Oct 2017 18:42:47 +0000 (UTC) Received: (qmail 93081 invoked by uid 99); 26 Oct 2017 18:42:47 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 26 Oct 2017 18:42:47 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 259FEDFBCA; Thu, 26 Oct 2017 18:42:47 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: dbirdsall@apache.org To: commits@trafodion.incubator.apache.org Date: Thu, 26 Oct 2017 18:42:47 -0000 Message-Id: <31a82f1b93e4465686422977afcbb823@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/2] incubator-trafodion git commit: fix TRAFODION-2714 archived-at: Thu, 26 Oct 2017 18:42:59 -0000 Repository: incubator-trafodion Updated Branches: refs/heads/master 9dd862c14 -> af54ea41f fix TRAFODION-2714 Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/04c0c46e Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/04c0c46e Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/04c0c46e Branch: refs/heads/master Commit: 04c0c46ee79a832c7e653bbd1b83fcc920dffb8f Parents: 0f8c3e2 Author: SuJinpei <873118042@qq.com> Authored: Thu Oct 19 11:25:47 2017 +0800 Committer: SuJinpei <873118042@qq.com> Committed: Thu Oct 19 11:25:47 2017 +0800 ---------------------------------------------------------------------- core/conn/odb/src/odb.c | 102 +++++++++++++++++++++++++++++-------------- 1 file changed, 69 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/04c0c46e/core/conn/odb/src/odb.c ---------------------------------------------------------------------- diff --git a/core/conn/odb/src/odb.c b/core/conn/odb/src/odb.c index 5d26c03..308edd1 100755 --- a/core/conn/odb/src/odb.c +++ b/core/conn/odb/src/odb.c @@ -39,7 +39,7 @@ char *odbauth = "Trafodion Dev "; #define TD_CHUNK 32 /* Granularity of td[] memory allocation */ #define MAX_ARGS 11 /* Max arguments for interactive mode */ #define ARG_LENGTH 128 /* Max argument length in interactive mode */ -#define LINE_CHUNK 512 /* size of memory chunks allocated to store lines */ +#define LINE_CHUNK 51200 /* size of memory chunks allocated to store lines */ #define MAX_VNLEN 32 /* Max variable name length */ #define MAX_PK_COLS 16 /* Max number of PK elements */ #define MAXCOL_LEN 128 /* Max table column name length */ @@ -6196,11 +6196,11 @@ static void Oload(int eid) 0100 = escape flag 0200 = embed file read */ ccl = 0, /* Continue cleaning to RS in the next buffer */ pstats = 0, /* Error flag: 0 = print stats, 1 = don't print stats */ - lfs = etab[eid].fs, /* local field separator */ - lrs = etab[eid].rs, /* local record separator */ - lsq = etab[eid].sq, /* local string qualifier */ - lem = etab[eid].em, /* local embed character */ - lec = etab[eid].ec; /* local escape character */ + lfs = etab[eid].fs, /* local field separator */ + lrs = etab[eid].rs, /* local record separator */ + lsq = etab[eid].sq ? etab[eid].sq : '"', /* local string qualifier, deafult is '"' */ + lem = etab[eid].em, /* local embed character */ + lec = etab[eid].ec; /* local escape character */ int *ldrs=0, /* pointer to array containing loaders EIDs */ *rmap=0; /* Input File Fields Map (reverse map): >=0 maps the correspoding Table Column Number @@ -6778,10 +6778,14 @@ static void Oload(int eid) map[j].max = (int) strtol ( bp, NULL, 10); } else { fg |= 0010; /* delimited format flag */ - map[j].idx = (int)strtol( bp, NULL, 10); + map[j].idx = (int)strtol( bp, NULL, 10) - 1; + if (map[j].idx < 0) { + fprintf(stderr, "odb [Oload(%d)] - Error: Field index should should start from 1\n", __LINE__); + goto oload_exit; + } } while ( *bp && *bp++ != ':' ); - if ( bp ) { + if ( *bp ) { if ( !strmicmp ( "substr", bp, 6 ) ) { map[j].op = 1; while ( *bp && *bp++ != ':' ); @@ -7301,13 +7305,13 @@ static void Oload(int eid) } nb += len; /* update bytes read from file */ p = 0; /* reset buffer index */ - while ( lts ) { /* skip initial lines */ - if ( buff[p++] == lrs ) { - lts-- ; + while (lts && p < len) { /* skip initial lines */ + if (buff[p++] == lrs) { + --lts; } } if ( ccl ) { /* continue cleaning rest of line */ - while ( p < etab[eid].buffsz && buff[p] != lrs ) /* ... skip the rest of the line */ + while ( p < len && buff[p] != lrs ) /* ... skip the rest of the line */ p++; if ( buff[p] == lrs ) { /* if a record separator has been found */ ccl = 0; /* switch the continue cleaning flag off */ @@ -7947,8 +7951,6 @@ static void Oload2(int eid) j=0, /* loop variable */ lts = etab[eid].k , /* lines to skip */ isgz=0; /* input file is gzipped: 0=no , 1=yes */ - int lfs = etab[eid].fs, /* local field separator */ - lrs = etab[eid].rs; /* local record separator */ int *ldrs=0; /* pointer to array containing loaders EIDs */ FILE *fl=0; /* data to load file pointer */ #ifdef HDFS @@ -7957,14 +7959,21 @@ static void Oload2(int eid) char *buff = 0, /* IO buffer */ *buff_save = 0, /* save original IO buffer pointer */ *gzbuff = 0, /* GZIP IO buffer */ - *bc = 0, /* current field buffer size pointer */ - *bn = 0; /* next field buffer size pointer */ + *str = 0, /* field buffer */ + *bc = 0; /* current field buffer size pointer */ char num[32]; /* Formatted Number String */ char tim[15]; /* Formatted Time String */ struct timeval tve; /* timeval struct to define elapesd/timelines */ SQLCHAR *Odp = 0; /* rowset buffer data pointer */ double seconds = 0; /* seconds used for timings */ z_stream gzstream = { 0 } ; /* zlib structure for gziped files */ + unsigned char fg = 0, /* Oload flags: + 0001 = in a quoted string 0020 = field ready + 0010 = delimited fields 0040 = record ready + 0100 = escape flag */ + lfs = etab[eid].fs, /* local field separator */ + lrs = etab[eid].rs, /* local record separator */ + lsq = etab[eid].sq ? etab[eid].sq : '"'; /* local string qualifier, deafult is '"' */ /* Check if we have to use another ODBC connection */ if ( thps[tid].cr > 0 ) { @@ -8121,6 +8130,13 @@ static void Oload2(int eid) etab[eid].s += ( etab[eid].td[j].Osize + etab[eid].td[j].pad + sizeof(SQLLEN) ); /* space for length indicator */ } + /* Allocate field buffer */ + if ((str = calloc(1, etab[eid].buffsz + 1)) == (void *)NULL) { + fprintf(stderr, "odb [Oload2(%d)] - Error allocating field buffer: [%d] %s\n", + __LINE__, errno, strerror(errno)); + goto oload2_exit; + } + /* Truncate target table */ if ( etab[eid].flg & 0002 ) { /* truncate target table */ if ( f & 020000 ) /* if verbose */ @@ -8404,23 +8420,44 @@ static void Oload2(int eid) } } nb += len; /* update bytes read from file */ - while ( lts ) { /* skip initial lines */ + while ( lts && len ) { /* skip initial lines */ if ( *buff++ == lrs ) { lts-- ; } len-- ; } bc = buff ; /* Initialize current field position */ - while ( 1 ) { - if ( k < l - 1 ) { - bn = memchr ( bc , lfs , len ) ; /* identify next field sep */ - } else { - bn = memchr ( bc , lrs , len ) ; /* identofy next record sep */ + size_t i = 0; + while (i < len) { + ifl = 0; + for (; i < len; ++i) { + if ((fg & 0001) && (fg & 0100) && (bc[i] == lfs || bc[i] == lrs)) { /* treat string qualifier before as end quote */ + fg &= ~0101; + } + if (bc[i] == lfs && !(fg & 0001)) { /* if field sep... */ + fg |= 0020; /* set field complete flag on */ + ++i; + break; + } + else if (bc[i] == lrs && !(fg & 0001)) { /* if record sep... */ + fg |= 0040; /* set record complete flag on */ + ++i; + break; + } + else if ((bc[i] == lsq) && (fg & 0001) && !(fg & 0100)) { /* if string qualifier char and in string qualifier */ + fg |= 0100; + } + else if ((bc[i] == lsq) && !(fg & 0100)) { + fg ^= 0001; /* flip quoted string flag */ + } + else { /* add new character to field buffer */ + str[ifl++] = bc[i]; + fg &= ~0100; /* set escape flag off */ + } } - if ( bn ) { /* field complete */ - ifl = bn - bc ; /* determine field length */ + if ( fg & 0060 ) { /* field complete */ if ( ifl ) { - MEMCPY(Odp, bc, ifl); + MEMCPY(Odp, str, ifl); Odp += etab[eid].td[k].Osize + etab[eid].td[k].pad - rl ; *((SQLLEN *)(Odp)) = (SQLLEN)(ifl+rl) ; } else if ( rl ) { @@ -8431,8 +8468,6 @@ static void Oload2(int eid) *((SQLLEN *)(Odp)) = (SQLLEN)SQL_NULL_DATA ; } rl = 0 ; - len -= ( ifl + 1 ); - bc = bn + 1 ; Odp += sizeof(SQLLEN) ; if ( ++k == l ) { /* row completed */ k = 0 ; @@ -8440,10 +8475,10 @@ static void Oload2(int eid) nrf++ ; } } else { /* field incomplete */ - rl = ifl = len ; + rl = ifl; if ( ifl ) - MEMCPY(Odp, bc, ifl); - len = 0 ; + MEMCPY(Odp, str, ifl); + memset(str, '\0', ifl); Odp += ifl ; break; } @@ -8490,8 +8525,6 @@ static void Oload2(int eid) k = 0; Odp = etab[eid].Orowsetl ; } - if ( len == 0 ) - break; } } oload2_exit: @@ -8586,6 +8619,9 @@ static void Oload2(int eid) free ( etab[eid].td[i].Oname ); free(etab[eid].td); } + if (str) { + free(str); + } /* Close bad file */ if ( etab[eid].fo != stderr ) @@ -9659,7 +9695,7 @@ static void OloadJson(int eid) } else { // read rows - while (pJsonReader->errorCode == JSON_SUCCESS) { + while (pJsonReader->errorCode == JSON_SUCCESS) { if (pJsonReader->state == JSON_STATE_MEMBER_KEY) { jsonReadKey(pJsonReader, keybuf, sizeof(keybuf)); }