hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From h...@apache.org
Subject incubator-hawq git commit: HAWQ-280. Fix error accessing external table or copying from file with bad rows
Date Fri, 25 Dec 2015 06:17:41 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master d926fbf42 -> fc89b4519


HAWQ-280. Fix error accessing external table or copying from file with bad rows


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/fc89b451
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/fc89b451
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/fc89b451

Branch: refs/heads/master
Commit: fc89b45197d6bd4c4abb6b2442627c200ecee5dc
Parents: d926fbf
Author: Ruilong Huo <rhuo@pivotal.io>
Authored: Thu Dec 24 20:24:43 2015 -0800
Committer: Ruilong Huo <rhuo@pivotal.io>
Committed: Fri Dec 25 14:17:27 2015 +0800

----------------------------------------------------------------------
 src/backend/access/external/fileam.c | 11 +----------
 src/backend/cdb/cdbsreh.c            |  4 ----
 src/backend/commands/copy.c          | 11 +----------
 src/include/cdb/cdbsreh.h            |  7 -------
 4 files changed, 2 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fc89b451/src/backend/access/external/fileam.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/fileam.c b/src/backend/access/external/fileam.c
index 82abb9d..645e6dc 100644
--- a/src/backend/access/external/fileam.c
+++ b/src/backend/access/external/fileam.c
@@ -824,19 +824,10 @@ else \
 if (IsRejectLimitReached(pstate->cdbsreh)) \
 { \
 	char *rejectmsg_normal = "Segment reject limit reached. Aborting operation. Last error was:";\
-	char *rejectmsg_allbad = "All 1000 first rows in this segment were rejected. Aborting operation
regardless of REJECT LIMIT value. Last error was:";\
  	char *rejectmsg_csv_unparsable = "Input includes invalid CSV data that corrupts the ability
to parse data rows. This usually means several unescaped embedded QUOTE characters. Data is
not parsable.Last error was:";\
 	char *finalmsg;\
 \
- 	if (FIRST_1000_BAD(pstate->cdbsreh))\
- 	{\
- 		/* the special "first 1000 are bad" case */\
- 		finalmsg = (char *) palloc((strlen(pstate->cdbsreh->errmsg) + \
- 									strlen(rejectmsg_allbad) + 12 + 1)\
- 								   * sizeof(char)); \
- 		sprintf(finalmsg, "%s %s", rejectmsg_allbad, pstate->cdbsreh->errmsg);\
- 	}\
- 	else if (CSV_IS_UNPARSABLE(pstate->cdbsreh))\
+ 	if (CSV_IS_UNPARSABLE(pstate->cdbsreh))\
   	{\
  		/* the special "csv un-parsable" case */\
   		finalmsg = (char *) palloc((strlen(pstate->cdbsreh->errmsg) + \

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fc89b451/src/backend/cdb/cdbsreh.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbsreh.c b/src/backend/cdb/cdbsreh.c
index 4e08967..87048c3 100644
--- a/src/backend/cdb/cdbsreh.c
+++ b/src/backend/cdb/cdbsreh.c
@@ -581,10 +581,6 @@ bool IsRejectLimitReached(CdbSreh *cdbsreh)
 {
 	bool	limit_reached = false;
 	
-	/* special case: check for the case that we are rejecting every single row */
-	if(FIRST_1000_BAD(cdbsreh))
-		return true;
-	
 	/* special case: check for un-parsable csv format errors */
 	if(CSV_IS_UNPARSABLE(cdbsreh))
 		return true;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fc89b451/src/backend/commands/copy.c
----------------------------------------------------------------------
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 19d388d..d454d80 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -277,19 +277,10 @@ continue; /* move on to the next data line */
 if (IsRejectLimitReached(cstate->cdbsreh)) \
 {\
 	char *rejectmsg_normal = "Segment reject limit reached. Aborting operation. Last error was:";\
-	char *rejectmsg_allbad = "All 1000 first rows in this segment were rejected. Aborting operation
regardless of REJECT LIMIT value. Last error was:";\
 	char *rejectmsg_csv_unparsable = "Input includes invalid CSV data that corrupts the ability
to parse data rows. This usually means several unescaped embedded QUOTE characters. Data is
not parsable.Last error was:";\
 	char *finalmsg;\
 \
-	if (FIRST_1000_BAD(cstate->cdbsreh))\
-	{\
-		/* the special "first 1000 are bad" case */\
-		finalmsg = (char *) palloc((strlen(cstate->cdbsreh->errmsg) + \
-									strlen(rejectmsg_allbad) + 12 + 1)\
-									* sizeof(char)); \
-		sprintf(finalmsg, "%s %s", rejectmsg_allbad, cstate->cdbsreh->errmsg);\
-	}\
-	else if (CSV_IS_UNPARSABLE(cstate->cdbsreh))\
+	if (CSV_IS_UNPARSABLE(cstate->cdbsreh))\
 	{\
 		/* the special "csv un-parsable" case */\
 		finalmsg = (char *) palloc((strlen(cstate->cdbsreh->errmsg) + \

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fc89b451/src/include/cdb/cdbsreh.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/cdbsreh.h b/src/include/cdb/cdbsreh.h
index dfe5e2d..85c15c2 100644
--- a/src/include/cdb/cdbsreh.h
+++ b/src/include/cdb/cdbsreh.h
@@ -56,13 +56,6 @@
 #define errtable_rawbytes 8   
 
 /*
- * If the 1st 1000 rows got rejected (ALL of them) - there's no point
- * in resuming the load. Better if we abort even if REJECT LIMIT is more
- * than 1000.
- */
-#define FIRST_1000_BAD(sreh) ((sreh->processed == 1000 && sreh->rejectcount
>= 1000) ? (true) : (false))
-
-/*
  * In cases of invalid csv input data we end up with not being able to parse the
  * data, resulting in very large data rows. In copy.c we throw an error ("line
  * too long") and continue to try and parse. In some cases this is enough to 


Mime
View raw message