hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r...@apache.org
Subject incubator-hawq git commit: HAWQ-804. Add feature test for error table with new framework
Date Wed, 15 Jun 2016 05:37:10 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 24a1d0114 -> 09c8b8589


HAWQ-804. Add feature test for error table with new framework


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/09c8b858
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/09c8b858
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/09c8b858

Branch: refs/heads/master
Commit: 09c8b85896e00e23ab98599df3d1a54e435c76ec
Parents: 24a1d01
Author: xunzhang <xunzhangthu@gmail.com>
Authored: Wed Jun 15 01:06:49 2016 +0800
Committer: xunzhang <xunzhangthu@gmail.com>
Committed: Wed Jun 15 13:31:13 2016 +0800

----------------------------------------------------------------------
 src/test/feature/ExternalSource/data/nation.tbl |  25 ++++
 .../ExternalSource/data/nation_error50.tbl      |  75 +++++++++++
 .../ExternalSource/sql_source/errortbl.source   |  93 ++++++++++++++
 .../feature/ExternalSource/test_errortbl.cpp    | 125 +++++++++++++++++++
 4 files changed, 318 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/09c8b858/src/test/feature/ExternalSource/data/nation.tbl
----------------------------------------------------------------------
diff --git a/src/test/feature/ExternalSource/data/nation.tbl b/src/test/feature/ExternalSource/data/nation.tbl
new file mode 100755
index 0000000..ee71b02
--- /dev/null
+++ b/src/test/feature/ExternalSource/data/nation.tbl
@@ -0,0 +1,25 @@
+0|ALGERIA|0| haggle. carefully final deposits detect slyly agai
+1|ARGENTINA|1|al foxes promise slyly according to the regular accounts. bold requests alon
+2|BRAZIL|1|y alongside of the pending deposits. carefully special packages are about the
ironic forges. slyly special 
+3|CANADA|1|eas hang ironic, silent packages. slyly regular packages are furiously over the
tithes. fluffily bold
+4|EGYPT|4|y above the carefully unusual theodolites. final dugouts are quickly across the
furiously regular d
+5|ETHIOPIA|0|ven packages wake quickly. regu
+6|FRANCE|3|refully final requests. regular, ironi
+7|GERMANY|3|l platelets. regular accounts x-ray: unusual, regular acco
+8|INDIA|2|ss excuses cajole slyly across the packages. deposits print aroun
+9|INDONESIA|2| slyly express asymptotes. regular deposits haggle slyly. carefully ironic
hockey players sleep blithely. carefull
+10|IRAN|4|efully alongside of the slyly final dependencies. 
+11|IRAQ|4|nic deposits boost atop the quickly final requests? quickly regula
+12|JAPAN|2|ously. final, express gifts cajole a
+13|JORDAN|4|ic deposits are blithely about the carefully regular pa
+14|KENYA|0| pending excuses haggle furiously deposits. pending, express pinto beans wake
fluffily past t
+15|MOROCCO|0|rns. blithely bold courts among the closely regular packages use furiously bold
platelets?
+16|MOZAMBIQUE|0|s. ironic, unusual asymptotes wake blithely r
+17|PERU|1|platelets. blithely pending dependencies use fluffily across the even pinto beans.
carefully silent accoun
+18|CHINA|2|c dependencies. furiously express notornis sleep slyly regular accounts. ideas
sleep. depos
+19|ROMANIA|3|ular asymptotes are about the furious multipliers. express dependencies nag
above the ironically ironic account
+20|SAUDI ARABIA|4|ts. silent requests haggle. closely express packages sleep across the blithely
+21|VIETNAM|2|hely enticingly express accounts. even, final 
+22|RUSSIA|3| requests against the platelets use never according to the quickly regular pint
+23|UNITED KINGDOM|3|eans boost carefully special requests. accounts are. carefull
+24|UNITED STATES|1|y final packages. slow foxes cajole quickly. quickly silent platelets
breach ironic accounts. unusual pinto be

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/09c8b858/src/test/feature/ExternalSource/data/nation_error50.tbl
----------------------------------------------------------------------
diff --git a/src/test/feature/ExternalSource/data/nation_error50.tbl b/src/test/feature/ExternalSource/data/nation_error50.tbl
new file mode 100755
index 0000000..6c82f16
--- /dev/null
+++ b/src/test/feature/ExternalSource/data/nation_error50.tbl
@@ -0,0 +1,75 @@
+0|ALGERIA|0| haggle. carefully final deposits detect slyly agai
+invalid format
+1|ARGENTINA|1|al foxes promise slyly according to the regular accounts. bold requests alon
+invalid format
+asfinvalid format
+invalid format
+invalid format
+2|BRAZIL|1|y alongside of the pending deposits. carefully special packages are about the
ironic forges. slyly special 
+invalid format
+thwdinvalid format
+invalid format
+3|CANADA|1|eas hang ironic, silent packages. slyly regular packages are furiously over the
tithes. fluffily bold
+sdhinvalid format
+tjinvalid format
+345invalid format
+4|EGYPT|4|y above the carefully unusual theodolites. final dugouts are quickly across the
furiously regular d
+invalid format
+thinvalid format
+oinvalid format
+5|ETHIOPIA|0|ven packages wake quickly. regu
+6|FRANCE|3|refully final requests. regular, ironi
+7|GERMANY|3|l platelets. regular accounts x-ray: unusual, regular acco
+invalid format
+utikinvalid format
+invalid format
+djinvalid format
+invalid format
+ykinvalid format
+invalid format
+yukyinvalid formatkt
+yukinvalid format
+kyinvalityukd format
+invtykalid format
+tykinvalid format
+yukinvalid format
+8|INDIA|2|ss excuses cajole slyly across the packages. deposits print aroun
+9|INDONESIA|2| slyly express asymptotes. regular deposits haggle slyly. carefully ironic
hockey players sleep blithely. carefull
+10|IRAN|4|efully alongside of the slyly final dependencies. 
+invalid format
+invalid format
+invalid format
+11|IRAQ|4|nic deposits boost atop the quickly final requests? quickly regula
+12|JAPAN|2|ously. final, express gifts cajole a
+invalid format
+invalid format
+invalid format
+13|JORDAN|4|ic deposits are blithely about the carefully regular pa
+14|KENYA|0| pending excuses haggle furiously deposits. pending, express pinto beans wake
fluffily past t
+kinvalid format
+kinvayklid format
+invalid format
+15|MOROCCO|0|rns. blithely bold courts among the closely regular packages use furiously bold
platelets?
+16|MOZAMBIQUE|0|s. ironic, unusual asymptotes wake blithely r
+invalidk format
+invalikyd format
+invalid format
+17|PERU|1|platelets. blithely pending dependencies use fluffily across the even pinto beans.
carefully silent accoun
+18|CHINA|2|c dependencies. furiously express notornis sleep slyly regular accounts. ideas
sleep. depos
+ktyinvalid format
+invalyukid format
+inykyvalid format
+19|ROMANIA|3|ular asymptotes are about the furious multipliers. express dependencies nag
above the ironically ironic account
+20|SAUDI ARABIA|4|ts. silent requests haggle. closely express packages sleep across the blithely
+invalid format
+21|VIETNAM|2|hely enticingly express accounts. even, final 
+22|RUSSIA|3| requests against the platelets use never according to the quickly regular pint
+iknvalid format
+invalid format
+ryukinvalid format
+invalid formatk
+invalid format
+rykinvalid format
+23|UNITED KINGDOM|3|eans boost carefully special requests. accounts are. carefull
+24|UNITED STATES|1|y final packages. slow foxes cajole quickly. quickly silent platelets
breach ironic accounts. unusual pinto be
+invalid format
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/09c8b858/src/test/feature/ExternalSource/sql_source/errortbl.source
----------------------------------------------------------------------
diff --git a/src/test/feature/ExternalSource/sql_source/errortbl.source b/src/test/feature/ExternalSource/sql_source/errortbl.source
new file mode 100644
index 0000000..aeaaff4
--- /dev/null
+++ b/src/test/feature/ExternalSource/sql_source/errortbl.source
@@ -0,0 +1,93 @@
+-- --------------------------------------
+-- 'gpfdist' protocol
+-- --------------------------------------
+
+CREATE EXTERNAL WEB TABLE gpfdist_status (x text)
+execute E'( python $GPHOME/bin/lib/gppinggpfdist.py @hostname@:7070 2>&1 || echo)
'
+on SEGMENT 0
+FORMAT 'text' (delimiter '|');
+
+CREATE EXTERNAL WEB TABLE gpfdist_start (x text)
+execute E'((@gpwhich_gpfdist@ -p 7070 -d @abs_srcdir@/data  </dev/null >/dev/null 2>&1
&); sleep 2; echo "starting...") '
+on SEGMENT 0
+FORMAT 'text' (delimiter '|');
+
+CREATE EXTERNAL WEB TABLE gpfdist_stop (x text)
+execute E'(/bin/pkill gpfdist || killall gpfdist) > /dev/null 2>&1; echo "stopping..."'
+on SEGMENT 0
+FORMAT 'text' (delimiter '|');
+-- start_ignore
+select * from gpfdist_stop;
+select * from gpfdist_status;
+select * from gpfdist_start;
+select * from gpfdist_status;
+-- end_ignore
+
+-- readable external table with error table
+
+CREATE EXTERNAL TABLE EXT_NATION1 ( N_NATIONKEY  INTEGER ,
+                            N_NAME       CHAR(25) ,
+                            N_REGIONKEY  INTEGER ,
+                            N_COMMENT    VARCHAR(152))
+location ('gpfdist://@hostname@:7070/nation_error50.tbl')
+FORMAT 'text' (delimiter '|')
+LOG ERRORS INTO EXT_NATION_ERROR1 SEGMENT REJECT LIMIT 51;
+
+CREATE EXTERNAL TABLE EXT_NATION2 ( N_NATIONKEY  INTEGER ,
+                            N_NAME       CHAR(25) ,
+                            N_REGIONKEY  INTEGER ,
+                            N_COMMENT    VARCHAR(152))
+location ('gpfdist://@hostname@:7070/nation_error50.tbl')
+FORMAT 'text' (delimiter '|')
+LOG ERRORS INTO EXT_NATION_ERROR2 SEGMENT REJECT LIMIT 50;
+
+CREATE EXTERNAL TABLE EXT_NATION3 ( N_NATIONKEY  INTEGER ,
+                            N_NAME       CHAR(25) ,
+                            N_REGIONKEY  INTEGER ,
+                            N_COMMENT    VARCHAR(152))
+location ('gpfdist://@hostname@:7070/nation.tbl')
+FORMAT 'text' (delimiter '|')
+LOG ERRORS INTO EXT_NATION_ERROR3 SEGMENT REJECT LIMIT 50;
+
+-- use existing error table
+CREATE EXTERNAL TABLE EXT_NATION_WITH_EXIST_ERROR_TABLE ( N_NATIONKEY  INTEGER ,
+                            N_NAME       CHAR(25) ,
+                            N_REGIONKEY  INTEGER ,
+                            N_COMMENT    VARCHAR(152))
+location ('gpfdist://@hostname@:7070/nation_error50.tbl')
+FORMAT 'text' (delimiter '|')
+LOG ERRORS INTO EXT_NATION_ERROR1 SEGMENT REJECT LIMIT 51;
+
+select * from EXT_NATION1;
+select count(*) from EXT_NATION_ERROR1; -- should be 50
+select * from EXT_NATION_WITH_EXIST_ERROR_TABLE;
+select count(*) from EXT_NATION_ERROR1; -- should be 100
+select * from EXT_NATION2; -- should fail
+select count(*) from EXT_NATION_ERROR2; -- should be empty
+select * from EXT_NATION3;
+select count(*) from EXT_NATION_ERROR3; -- should be empty
+
+truncate EXT_NATION_ERROR1;
+select * from EXT_NATION1 as x, EXT_NATION3 as y where x.n_nationkey = y.n_nationkey;
+select count(*) from EXT_NATION_ERROR1; -- should be 50
+
+select * from EXT_NATION1 as x, EXT_NATION1 as y where x.n_nationkey = y.n_nationkey; --should
fail on self join
+select * from EXT_NATION1 as x, EXT_NATION_WITH_EXIST_ERROR_TABLE as y where x.n_nationkey
= y.n_nationkey; --should fail with the same error table
+
+-- should fail on writable external table
+CREATE WRITABLE EXTERNAL TABLE EXT_NATION_WRITABLE ( N_NATIONKEY  INTEGER ,
+                            N_NAME       CHAR(25) ,
+                            N_REGIONKEY  INTEGER ,
+                            N_COMMENT    VARCHAR(152))
+LOCATION ('gpfdist://@hostname@:7070/nation_error50.tbl') 
+FORMAT 'text' (delimiter '|')
+LOG ERRORS INTO EXT_NATION_ERROR_WRITABLE SEGMENT REJECT LIMIT 5;
+
+-- start_ignore
+select * from gpfdist_stop;
+select * from gpfdist_status;
+-- end_ignore
+
+drop external table gpfdist_status;
+drop external table gpfdist_start;
+drop external table gpfdist_stop;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/09c8b858/src/test/feature/ExternalSource/test_errortbl.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/ExternalSource/test_errortbl.cpp b/src/test/feature/ExternalSource/test_errortbl.cpp
new file mode 100644
index 0000000..8bd5f51
--- /dev/null
+++ b/src/test/feature/ExternalSource/test_errortbl.cpp
@@ -0,0 +1,125 @@
+#include "gtest/gtest.h"
+
+#include "lib/sql_util.h"
+#include "lib/string_util.h"
+
+using hawq::test::SQLUtility;
+
+class TestErrorTable : public ::testing::Test {
+  public:
+   TestErrorTable() {}
+   ~TestErrorTable() {}
+};
+
+TEST_F(TestErrorTable, TestErrorTableAll) {
+  
+  SQLUtility util;
+  auto init_gpfdist = [&] () {
+    auto sql = "CREATE EXTERNAL WEB TABLE gpfdist_status (x text) "
+        "execute E'( python %s/bin/lib/gppinggpfdist.py localhost:7070 2>&1 || echo)
' "
+        "on SEGMENT 0 "
+        "FORMAT 'text' (delimiter '|');";
+    auto GPHOME = getenv("GPHOME");
+    util.execute(hawq::test::stringFormat(sql, GPHOME));
+
+    sql = "CREATE EXTERNAL WEB TABLE gpfdist_start (x text) "
+        "execute E'((%s/bin/gpfdist -p 7070 -d %s  </dev/null >/dev/null 2>&1
&); sleep 2; echo \"starting\"...) ' "
+        "on SEGMENT 0 "
+        "FORMAT 'text' (delimiter '|');";
+    std::string path = util.getTestRootPath() + "/ExternalSource/data";
+    util.execute(hawq::test::stringFormat(sql, GPHOME, path.c_str()));
+  
+    util.execute(
+        "CREATE EXTERNAL WEB TABLE gpfdist_stop (x text) "
+        "execute E'(/bin/pkill gpfdist || killall gpfdist) > /dev/null 2>&1; echo
stopping...' "
+        "on SEGMENT 0 "
+        "FORMAT 'text' (delimiter '|');");
+    util.execute("select * from gpfdist_stop;");
+    util.execute("select * from gpfdist_status;");
+    util.execute("select * from gpfdist_start;");
+    util.execute("select * from gpfdist_status;");
+  };
+
+  auto finalize_gpfdist = [&] () {
+    util.execute("drop external table EXT_NATION_WITH_EXIST_ERROR_TABLE;");
+    util.execute("drop external table EXT_NATION1;");
+    util.execute("drop table EXT_NATION_ERROR1 CASCADE;");
+    util.execute("drop external table EXT_NATION2;");
+    util.execute("drop table EXT_NATION_ERROR2 CASCADE;");
+    util.execute("drop external table EXT_NATION3;");
+    util.execute("drop table EXT_NATION_ERROR3 CASCADE;");
+    util.execute("select * from gpfdist_stop;");
+    util.execute("select * from gpfdist_status;");
+    util.execute("drop external table gpfdist_status;");
+    util.execute("drop external table gpfdist_start;");
+    util.execute("drop external table gpfdist_stop;");
+  };
+
+  init_gpfdist();
+  
+  // readable external table with error table
+  util.execute(
+      "CREATE EXTERNAL TABLE EXT_NATION1 ( N_NATIONKEY  INTEGER ,"
+      "N_NAME       CHAR(25) ,"
+      "N_REGIONKEY  INTEGER ,"
+      "N_COMMENT    VARCHAR(152))"
+      "location ('gpfdist://localhost:7070/nation_error50.tbl')"
+      "FORMAT 'text' (delimiter '|')"
+      "LOG ERRORS INTO EXT_NATION_ERROR1 SEGMENT REJECT LIMIT 51;");
+
+  util.execute(
+      "CREATE EXTERNAL TABLE EXT_NATION2 ( N_NATIONKEY  INTEGER ,"
+      "N_NAME       CHAR(25) ,"
+      "N_REGIONKEY  INTEGER ,"
+      "N_COMMENT    VARCHAR(152))"
+      "location ('gpfdist://localhost:7070/nation_error50.tbl')"
+      "FORMAT 'text' (delimiter '|')"
+      "LOG ERRORS INTO EXT_NATION_ERROR2 SEGMENT REJECT LIMIT 50;");
+
+  util.execute(
+      "CREATE EXTERNAL TABLE EXT_NATION3 ( N_NATIONKEY  INTEGER ,"
+      "N_NAME       CHAR(25) ,"
+      "N_REGIONKEY  INTEGER ,"
+      "N_COMMENT    VARCHAR(152))"
+      "location ('gpfdist://localhost:7070/nation.tbl')"
+      "FORMAT 'text' (delimiter '|')"
+      "LOG ERRORS INTO EXT_NATION_ERROR3 SEGMENT REJECT LIMIT 50;");
+
+  // use existing error table
+  util.execute(
+      "CREATE EXTERNAL TABLE EXT_NATION_WITH_EXIST_ERROR_TABLE ( N_NATIONKEY  INTEGER ,"
+      "N_NAME       CHAR(25) ,"
+      "N_REGIONKEY  INTEGER ,"
+      "N_COMMENT    VARCHAR(152))"
+      "location ('gpfdist://localhost:7070/nation_error50.tbl')"
+      "FORMAT 'text' (delimiter '|')"
+      "LOG ERRORS INTO EXT_NATION_ERROR1 SEGMENT REJECT LIMIT 51;");
+
+  util.query("select * from EXT_NATION1;", 25);
+  util.query("select * from EXT_NATION_ERROR1;", 50);
+  util.query("select * from EXT_NATION_WITH_EXIST_ERROR_TABLE;", 25);
+  util.query("select * from EXT_NATION_ERROR1;", 100);
+  util.execute("select * from EXT_NATION2;", false);
+  util.query("select * from EXT_NATION_ERROR2;", 0);
+  util.query("select * from EXT_NATION3;", 25);
+  util.query("select * from EXT_NATION_ERROR3;", 0);
+  
+  util.execute("truncate EXT_NATION_ERROR1;");
+  util.query("select * from EXT_NATION1 as x, EXT_NATION3 as y where x.n_nationkey = y.n_nationkey;",
25);
+  util.query("select * from EXT_NATION_ERROR1;", 50);
+  
+  util.execute("select * from EXT_NATION1 as x, EXT_NATION1 as y where x.n_nationkey = y.n_nationkey;",
false);
+  util.execute("select * from EXT_NATION1 as x, EXT_NATION_WITH_EXIST_ERROR_TABLE as y where
x.n_nationkey = y.n_nationkey;", false);
+
+  util.execute(
+      "CREATE WRITABLE EXTERNAL TABLE EXT_NATION_WRITABLE ( N_NATIONKEY  INTEGER , "
+      "N_NAME       CHAR(25) , "
+      "N_REGIONKEY  INTEGER , "
+      "N_COMMENT    VARCHAR(152)) "
+      "LOCATION ('gpfdist://localhost:7070/nation_error50.tbl') " 
+      "FORMAT 'text' (delimiter '|') "
+      "LOG ERRORS INTO EXT_NATION_ERROR_WRITABLE SEGMENT REJECT LIMIT 5;",
+      false);
+
+  finalize_gpfdist();
+}


Mime
View raw message