hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ztao1...@apache.org
Subject incubator-hawq git commit: HAWQ-753. Add parquet test using new test library.
Date Wed, 25 May 2016 01:46:05 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master b8c9ca9c3 -> 14cb8e091


HAWQ-753. Add parquet test using new test library.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/14cb8e09
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/14cb8e09
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/14cb8e09

Branch: refs/heads/master
Commit: 14cb8e091010e6b55db4b6e1bfe3551173709320
Parents: b8c9ca9
Author: ztao1987 <zhenglin.taozl@gmail.com>
Authored: Tue May 24 14:50:45 2016 +0800
Committer: ztao1987 <zhenglin.taozl@gmail.com>
Committed: Wed May 25 09:45:52 2016 +0800

----------------------------------------------------------------------
 src/test/feature/lib/data-gen.cpp         |   5 +-
 src/test/feature/lib/data-gen.h           |   9 +-
 src/test/feature/lib/sql-util.cpp         |   8 +-
 src/test/feature/lib/sql-util.h           |   2 +-
 src/test/feature/parquet/test-parquet.cpp | 138 +++++++++++++++++++++++++
 5 files changed, 152 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/14cb8e09/src/test/feature/lib/data-gen.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/data-gen.cpp b/src/test/feature/lib/data-gen.cpp
index e2e2f71..5ec806d 100644
--- a/src/test/feature/lib/data-gen.cpp
+++ b/src/test/feature/lib/data-gen.cpp
@@ -135,6 +135,9 @@ std::string DataGenerator::genTableDesc(bool appendonly,
   std::string desc = (appendonly ? "with (appendonly = true, orientation = "
                                  : "with (appendonly = false, orientation = ") +
                      orientation + ", compresstype = " + compresstype +
-                     ", compresslevel = " + std::to_string(compresslevel) + ")";
+                     (compresslevel != -1
+                          ? ", compresslevel = " + std::to_string(compresslevel)
+                          : "") +
+                     ")";
   return desc;
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/14cb8e09/src/test/feature/lib/data-gen.h
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/data-gen.h b/src/test/feature/lib/data-gen.h
index 25aac5a..3df0ac0 100644
--- a/src/test/feature/lib/data-gen.h
+++ b/src/test/feature/lib/data-gen.h
@@ -21,22 +21,23 @@ class DataGenerator {
 
   void genSimpleTable(std::string tableName, bool appendonly = true,
                       std::string orientation = "row",
-                      std::string compresstype = "none", int compresslevel = 0);
+                      std::string compresstype = "none",
+                      int compresslevel = -1);
 
   void genTableWithFullTypes(std::string tableName, bool appendonly = true,
                              std::string orientation = "row",
                              std::string compresstype = "none",
-                             int compresslevel = 0);
+                             int compresslevel = -1);
 
   void genTableWithSeries(std::string tableName, bool appendonly = true,
                           std::string orientation = "row",
                           std::string compresstype = "none",
-                          int compresslevel = 0);
+                          int compresslevel = -1);
 
   void genTableWithNull(std::string tableName, bool appendonly = true,
                         std::string orientation = "row",
                         std::string compresstype = "none",
-                        int compresslevel = 0);
+                        int compresslevel = -1);
 
  private:
   std::string genTableDesc(bool appendonly, std::string orientation,

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/14cb8e09/src/test/feature/lib/sql-util.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/sql-util.cpp b/src/test/feature/lib/sql-util.cpp
index 0714e22..6961983 100644
--- a/src/test/feature/lib/sql-util.cpp
+++ b/src/test/feature/lib/sql-util.cpp
@@ -33,14 +33,14 @@ SQLUtility::~SQLUtility() {
 }
 
 void SQLUtility::exec(const std::string &sql) {
-  EXPECT_EQ((conn->runSQLCommand(sql)).getLastStatus(), 0)
+  EXPECT_EQ(0, (conn->runSQLCommand(sql)).getLastStatus())
       << conn->getLastResult();
 }
 
 void SQLUtility::execute(const std::string &sql, bool check) {
   conn->runSQLCommand("SET SEARCH_PATH=" + schemaName + ";" + sql);
   EXPECT_NE(conn.get(), nullptr);
-  if (check) EXPECT_EQ(conn->getLastStatus(), 0) << conn->getLastResult();
+  if (check) EXPECT_EQ(0, conn->getLastStatus()) << conn->getLastResult();
 }
 
 void SQLUtility::query(const std::string &sql, int expectNum) {
@@ -82,7 +82,7 @@ void SQLUtility::execSQLFile(const std::string &sqlFile,
   // outFile is located in the same folder with ansFile
   std::string outFileAbsPath = fp.path + "/" + fp.fileBaseName + ".out";
   conn->setOutputFile(outFileAbsPath);
-  EXPECT_EQ(conn->runSQLFile(newSqlFile).getLastStatus(), 0);
+  EXPECT_EQ(0, conn->runSQLFile(newSqlFile).getLastStatus());
   conn->resetOutput();
   EXPECT_FALSE(conn->checkDiff(ansFileAbsPath, outFileAbsPath, true));
   if (conn->checkDiff(ansFileAbsPath, outFileAbsPath, true) == false) {
@@ -138,7 +138,7 @@ const PSQLQueryResult &SQLUtility::executeQuery(const std::string
&sql) {
   return result;
 }
 
-PSQL *SQLUtility::getPSQL() const { return conn.get(); }
+const PSQL *SQLUtility::getPSQL() const { return conn.get(); }
 
 std::string SQLUtility::getTestRootPath() const {
   std::string result;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/14cb8e09/src/test/feature/lib/sql-util.h
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/sql-util.h b/src/test/feature/lib/sql-util.h
index b5cba5c..9fb8994 100644
--- a/src/test/feature/lib/sql-util.h
+++ b/src/test/feature/lib/sql-util.h
@@ -50,7 +50,7 @@ class SQLUtility {
 
   // Get PSQL connection
   // @return PSQL raw pointer
-  PSQL *getPSQL() const;
+  const PSQL *getPSQL() const;
 
   // Get test root dir abs path
   // @return path string

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/14cb8e09/src/test/feature/parquet/test-parquet.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/parquet/test-parquet.cpp b/src/test/feature/parquet/test-parquet.cpp
new file mode 100644
index 0000000..2e65f0d
--- /dev/null
+++ b/src/test/feature/parquet/test-parquet.cpp
@@ -0,0 +1,138 @@
+#include "gtest/gtest.h"
+
+#include "lib/data-gen.h"
+#include "lib/sql-util.h"
+
+class TestParquet : public ::testing::Test {
+ public:
+  TestParquet() {}
+  ~TestParquet() {}
+};
+
+TEST_F(TestParquet, TestMultipleType) {
+  SQLUtility util;
+  util.execute("drop table if exists t1");
+  DataGenerator dGen(&util);
+  dGen.genTableWithFullTypes("t1", true, "parquet");
+  util.query(
+      "select * from t1",
+      "2147483647|||00:00:00||ff:89:71:45:ae:01|2000-01-01 "
+      "07:00:00+08||32767|t|192.168.1.255/"
+      "32|<(1,2),3>|[(0,0),(6,6)]|-178000000 "
+      "years|0|-$21,474,836.48|(200,400),(100,200)||<aa>bb</"
+      "aa>||123456789a|2001:db8:85a3:8d3:1319:8a2e:370:7344/"
+      "64|||1|0|(1,2)|4277-12-31|128|\n0|((100,123),(5,10),(7,2),(4,5))|hello "
+      "world||04:45:05.0012+08:40|||bbccddeeff|128||2001:db8:85a3:8d3:1319:"
+      "8a2e:370:7344/128|<(1,2),3>|[(0,0),(6,6)]||1|$0.00|(2,3),(0,1)|hello "
+      "world|<aa>bb</aa>||aaaa|2001:db8:85a3:8d3:1319:8a2e:370:7344/"
+      "64|0||2147483647|-Infinity|(1,2)|4277-12-31|Infinity|\n||abcd|15:01:03||"
+      "|2000-01-01 07:00:00+08|||t|||[(0,0),(6,6)]|-178000000 "
+      "years|0|-$21,474,836.48|(200,400),(100,200)||<aa>bb</"
+      "aa>||123456789a|2001:db8:85a3:8d3:1319:8a2e:370:7344/"
+      "64|||1|0|(1,2)|4277-12-31|128|\n0|((100,123),(5,10),(7,2),(4,5))|hello "
+      "world||04:45:05.0012+08:40|||bbccddeeff|128||2001:db8:85a3:8d3:1319:"
+      "8a2e:370:7344/128|<(1,2),3>||||$0.00||hello "
+      "world|||aaaa|2001:db8:85a3:8d3:1319:8a2e:370:7344/"
+      "64|0||2147483647|-Infinity|(1,2)|4277-12-31|Infinity|\n0|((100,123),(5,"
+      "10),(7,2),(4,5))|hello "
+      "world||04:45:05.0012+08:40|||bbccddeeff|128||2001:db8:85a3:8d3:1319:"
+      "8a2e:370:7344/128|<(1,2),3>||||$0.00||hello "
+      "world|||||0||2147483647|-Infinity|(1,2)|4277-12-31|Infinity|\n0|((100,"
+      "123),(5,10),(7,2),(4,5))|hello "
+      "world||04:45:05.0012+08:40|||bbccddeeff|128||2001:db8:85a3:8d3:1319:"
+      "8a2e:370:7344/128|<(1,2),3>||||||hello world|||||0||34|||||\n");
+}
+
+TEST_F(TestParquet, TestCompression) {
+  SQLUtility util;
+  util.execute("drop table if exists t21");
+  util.execute("drop table if exists t22");
+  DataGenerator dGen(&util);
+  dGen.genTableWithNull("t21", true, "parquet", "gzip", 9);
+  dGen.genTableWithNull("t22", true, "parquet", "snappy");
+  util.query("select * from t21,t22",
+             "15||aa|15||aa|\n15||aa|||WET|\n15||aa||51||\n||WET|15||aa|\n||"
+             "WET|||WET|\n||WET||51||\n|51||15||aa|\n|51||||WET|\n|51|||51||"
+             "\n");
+}
+
+TEST_F(TestParquet, TestSize) {
+  SQLUtility util;
+  // value/record size equal to pagesize/rowgroupsize
+  util.execute("drop table if exists t31");
+  util.execute(
+      "create table t31 (a1 char(10485760), a2 char(10485760), a3 "
+      "char(10485760), a4 char(10485760), a5 char(10485760), a6 "
+      "char(10485760), a7 char(10485760), a8 char(10485760), a9 "
+      "char(10485760), a10 char(10485760)) with(appendonly=true, "
+      "orientation=parquet, pagesize=10485760, rowgroupsize=104857600)");
+  util.execute(
+      "insert into t31 values ( ('a'::char(10485760)), "
+      "('a'::char(10485760)), ('a'::char(10485760)), ('a'::char(10485760)), "
+      "('a'::char(10485760)), ('a'::char(10485760)), ('a'::char(10485760)), "
+      "('a'::char(10485760)), ('a'::char(10485760)), ('a'::char(10485760)) );",
+      false);
+  EXPECT_STREQ("ERROR:  value for column \"a1\" exceeds pagesize 10485760!",
+               util.getPSQL()->getLastResult().substr(0, 56).c_str());
+
+  // single column, one data page contains several values, one rwo group
+  // contains several groups
+  util.execute("drop table if exists t32");
+  util.execute("drop table if exists t33");
+  util.execute(
+      "create table t32 ( a1 text ) with(appendonly=true, "
+      "orientation=parquet)");
+  util.execute("insert into t32 values(repeat('parquet',100))");
+  util.execute("insert into t32 values(repeat('parquet',20))");
+  util.execute("insert into t32 values(repeat('parquet',30))");
+  util.execute(
+      "create table t33 ( a1 text ) with(appendonly=true, orientation=parquet, "
+      "pagesize=1024, rowgroupsize=1025)");
+  util.execute("insert into t33 select * from t32");
+  util.query("select * from t33", 3);
+
+  // large data insert, several column values in one page, several rows in one
+  // rowgroup
+  util.execute("drop table if exists t34");
+  util.execute("drop table if exists t35");
+  util.execute(
+      "create table t34 (a1 char(1048576), a2 char(2048576), a3 "
+      "char(3048576), a4 char(4048576), a5 char(5048576), a6 char(6048576), a7 "
+      "char(7048576), a8 char(8048576), a9 char(9048576), a10 char(9)) "
+      "with(appendonly=true, orientation=parquet, pagesize=10485760, "
+      "rowgroupsize=90874386)");
+  util.execute(
+      "insert into t34 values ( ('a'::char(1048576)), "
+      "('a'::char(2048576)), ('a'::char(3048576)), ('a'::char(4048576)), "
+      "('a'::char(5048576)), ('a'::char(6048576)), ('a'::char(7048576)), "
+      "('a'::char(8048576)), ('a'::char(9048576)), ('a'::char(9)) )");
+  util.execute(
+      "create table t35 (a1 char(1048576), a2 char(2048576), a3 "
+      "char(3048576), a4 char(4048576), a5 char(5048576), a6 char(6048576), a7 "
+      "char(7048576), a8 char(8048576), a9 char(9048576), a10 char(9)) "
+      "with(appendonly=true, orientation=parquet, pagesize=10485760, "
+      "rowgroupsize=17437200)");
+  util.execute("insert into t35 select * from t34");
+  util.query("select * from t35", 1);
+}
+
+TEST_F(TestParquet, TestPartition) {
+  SQLUtility util;
+  util.execute("drop table if exists t4");
+  util.execute(
+      "create table t4 (id SERIAL,a1 int,a2 char(5)) WITH (appendonly=true, "
+      "orientation=parquet,compresstype=gzip,compresslevel=1) distributed "
+      "randomly Partition by range(a1) (start(1)  end(16) every(8)"
+      "WITH(appendonly = true, orientation = parquet, compresstype = "
+      "snappy))");
+  util.execute("insert into t4(a1,a2) values(generate_series(1,5),'M')");
+  util.execute(
+      "alter table t4 add partition new_p start(17) end (20) WITH "
+      "(appendonly=true, orientation=parquet, compresstype=snappy)");
+  util.execute(
+      "alter table t4 add default partition df_p WITH "
+      "(appendonly=true, orientation=parquet, compresstype=gzip, "
+      "compresslevel=3)");
+  util.execute("insert into t4(a1,a2) values(generate_series(6,25),'F')");
+  util.query("select * from t4", 25);
+}


Mime
View raw message