hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject incubator-hawq git commit: HAWQ-1112. Error message is not accurate when hawq register with single file and the size is larger than real size.
Date Wed, 19 Oct 2016 03:12:46 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 6e921df29 -> 4af3768cd


HAWQ-1112. Error message is not accurate when hawq register with single file and the size
is larger than real size.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/4af3768c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/4af3768c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/4af3768c

Branch: refs/heads/master
Commit: 4af3768cd0b44807ac1cb2859ac5a1e16a03d618
Parents: 6e921df
Author: Wen Lin <wlin@pivotal.io>
Authored: Wed Oct 19 11:11:13 2016 +0800
Committer: Wen Lin <wlin@pivotal.io>
Committed: Wed Oct 19 11:11:13 2016 +0800

----------------------------------------------------------------------
 .../ManagementTool/test_hawq_register.cpp       | 243 --------------
 .../feature/ManagementTool/test_hawq_register.h |  12 +
 .../test_hawq_register_usage1.cpp               | 332 +++++++++++++++++++
 .../test_hawq_register_usage2_case1.cpp         |  32 ++
 tools/bin/hawqregister                          |   5 +-
 5 files changed, 380 insertions(+), 244 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4af3768c/src/test/feature/ManagementTool/test_hawq_register.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/test_hawq_register.cpp b/src/test/feature/ManagementTool/test_hawq_register.cpp
deleted file mode 100644
index 123f73c..0000000
--- a/src/test/feature/ManagementTool/test_hawq_register.cpp
+++ /dev/null
@@ -1,243 +0,0 @@
-#include <string>
-
-#include "gtest/gtest.h"
-#include "lib/command.h"
-#include "lib/sql_util.h"
-#include "lib/string_util.h"
-#include "lib/hdfs_config.h"
-#include "test_hawq_register.h"
-
-using std::string;
-using hawq::test::SQLUtility;
-using hawq::test::Command;
-using hawq::test::HdfsConfig;
-
-TEST_F(TestHawqRegister, TestUsage1SingleHawqFile) {
-	SQLUtility util;
-	string rootPath(util.getTestRootPath());
-	string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
-	string filePath = rootPath + relativePath;
-
-    auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq",
filePath.c_str(), getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-
-	util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
-	util.query("select * from hawqregister;", 0);
-
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-
-	util.query("select * from hawqregister;", 3);
-	util.execute("insert into hawqregister values(1);");
-	util.query("select * from hawqregister;", 4);
-	util.execute("drop table hawqregister;");
-}
-
-TEST_F(TestHawqRegister, TestUsage1SingleHiveFile) {
-	SQLUtility util;
-	string rootPath(util.getTestRootPath());
-	string relativePath("/ManagementTool/test_hawq_register_hive.paq");
-	string filePath = rootPath + relativePath;
-
-    auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hive.paq",
filePath.c_str(), getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-
-	util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
-	util.query("select * from hawqregister;", 0);
-
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hive.paq hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-
-	util.query("select * from hawqregister;", 1);
-	util.execute("insert into hawqregister values(1);");
-	util.query("select * from hawqregister;", 2);
-	util.execute("drop table hawqregister;");
-}
-
-TEST_F(TestHawqRegister, TestDataTypes) {
-	SQLUtility util;
-	string rootPath(util.getTestRootPath());
-	/* This parquet file is generate by HIVE, using the table created by */
-	/* 'create table parquet(a boolean, b tinyint, c smallint, d int, e bigint, f date, g float,
h double, i string, j binary, k char(10), l varchar(10)) stored as parquet;' */
-	string relativePath("/ManagementTool/test_hawq_register_data_types.paq");
-	string filePath = rootPath + relativePath;
-
-    auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_data_types.paq",
filePath.c_str(), getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-
-	util.execute("create table hawqregister(a bool, b int2, c int2, d int4, e int8, f date,
g float4, h float8, i varchar, j bytea, k char, l varchar) with (appendonly=true, orientation=parquet);");
-	util.query("select * from hawqregister;", 0);
-
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_data_types.paq
hawqregister", HAWQ_DB, getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-
-	util.query("select * from hawqregister;", 1);
-	util.execute("drop table hawqregister;");
-}
-
-TEST_F(TestHawqRegister, TestAllNULL) {
-	SQLUtility util;
-	string rootPath(util.getTestRootPath());
-	/* This parquet file is generate by HIVE, using the table created by */
-	/* 'create table parquet(a boolean, b tinyint, c smallint, d int, e bigint, f date, g float,
h double, i string, j binary, k char(10), l varchar(10)) stored as parquet;' */
-	/* with all the values set to NULL */
-	string relativePath("/ManagementTool/test_hawq_register_null.paq");
-	string filePath = rootPath + relativePath;
-
-    auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_data_types.paq",
filePath.c_str(), getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-
-	util.execute("create table hawqregister(a bool, b int2, c int2, d int4, e int8, f date,
g float4, h float8, i varchar, j bytea, k char, l varchar) with (appendonly=true, orientation=parquet);");
-	util.query("select * from hawqregister;", 0);
-
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_data_types.paq
hawqregister", HAWQ_DB, getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-
-	util.query("select * from hawqregister;", 1);
-	util.execute("drop table hawqregister;");
-}
-
-TEST_F(TestHawqRegister, TestFiles) {
-	SQLUtility util;
-	string rootPath(util.getTestRootPath());
-	string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
-	string filePath1 = rootPath + relativePath;
-	relativePath = "/ManagementTool/test_hawq_register_hive.paq";
-	string filePath2 = rootPath + relativePath;
-
-    auto cmd = hawq::test::stringFormat("hadoop fs -mkdir -p %s %s/hawq_register_test/t/t",
filePath1.c_str(), getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-    cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hawq1.paq",
filePath1.c_str(), getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-    cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hawq2.paq",
filePath1.c_str(), getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-    cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/t/hawq.paq",
filePath1.c_str(), getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-
-    cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hive1.paq",
filePath2.c_str(), getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-    cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hive2.paq",
filePath2.c_str(), getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-    cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/t/hive.paq",
filePath2.c_str(), getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-
-	util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
-	util.query("select * from hawqregister;", 0);
-
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_test hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-
-	util.query("select * from hawqregister;", 12);
-	util.execute("insert into hawqregister values(1);");
-	util.query("select * from hawqregister;", 13);
-    cmd = hawq::test::stringFormat("hadoop fs -rm -r %s/hawq_register_test", getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-	util.execute("drop table hawqregister;");
-}
-
-TEST_F(TestHawqRegister, TestUsage1NotParquetFile) {
-	SQLUtility util;
-	string rootPath(util.getTestRootPath());
-	string relativePath("/ManagementTool/test_hawq_register_not_paq");
-	string filePath = rootPath + relativePath;
-
-    auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test_not_paq",
filePath.c_str(), getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-
-	util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
-	util.query("select * from hawqregister;", 0);
-
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_test_not_paq
hawqregister", HAWQ_DB, getHdfsLocation().c_str());
-	EXPECT_EQ(1, Command::getCommandStatus(cmd));
-	util.query("select * from hawqregister;", 0);
-
-    cmd = hawq::test::stringFormat("hadoop fs -rm %s/hawq_register_test_not_paq", getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-	util.execute("drop table hawqregister;");
-}
-
-TEST_F(TestHawqRegister, TestUsage1NotParquetTable) {
-	SQLUtility util;
-	string rootPath(util.getTestRootPath());
-	string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
-	string filePath = rootPath + relativePath;
-
-    auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq",
filePath.c_str(), getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-
-	util.execute("create table hawqregister(i int);");
-	util.query("select * from hawqregister;", 0);
-
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
-	EXPECT_EQ(1, Command::getCommandStatus(cmd));
-	util.query("select * from hawqregister;", 0);
-
-    cmd = hawq::test::stringFormat("hadoop fs -rm  %s/hawq_register_hawq.paq", getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-	util.execute("drop table hawqregister;");
-}
-
-TEST_F(TestHawqRegister, TestUsage1FileNotExist) {
-	SQLUtility util;
-
-	util.execute("create table hawqregister(i int);");
-	util.query("select * from hawqregister;", 0);
-
-    auto cmd = hawq::test::stringFormat("hawq register -d %s -f %shawq_register_file_not_exist
hawqregister", HAWQ_DB, getHdfsLocation().c_str());
-	EXPECT_EQ(1, Command::getCommandStatus(cmd));
-	util.query("select * from hawqregister;", 0);
-
-	util.execute("drop table hawqregister;");
-}
-
-TEST_F(TestHawqRegister, TestUsage1NotHDFSPath) {
-	SQLUtility util;
-	string rootPath(util.getTestRootPath());
-	string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
-	string filePath = rootPath + relativePath;
-
-    auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq",
filePath.c_str(), getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-
-	util.execute("create table hawqregister(i int);");
-	util.query("select * from hawqregister;", 0);
-
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
-	EXPECT_EQ(1, Command::getCommandStatus(cmd));
-	util.query("select * from hawqregister;", 0);
-
-    cmd = hawq::test::stringFormat("hadoop fs -rm %s/hawq_register_hawq.paq", getHdfsLocation().c_str());
-	EXPECT_EQ(0, Command::getCommandStatus(cmd));
-	util.execute("drop table hawqregister;");
-}
-
-TEST_F(TestHawqRegister, TestEmptyTable) {
-  SQLUtility util;
-  util.execute("drop table if exists t9;");
-  util.execute("create table t9(i int) with (appendonly=true, orientation=row) distributed
randomly;");
-  EXPECT_EQ(0, Command::getCommandStatus("hawq extract -d " + (string) HAWQ_DB + " -o t9.yml
testhawqregister_testemptytable.t9"));
-  EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c t9.yml
testhawqregister_testemptytable.nt9"));
-  util.query("select * from nt9;", 0);
-  EXPECT_EQ(0, Command::getCommandStatus("rm -rf t9.yml"));
-  util.execute("drop table t9;");
-  util.execute("drop table nt9;");
-}
-
-TEST_F(TestHawqRegister, TestIncorrectYaml) {
-  SQLUtility util;
-  string filePath = util.getTestRootPath() + "/ManagementTool/";
-
-  EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c "
+ filePath + "missing_pagesize.yml xx"));
-  EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c "
+ filePath + "missing_rowgroupsize.yml xx"));
-  EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c "
+ filePath + "missing_filesize.yml xx"));
-  EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c "
+ filePath + "wrong_schema.yml xx"));
-  EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c "
+ filePath + "missing_checksum.yml xx"));
-  EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c "
+ filePath + "wrong_dfs_url.yml xx"));
-  EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c "
+ filePath + "missing_bucketnum.yml xx"));
-}
-
-TEST_F(TestHawqRegister, TestDismatchFileNumber) {
-  SQLUtility util;
-  string filePath = util.getTestRootPath() + "/ManagementTool/";
-  EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c "
+ filePath + "files_incomplete.yml xx"));
-}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4af3768c/src/test/feature/ManagementTool/test_hawq_register.h
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/test_hawq_register.h b/src/test/feature/ManagementTool/test_hawq_register.h
index f033f6d..dfb041d 100644
--- a/src/test/feature/ManagementTool/test_hawq_register.h
+++ b/src/test/feature/ManagementTool/test_hawq_register.h
@@ -3,6 +3,7 @@
 
 #include <string>
 #include <pwd.h>
+#include <fstream>
 #include "lib/hdfs_config.h"
 #include "gtest/gtest.h"
 
@@ -49,6 +50,17 @@ class TestHawqRegister : public ::testing::Test {
             return "";
         }
 
+        int getFileSize(const char *file){
+            if (file == NULL)
+                return -1;
+            std::ifstream stream;
+            stream.open(file, std::ios_base::binary);
+            stream.seekg(0, std::ios_base::end);
+            int size = stream.tellg();
+            stream.close();
+            return size;
+        }
+
         void checkPgAOSegValue(std::string relname, std::string value, std::string fmt) {
             std::string reloid = getTableOid(relname);
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4af3768c/src/test/feature/ManagementTool/test_hawq_register_usage1.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/test_hawq_register_usage1.cpp b/src/test/feature/ManagementTool/test_hawq_register_usage1.cpp
index 1843876..a2c4a69 100644
--- a/src/test/feature/ManagementTool/test_hawq_register_usage1.cpp
+++ b/src/test/feature/ManagementTool/test_hawq_register_usage1.cpp
@@ -109,3 +109,335 @@ TEST_F(TestHawqRegister, TestUsage1ExpectSuccess) {
   register_lambda();
   gc_lambda();
 }
+
+TEST_F(TestHawqRegister, TestUsage1SingleHawqFile) {
+    SQLUtility util;
+    string rootPath(util.getTestRootPath());
+    string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
+    string filePath = rootPath + relativePath;
+
+    auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq",
filePath.c_str(), getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
+    util.query("select * from hawqregister;", 0);
+
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    util.query("select * from hawqregister;", 3);
+    util.execute("insert into hawqregister values(1);");
+    util.query("select * from hawqregister;", 4);
+    util.execute("drop table hawqregister;");
+}
+
+TEST_F(TestHawqRegister, TestUsage1SingleHiveFile) {
+    SQLUtility util;
+    string rootPath(util.getTestRootPath());
+    string relativePath("/ManagementTool/test_hawq_register_hive.paq");
+    string filePath = rootPath + relativePath;
+
+    auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hive.paq",
filePath.c_str(), getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
+    util.query("select * from hawqregister;", 0);
+
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hive.paq hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    util.query("select * from hawqregister;", 1);
+    util.execute("insert into hawqregister values(1);");
+    util.query("select * from hawqregister;", 2);
+    util.execute("drop table hawqregister;");
+}
+
+TEST_F(TestHawqRegister, TestDataTypes) {
+    SQLUtility util;
+    string rootPath(util.getTestRootPath());
+    /* This parquet file is generate by HIVE, using the table created by */
+    /* 'create table parquet(a boolean, b tinyint, c smallint, d int, e bigint, f date, g
float, h double, i string, j binary, k char(10), l varchar(10)) stored as parquet;' */
+    string relativePath("/ManagementTool/test_hawq_register_data_types.paq");
+    string filePath = rootPath + relativePath;
+
+    auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_data_types.paq",
filePath.c_str(), getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    util.execute("create table hawqregister(a bool, b int2, c int2, d int4, e int8, f date,
g float4, h float8, i varchar, j bytea, k char, l varchar) with (appendonly=true, orientation=parquet);");
+    util.query("select * from hawqregister;", 0);
+
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_data_types.paq
hawqregister", HAWQ_DB, getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    util.query("select * from hawqregister;", 1);
+    util.execute("drop table hawqregister;");
+}
+
+TEST_F(TestHawqRegister, TestAllNULL) {
+    SQLUtility util;
+    string rootPath(util.getTestRootPath());
+    /* This parquet file is generate by HIVE, using the table created by */
+    /* 'create table parquet(a boolean, b tinyint, c smallint, d int, e bigint, f date, g
float, h double, i string, j binary, k char(10), l varchar(10)) stored as parquet;' */
+    /* with all the values set to NULL */
+    string relativePath("/ManagementTool/test_hawq_register_null.paq");
+    string filePath = rootPath + relativePath;
+
+    auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_data_types.paq",
filePath.c_str(), getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    util.execute("create table hawqregister(a bool, b int2, c int2, d int4, e int8, f date,
g float4, h float8, i varchar, j bytea, k char, l varchar) with (appendonly=true, orientation=parquet);");
+    util.query("select * from hawqregister;", 0);
+
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_data_types.paq
hawqregister", HAWQ_DB, getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    util.query("select * from hawqregister;", 1);
+    util.execute("drop table hawqregister;");
+}
+
+TEST_F(TestHawqRegister, TestFiles) {
+    SQLUtility util;
+    string rootPath(util.getTestRootPath());
+    string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
+    string filePath1 = rootPath + relativePath;
+    relativePath = "/ManagementTool/test_hawq_register_hive.paq";
+    string filePath2 = rootPath + relativePath;
+
+    auto cmd = hawq::test::stringFormat("hadoop fs -mkdir -p %s/hawq_register_test/t", getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+    cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hawq1.paq",
filePath1.c_str(), getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+    cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hawq2.paq",
filePath1.c_str(), getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+    cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/t/hawq.paq",
filePath1.c_str(), getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hive1.paq",
filePath2.c_str(), getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+    cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hive2.paq",
filePath2.c_str(), getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+    cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/t/hive.paq",
filePath2.c_str(), getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
+    util.query("select * from hawqregister;", 0);
+
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_test hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    util.query("select * from hawqregister;", 12);
+    util.execute("insert into hawqregister values(1);");
+    util.query("select * from hawqregister;", 13);
+    cmd = hawq::test::stringFormat("hadoop fs -rm -r %s/hawq_register_test", getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+    util.execute("drop table hawqregister;");
+}
+
+TEST_F(TestHawqRegister, TestUsage1NotParquetFile) {
+    SQLUtility util;
+    string rootPath(util.getTestRootPath());
+    string relativePath("/ManagementTool/test_hawq_register_not_paq");
+    string filePath = rootPath + relativePath;
+
+    auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test_not_paq",
filePath.c_str(), getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
+    util.query("select * from hawqregister;", 0);
+
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_test_not_paq
hawqregister", HAWQ_DB, getHdfsLocation().c_str());
+    EXPECT_EQ(1, Command::getCommandStatus(cmd));
+    util.query("select * from hawqregister;", 0);
+
+    cmd = hawq::test::stringFormat("hadoop fs -rm %s/hawq_register_test_not_paq", getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+    util.execute("drop table hawqregister;");
+}
+
+TEST_F(TestHawqRegister, TestUsage1NotParquetTable) {
+    SQLUtility util;
+    string rootPath(util.getTestRootPath());
+    string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
+    string filePath = rootPath + relativePath;
+
+    /* register a parquet file to a row random table, should fail */
+    auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq",
filePath.c_str(), getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    util.execute("create table hawqregister(i int);");
+    util.query("select * from hawqregister;", 0);
+
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
+    EXPECT_EQ(1, Command::getCommandStatus(cmd));
+    util.query("select * from hawqregister;", 0);
+    util.execute("drop table hawqregister;");
+
+    /* register a parquet file to a row hash table, should fail */
+    util.execute("create table hawqregister(i int) distributed by (i);");
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
+    EXPECT_EQ(1, Command::getCommandStatus(cmd));
+    util.query("select * from hawqregister;", 0);
+    util.execute("drop table hawqregister;");
+
+    /* register a parquet file to a parquet hash table, should fail */
+    util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet)
distributed by (i);");
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
+    EXPECT_EQ(1, Command::getCommandStatus(cmd));
+    util.query("select * from hawqregister;", 0);
+    util.execute("drop table hawqregister;");
+
+    cmd = hawq::test::stringFormat("hadoop fs -rm  %s/hawq_register_hawq.paq", getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+}
+
+TEST_F(TestHawqRegister, TestUsage1FileNotExist) {
+    SQLUtility util;
+
+    util.execute("create table hawqregister(i int);");
+    util.query("select * from hawqregister;", 0);
+
+    auto cmd = hawq::test::stringFormat("hawq register -d %s -f %shawq_register_file_not_exist
hawqregister", HAWQ_DB, getHdfsLocation().c_str());
+    EXPECT_EQ(1, Command::getCommandStatus(cmd));
+    util.query("select * from hawqregister;", 0);
+
+    util.execute("drop table hawqregister;");
+}
+
+TEST_F(TestHawqRegister, TestUsage1NoTable) {
+    SQLUtility util;
+    string rootPath(util.getTestRootPath());
+    string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
+    string filePath = rootPath + relativePath;
+
+    auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq",
filePath.c_str(), getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    /* register a parquet file to a table not exist, should fail */
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s hawqregister", HAWQ_DB, getHdfsLocation().c_str());
+    EXPECT_EQ(1, Command::getCommandStatus(cmd));
+}
+
+TEST_F(TestHawqRegister, TestUsage1NotHDFSPath) {
+    SQLUtility util;
+    string rootPath(util.getTestRootPath());
+    string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
+    string filePath = rootPath + relativePath;
+
+    /* register a non-hdfs parquet file, should failed */
+    util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
+    util.query("select * from hawqregister;", 0);
+
+    auto cmd = hawq::test::stringFormat("hawq register -d %s -f %s hawqregister", HAWQ_DB,
filePath.c_str());
+    EXPECT_EQ(1, Command::getCommandStatus(cmd));
+    util.query("select * from hawqregister;", 0);
+
+    util.execute("drop table hawqregister;");
+}
+
+TEST_F(TestHawqRegister, TestUsage1EofSuccess) {
+    SQLUtility util;
+    string rootPath(util.getTestRootPath());
+    string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
+    string filePath = rootPath + relativePath;
+
+    /* register a parquet file with eof=0, should success */
+    auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq",
filePath.c_str(), getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
+    util.query("select * from hawqregister;", 0); 
+
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister
-e 0", HAWQ_DB, getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+    util.query("select * from hawqregister;", 0);
+
+    /* register a parquet file with eof=filesize, should success */
+    int size = getFileSize(filePath.c_str());
+    cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(),
getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister
-e %d", HAWQ_DB, getHdfsLocation().c_str(), size);
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+    util.query("select * from hawqregister;", 3);
+    util.execute("drop table hawqregister;");
+
+    /* register a parquet file with eof<filesize, should success */
+    cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(),
getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister
-e %d", HAWQ_DB, getHdfsLocation().c_str(), size/2);
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+    util.execute("drop table hawqregister;");
+}
+
+TEST_F(TestHawqRegister, TestUsage1EofFailure) {
+    SQLUtility util;
+    string rootPath(util.getTestRootPath());
+    string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
+    string filePath = rootPath + relativePath;
+
+    auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq",
filePath.c_str(), getHdfsLocation().c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    /* register a parquet file with eof=-1, should success */
+    util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
+    util.query("select * from hawqregister;", 0);
+
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister
-e -1", HAWQ_DB, getHdfsLocation().c_str());
+    EXPECT_EQ(1, Command::getCommandStatus(cmd));
+    util.query("select * from hawqregister;", 0);
+
+    /* register a parquet file with eof > filesize, should success */
+    int size = getFileSize(filePath.c_str());
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister
-e %d", HAWQ_DB, getHdfsLocation().c_str(), size+1);
+    EXPECT_EQ(1, Command::getCommandStatus(cmd));
+    util.query("select * from hawqregister;", 0);
+
+    /* register a parquet file with eof=float, should success */
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister
-e 11.1", HAWQ_DB, getHdfsLocation().c_str());
+    EXPECT_EQ(1, Command::getCommandStatus(cmd));
+    util.execute("drop table hawqregister;");
+}
+
+TEST_F(TestHawqRegister, TestUsage1FolderFailure) {
+    SQLUtility util;
+    string folderName = "usage1tmp";
+    string folderNameNotExist = "usage1tmpNotExist";
+    string rootPath(util.getTestRootPath());
+    string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
+    string filePath = rootPath + relativePath;
+    string relativePath2("/ManagementTool/files_incomplete.yml");
+    string filePath2 = rootPath + relativePath2;
+
+    auto cmd = hawq::test::stringFormat("hdfs dfs -mkdir %s/usage1tmp", getHdfsLocation().c_str(),
"");
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
+
+    /* register a empty folder, should success */
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister", HAWQ_DB,
getHdfsLocation().c_str(), folderName.c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    /* register a not exist folder, should fail */
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister", HAWQ_DB,
getHdfsLocation().c_str(), folderNameNotExist.c_str());
+    EXPECT_EQ(1, Command::getCommandStatus(cmd));
+
+    /* register a folder with eof, should fail */
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister -e 100", HAWQ_DB,
getHdfsLocation().c_str(), folderName.c_str());
+    EXPECT_EQ(1, Command::getCommandStatus(cmd));
+
+    /* register a folder containing parquet and non-parquet files, should fail */
+    cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/%s/hawq_register_hawq.paq", filePath.c_str(),
getHdfsLocation().c_str(), folderName.c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/%s/", filePath2.c_str(), getHdfsLocation().c_str(),
folderName.c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister", HAWQ_DB,
getHdfsLocation().c_str(), folderName.c_str());
+    EXPECT_EQ(1, Command::getCommandStatus(cmd));
+
+    util.execute("drop table hawqregister;");
+    cmd = hawq::test::stringFormat("hdfs dfs -rm -r %s/%s", getHdfsLocation().c_str(), folderName.c_str());
+    EXPECT_EQ(0, Command::getCommandStatus(cmd));
+}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4af3768c/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
index 6640775..6d44ca0 100644
--- a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
+++ b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
@@ -14,6 +14,38 @@ using hawq::test::SQLUtility;
 using hawq::test::Command;
 using hawq::test::HdfsConfig;
 
+TEST_F(TestHawqRegister, TestUsage2Case1EmptyTable) {
+  SQLUtility util;
+  util.execute("drop table if exists t9;");
+  util.execute("create table t9(i int) with (appendonly=true, orientation=row) distributed
randomly;");
+  EXPECT_EQ(0, Command::getCommandStatus("hawq extract -d " + (string) HAWQ_DB + " -o t9.yml
testhawqregister_testusage2case1emptytable.t9"));
+  EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c t9.yml
testhawqregister_testusage2case1emptytable.nt9"));
+  util.query("select * from nt9;", 0);
+  EXPECT_EQ(0, Command::getCommandStatus("rm -rf t9.yml"));
+  util.execute("drop table t9;");
+  util.execute("drop table nt9;");
+}
+
+TEST_F(TestHawqRegister, TestUsage2Case1IncorrectYaml) {
+  SQLUtility util;
+  string filePath = util.getTestRootPath() + "/ManagementTool/";
+
+  EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c "
+ filePath + "missing_pagesize.yml xx"));
+  EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c "
+ filePath + "missing_rowgroupsize.yml xx"));
+  EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c "
+ filePath + "missing_filesize.yml xx"));
+  EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c "
+ filePath + "wrong_schema.yml xx"));
+  EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c "
+ filePath + "missing_checksum.yml xx"));
+  EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c "
+ filePath + "wrong_dfs_url.yml xx"));
+  EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c "
+ filePath + "missing_bucketnum.yml xx"));
+}
+
+TEST_F(TestHawqRegister, TestUsage2Case1MismatchFileNumber) {
+  SQLUtility util;
+  string filePath = util.getTestRootPath() + "/ManagementTool/";
+  EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c "
+ filePath + "files_incomplete.yml xx"));
+}
+
+
 TEST_F(TestHawqRegister, TestUsage2Case1Expected) {
     SQLUtility util;
     std::vector<string> create_table_matrix = {"distributed by (i)", "distributed randomly"};

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4af3768c/tools/bin/hawqregister
----------------------------------------------------------------------
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index a1c8e29..245d4d8 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -498,7 +498,10 @@ class HawqRegister(object):
             hdfscmd = 'hadoop fs -du %s' % fn
             _, out, _ = local_ssh_output(hdfscmd)
             if self.sizes[k] > int(out.strip().split()[0]):
-                logger.error('File size(%s) in yaml configuration file should not exceed
actual length(%s) of file %s.' % (self.sizes[k], out.strip().split()[0], fn))
+                if self.mode == 'usage1':
+                    logger.error('Specified file size(%s) should not exceed actual length(%s)
of file %s.' % (self.sizes[k], out.strip().split()[0], fn))
+                else:
+                    logger.error('File size(%s) in yaml configuration file should not exceed
actual length(%s) of file %s.' % (self.sizes[k], out.strip().split()[0], fn))    
                 self.failure_handler.rollback()
                 sys.exit(1)
 



Mime
View raw message