hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject incubator-hawq git commit: HAWQ-1128. Support HAWQ register tables with same file name in different schema.
Date Wed, 02 Nov 2016 03:49:54 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 56e21fce3 -> 7363ea75c


HAWQ-1128. Support HAWQ register tables with same file name in different schema.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/7363ea75
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/7363ea75
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/7363ea75

Branch: refs/heads/master
Commit: 7363ea75cbb900e8aefd5470b94c81bf1714e659
Parents: 56e21fc
Author: xunzhang <xunzhangthu@gmail.com>
Authored: Mon Oct 31 21:34:11 2016 +0800
Committer: Wen Lin <wlin@pivotal.io>
Committed: Wed Nov 2 11:49:09 2016 +0800

----------------------------------------------------------------------
 .../test_hawq_register_usage1.cpp               | 76 +++++++++++++-------
 .../test_hawq_register_usage2_case1.cpp         | 20 +++---
 .../test_hawq_register_usage2_case2.cpp         |  4 +-
 tools/bin/hawqregister                          | 73 +++++++++----------
 4 files changed, 100 insertions(+), 73 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7363ea75/src/test/feature/ManagementTool/test_hawq_register_usage1.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/test_hawq_register_usage1.cpp b/src/test/feature/ManagementTool/test_hawq_register_usage1.cpp
index a2c4a69..ce6ed08 100644
--- a/src/test/feature/ManagementTool/test_hawq_register_usage1.cpp
+++ b/src/test/feature/ManagementTool/test_hawq_register_usage1.cpp
@@ -14,6 +14,32 @@ using hawq::test::SQLUtility;
 using hawq::test::Command;
 using hawq::test::HdfsConfig;
 
+TEST_F(TestHawqRegister, TestUsage1ExpectSuccessDifferentSchema) {
+  SQLUtility util;
+  string rootPath(util.getTestRootPath());
+  string filePath = rootPath + "/ManagementTool/data/parquet200/dat.paq";
+
+  util.execute("drop table if exists xz.t;");
+  util.execute("drop schema if exists xz;");
+  util.execute("create schema xz;");
+  util.execute("drop table if exists t;");
+  auto cmd = hawq::test::stringFormat("hdfs dfs -put -f %s %s/usage1dat.paq", filePath.c_str(),
getHdfsLocation().c_str());
+  EXPECT_EQ(0, Command::getCommandStatus(cmd));
+  std::string policy = "DISTRIBUTED RANDOMLY";;
+  auto sql1 = hawq::test::stringFormat("CREATE TABLE t(i int) with (appendonly=true, orientation=parquet)
%s;", policy.c_str());
+  auto sql2 = hawq::test::stringFormat("CREATE TABLE xz.t(i int) with (appendonly=true, orientation=parquet)
%s;", policy.c_str());
+  util.execute(sql1); util.execute(sql2);
+  util.query("SELECT * from t", 0);
+  cmd = hawq::test::stringFormat("hawq register -d %s -f %s/usage1dat.paq testhawqregister_testusage1expectsuccessdifferentschema.t",
HAWQ_DB, getHdfsLocation().c_str());
+  EXPECT_EQ(0, Command::getCommandStatus(cmd));
+  util.query("select * from t;", 200);
+  util.execute("insert into t values(201);");
+  util.query("select * from t;", 201);
+  util.execute("drop table xz.t;");
+  util.execute("drop table t;");
+  util.execute("drop schema xz;");
+}
+
 TEST_F(TestHawqRegister, TestUsage1ExpectSuccess) {
   // Register file/folder into HAWQ by specific file/folder name
 
@@ -40,7 +66,7 @@ TEST_F(TestHawqRegister, TestUsage1ExpectSuccess) {
         auto sql = hawq::test::stringFormat("CREATE TABLE t_%s(i int) with (appendonly=true,
orientation=%s) %s;", std::to_string(suffix).c_str(), ddl.c_str(), policy.c_str());
         util.execute(sql); util.query(hawq::test::stringFormat("SELECT * from t_%s", std::to_string(suffix).c_str()),
0);
         
-        cmd = hawq::test::stringFormat("hawq register -d %s -f %s/usage1dat.paq t_%s", HAWQ_DB,
getHdfsLocation().c_str(), std::to_string(suffix).c_str());
+        cmd = hawq::test::stringFormat("hawq register -d %s -f %s/usage1dat.paq testhawqregister_testusage1expectsuccess.t_%s",
HAWQ_DB, getHdfsLocation().c_str(), std::to_string(suffix).c_str());
         EXPECT_EQ(0, Command::getCommandStatus(cmd));
         
         util.query(hawq::test::stringFormat("select * from t_%s;", std::to_string(suffix).c_str()),
200);
@@ -60,7 +86,7 @@ TEST_F(TestHawqRegister, TestUsage1ExpectSuccess) {
         auto sql = hawq::test::stringFormat("CREATE TABLE t_%s(i int) with (appendonly=true,
orientation=%s) %s;", std::to_string(suffix).c_str(), ddl.c_str(), policy.c_str());
         util.execute(sql); util.query(hawq::test::stringFormat("SELECT * from t_%s", std::to_string(suffix).c_str()),
0);
         
-        cmd = hawq::test::stringFormat("hawq register -d %s -f %s/usage1dat.paq -e 596 t_%s",
HAWQ_DB, getHdfsLocation().c_str(), std::to_string(suffix).c_str());
+        cmd = hawq::test::stringFormat("hawq register -d %s -f %s/usage1dat.paq -e 596 testhawqregister_testusage1expectsuccess.t_%s",
HAWQ_DB, getHdfsLocation().c_str(), std::to_string(suffix).c_str());
         EXPECT_EQ(0, Command::getCommandStatus(cmd));
         
         util.query(hawq::test::stringFormat("select * from t_%s;", std::to_string(suffix).c_str()),
100);
@@ -84,7 +110,7 @@ TEST_F(TestHawqRegister, TestUsage1ExpectSuccess) {
           auto sql = hawq::test::stringFormat("CREATE TABLE t_%s(i int) with (appendonly=true,
orientation=%s) %s;", std::to_string(suffix).c_str(), ddl.c_str(), policy.c_str());
           util.execute(sql); util.query(hawq::test::stringFormat("SELECT * from t_%s", std::to_string(suffix).c_str()),
0);
         
-          cmd = hawq::test::stringFormat("hawq register -d %s -f %s%s t_%s", HAWQ_DB, getHdfsLocation().c_str(),
folder.c_str(), std::to_string(suffix).c_str());
+          cmd = hawq::test::stringFormat("hawq register -d %s -f %s%s testhawqregister_testusage1expectsuccess.t_%s",
HAWQ_DB, getHdfsLocation().c_str(), folder.c_str(), std::to_string(suffix).c_str());
           EXPECT_EQ(0, Command::getCommandStatus(cmd));
     
           util.query(hawq::test::stringFormat("select * from t_%s;", std::to_string(suffix).c_str()),
200);
@@ -122,7 +148,7 @@ TEST_F(TestHawqRegister, TestUsage1SingleHawqFile) {
     util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
     util.query("select * from hawqregister;", 0);
 
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq testhawqregister_testusage1singlehawqfile.hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
     EXPECT_EQ(0, Command::getCommandStatus(cmd));
 
     util.query("select * from hawqregister;", 3);
@@ -143,7 +169,7 @@ TEST_F(TestHawqRegister, TestUsage1SingleHiveFile) {
     util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
     util.query("select * from hawqregister;", 0);
 
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hive.paq hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hive.paq testhawqregister_testusage1singlehivefile.hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
     EXPECT_EQ(0, Command::getCommandStatus(cmd));
 
     util.query("select * from hawqregister;", 1);
@@ -166,7 +192,7 @@ TEST_F(TestHawqRegister, TestDataTypes) {
     util.execute("create table hawqregister(a bool, b int2, c int2, d int4, e int8, f date,
g float4, h float8, i varchar, j bytea, k char, l varchar) with (appendonly=true, orientation=parquet);");
     util.query("select * from hawqregister;", 0);
 
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_data_types.paq
hawqregister", HAWQ_DB, getHdfsLocation().c_str());
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_data_types.paq
testhawqregister_testdatatypes.hawqregister", HAWQ_DB, getHdfsLocation().c_str());
     EXPECT_EQ(0, Command::getCommandStatus(cmd));
 
     util.query("select * from hawqregister;", 1);
@@ -188,7 +214,7 @@ TEST_F(TestHawqRegister, TestAllNULL) {
     util.execute("create table hawqregister(a bool, b int2, c int2, d int4, e int8, f date,
g float4, h float8, i varchar, j bytea, k char, l varchar) with (appendonly=true, orientation=parquet);");
     util.query("select * from hawqregister;", 0);
 
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_data_types.paq
hawqregister", HAWQ_DB, getHdfsLocation().c_str());
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_data_types.paq
testhawqregister_testallnull.hawqregister", HAWQ_DB, getHdfsLocation().c_str());
     EXPECT_EQ(0, Command::getCommandStatus(cmd));
 
     util.query("select * from hawqregister;", 1);
@@ -222,7 +248,7 @@ TEST_F(TestHawqRegister, TestFiles) {
     util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
     util.query("select * from hawqregister;", 0);
 
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_test hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_test testhawqregister_testfiles.hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
     EXPECT_EQ(0, Command::getCommandStatus(cmd));
 
     util.query("select * from hawqregister;", 12);
@@ -245,7 +271,7 @@ TEST_F(TestHawqRegister, TestUsage1NotParquetFile) {
     util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
     util.query("select * from hawqregister;", 0);
 
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_test_not_paq
hawqregister", HAWQ_DB, getHdfsLocation().c_str());
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_test_not_paq
testhawqregister_testusage1notparquetfile.hawqregister", HAWQ_DB, getHdfsLocation().c_str());
     EXPECT_EQ(1, Command::getCommandStatus(cmd));
     util.query("select * from hawqregister;", 0);
 
@@ -267,21 +293,21 @@ TEST_F(TestHawqRegister, TestUsage1NotParquetTable) {
     util.execute("create table hawqregister(i int);");
     util.query("select * from hawqregister;", 0);
 
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq testhawqregister_testusage1notparquettable.hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
     EXPECT_EQ(1, Command::getCommandStatus(cmd));
     util.query("select * from hawqregister;", 0);
     util.execute("drop table hawqregister;");
 
     /* register a parquet file to a row hash table, should fail */
     util.execute("create table hawqregister(i int) distributed by (i);");
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq testhawqregister_testusage1notparquettable.hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
     EXPECT_EQ(1, Command::getCommandStatus(cmd));
     util.query("select * from hawqregister;", 0);
     util.execute("drop table hawqregister;");
 
     /* register a parquet file to a parquet hash table, should fail */
     util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet)
distributed by (i);");
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq testhawqregister_testusage1notparquettable.hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
     EXPECT_EQ(1, Command::getCommandStatus(cmd));
     util.query("select * from hawqregister;", 0);
     util.execute("drop table hawqregister;");
@@ -296,7 +322,7 @@ TEST_F(TestHawqRegister, TestUsage1FileNotExist) {
     util.execute("create table hawqregister(i int);");
     util.query("select * from hawqregister;", 0);
 
-    auto cmd = hawq::test::stringFormat("hawq register -d %s -f %shawq_register_file_not_exist
hawqregister", HAWQ_DB, getHdfsLocation().c_str());
+    auto cmd = hawq::test::stringFormat("hawq register -d %s -f %shawq_register_file_not_exist
testhawqregister_testusage1notparquettable.hawqregister", HAWQ_DB, getHdfsLocation().c_str());
     EXPECT_EQ(1, Command::getCommandStatus(cmd));
     util.query("select * from hawqregister;", 0);
 
@@ -313,7 +339,7 @@ TEST_F(TestHawqRegister, TestUsage1NoTable) {
     EXPECT_EQ(0, Command::getCommandStatus(cmd));
 
     /* register a parquet file to a table not exist, should fail */
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s hawqregister", HAWQ_DB, getHdfsLocation().c_str());
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s testhawqregister_testusage1notable.hawqregister",
HAWQ_DB, getHdfsLocation().c_str());
     EXPECT_EQ(1, Command::getCommandStatus(cmd));
 }
 
@@ -327,7 +353,7 @@ TEST_F(TestHawqRegister, TestUsage1NotHDFSPath) {
     util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
     util.query("select * from hawqregister;", 0);
 
-    auto cmd = hawq::test::stringFormat("hawq register -d %s -f %s hawqregister", HAWQ_DB,
filePath.c_str());
+    auto cmd = hawq::test::stringFormat("hawq register -d %s -f %s testhawqregister_testusage1nothdfspath.hawqregister",
HAWQ_DB, filePath.c_str());
     EXPECT_EQ(1, Command::getCommandStatus(cmd));
     util.query("select * from hawqregister;", 0);
 
@@ -347,7 +373,7 @@ TEST_F(TestHawqRegister, TestUsage1EofSuccess) {
     util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
     util.query("select * from hawqregister;", 0); 
 
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister
-e 0", HAWQ_DB, getHdfsLocation().c_str());
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq testhawqregister_testusage1eofsuccess.hawqregister
-e 0", HAWQ_DB, getHdfsLocation().c_str());
     EXPECT_EQ(0, Command::getCommandStatus(cmd));
     util.query("select * from hawqregister;", 0);
 
@@ -356,7 +382,7 @@ TEST_F(TestHawqRegister, TestUsage1EofSuccess) {
     cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(),
getHdfsLocation().c_str());
     EXPECT_EQ(0, Command::getCommandStatus(cmd));
 
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister
-e %d", HAWQ_DB, getHdfsLocation().c_str(), size);
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq testhawqregister_testusage1eofsuccess.hawqregister
-e %d", HAWQ_DB, getHdfsLocation().c_str(), size);
     EXPECT_EQ(0, Command::getCommandStatus(cmd));
     util.query("select * from hawqregister;", 3);
     util.execute("drop table hawqregister;");
@@ -366,7 +392,7 @@ TEST_F(TestHawqRegister, TestUsage1EofSuccess) {
     EXPECT_EQ(0, Command::getCommandStatus(cmd));
 
     util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister
-e %d", HAWQ_DB, getHdfsLocation().c_str(), size/2);
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq testhawqregister_testusage1eofsuccess.hawqregister
-e %d", HAWQ_DB, getHdfsLocation().c_str(), size/2);
     EXPECT_EQ(0, Command::getCommandStatus(cmd));
     util.execute("drop table hawqregister;");
 }
@@ -384,18 +410,18 @@ TEST_F(TestHawqRegister, TestUsage1EofFailure) {
     util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
     util.query("select * from hawqregister;", 0);
 
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister
-e -1", HAWQ_DB, getHdfsLocation().c_str());
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq testhawqregister_testusage1eoffailure.hawqregister
-e -1", HAWQ_DB, getHdfsLocation().c_str());
     EXPECT_EQ(1, Command::getCommandStatus(cmd));
     util.query("select * from hawqregister;", 0);
 
     /* register a parquet file with eof > filesize, should success */
     int size = getFileSize(filePath.c_str());
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister
-e %d", HAWQ_DB, getHdfsLocation().c_str(), size+1);
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq testhawqregister_testusage1eoffailure.hawqregister
-e %d", HAWQ_DB, getHdfsLocation().c_str(), size+1);
     EXPECT_EQ(1, Command::getCommandStatus(cmd));
     util.query("select * from hawqregister;", 0);
 
     /* register a parquet file with eof=float, should success */
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister
-e 11.1", HAWQ_DB, getHdfsLocation().c_str());
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq testhawqregister_testusage1eoffailure.hawqregister
-e 11.1", HAWQ_DB, getHdfsLocation().c_str());
     EXPECT_EQ(1, Command::getCommandStatus(cmd));
     util.execute("drop table hawqregister;");
 }
@@ -416,15 +442,15 @@ TEST_F(TestHawqRegister, TestUsage1FolderFailure) {
     util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
 
     /* register a empty folder, should success */
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister", HAWQ_DB,
getHdfsLocation().c_str(), folderName.c_str());
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s testhawqregister_testUsage1folderfailure.hawqregister",
HAWQ_DB, getHdfsLocation().c_str(), folderName.c_str());
     EXPECT_EQ(0, Command::getCommandStatus(cmd));
 
     /* register a not exist folder, should fail */
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister", HAWQ_DB,
getHdfsLocation().c_str(), folderNameNotExist.c_str());
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s testhawqregister_testUsage1folderfailure.hawqregister",
HAWQ_DB, getHdfsLocation().c_str(), folderNameNotExist.c_str());
     EXPECT_EQ(1, Command::getCommandStatus(cmd));
 
     /* register a folder with eof, should fail */
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister -e 100", HAWQ_DB,
getHdfsLocation().c_str(), folderName.c_str());
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s testhawqregister_testUsage1folderfailure.hawqregister
-e 100", HAWQ_DB, getHdfsLocation().c_str(), folderName.c_str());
     EXPECT_EQ(1, Command::getCommandStatus(cmd));
 
     /* register a folder containing parquet and non-parquet files, should fail */
@@ -434,7 +460,7 @@ TEST_F(TestHawqRegister, TestUsage1FolderFailure) {
     cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/%s/", filePath2.c_str(), getHdfsLocation().c_str(),
folderName.c_str());
     EXPECT_EQ(0, Command::getCommandStatus(cmd));
 
-    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s hawqregister", HAWQ_DB,
getHdfsLocation().c_str(), folderName.c_str());
+    cmd = hawq::test::stringFormat("hawq register -d %s -f %s/%s testhawqregister_testUsage1folderfailure.hawqregister",
HAWQ_DB, getHdfsLocation().c_str(), folderName.c_str());
     EXPECT_EQ(1, Command::getCommandStatus(cmd));
 
     util.execute("drop table hawqregister;");

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7363ea75/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
index 90aa8f0..c599f7c 100644
--- a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
+++ b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
@@ -33,20 +33,20 @@ TEST_F(TestHawqRegister, TestUsage2Case1IncorrectYaml) {
     SQLUtility util;
     string filePath = util.getTestRootPath() + "/ManagementTool/";
 
-    EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "missing_pagesize.yml xx"));
-    EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "missing_rowgroupsize.yml xx"));
-    EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "missing_filesize.yml xx"));
-    EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "wrong_schema.yml xx"));
-    EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "missing_checksum.yml xx"));
-    EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "wrong_dfs_url.yml xx"));
-    EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "missing_bucketnum.yml xx"));
-    EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "missing_constraint_partition.yml xx"));
+    EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "missing_pagesize.yml testhawqregister_testusage2case1incorrectyaml.xx"));
+    EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "missing_rowgroupsize.yml testhawqregister_testusage2case1incorrectyaml.xx"));
+    EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "missing_filesize.yml testhawqregister_testusage2case1incorrectyaml.xx"));
+    EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "wrong_schema.yml testhawqregister_testusage2case1incorrectyaml.xx"));
+    EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "missing_checksum.yml testhawqregister_testusage2case1incorrectyaml.xx"));
+    EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "wrong_dfs_url.yml testhawqregister_testusage2case1incorrectyaml.xx"));
+    EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "missing_bucketnum.yml testhawqregister_testusage2case1incorrectyaml.xx"));
+    EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "missing_constraint_partition.yml testhawqregister_testusage2case1incorrectyaml.xx"));
 }
 
 TEST_F(TestHawqRegister, TestUsage2Case1MismatchFileNumber) {
     SQLUtility util;
     string filePath = util.getTestRootPath() + "/ManagementTool/";
-    EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "files_incomplete.yml xx"));
+    EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "files_incomplete.yml testhawqregister_testusage2case1mismatchfilenumber.xx"));
 }
 
 
@@ -317,7 +317,7 @@ TEST_F(TestHawqRegister, DISABLED_TestUsage2Case1ErrorRowgroupSize) {
 TEST_F(TestHawqRegister, TestUsage2Case1IncludeDirectory) {
     SQLUtility util;
     string test_root(util.getTestRootPath());
-    EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hdfs dfs -put -f %s/ManagementTool/usage2case1
%s/", test_root.c_str(), getHdfsLocation().c_str())));
+    EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hadoop fs -put -f %s/ManagementTool/usage2case1
%s/", test_root.c_str(), getHdfsLocation().c_str())));
     string t_yml(hawq::test::stringFormat("%s/ManagementTool/usage2case1/includedirectory.yml",
test_root.c_str()));
     EXPECT_EQ(1, Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s
-c %s testhawqregister_testusage2case1includedirectory.nt", HAWQ_DB, t_yml.c_str())));
     EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hdfs dfs -rm -r %s/usage2case1",
getHdfsLocation().c_str())));

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7363ea75/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp b/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp
index ad28a0a..14ea48e 100644
--- a/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp
+++ b/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp
@@ -111,7 +111,7 @@ void TestHawqRegister::runYamlCaseForceMode(std::string casename, std::string
ym
         for(int i=8; i<=13; i++){
           string file(hawq::test::stringFormat("%s/ManagementTool/usage2case2/", test_root.c_str()));
           file.append(std::to_string(i));
-          EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hdfs dfs -put
%s /hawq_default/16385/%s/%s/", file.c_str(), strs_src_dst["@DATABASE_OID@"].c_str(),strs_src_dst["@TABLE_OID_OLD@"].c_str())));
+          EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hadoop fs -put
%s /hawq_default/16385/%s/%s/", file.c_str(), strs_src_dst["@DATABASE_OID@"].c_str(),strs_src_dst["@TABLE_OID_OLD@"].c_str())));
         }
     }
     //printf("%s\n", hawq::test::stringFormat("hawq register --force -d %s -c %s testhawqregister_%s.nt",
HAWQ_DB, t_yml.c_str(), casename.c_str()).c_str());
@@ -366,7 +366,7 @@ TEST_F(TestHawqRegister, DISABLED_TestUsage2Case2ErrorRowgroupSize) {
 TEST_F(TestHawqRegister, TestUsage2Case2IncludeDirectory) {
     SQLUtility util;
     string test_root(util.getTestRootPath());
-    EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hdfs dfs -put -f %s/ManagementTool/usage2case2
%s/", test_root.c_str(), getHdfsLocation().c_str())));
+    EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hadoop fs -put -f %s/ManagementTool/usage2case2
%s/", test_root.c_str(), getHdfsLocation().c_str())));
     string t_yml(hawq::test::stringFormat("%s/ManagementTool/usage2case2/includedirectory.yml",
test_root.c_str()));
     EXPECT_EQ(1, Command::getCommandStatus(hawq::test::stringFormat("hawq register --force
-d %s -c %s testhawqregister_testusage2case2includedirectory.nt", HAWQ_DB, t_yml.c_str())));
     EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hdfs dfs -rm -r %s/usage2case2",
getHdfsLocation().c_str())));

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7363ea75/tools/bin/hawqregister
----------------------------------------------------------------------
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index 356aef1..3477e81 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -142,6 +142,12 @@ def ispartition(yml_file):
             return True
     return False
 
+def tablename_handler(tablename):
+    tmp_lst = tablename.split('.')
+    if len(tmp_lst) > 1:
+        return tmp_lst[-2], tmp_lst[-1]
+    return 'public', tablename
+
 class FailureHandler(object):
     def __init__(self, conn):
         self.operations = []
@@ -197,12 +203,18 @@ class GpRegisterAccessor(object):
         '''execute query and return dict result'''
         return self.conn.query(sql).dictresult()
 
+    def get_schema_id(self, schemaname):
+        qry = """select oid from pg_namespace where nspname='%s';""" % schemaname
+        return self.exec_query(qry)[0]['oid']
+
     def get_table_existed(self, tablename):
-        qry = """select count(*) from pg_class where relname = '%s';""" % tablename.split('.')[-1].lower()
+        schemaname, tablename = tablename_handler(tablename)
+        qry = """select count(*) from pg_class where relnamespace = '%s' and relname = '%s';"""
% (self.get_schema_id(schemaname), tablename)
         return self.exec_query(qry)[0]['count'] == 1
 
     def get_table_column_num(self, tablename):
-        qry = """select count(*) from pg_attribute ,pg_class where pg_class.relname = '%s'
and pg_class.oid = pg_attribute.attrelid and attnum > 0;""" % tablename.split('.')[-1].lower()
+        schemaname, tablename = tablename_handler(tablename)
+        qry = """select count(*) from pg_attribute ,pg_class where pg_class.relnamespace
= '%s' and pg_class.relname = '%s' and pg_class.oid = pg_attribute.attrelid and attnum >
0;""" % (self.get_schema_id(schemaname), tablename)
         return self.exec_query(qry)[0]['count']
 
     def do_create_table(self, src_table_name, tablename, schema_info, fmt, distrbution_policy,
file_locations, bucket_number, partitionby, partitions_constraint, partitions_name):
@@ -241,7 +253,8 @@ class GpRegisterAccessor(object):
         return True, query
 
     def check_hash_type(self, tablename):
-        qry = """select attrnums from gp_distribution_policy, pg_class where pg_class.relname
= '%s' and pg_class.oid = gp_distribution_policy.localoid;""" % tablename
+        schemaname, tablename = tablename_handler(tablename)
+        qry = """select attrnums from gp_distribution_policy, pg_class where pg_class.relnamespace
= '%s' and pg_class.relname = '%s' and pg_class.oid = gp_distribution_policy.localoid;"""
% (self.get_schema_id(schemaname), tablename)
         rows = self.exec_query(qry)
         if len(rows) == 0:
             logger.error('Table %s is not an append-only table. There is no record in gp_distribution_policy
table.' % tablename)
@@ -252,9 +265,9 @@ class GpRegisterAccessor(object):
 
     # pg_paqseg_#
     def get_seg_name(self, tablename, database, fmt):
-        tablename = tablename.split('.')[-1]
+        schemaname, tablename = tablename_handler(tablename)
         query = ("select pg_class2.relname from pg_class as pg_class1, pg_appendonly, pg_class
as pg_class2 "
-                 "where pg_class1.relname ='%s' and pg_class1.oid = pg_appendonly.relid and
pg_appendonly.segrelid = pg_class2.oid;") % tablename
+                 "where pg_class1.relname ='%s' and pg_class1.oid = pg_appendonly.relid and
pg_appendonly.segrelid = pg_class2.oid and pg_class1.relnamespace = '%s';") % (tablename,
self.get_schema_id(schemaname))
         rows = self.exec_query(query)
         if len(rows) == 0:
             logger.error('table "%s" not found in db "%s"' % (tablename, database))
@@ -267,7 +280,8 @@ class GpRegisterAccessor(object):
         return (relname, True)
 
     def get_distribution_policy_info(self, tablename):
-        query = "select oid from pg_class where relname = '%s';" % tablename.split('.')[-1].lower()
+        schemaname, tablename = tablename_handler(tablename)
+        query = "select oid from pg_class where relnamespace = '%s' and relname = '%s';"
% (self.get_schema_id(schemaname), tablename)
         rows = self.exec_query(query)
         oid = rows[0]['oid']
         query = "select * from gp_distribution_policy where localoid = '%s';" % oid
@@ -276,23 +290,27 @@ class GpRegisterAccessor(object):
 
     def get_partition_info(self, tablename):
         ''' Get partition information from pg_partitions, return a constraint-tablename dictionary
'''
-        query = "SELECT partitiontablename, partitionboundary FROM pg_partitions WHERE tablename
= '%s'" % tablename
+        schemaname, tablename = tablename_handler(tablename)
+        query = "SELECT partitiontablename, partitionboundary FROM pg_partitions WHERE partitionschemaname
= '%s' and tablename = '%s'" % (schemaname, tablename)
         return self.exec_query(query)
 
     def get_partitionby(self, tablename):
-        query = "SELECT partitionschemaname, partitiontablename, partitionname, partitiontype,
parentpartitiontablename, partitionboundary FROM pg_partitions WHERE tablename='%s';" % tablename
+        schemaname, tablename = tablename_handler(tablename)
+        query = "SELECT partitionschemaname, partitiontablename, partitionname, partitiontype,
parentpartitiontablename, partitionboundary FROM pg_partitions WHERE schemaname = '%s' and
tablename='%s';" % (schemaname, tablename)
         parition_type = self.exec_query(query)[0]['partitiontype']
-        query = "SELECT columnname, partitionlevel FROM pg_partition_columns WHERE tablename='%s'
ORDER BY position_in_partition_key;" % tablename
+        query = "SELECT columnname, partitionlevel FROM pg_partition_columns WHERE schemaname
= '%s' and tablename='%s' ORDER BY position_in_partition_key;" % (schemaname, tablename)
         partition_columnname = self.exec_query(query)[0]['columnname']
         partitionby = 'PARTITION BY %s (%s)' % (parition_type, partition_columnname)
         return partitionby
 
     def get_partition_num(self, tablename):
-        query = "SELECT partitionschemaname from pg_partitions WHERE tablename='%s';" % tablename
+        schemaname, tablename = tablename_handler(tablename)
+        query = "SELECT partitionschemaname from pg_partitions WHERE schemaname = '%s' and
tablename='%s';" % (schemaname, tablename)
         return len(self.exec_query(query))
 
     def get_bucket_number(self, tablename):
-        query = "select oid from pg_class where relname = '%s';" % tablename.split('.')[-1].lower()
+        schemaname, tablename = tablename_handler(tablename)
+        query = "select oid from pg_class where relnamespace = '%s' and relname = '%s';"
% (self.get_schema_id(schemaname), tablename)
         rows = self.exec_query(query)
         oid = rows[0]['oid']
         query = "select * from gp_distribution_policy where localoid = '%s';" % oid
@@ -300,13 +318,14 @@ class GpRegisterAccessor(object):
         return rows[0]['bucketnum']
 
     def get_metadata_from_database(self, tablename, seg_name):
+        schemaname, tablename = tablename_handler(tablename) 
         query = 'select segno from pg_aoseg.%s;' % seg_name
         firstsegno = len(self.exec_query(query)) + 1
         # get the full path of correspoding file for target table
         query = ("select location, gp_persistent_tablespace_node.tablespace_oid, database_oid,
relfilenode from pg_class, gp_persistent_relation_node, "
-                 "gp_persistent_tablespace_node, gp_persistent_filespace_node where relname
= '%s' and pg_class.relfilenode = "
+                 "gp_persistent_tablespace_node, gp_persistent_filespace_node where relnamespace
= '%s' and relname = '%s' and pg_class.relfilenode = "
                  "gp_persistent_relation_node.relfilenode_oid and gp_persistent_relation_node.tablespace_oid
= gp_persistent_tablespace_node.tablespace_oid "
-                 "and gp_persistent_filespace_node.filespace_oid = gp_persistent_filespace_node.filespace_oid;")
% tablename.split('.')[-1]
+                 "and gp_persistent_filespace_node.filespace_oid = gp_persistent_filespace_node.filespace_oid;")
% (self.get_schema_id(schemaname), tablename)
         D = self.exec_query(query)[0]
         tabledir = '/'.join([D['location'].strip(), str(D['tablespace_oid']), str(D['database_oid']),
str(D['relfilenode']), ''])
         return firstsegno, tabledir
@@ -837,16 +856,6 @@ class HawqRegister(object):
                     sys.exit(1)
                 self.failure_handler.commit(('HDFSCMD', hdfscmd))
 
-    def _delete_files_in_hdfs(self):
-        for fn in self.files_delete:
-            hdfscmd = 'hadoop fs -rm %s' % fn
-            sys.stdout.write('hdfscmd: "%s"\n' % hdfscmd)
-            result = local_ssh(hdfscmd, logger)
-            if result != 0:
-                logger.error('Fail to delete %s ' % fn)
-                self.failure_handler.rollback()
-                sys.exit(1)
-
     def _set_modify_metadata(self, mode):
         segno = self.firstsegno
         append_eofs = self.sizes_append
@@ -924,15 +933,6 @@ class HawqRegister(object):
             self.failure_handler.rollback()
             sys.exit(1)
 
-    def _delete_metadata(self):
-        query = "set allow_system_table_mods='dml';"
-        query += "begin transaction;"
-        segno_lst = [fn.strip().split('/')[-1] for fn in self.files_delete]
-        for seg in segno_lst:
-            query += "delete from pg_aoseg.%s where segno = '%s';" % (self.seg_name, seg)
-        query += "end transaction;"
-        return self.utility_accessor.update_catalog(query)
-
     def _prepare_register(self):
         if not self.do_not_move:
             self._set_move_files_in_hdfs()
@@ -954,13 +954,13 @@ class HawqRegisterPartition(HawqRegister):
 
     def _get_partition_info(self):
         dic = {}
-        for ele in self.accessor.get_partition_info(self.dst_table_name.split('.')[-1]):
+        for ele in self.accessor.get_partition_info(self.dst_table_name):
             dic[ele['partitionboundary']] = ele['partitiontablename']
         return dic
 
     def _check_partitionby(self):
         def get_partitionby():
-            return self.accessor.get_partitionby(self.dst_table_name.split('.')[-1])
+            return self.accessor.get_partitionby(self.dst_table_name)
 
         if self.partitionby != get_partitionby():
             logger.error('PartitionBy of %s is not consistent with previous partitionby.'
% self.tablename)
@@ -969,7 +969,7 @@ class HawqRegisterPartition(HawqRegister):
 
     def _check_partition_num(self):
         def get_partition_num():
-            return self.accessor.get_partition_num(self.dst_table_name.split('.')[-1])
+            return self.accessor.get_partition_num(self.dst_table_name)
 
         if get_partition_num() < len(self.partitions_name):
             logger.error('Partition Number of %s is not consistent with previous partition
number.' % self.tablename)
@@ -1010,13 +1010,14 @@ class HawqRegisterPartition(HawqRegister):
         self.queries = "set allow_system_table_mods='dml';"
         self.queries += "begin transaction;"
         self._check_duplicate_constraint()
+        schemaname, _ = tablename_handler(self.dst_table_name)
         for k, pn in enumerate(self.partitions_name):
             self.constraint = self.partitions_constraint[k]
             if not partitions.has_key(self.constraint):
                 logger.error('Partition Constraint "%s" is not in table %s' % (self.constraint,
self.tablename))
                 self.failure_handler.rollback()
                 sys.exit(1)
-            self.tablename = partitions[self.constraint]
+            self.tablename = schemaname + '.' + partitions[self.constraint]
             self.files = self.partitions_filepaths[k]
             self.sizes = self.partitions_filesizes[k]
             self.tupcounts = self.partitions_tupcounts[k]



Mime
View raw message