hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hubertzh...@apache.org
Subject incubator-hawq git commit: HAWQ-1113. Fix bug and add test for hawq register force mode.
Date Fri, 28 Oct 2016 00:22:04 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master ced05d56f -> 6cefb5529


HAWQ-1113. Fix bug and add test for hawq register force mode.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/6cefb552
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/6cefb552
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/6cefb552

Branch: refs/heads/master
Commit: 6cefb5529c536df99fb864105e31e083c37741ea
Parents: ced05d5
Author: hzhang2 <zhanghuan929@163.com>
Authored: Fri Oct 28 08:22:48 2016 +0800
Committer: hzhang2 <zhanghuan929@163.com>
Committed: Fri Oct 28 08:22:48 2016 +0800

----------------------------------------------------------------------
 .../feature/ManagementTool/test_hawq_register.h |   2 +-
 .../test_hawq_register_usage2_case2.cpp         |  14 ++++-
 src/test/feature/ManagementTool/usage2case2/10  | Bin 0 -> 535 bytes
 src/test/feature/ManagementTool/usage2case2/11  | Bin 0 -> 535 bytes
 src/test/feature/ManagementTool/usage2case2/12  | Bin 0 -> 535 bytes
 src/test/feature/ManagementTool/usage2case2/13  | Bin 0 -> 535 bytes
 src/test/feature/ManagementTool/usage2case2/8   | Bin 0 -> 535 bytes
 src/test/feature/ManagementTool/usage2case2/9   | Bin 0 -> 543 bytes
 .../usage2case2/same_path_yaml_config_tpl.yml   |  57 +++++++++++++++++++
 tools/bin/hawqregister                          |  35 +++++++++---
 10 files changed, 97 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/test_hawq_register.h
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/test_hawq_register.h b/src/test/feature/ManagementTool/test_hawq_register.h
index 4edf2aa..dce1c8c 100644
--- a/src/test/feature/ManagementTool/test_hawq_register.h
+++ b/src/test/feature/ManagementTool/test_hawq_register.h
@@ -94,7 +94,7 @@ class TestHawqRegister : public ::testing::Test {
 
         void runYamlCaseTableNotExists(std::string casename, std::string ymlname, int expectederror,
int checknum);
         void runYamlCaseTableExists(std::string casename, std::string ymlname, int isexpectederror,
int checknum);
-        void runYamlCaseForceMode(std::string casename, std::string ymlname, int isexpectederror,
int rows, int checknum);
+        void runYamlCaseForceMode(std::string casename, std::string ymlname, int isexpectederror,
int rows, int checknum, bool samepath);
 
     private:
         std::unique_ptr<hawq::test::PSQL> conn;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp b/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp
index 492dbd3..ad28a0a 100644
--- a/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp
+++ b/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp
@@ -80,7 +80,7 @@ TEST_F(TestHawqRegister, TestUsage2Case2Expected) {
     }
 }
 
-void TestHawqRegister::runYamlCaseForceMode(std::string casename, std::string ymlname, int
isexpectederror = 1, int rows = 50, int checknum = 200) {
+void TestHawqRegister::runYamlCaseForceMode(std::string casename, std::string ymlname, int
isexpectederror = 1, int rows = 50, int checknum = 200, bool samepathfile = false) {
     SQLUtility util;
     string test_root(util.getTestRootPath());
     string t_yml_tpl(hawq::test::stringFormat("%s/ManagementTool/%s_tpl.yml", test_root.c_str(),
ymlname.c_str()));
@@ -107,11 +107,17 @@ void TestHawqRegister::runYamlCaseForceMode(std::string casename, std::string
ym
     hc.getNamenodeHost(hdfs_prefix);
     strs_src_dst["@PORT@"]= hdfs_prefix;
     frep.replace(t_yml_tpl, t_yml, strs_src_dst);
+    if(samepathfile) {
+        for(int i=8; i<=13; i++){
+          string file(hawq::test::stringFormat("%s/ManagementTool/usage2case2/", test_root.c_str()));
+          file.append(std::to_string(i));
+          EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hdfs dfs -put
%s /hawq_default/16385/%s/%s/", file.c_str(), strs_src_dst["@DATABASE_OID@"].c_str(),strs_src_dst["@TABLE_OID_OLD@"].c_str())));
+        }
+    }
     //printf("%s\n", hawq::test::stringFormat("hawq register --force -d %s -c %s testhawqregister_%s.nt",
HAWQ_DB, t_yml.c_str(), casename.c_str()).c_str());
     //sleep(60);
     EXPECT_EQ(isexpectederror, Command::getCommandStatus(hawq::test::stringFormat("hawq register
--force -d %s -c %s testhawqregister_%s.nt", HAWQ_DB, t_yml.c_str(), casename.c_str())));
     util.query("select * from nt;", checknum);
-
     EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf %s", t_yml.c_str())));
     util.execute("drop table t;");
     util.execute("drop table nt;");
@@ -250,6 +256,10 @@ TEST_F(TestHawqRegister, TestUsage2Case2NormalYamlConfig) {
     runYamlCaseForceMode("testusage2case2normalyamlconfig", "usage2case2/normal_yaml_config",
0, 50, 150);
 }
 
+TEST_F(TestHawqRegister, TestUsage2Case2SamePathYamlConfig) {
+    runYamlCaseForceMode("testusage2case2samepathyamlconfig", "usage2case2/same_path_yaml_config",
0, 50, 332, true);
+}
+
 TEST_F(TestHawqRegister, TestUsage2Case2NormalYamlNoUpdateConfig) {
     runYamlCaseForceMode("testusage2case2normalyamlnoupdateconfig", "usage2case2/normal_yaml_no_update_config",
0, 50, 100);
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/usage2case2/10
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case2/10 b/src/test/feature/ManagementTool/usage2case2/10
new file mode 100644
index 0000000..1aceb3f
Binary files /dev/null and b/src/test/feature/ManagementTool/usage2case2/10 differ

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/usage2case2/11
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case2/11 b/src/test/feature/ManagementTool/usage2case2/11
new file mode 100644
index 0000000..1aceb3f
Binary files /dev/null and b/src/test/feature/ManagementTool/usage2case2/11 differ

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/usage2case2/12
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case2/12 b/src/test/feature/ManagementTool/usage2case2/12
new file mode 100644
index 0000000..1aceb3f
Binary files /dev/null and b/src/test/feature/ManagementTool/usage2case2/12 differ

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/usage2case2/13
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case2/13 b/src/test/feature/ManagementTool/usage2case2/13
new file mode 100644
index 0000000..1aceb3f
Binary files /dev/null and b/src/test/feature/ManagementTool/usage2case2/13 differ

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/usage2case2/8
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case2/8 b/src/test/feature/ManagementTool/usage2case2/8
new file mode 100644
index 0000000..1aceb3f
Binary files /dev/null and b/src/test/feature/ManagementTool/usage2case2/8 differ

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/usage2case2/9
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case2/9 b/src/test/feature/ManagementTool/usage2case2/9
new file mode 100644
index 0000000..1155c33
Binary files /dev/null and b/src/test/feature/ManagementTool/usage2case2/9 differ

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/usage2case2/same_path_yaml_config_tpl.yml
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case2/same_path_yaml_config_tpl.yml b/src/test/feature/ManagementTool/usage2case2/same_path_yaml_config_tpl.yml
new file mode 100644
index 0000000..9bba0aa
--- /dev/null
+++ b/src/test/feature/ManagementTool/usage2case2/same_path_yaml_config_tpl.yml
@@ -0,0 +1,57 @@
+Bucketnum: 6
+DBVersion: PostgreSQL 8.2.15 (Greenplum Database 4.2.0 build 1) (HAWQ 2.0.1.0 build
+  dev) on x86_64-apple-darwin14.5.0, compiled by GCC Apple LLVM version 6.1.0 (clang-602.0.53)
+  (based on LLVM 3.6.0svn) compiled on Jul  6 2016 10:22:33
+DFS_URL: hdfs://@PORT@
+Distribution_Policy: DISTRIBUTED BY (i)
+Encoding: UTF8
+FileFormat: Parquet
+Parquet_FileLocations:
+  Checksum: false
+  CompressionLevel: 0
+  CompressionType: null
+  EnableDictionary: false
+  Files:
+  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/2
+    size: 250
+  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/4
+    size: 250
+  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/5
+    size: 258
+  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/6
+    size: 270 
+  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/3
+    size: 258
+  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/1
+    size: 228
+  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/2
+    size: 215
+  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/3
+    size: 215
+  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/4
+    size: 220
+  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/1
+    size: 254
+  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/6
+    size: 215
+  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/5
+    size: 210
+  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/9
+    size: 543
+  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/8
+    size: 535
+  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/10
+    size: 535
+  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/11
+    size: 535
+  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/12
+    size: 535
+  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/13
+    size: 535
+  PageSize: 1048576
+  RowGroupSize: 8388608
+Parquet_Schema:
+- name: i
+  type: int4
+TableName: public.nt
+Version: 1.0.0

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/tools/bin/hawqregister
----------------------------------------------------------------------
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index 46a9a8f..492b151 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -574,8 +574,10 @@ class HawqRegister(object):
             sys.exit(1)
         self.firstsegno, self.tabledir = self._get_metadata()
         
-        seg_list, existed_sizes = self._get_metadata_from_table()
-        existed_files = [self.tabledir + seg for seg in seg_list]
+        if self.yml and self.mode == 'force':
+            existed_files, existed_sizes = self._get_files_in_hdfs(self.tabledir)
+        else:
+            existed_files, existed_sizes = self._get_files_in_hdfs(self.filepath)
         # check if file numbers in hdfs is consistent with the record count of pg_aoseg.
         hdfs_file_no_lst = [f.split('/')[-1] for f in existed_files]
         for k in range(1, self.firstsegno - 1):
@@ -740,13 +742,28 @@ class HawqRegister(object):
         catalog_lst = [str(i) for i in range(1, segno)]
         new_catalog_lst = [str(i) for i in range(segno, len(self.files_update) + 1)]
         exist_catalog_lst = []
+        tmp_files_update = [f for f in self.files_update]
+        tmp_sizes_update = [f for f in self.sizes_update]
+
+        tmp_tupcounts_update = [f for f in self.tupcounts_update]
+        tmp_eofuncompresseds_update = [f for f in self.eofuncompresseds_update]
+        tmp_varblockcounts_update = [f for f in self.varblockcounts_update]
+         
         for k, seg in enumerate(update_segno_lst):
             if seg not in catalog_lst:
-                self.files_same_path.append(self.files_update[k])
-                self.sizes_same_path.append(self.sizes_update[k])
-                self.tupcounts_same_path.append(self.tupcounts_update[k])
-                self.eofuncompresseds_same_path.append(self.eofuncompresseds_update[k])
-                self.varblockcounts_same_path.append(self.varblockcounts_update[k])
+
+                self.files_same_path.append(tmp_files_update[k])
+                self.sizes_same_path.append(tmp_sizes_update[k])
+                self.tupcounts_same_path.append(tmp_tupcounts_update[k])
+                self.eofuncompresseds_same_path.append(tmp_eofuncompresseds_update[k])
+                self.varblockcounts_same_path.append(tmp_varblockcounts_update[k])
+           
+                self.files_update.remove(tmp_files_update[k])
+                self.sizes_update.remove(tmp_sizes_update[k])
+                self.tupcounts_update.remove(tmp_tupcounts_update[k])
+                self.eofuncompresseds_update.remove(tmp_eofuncompresseds_update[k])
+                self.varblockcounts_update.remove(tmp_varblockcounts_update[k])
+
             if seg in new_catalog_lst:
                 exist_catalog_lst.append(seg)
         for seg in update_segno_lst:
@@ -757,11 +774,13 @@ class HawqRegister(object):
                     while (str(segno) in exist_catalog_lst):
                         segno += 1
                     self.segnos_same_path.append(segno)
+                    segno += 1
 
         for k, f in enumerate(self.files_same_path):
             self.srcfiles.append(f)
             self.dstfiles.append(self.tabledir + str(self.segnos_same_path[k]))
 
+        segno = self.firstsegno + len(self.files_same_path)
         for f in self.files_append:
             self.srcfiles.append(f)
             self.dstfiles.append(self.tabledir + str(segno))
@@ -823,7 +842,7 @@ class HawqRegister(object):
             if len(same_path_eofs) > 0:
                 query += 'insert into pg_aoseg.%s values(%d, %d, %d, %d)' % (self.seg_name,
same_path_segno_lst[0], same_path_eofs[0], same_path_tupcounts[0], same_path_eofuncompresseds[0])
                 k = 0
-                for same_path_eof, same_path_tupcount, same_path_eofuncompressed in zip(same_path_eofs[1:],
same_path_tupcounts[1:], same_path_eofuncompresseds[1]):
+                for same_path_eof, same_path_tupcount, same_path_eofuncompressed in zip(same_path_eofs[1:],
same_path_tupcounts[1:], same_path_eofuncompresseds[1:]):
                     query += ',(%d, %d, %d, %d)' % (same_path_segno_lst[k + 1], same_path_eof,
same_path_tupcount, same_path_eofuncompressed)
                     k += 1
                 query += ';'


Mime
View raw message