hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject incubator-hawq git commit: HAWQ-1034. Delete code for hawq register repair mode
Date Mon, 31 Oct 2016 06:18:51 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 45adcbdb3 -> 778ab999a


HAWQ-1034. Delete code for hawq register repair mode


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/778ab999
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/778ab999
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/778ab999

Branch: refs/heads/master
Commit: 778ab999a95c36999430fea6ea29e7839106d9d1
Parents: 45adcbd
Author: Chunling Wang <wangchunling14@126.com>
Authored: Mon Oct 31 13:50:14 2016 +0800
Committer: Chunling Wang <wangchunling14@126.com>
Committed: Mon Oct 31 13:50:14 2016 +0800

----------------------------------------------------------------------
 .../test_hawq_register_usage2_case3.cpp         | 81 --------------------
 .../ManagementTool/usage2case3/t_tpl_old_1.yml  | 31 --------
 .../ManagementTool/usage2case3/t_tpl_old_2.yml  | 33 --------
 .../ManagementTool/usage2case3/t_tpl_old_3.yml  | 21 -----
 .../ManagementTool/usage2case3/t_tpl_old_4.yml  | 23 ------
 tools/bin/hawqregister                          | 63 +++------------
 6 files changed, 12 insertions(+), 240 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/778ab999/src/test/feature/ManagementTool/test_hawq_register_usage2_case3.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/test_hawq_register_usage2_case3.cpp b/src/test/feature/ManagementTool/test_hawq_register_usage2_case3.cpp
deleted file mode 100644
index 2b4d07b..0000000
--- a/src/test/feature/ManagementTool/test_hawq_register_usage2_case3.cpp
+++ /dev/null
@@ -1,81 +0,0 @@
-#include <string>
-
-#include "lib/command.h"
-#include "lib/sql_util.h"
-#include "lib/string_util.h"
-#include "lib/hdfs_config.h"
-#include "lib/file_replace.h"
-#include "test_hawq_register.h"
-
-#include "gtest/gtest.h"
-
-using std::string;
-using hawq::test::SQLUtility;
-using hawq::test::Command;
-using hawq::test::HdfsConfig;
-
-TEST_F(TestHawqRegister, DISABLED_TestUsage2Case3Expected) {
-    SQLUtility util;
-    string test_root(util.getTestRootPath());
-    std::vector<string> create_table_matrix = {"distributed by (i)", "distributed randomly"};
-    std::vector<string> fmt_matrix = {"row", "parquet"};
-    std::vector<string> option_matrix = {"--repair", "-R"};
-    int suffix=0;
-
-    for (auto & opt : option_matrix) {
-        suffix = 0;
-        for (auto & ddl : create_table_matrix) {
-            for (auto & fmt : fmt_matrix) {
-                suffix++;
-                string t_yml_tpl_old(hawq::test::stringFormat("%s/ManagementTool/usage2case3/t_tpl_old_%s.yml",
test_root.c_str(), std::to_string(suffix).c_str()));
-                string t_yml_old(hawq::test::stringFormat("%s/ManagementTool/usage2case3/t_old_%s.yml",
test_root.c_str(), std::to_string(suffix).c_str()));
-                auto t = hawq::test::stringFormat("t_usage2_case3_%s", std::to_string(suffix).c_str());
-                auto nt = hawq::test::stringFormat("nt_usage2_case3_%s", std::to_string(suffix).c_str());
-                util.execute(hawq::test::stringFormat("drop table if exists %s;", t.c_str()));
-                util.execute(hawq::test::stringFormat("drop table if exists %s;", nt.c_str()));
-
-                // hawq register -d hawq_feature_test -c t_usage2_case3_#.yml nt_usage2_case3_#,
where nt_usage2_case3_# does not exist
-                util.execute(hawq::test::stringFormat("create table %s(i int) with (appendonly=true,
orientation=%s) %s;", t.c_str(), fmt.c_str(), ddl.c_str()));
-                util.execute(hawq::test::stringFormat("insert into %s select generate_series(1,
100);", t.c_str()));
-                util.execute(hawq::test::stringFormat("insert into %s select generate_series(101,
200);", t.c_str()));
-                util.query(hawq::test::stringFormat("select * from %s;", t.c_str()), 200);
-                EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq extract
-d %s -o t_%s.yml testhawqregister_testusage2case3expected.%s", HAWQ_DB, std::to_string(suffix).c_str(),
t.c_str())));
-                EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq register
-d %s -c t_%s.yml testhawqregister_testusage2case3expected.%s", HAWQ_DB, std::to_string(suffix).c_str(),
nt.c_str())));
-                util.query(hawq::test::stringFormat("select * from %s;", nt.c_str()), 200);
-
-                // hawq register -d hawq_feature_test -c t_usage2_case3_#.yml nt_usage2_case3_#,
where nt_usage2_case3_# exists
-                util.execute(hawq::test::stringFormat("drop table if exists %s;", t.c_str()));
-                util.execute(hawq::test::stringFormat("create table %s(i int) with (appendonly=true,
orientation=%s) %s;", t.c_str(), fmt.c_str(), ddl.c_str()));
-                util.execute(hawq::test::stringFormat("insert into %s select generate_series(101,
150);", t.c_str()));
-                util.query(hawq::test::stringFormat("select * from %s;", t.c_str()), 50);
-                EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq extract
-d %s -o t_%s.yml testhawqregister_testusage2case3expected.%s", HAWQ_DB, std::to_string(suffix).c_str(),
t.c_str())));
-                EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq register
-d %s -c t_%s.yml testhawqregister_testusage2case3expected.%s", HAWQ_DB, std::to_string(suffix).c_str(),
nt.c_str())));
-                util.query(hawq::test::stringFormat("select * from %s;", nt.c_str()), 250);
-
-                // hawq register --repair/-R -d hawq_feature_test -c t_old_#.yml nt_usage2_case3_#
-                hawq::test::FileReplace frep;
-                std::unordered_map<std::string, std::string> strs_src_dst;
-                strs_src_dst["@DATABASE_OID@"]= getDatabaseOid();
-                strs_src_dst["@TABLE_OID@"]= getTableOid(nt);
-                string hdfs_prefix;
-                hawq::test::HdfsConfig hc;
-                hc.getNamenodeHost(hdfs_prefix);
-                strs_src_dst["@PORT@"]= hdfs_prefix;
-                frep.replace(t_yml_tpl_old, t_yml_old, strs_src_dst);
-                EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq register
%s -d %s -c %s testhawqregister_testusage2case3expected.%s", opt.c_str(), HAWQ_DB, t_yml_old.c_str(),
nt.c_str())));
-                util.query(hawq::test::stringFormat("select * from %s;", nt.c_str()), 100);
-
-                if (fmt == "row")
-                   checkPgAOSegValue(nt, "-1", "aoseg");
-                else 
-                   checkPgAOSegValue(nt, "-1", "paqseg");
-                
-                EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf t_%s.yml",
std::to_string(suffix).c_str())));
-                EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf %s",
t_yml_old.c_str())));
-                util.execute(hawq::test::stringFormat("drop table %s;", t.c_str()));
-                util.execute(hawq::test::stringFormat("drop table %s;", nt.c_str()));
-            }
-        }
-    }
-}
-

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/778ab999/src/test/feature/ManagementTool/usage2case3/t_tpl_old_1.yml
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_1.yml b/src/test/feature/ManagementTool/usage2case3/t_tpl_old_1.yml
deleted file mode 100644
index 3ec2f4e..0000000
--- a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_1.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-AO_FileLocations:
-  Blocksize: 32768
-  Checksum: false
-  CompressionLevel: 0
-  CompressionType: null
-  Files:
-  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/1
-    size: 176
-  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/2
-    size: 168
-  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/3
-    size: 192
-  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/4
-    size: 168
-  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/5
-    size: 192
-  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/6
-    size: 216
-AO_Schema:
-- name: i
-  type: int4
-Bucketnum: 6
-DBVersion: PostgreSQL 8.2.15 (Greenplum Database 4.2.0 build 1) (HAWQ 2.0.1.0 build
-  dev) on x86_64-apple-darwin14.5.0, compiled by GCC Apple LLVM version 6.1.0 (clang-602.0.53)
-  (based on LLVM 3.6.0svn) compiled on Jul  6 2016 10:22:33
-DFS_URL: hdfs://@PORT@
-Distribution_Policy: DISTRIBUTED BY (i)
-Encoding: UTF8
-FileFormat: AO
-TableName: public.t_1
-Version: 1.0.0

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/778ab999/src/test/feature/ManagementTool/usage2case3/t_tpl_old_2.yml
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_2.yml b/src/test/feature/ManagementTool/usage2case3/t_tpl_old_2.yml
deleted file mode 100644
index 1a2a459..0000000
--- a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_2.yml
+++ /dev/null
@@ -1,33 +0,0 @@
-Bucketnum: 6
-DBVersion: PostgreSQL 8.2.15 (Greenplum Database 4.2.0 build 1) (HAWQ 2.0.1.0 build
-  dev) on x86_64-apple-darwin14.5.0, compiled by GCC Apple LLVM version 6.1.0 (clang-602.0.53)
-  (based on LLVM 3.6.0svn) compiled on Jul  6 2016 10:22:33
-DFS_URL: hdfs://@PORT@
-Distribution_Policy: DISTRIBUTED BY (i)
-Encoding: UTF8
-FileFormat: Parquet
-Parquet_FileLocations:
-  Checksum: false
-  CompressionLevel: 0
-  CompressionType: null
-  EnableDictionary: false
-  Files:
-  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/1
-    size: 254
-  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/2
-    size: 250
-  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/3
-    size: 258
-  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/4
-    size: 250
-  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/5
-    size: 258
-  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/6
-    size: 270
-  PageSize: 1048576
-  RowGroupSize: 8388608
-Parquet_Schema:
-- name: i
-  type: int4
-TableName: public.t_2
-Version: 1.0.0

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/778ab999/src/test/feature/ManagementTool/usage2case3/t_tpl_old_3.yml
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_3.yml b/src/test/feature/ManagementTool/usage2case3/t_tpl_old_3.yml
deleted file mode 100644
index df6730e..0000000
--- a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_3.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-AO_FileLocations:
-  Blocksize: 32768
-  Checksum: false
-  CompressionLevel: 0
-  CompressionType: null
-  Files:
-  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/1
-    size: 1016
-AO_Schema:
-- name: i
-  type: int4
-Bucketnum: 6
-DBVersion: PostgreSQL 8.2.15 (Greenplum Database 4.2.0 build 1) (HAWQ 2.0.1.0 build
-  dev) on x86_64-apple-darwin14.5.0, compiled by GCC Apple LLVM version 6.1.0 (clang-602.0.53)
-  (based on LLVM 3.6.0svn) compiled on Jul  6 2016 10:22:33
-DFS_URL: hdfs://@PORT@
-Distribution_Policy: DISTRIBUTED RANDOMLY
-Encoding: UTF8
-FileFormat: AO
-TableName: public.t_3
-Version: 1.0.0

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/778ab999/src/test/feature/ManagementTool/usage2case3/t_tpl_old_4.yml
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_4.yml b/src/test/feature/ManagementTool/usage2case3/t_tpl_old_4.yml
deleted file mode 100644
index 8df2afc..0000000
--- a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_4.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-Bucketnum: 6
-DBVersion: PostgreSQL 8.2.15 (Greenplum Database 4.2.0 build 1) (HAWQ 2.0.1.0 build
-  dev) on x86_64-apple-darwin14.5.0, compiled by GCC Apple LLVM version 6.1.0 (clang-602.0.53)
-  (based on LLVM 3.6.0svn) compiled on Jul  6 2016 10:22:33
-DFS_URL: hdfs://@PORT@
-Distribution_Policy: DISTRIBUTED RANDOMLY
-Encoding: UTF8
-FileFormat: Parquet
-Parquet_FileLocations:
-  Checksum: false
-  CompressionLevel: 0
-  CompressionType: null
-  EnableDictionary: false
-  Files:
-  - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/1
-    size: 596
-  PageSize: 1048576
-  RowGroupSize: 8388608
-Parquet_Schema:
-- name: i
-  type: int4
-TableName: public.t_4
-Version: 1.0.0

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/778ab999/tools/bin/hawqregister
----------------------------------------------------------------------
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index 67a188d..b9b759e 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -18,7 +18,7 @@
 # under the License.
 
 # Usage1: hawq register [-h hostname] [-p port] [-U username] [-d database] [-f filepath]
[-e eof] <tablename>
-# Usage2: hawq register [-h hostname] [-p port] [-U username] [-d database] [-c config] [--force]
[--repair] <tablename>
+# Usage2: hawq register [-h hostname] [-p port] [-U username] [-d database] [-c config] [--force]
<tablename>
 
 import os
 import sys
@@ -54,7 +54,6 @@ def option_parser():
     parser.add_option('-e', '--eof', dest='filesize', type='int', default=None, help='eof
of the file to be registered')
     parser.add_option('-c', '--config', dest='yml_config', default='', help='configuration
file in YAML format')
     parser.add_option('-F', '--force', dest='force', action='store_true', default=False)
-    parser.add_option('-R', '--repair', dest='repair', action='store_true', default=False)
     return parser
 
 
@@ -340,7 +339,7 @@ class HawqRegister(object):
         self.accessor = GpRegisterAccessor(conn)
         self.utility_accessor = GpRegisterAccessor(utility_conn)
         self.failure_handler = failure_handler
-        self.mode = self._init_mode(options.force, options.repair)
+        self.mode = self._init_mode(options.force)
         self.srcfiles = []
         self.dstfiles = []
         self.files_same_path = []
@@ -351,20 +350,13 @@ class HawqRegister(object):
         self.eofuncompresseds_same_path = []
         self.segnos_same_path = []
 
-    def _init_mode(self, force, repair):
+    def _init_mode(self, force):
         def table_existed():
             return self.accessor.get_table_existed(self.dst_table_name)
 
         if self.yml:
             if force:
                 return 'force'
-            elif repair:
-                logger.info('--repair mode is not supported in current release of hawq register.')
-                sys.exit(0)
-                if not table_existed():
-                    logger.error('--repair mode asserts the table has been already created.')
-                    sys.exit(1)
-                return 'repair'
             else:
                 return 'usage2_table_not_exist'
         else:
@@ -583,9 +575,8 @@ class HawqRegister(object):
             self.filepath = self.files[0][:self.files[0].rfind('/')] if self.files else ''
             self._check_file_not_folder()
             self._check_database_encoding()
-            if self.mode != 'repair':
-                if not self._create_table() and self.mode != 'force':
-                    self.mode = 'usage2_table_exist'
+            if not self._create_table() and self.mode != 'force':
+                self.mode = 'usage2_table_exist'
         else:
             if self._is_folder(self.filepath) and self.filesize:
                 logger.error('-e option is only supported with single file case.')
@@ -628,23 +619,6 @@ class HawqRegister(object):
                 logger.error("Hawq aoseg metadata doesn't consistent with file numbers in
hdfs.")
                 self.failure_handler.rollback()
                 sys.exit(1)
-        if self.mode == 'repair':
-            if self.tabledir.strip('/') != self.filepath.strip('/'):
-                logger.error("In repair mode, file path from yaml file should be the same
with table's path.")
-                self.failure_handler.rollback()
-                sys.exit(1)
-            existed_info = {}
-            for k, fn in enumerate(existed_files):
-                existed_info[fn] = existed_sizes[k]
-            for k, fn in enumerate(self.files):
-                if fn not in existed_files:
-                    logger.error('Can not register in repair mode since giving non-existing
file: %s.' % fn)
-                    self.failure_handler.rollback()
-                    sys.exit(1)
-                if self.sizes[k] > existed_info[fn]:
-                    logger.error('Can not register in repair mode since giving larger file
size: %s' % self.sizes[k])
-                    self.failure_handler.rollback()
-                    sys.exit(1)
 
         if self.mode == 'usage2_table_exist':
             if self.tabledir.strip('/') == self.filepath.strip('/'):
@@ -689,14 +663,6 @@ class HawqRegister(object):
                     logger.error('In force mode, you should include existing table files
in yaml configuration file. Otherwise you should drop the previous table before register --force.')
                     self.failure_handler.rollback()
                     sys.exit(1)
-        elif self.mode == 'repair':
-            self.do_not_move = True
-            self.files_update, self.sizes_update = [fn for fn in self.files], [sz for sz
in self.sizes]
-            self.files_delete = []
-            for fn in existed_files:
-                if fn not in self.files:
-                    self.files_delete.append(fn)
-            self.files_append, self.sizes_append = [], []
 
         self._check_files_and_table_in_same_hdfs_cluster(self.filepath, self.tabledir)
 
@@ -963,7 +929,7 @@ class HawqRegister(object):
     def _prepare_register(self):
         if not self.do_not_move:
             self._set_move_files_in_hdfs()
-        if self.mode == 'force' or self.mode == 'repair':
+        if self.mode == 'force':
             self._set_modify_metadata('force')
         else:
             self._set_modify_metadata('insert')
@@ -972,10 +938,6 @@ class HawqRegister(object):
         if not self.do_not_move:
             self._move_files_in_hdfs()
         self._modify_metadata()
-        if self.mode == 'repair':
-            if self.files_delete:
-                self._delete_files_in_hdfs()
-                self._delete_metadata()
         logger.info('Hawq Register Succeed.')
 
 
@@ -1033,11 +995,10 @@ class HawqRegisterPartition(HawqRegister):
                 self._check_file_not_folder()
         if self.yml:
             self._check_database_encoding()
-            if self.mode != 'repair':
-                if not self._create_table() and self.mode != 'force':
-                    self.mode = 'usage2_table_exist'
-                    self._check_partitionby()
-                    self._check_partition_num()
+            if not self._create_table() and self.mode != 'force':
+                self.mode = 'usage2_table_exist'
+                self._check_partitionby()
+                self._check_partition_num()
         partitions = self._get_partition_info()
         self.queries = "set allow_system_table_mods='dml';"
         self.queries += "begin transaction;"
@@ -1124,10 +1085,10 @@ def test(options, args):
 if __name__ == '__main__':
     parser = option_parser()
     options, args = parser.parse_args()
-    if len(args) != 1 or (options.force and options.repair):
+    if len(args) != 1:
         parser.print_help(sys.stderr)
         sys.exit(1)
-    if (options.yml_config or options.force or options.repair) and options.filepath:
+    if (options.yml_config or options.force) and options.filepath:
         parser.print_help(sys.stderr)
         sys.exit(1)
     if local_ssh('hdfs'):


Mime
View raw message