hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r...@apache.org
Subject [2/2] incubator-hawq git commit: HAWQ-560. Set 'output.replace-datanode-on-failure' depend on datanodes number
Date Mon, 28 Mar 2016 03:07:34 GMT
HAWQ-560. Set 'output.replace-datanode-on-failure' depend on datanodes number


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/a4969d26
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/a4969d26
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/a4969d26

Branch: refs/heads/master
Commit: a4969d26c70bc4b675763ebb7287674bfb7fc408
Parents: 3db0b6f
Author: rlei <rlei@pivotal.io>
Authored: Fri Mar 18 18:01:46 2016 +0800
Committer: rlei <rlei@pivotal.io>
Committed: Mon Mar 28 11:07:03 2016 +0800

----------------------------------------------------------------------
 tools/bin/hawq_ctl             | 24 +++++++++++++--
 tools/bin/hawqpylib/hawqlib.py | 61 +++++++++++++++++++++++++++++++++++++
 2 files changed, 83 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a4969d26/tools/bin/hawq_ctl
----------------------------------------------------------------------
diff --git a/tools/bin/hawq_ctl b/tools/bin/hawq_ctl
index 0d2aa09..ce9fa7d 100755
--- a/tools/bin/hawq_ctl
+++ b/tools/bin/hawq_ctl
@@ -182,6 +182,18 @@ class HawqInit:
             logger.warn("Set standby host name failed")
         return result
 
+    def set_number_by_datanodes_num(self):
+        reuslt = 0
+        if self.hosts_count_number < 4:
+            # Set output.replace-datanode-on-failure to false if datanode number less than
4.
+            logger.info("Set output.replace-datanode-on-failure to false as %s datanode detected"
% self.hosts_count_number)
+            logger.info("If you actually have more than 4 datanodes, please set it to 'true'")
+            cmd = "hawq config -c 'output.replace-datanode-on-failure'  -v 'false' --skipvalidation
-q > /dev/null"
+            result = local_ssh(cmd, logger)
+            if result != 0:
+                logger.warn("Set output.replace-datanode-on-failure failed")
+        return result
+
     def set_total_vsegment_num(self):
         cmd = "%s; hawq config -c default_hash_table_bucket_number -v %s --skipvalidation
-q > /dev/null" % \
                (source_hawq_env, self.total_vseg_num)
@@ -337,7 +349,9 @@ class HawqInit:
         logger.info("%s segment hosts defined" % self.hosts_count_number)
         logger.info("Set default_hash_table_bucket_number as: %s" % self.total_vseg_num)
         check_return_code(self.set_total_vsegment_num())
+        logger.info("Set hawq_rm_nvseg_perquery_perseg_limit as: %s" % self.vseg_num_per_node)
         check_return_code(self.set_vsegment_num_per_node())
+        check_return_code(self.set_number_by_datanodes_num())
 
         master_cmd = self._get_master_init_cmd()
         logger.info("Start to init master node: '%s'" % self.master_host_name)
@@ -380,7 +394,9 @@ class HawqInit:
             logger.info("%s segment hosts defined" % self.hosts_count_number)
             logger.info("Set default_hash_table_bucket_number as: %s" % self.total_vseg_num)
             check_return_code(self.set_total_vsegment_num())
+            logger.info("Set hawq_rm_nvseg_perquery_perseg_limit as: %s" % self.vseg_num_per_node)
             check_return_code(self.set_vsegment_num_per_node())
+            check_return_code(self.set_number_by_datanodes_num())
             logger.info("Start to init master")
             cmd = self._get_master_init_cmd()
             check_return_code(local_ssh(cmd, logger, warning = True), logger, \
@@ -931,7 +947,8 @@ def get_args():
             quiet_stdout_logging()
         logger, log_filename = setup_hawq_tool_logging('hawq_%s' % opts.hawq_command,getLocalHostname(),getUserName(),
opts.log_dir)
         logger.info("Prepare to do 'hawq %s'" % opts.hawq_command)
-        logger.info("You can check log in %s" % log_filename)
+        logger.info("You can find log in:")
+        logger.info(log_filename)
     else:
         print COMMON_HELP
         sys.exit(1)
@@ -953,7 +970,8 @@ def get_args():
     if not opts.GPHOME:
         logger.error("Didn't get GPHOME value, exit")
         sys.exit()
-    logger.debug("GPHOME is %s" % opts.GPHOME)
+    logger.info("GPHOME is set to:")
+    logger.info(opts.GPHOME)
     global source_hawq_env
     source_hawq_env = "source %s/greenplum_path.sh" % opts.GPHOME 
 
@@ -963,6 +981,8 @@ def get_args():
         logger.error("'root' user is not allowed")
         sys.exit()
     logger.debug("Current user is '%s'" % opts.user)
+    logger.debug("Parsing config file:")
+    logger.debug("%s/etc/hawq-site.xml" % opts.GPHOME)
     hawqsite = HawqXMLParser(opts.GPHOME)
     hawqsite.get_all_values()
     hawq_dict = hawqsite.hawq_dict

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a4969d26/tools/bin/hawqpylib/hawqlib.py
----------------------------------------------------------------------
diff --git a/tools/bin/hawqpylib/hawqlib.py b/tools/bin/hawqpylib/hawqlib.py
index 11a9dfa..cc9e83c 100755
--- a/tools/bin/hawqpylib/hawqlib.py
+++ b/tools/bin/hawqpylib/hawqlib.py
@@ -22,7 +22,9 @@ import threading
 import Queue
 from xml.dom import minidom
 from xml.etree.ElementTree import ElementTree
+import shutil
 from gppylib.db import dbconn
+import re
 
 
 class HawqCommands(object):
@@ -355,6 +357,65 @@ def parse_hosts_file(GPHOME):
     return host_list
 
 
+def update_xml_property(xmlfile, property_name, property_value):
+    file_path, filename = os.path.split(xmlfile)
+    xmlfile_backup = os.path.join(file_path, '.bak.' + filename)
+    xmlfile_swap = os.path.join(file_path, '.swp.' + filename)
+    shutil.copyfile(xmlfile, xmlfile_backup)
+    with open(xmlfile_backup) as f:
+        f_tmp = open(xmlfile_swap, 'w')
+        while 1:
+            line = f.readline()
+
+            if not line:
+                break
+
+            m = re.match('\s*<name>%s' % property_name, line)
+            if m:
+                while 1:
+                    next_line = f.readline()
+                    m2 = re.match('\s*<value>', next_line)
+                    if m2:
+                        f_tmp.write(line)
+                        p = re.compile('\s*<value>(.*)</value>')
+                        p_value = p.match(next_line).group(1)
+                        next_line_new = re.sub(p_value, property_value, next_line)
+                        f_tmp.write(next_line_new)
+                        break
+            else:
+                f_tmp.write(line)
+        f_tmp.close()
+
+    shutil.move(xmlfile_swap, xmlfile)
+
+
+def append_xml_property(xmlfile, property_name, property_value):
+    file_path, filename = os.path.split(xmlfile)
+    xmlfile_backup = os.path.join(file_path, '.bak.' + filename)
+    xmlfile_swap = os.path.join(file_path, '.swp.' + filename)
+    shutil.copyfile(xmlfile, xmlfile_backup)
+    with open(xmlfile_backup) as f:
+        f_tmp = open(xmlfile_swap, 'w')
+        while 1:
+            line = f.readline()
+
+            if not line:
+                break
+
+            m = re.match('\s*</configuration>', line)
+            if m:
+                f_tmp.write('    <property>\n')
+                f_tmp.write('        <name>%s</name>\n' % property_name)
+                f_tmp.write('       <value>%s</value>\n' % property_value)
+                f_tmp.write('    </property>\n')
+                f_tmp.write('</configuration>\n')
+            else:
+                f_tmp.write(line)
+        f_tmp.close()
+
+    shutil.move(xmlfile_swap, xmlfile)
+
+
 def remove_property_xml(property_name, org_config_file):
     tree = ElementTree()
     tree.parse(org_config_file)


Mime
View raw message