hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From odiache...@apache.org
Subject [33/38] incubator-hawq git commit: HAWQ-560. Set 'output.replace-datanode-on-failure' depend on datanodes number to hdfs-client.xml
Date Thu, 31 Mar 2016 00:24:26 GMT
HAWQ-560. Set 'output.replace-datanode-on-failure' depend on datanodes number to hdfs-client.xml


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2becc74d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2becc74d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2becc74d

Branch: refs/heads/HAWQ-546
Commit: 2becc74dc6a4612732c253c89a14aadcbe86a8a2
Parents: 312de8f
Author: rlei <rlei@pivotal.io>
Authored: Sun Mar 27 16:47:16 2016 +0800
Committer: Oleksandr Diachenko <odiachenko@pivotal.io>
Committed: Wed Mar 30 17:23:33 2016 -0700

----------------------------------------------------------------------
 tools/bin/hawq_ctl             | 60 +++++++++++++++++++++++++++++--------
 tools/bin/hawqconfig           | 60 +++----------------------------------
 tools/bin/hawqpylib/hawqlib.py | 19 ++++++++++--
 3 files changed, 68 insertions(+), 71 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2becc74d/tools/bin/hawq_ctl
----------------------------------------------------------------------
diff --git a/tools/bin/hawq_ctl b/tools/bin/hawq_ctl
index ce9fa7d..c567b09 100755
--- a/tools/bin/hawq_ctl
+++ b/tools/bin/hawq_ctl
@@ -182,16 +182,50 @@ class HawqInit:
             logger.warn("Set standby host name failed")
         return result
 
-    def set_number_by_datanodes_num(self):
-        reuslt = 0
-        if self.hosts_count_number < 4:
-            # Set output.replace-datanode-on-failure to false if datanode number less than
4.
-            logger.info("Set output.replace-datanode-on-failure to false as %s datanode detected"
% self.hosts_count_number)
-            logger.info("If you actually have more than 4 datanodes, please set it to 'true'")
-            cmd = "hawq config -c 'output.replace-datanode-on-failure'  -v 'false' --skipvalidation
-q > /dev/null"
-            result = local_ssh(cmd, logger)
-            if result != 0:
-                logger.warn("Set output.replace-datanode-on-failure failed")
+    def sync_hdfs_client(self):
+        sync_host_str = ""
+        for node in self.host_list:
+            sync_host_str += " -h %s" % node
+
+        if 'hawq_standby_address_host' in self.hawq_dict and self.standby_host_name.lower()
not in ('', 'none'):
+            sync_host_str += " -h %s" % self.standby_host_name
+
+        result = local_ssh("hawq scp %s %s/etc/hdfs-client.xml =:%s/etc/" % (sync_host_str,
self.GPHOME, self.GPHOME))
+        if result != 0:
+            logger.error("Sync hdfs-client.xml failed.")
+            sys.exit(1)
+
+    def set_replace_datanode_on_failure(self):
+        xml_file = "%s/etc/hdfs-client.xml" % self.GPHOME
+        property_name = 'output.replace-datanode-on-failure'
+        pexist, pname, pvalue = check_property_exist_xml(xml_file, property_name)
+        datanodes_threshold = 4
+
+        if self.hosts_count_number < datanodes_threshold:
+            property_value = 'false'
+        else:
+            property_value = 'true'
+
+        if pvalue != property_value:
+            logger.info("Set output.replace-datanode-on-failure to %s" % property_value)
+            if pexist:
+                logger.debug("Update output.replace-datanode-on-failure to %s" % property_value)
+                update_xml_property(xml_file, property_name, property_value)
+            else:
+                logger.debug("Add output.replace-datanode-on-failure as %s" % property_value)
+                append_xml_property(xml_file, property_name, property_value)
+
+            self.sync_hdfs_client()
+
+            # Check hdfs-client.xml again to validate changes
+            pexist, pname, pvalue = check_property_exist_xml(xml_file, property_name)
+            if pvalue == property_value:
+                result = 0
+            else:
+                result = 1
+        else:
+            result = 0
+
         return result
 
     def set_total_vsegment_num(self):
@@ -270,7 +304,7 @@ class HawqInit:
                 logger.error("Failed to connect to database, this script can only be run
when the database is up")
                 cmd = "%s; hawq stop master -a -M fast" % source_hawq_env
                 check_return_code(local_ssh(cmd, logger), logger, "Stop hawq master failed,
exit")
-            remove_property_xml("hawq_standby_address_host", "%s/etc/hawq-site.xml" % self.GPHOME)
+            remove_property_xml("hawq_standby_address_host", "%s/etc/hawq-site.xml" % self.GPHOME,
self.quiet)
             host_list = parse_hosts_file(self.GPHOME)
             sync_hawq_site(self.GPHOME, host_list)
             gpsyncmaster_pid = gp.getSyncmasterPID(running_standby_host, self.master_data_directory)
@@ -351,7 +385,7 @@ class HawqInit:
         check_return_code(self.set_total_vsegment_num())
         logger.info("Set hawq_rm_nvseg_perquery_perseg_limit as: %s" % self.vseg_num_per_node)
         check_return_code(self.set_vsegment_num_per_node())
-        check_return_code(self.set_number_by_datanodes_num())
+        check_return_code(self.set_replace_datanode_on_failure())
 
         master_cmd = self._get_master_init_cmd()
         logger.info("Start to init master node: '%s'" % self.master_host_name)
@@ -396,7 +430,7 @@ class HawqInit:
             check_return_code(self.set_total_vsegment_num())
             logger.info("Set hawq_rm_nvseg_perquery_perseg_limit as: %s" % self.vseg_num_per_node)
             check_return_code(self.set_vsegment_num_per_node())
-            check_return_code(self.set_number_by_datanodes_num())
+            check_return_code(self.set_replace_datanode_on_failure())
             logger.info("Start to init master")
             cmd = self._get_master_init_cmd()
             check_return_code(local_ssh(cmd, logger, warning = True), logger, \

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2becc74d/tools/bin/hawqconfig
----------------------------------------------------------------------
diff --git a/tools/bin/hawqconfig b/tools/bin/hawqconfig
index fda8782..6618bc3 100755
--- a/tools/bin/hawqconfig
+++ b/tools/bin/hawqconfig
@@ -18,7 +18,7 @@
 try:
     import os, sys, re
     from optparse import Option, OptionParser
-    from hawqpylib.hawqlib import HawqXMLParser, parse_hosts_file, remove_property_xml, local_ssh
+    from hawqpylib.hawqlib import HawqXMLParser, parse_hosts_file, remove_property_xml, update_xml_property,
append_xml_property, local_ssh
     from gppylib.commands.unix import getLocalHostname, getUserName
     from gppylib.db import dbconn
     from pg import DatabaseError
@@ -166,64 +166,12 @@ def show_property(hawq_site, property_name):
     return None
 
 
-def update_line(xmlfile, property_name, property_value):
-    with open(xmlfile) as f:
-        f_tmp = open(xmlfile + '.backup', 'w')
-        while 1:
-            line = f.readline()
-
-            if not line:
-                break
-
-            m = re.match('\s*<name>%s' % property_name, line)
-            if m:
-                while 1:
-                    next_line = f.readline()
-                    m2 = re.match('\s*<value>', next_line)
-                    if m2:
-                        f_tmp.write(line)
-                        p = re.compile('\s*<value>(.*)</value>')
-                        p_value = p.match(next_line).group(1)
-                        next_line_new = re.sub(p_value, property_value, next_line)
-                        f_tmp.write(next_line_new)
-                        break
-            else:
-                f_tmp.write(line)
-        f_tmp.close()
-    return None
-
-
-def add_property(xmlfile, property_name, property_value):
-    with open(xmlfile) as f:
-        f_tmp = open(xmlfile + '.backup', 'w')
-        while 1:
-            line = f.readline()
-
-            if not line:
-                break
-
-            m = re.match('\s*</configuration>', line)
-            if m:
-                f_tmp.write('    <property>\n')
-                f_tmp.write('        <name>%s</name>\n' % property_name)
-                f_tmp.write('       <value>%s</value>\n' % property_value)
-                f_tmp.write('    </property>\n')
-                f_tmp.write('</configuration>\n')
-            else:
-                f_tmp.write(line)
-        f_tmp.close()
-    return None
-
-
 def update_hawq_site(org_config_file, hawq_site, property_name, property_value):
-    os.system('cp %s %s' % (org_config_file, org_config_file + '.org'))
     result = check_property(hawq_site, property_name, property_value)
     if result:
-        update_line(org_config_file, property_name, property_value)
-        os.system('mv %s %s' % (org_config_file + '.backup', org_config_file))
+        update_xml_property(org_config_file, property_name, property_value)
     else:
-        add_property(org_config_file, property_name, property_value)
-        os.system('mv %s %s' % (org_config_file + '.backup', org_config_file))
+        append_xml_property(org_config_file, property_name, property_value)
 
 
 def sync_hawq_site(config_dir, host_list):
@@ -283,7 +231,7 @@ if __name__ == '__main__':
         if options.remove in except_list:
             print "Remove %s is not allowed" % options.remove
             sys.exit(1)
-        remove_property_xml(options.remove, org_config_file)
+        remove_property_xml(options.remove, org_config_file, options.quiet)
         sync_hawq_site(GPHOME, host_list)
     else:
         print "Please input correct options"

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2becc74d/tools/bin/hawqpylib/hawqlib.py
----------------------------------------------------------------------
diff --git a/tools/bin/hawqpylib/hawqlib.py b/tools/bin/hawqpylib/hawqlib.py
index cc9e83c..85354b4 100755
--- a/tools/bin/hawqpylib/hawqlib.py
+++ b/tools/bin/hawqpylib/hawqlib.py
@@ -115,6 +115,20 @@ class threads_with_return(object):
         return self.return_values
 
 
+def check_property_exist_xml(xml_file, property_name):
+    property_exist = False
+    property_value = ''
+    with open(xml_file) as f:
+        xmldoc = minidom.parse(f)
+    for node in xmldoc.getElementsByTagName('property'):
+        name, value = (node.getElementsByTagName('name')[0].childNodes[0].data,
+                       node.getElementsByTagName('value')[0].childNodes[0].data)
+        if name == property_name:
+            property_exist = True
+            property_value = value
+    return property_exist, property_name, property_value
+
+
 class HawqXMLParser:
     def __init__(self, GPHOME):
         self.GPHOME = GPHOME
@@ -416,14 +430,15 @@ def append_xml_property(xmlfile, property_name, property_value):
     shutil.move(xmlfile_swap, xmlfile)
 
 
-def remove_property_xml(property_name, org_config_file):
+def remove_property_xml(property_name, org_config_file, quiet = False):
     tree = ElementTree()
     tree.parse(org_config_file)
     root = tree.getroot()
     for child in root:
         for subet in child:
             if subet.text == property_name:
-                print "Remove property %s." % subet.text
+                if not quiet:
+                    print "Remove property %s." % subet.text
                 root.remove(child)
     tree.write(org_config_file, encoding="utf-8")
 


Mime
View raw message