hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bhuvnesh2...@apache.org
Subject incubator-hawq git commit: [#116576425] - Add option to ignore hosts on which SSH test fails to skip syncing configuration files
Date Fri, 01 Apr 2016 06:19:11 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/HAWQ-617 [created] 7e8331a7e


[#116576425] - Add option to ignore hosts on which SSH test fails to skip syncing configuration
files


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/7e8331a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/7e8331a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/7e8331a7

Branch: refs/heads/HAWQ-617
Commit: 7e8331a7e314c9d5b7f2ec65078fa069271a284d
Parents: 071f276
Author: Bhuvnesh Chaudhary <bchaudhary@pivotal.io>
Authored: Thu Mar 31 23:14:54 2016 -0700
Committer: Bhuvnesh Chaudhary <bchaudhary@pivotal.io>
Committed: Thu Mar 31 23:14:54 2016 -0700

----------------------------------------------------------------------
 .gitignore                          | 55 +-------------------------------
 tools/bin/gppylib/util/ssh_utils.py | 23 +++++++++++++
 tools/bin/gpscp                     | 36 +++++++++++++--------
 tools/bin/hawq_ctl                  | 13 +++++---
 tools/bin/hawqconfig                | 12 ++++---
 tools/bin/hawqpylib/HAWQ_HELP.py    |  1 +
 6 files changed, 64 insertions(+), 76 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e8331a7/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 8fd9023..485dee6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,54 +1 @@
-# Object files
-*.o
-*.ko
-*.obj
-*.elf
-.deps
-
-# Precompiled Headers
-*.gch
-*.pch
-
-# Libraries
-*.lib
-*.a
-*.la
-*.lo
-
-# Shared objects (inc. Windows DLLs)
-*.dll
-*.so
-*.so.*
-*.dylib
-
-# Executables
-*.exe
-*.app
-*.i*86
-*.x86_64
-*.hex
-
-# Debug files
-*.dSYM/
-*.o
-*.ko
-*.obj
-*.elf
-objfiles.txt
-
-# Eclipse Project
-.project
-.pydevproject
-.cproject
-.settings
-
-# Generated files
-BUILD_NUMBER
-GNUmakefile
-config.log
-config.status
-VERSION
-env.sh
-ext/
-plr.tgz
-autom4te.cache/
+.idea

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e8331a7/tools/bin/gppylib/util/ssh_utils.py
----------------------------------------------------------------------
diff --git a/tools/bin/gppylib/util/ssh_utils.py b/tools/bin/gppylib/util/ssh_utils.py
index 3194e11..853c0f5 100644
--- a/tools/bin/gppylib/util/ssh_utils.py
+++ b/tools/bin/gppylib/util/ssh_utils.py
@@ -160,6 +160,29 @@ class HostList():
         
         return self.list
 
+    def removeBadHosts(self):
+        ''' Update list of host to include only the host on which SSH was successful'''
+
+        pool = WorkerPool()
+
+        for h in self.list:
+            cmd = Echo('ssh test', '', ctxt=REMOTE, remoteHost=h)
+            pool.addCommand(cmd)
+
+        pool.join()
+        pool.haltWork()
+
+        bad_hosts = []
+        working_hosts = []
+        for cmd in pool.getCompletedItems():
+            if not cmd.get_results().wasSuccessful():
+                bad_hosts.append(cmd.remoteHost)
+            else:
+                working_hosts.append(cmd.remoteHost)
+
+        self.list = working_hosts[:]
+        return bad_hosts
+
 # Session is a command session, derived from a base class cmd.Cmd
 class Session(cmd.Cmd):
     '''Implements a list of open ssh sessions ready to execute commands'''

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e8331a7/tools/bin/gpscp
----------------------------------------------------------------------
diff --git a/tools/bin/gpscp b/tools/bin/gpscp
index d00f15d..c02d677 100755
--- a/tools/bin/gpscp
+++ b/tools/bin/gpscp
@@ -64,6 +64,7 @@ class Global:
     opt['-f'] = None
     opt['-J'] = '=:'
     opt['-r'] = False
+    opt['--ignore-bad-hosts'] = False
     filePath = []
 
 GV = Global()
@@ -86,18 +87,19 @@ def print_version():
 #############
 def parseCommandLine():
     try:
-        (options, args) = getopt.getopt(sys.argv[1:], '?vrJ:p:u:h:f:', ['version'])
+        (options, args) = getopt.getopt(sys.argv[1:], '?vrJ:p:u:h:f:', ['version', 'ignore-bad-hosts'])
     except Exception, e:
         usage('[ERROR] ' + str(e))
 
     for (switch, val) in options:
-	if (switch == '-?'): 	      usage(0)
-	elif (switch == '-v'):        GV.opt[switch] = True
-	elif (switch == '-f'):        GV.opt[switch] = val
-	elif (switch == '-h'):        GV.opt[switch].append(val)
-        elif (switch == '-J'):        GV.opt[switch] = val + ':'
-        elif (switch == '-r'):        GV.opt[switch] = True
-        elif (switch == '--version'): print_version()
+        if (switch == '-?'): 	                              usage(0)
+        elif (switch == '-v'):                              GV.opt[switch] = True
+        elif (switch == '-f'):                              GV.opt[switch] = val
+        elif (switch == '-h'):                              GV.opt[switch].append(val)
+        elif (switch == '-J'):                              GV.opt[switch] = val + ':'
+        elif (switch == '-r'):                              GV.opt[switch] = True
+        elif (switch == '--version'):                       print_version()
+        elif (switch == '--ignore-bad-hosts'):          GV.opt[switch] = True
 
     hf = (len(GV.opt['-h']) and 1 or 0) + (GV.opt['-f'] and 1 or 0)
     if hf != 1:
@@ -131,15 +133,23 @@ try:
     if GV.opt['-f']:
         hostlist.parseFile(GV.opt['-f'])
 
-    try:
-        hostlist.checkSSH()
-    except ssh_utils.SSHError, e:
-        sys.exit('[ERROR] ' + str(e))
+    if GV.opt['--ignore-bad-hosts']:
+        original_hostlist = hostlist.list
+        bad_hosts = hostlist.removeBadHosts()
+        if len(bad_hosts) == len(original_hostlist):
+            sys.exit('[ERROR]: Unable to SSH to any of the hosts {0}'.format(original_hostlist))
+        if len(bad_hosts) > 0:
+            print "[WARN]: Skipping syncing configuration file on hosts {0}, as ssh test
failed".format(bad_hosts)
+    else:
+        try:
+            hostlist.checkSSH()
+        except ssh_utils.SSHError, e:
+            sys.exit('[ERROR] ' + str(e))
 
     GV.opt['-h'] = hostlist.filterMultiHomedHosts()
+
     if len(GV.opt['-h']) == 0:
         usage('Error: missing hosts in -h and/or -f arguments')
-
     scp = 'scp -o "BatchMode yes" -o "StrictHostKeyChecking no"'
     if GV.opt['-r']:  scp += ' -r'
     

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e8331a7/tools/bin/hawq_ctl
----------------------------------------------------------------------
diff --git a/tools/bin/hawq_ctl b/tools/bin/hawq_ctl
index 7937ac6..c43dfd4 100755
--- a/tools/bin/hawq_ctl
+++ b/tools/bin/hawq_ctl
@@ -1145,7 +1145,7 @@ def hawq_activate_standby(opts, hawq_dict):
             logger.error("Stop master failed, try again with immediate mode")
             cmd = "%s; hawq stop master -a -M immediate -q;" % source_hawq_env
             return_result = remote_ssh(cmd, old_master_host_name, '')
-            if return_resutl != 0:
+            if return_result != 0:
                 logger.error("Stop master failed, abort")
                 logger.error("Please manually bring hawq cluster down, then do activate standby
again")
                 sys.exit(1)
@@ -1175,13 +1175,13 @@ def hawq_activate_standby(opts, hawq_dict):
     
     # Set current standby host name as the new master host name in configuration.
     logger.info("Update master host name in hawq-site.xml")
-    cmd = "%s; hawq config -c hawq_master_address_host -v %s --skipvalidation -q" % \
-           (source_hawq_env, hawq_dict['hawq_standby_address_host'])
+    ignore_bad_hosts = '--ignore-bad-hosts' if opts.ignore_bad_hosts else ''
+    cmd = "%s; hawq config -c hawq_master_address_host -v %s --skipvalidation -q %s" % (source_hawq_env,
hawq_dict['hawq_standby_address_host'], ignore_bad_hosts)
     check_return_code(remote_ssh(cmd, old_standby_host_name, ''), logger, "Set hawq_master_address_host
failed")
 
     # Remove the old standby host configuration from hawq-site.xml.
     logger.info("Remove current standby from hawq-site.xml")
-    cmd = "%s; hawq config -r hawq_standby_address_host --skipvalidation -q" % source_hawq_env
+    cmd = "%s; hawq config -r hawq_standby_address_host --skipvalidation -q %s" % (source_hawq_env,
ignore_bad_hosts)
     check_return_code(remote_ssh(cmd, old_standby_host_name, ''), logger, "Remove hawq_standby_address_host
from configuration failed")
 
     cmd = '''echo "gp_persistent_repair_global_sequence = true" >> %s/%s''' % (hawq_dict['hawq_master_directory'],
'postgresql.conf')
@@ -1279,6 +1279,10 @@ def create_parser():
     parser.add_option('-n', '--no-update', action='store_true',
                       dest='no_update', default=False,
                       help='Do not update system catalog tables.')
+    parser.add_option('-i', '--ignore-bad-hosts',
+                      dest='ignore_bad_hosts', action='store_true',
+                      default=False,
+                      help='Skips syncing configuration files on hosts on which SSH fails')
     parser.add_option("--bucket_number",
                       type="int",
                       dest="default_hash_table_bucket_number",
@@ -1319,6 +1323,7 @@ def create_parser():
                       dest="shared_buffers",
                       default="128000kB",
                       help="Sets the shared_buffers for formatting hawq database")
+
     (options, args) = parser.parse_args()
     if len(args) == 0:
         parser.print_help()

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e8331a7/tools/bin/hawqconfig
----------------------------------------------------------------------
diff --git a/tools/bin/hawqconfig b/tools/bin/hawqconfig
index 6618bc3..4a1e3c7 100755
--- a/tools/bin/hawqconfig
+++ b/tools/bin/hawqconfig
@@ -39,6 +39,8 @@ def parseargs():
     parser.add_option('-l', '--list', action='store_true',
                       help="List all HAWQ Properties.")
     parser.add_option('--skipvalidation', action='store_true', default=False)
+    parser.add_option('--ignore-bad-hosts', action='store_true', default=False,
+                      help='Skips copying configuration files on host on which SSH fails')
     parser.add_option('-q', '--quiet', action='store_true', default=False)
     parser.add_option("-v", "--value",
                       dest="property_value",
@@ -174,12 +176,11 @@ def update_hawq_site(org_config_file, hawq_site, property_name, property_value):
         append_xml_property(org_config_file, property_name, property_value)
 
 
-def sync_hawq_site(config_dir, host_list):
+def sync_hawq_site(config_dir, host_list, ignore_bad_hosts):
     sync_host_str = ""
     for node in host_list:
         sync_host_str += " -h %s" % node
-
-    result = local_ssh("hawq scp %s %s/etc/hawq-site.xml =:%s/etc/" % (sync_host_str, config_dir,
config_dir))
+    result = local_ssh("hawq scp %s %s %s/etc/hawq-site.xml =:%s/etc/" % (sync_host_str,
ignore_bad_hosts, config_dir, config_dir))
     if result != 0:
         sys.exit("sync hawq-site.xml failed.")
 
@@ -196,6 +197,7 @@ if __name__ == '__main__':
     segment_list = parse_hosts_file(GPHOME)
     master_host = hawq_site.hawq_dict['hawq_master_address_host']
     host_list = segment_list + [master_host]
+    ignore_bad_hosts = '--ignore-bad-hosts' if options.ignore_bad_hosts else ''
     if 'hawq_standby_address_host' in hawq_site.hawq_dict:
         standby_host = hawq_site.hawq_dict['hawq_standby_address_host']
         if standby_host not in ('None', 'none', ''):
@@ -212,7 +214,7 @@ if __name__ == '__main__':
             check_property_valid(hawq_site, options.change)
 
         update_hawq_site(org_config_file, hawq_site, options.change, options.property_value)
-        sync_hawq_site(GPHOME, host_list)
+        sync_hawq_site(GPHOME, host_list, ignore_bad_hosts)
 
         if not options.quiet:
             latest_hawq_site = HawqXMLParser(GPHOME)
@@ -232,7 +234,7 @@ if __name__ == '__main__':
             print "Remove %s is not allowed" % options.remove
             sys.exit(1)
         remove_property_xml(options.remove, org_config_file, options.quiet)
-        sync_hawq_site(GPHOME, host_list)
+        sync_hawq_site(GPHOME, host_list, ignore_bad_hosts)
     else:
         print "Please input correct options"
         sys.exit(1)

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e8331a7/tools/bin/hawqpylib/HAWQ_HELP.py
----------------------------------------------------------------------
diff --git a/tools/bin/hawqpylib/HAWQ_HELP.py b/tools/bin/hawqpylib/HAWQ_HELP.py
index 2c184ac..e0d901f 100755
--- a/tools/bin/hawqpylib/HAWQ_HELP.py
+++ b/tools/bin/hawqpylib/HAWQ_HELP.py
@@ -164,6 +164,7 @@ The "options" are:
    -v --verbose        Displays detailed status.
    -r --remove         HAWQ GUC name to be removed.
    --skipvalidation    Skip the system validation checks.
+   --ignore-bad-hosts  Skips copying configuration files on host on which SSH fails
 
 See 'hawq --help' for more information on other commands.
 """


Mime
View raw message