hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r...@apache.org
Subject [1/8] incubator-hawq git commit: HAWQ-121. Remove legacy command line tools.
Date Thu, 05 Nov 2015 03:09:57 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 4e392375e -> 9932786b8


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9932786b/tools/sbin/gpupgrademirror.py
----------------------------------------------------------------------
diff --git a/tools/sbin/gpupgrademirror.py b/tools/sbin/gpupgrademirror.py
deleted file mode 100755
index 0a8818d..0000000
--- a/tools/sbin/gpupgrademirror.py
+++ /dev/null
@@ -1,3968 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (c) Greenplum Inc 2010. All Rights Reserved.
-#
-
-from gppylib.commands.base import *
-from optparse import Option, OptionGroup, OptionParser, OptionValueError, SUPPRESS_USAGE
-from time import strftime, sleep
-import bisect
-import copy
-import datetime
-import os
-import pprint
-import signal
-import sys
-import tempfile
-import threading
-import traceback
-
-
-
-sys.path.append(os.path.join(os.path.dirname(__file__), "../bin"))
-sys.path.append(os.path.join(os.path.dirname(__file__), "../bin/lib"))
-
-
-try:
-    from pysync import *
-    from gppylib.commands.unix import *
-    from gppylib.commands.gp import *
-    from gppylib.commands.pg import PgControlData
-    from gppylib.gparray import GpArray, get_host_interface
-    from gppylib.gpparseopts import OptParser, OptChecker
-    from gppylib.gplog import *
-    from gppylib.db import dbconn
-    from gppylib.db import catalog
-    from gppylib.userinput import *
-    from pygresql.pgdb import DatabaseError
-    from pygresql import pg
-    from gppylib.gpcoverage import GpCoverage
-except ImportError, e:
-    sys.exit('ERROR: Cannot import modules.  Please check that you have sourced greenplum_path.sh.  Detail: ' + str(e))
-
-#
-# Constants
-#
-
-EXECNAME = os.path.split(__file__)[-1]
-FULL_EXECNAME = os.path.abspath( __file__ )
-
-DESCRIPTION = ("""Upgrades 3.3.x.x mirrors to 4.0 mirrors.""")
-
-_help  = [""" TODO add help """]
-
-_usage = """ TODO add usage """
-
-TEN_MEG = 10485760
-ONE_GIG = 128  * TEN_MEG
-TEN_GIG = 1024 * TEN_MEG
-
-INFO_DIR = "gpupgrademirrorinfodir"
-
-PID_FILE = "gpupgrademirror.pid"
-
-GPUPGRADEMIRROR_TEMP_DIR = "gpupgrademirrortempdir"
-
-GPMIGRATOR_TEMP_DIR = "gpmigrator"
-
-
-#-----------------------------------------------------------                                                                             
-def Shutdown(env, utility=False):
-    """
-      This function will attempt to stop a database system.
-    """
-
-    try:
-        if utility:
-           cmd = 'gpstop -a -m -f'
-        else:
-           cmd = 'gpstop -a -f'
-
-        logger.debug("Stopping cluster with env = %s" % str(env))
-        pid = subprocess.Popen( args = cmd
-                              , preexec_fn = os.setpgrp
-                              , env = env
-                              , shell = True
-                              , stdout = subprocess.PIPE
-                              , stderr = subprocess.PIPE
-                              , close_fds = True
-                              )
-                                                                            
-        retcode = None
-        
-        """ KAS note to self. Could the output be so big that it uses up too much memory? """
-        stdoutdata, stderrdata = pid.communicate()
-        logger.debug("gpstop for master output = \n" + stdoutdata)
-        if len(stderrdata) > 0:
-           logger.debug("gpstop for master error  = " + stderrdata)
-        retcode = pid.returncode
-        
-        if retcode < 0:
-            raise Exception("gpstop terminated by signal.");
-
-        if retcode == 1:
-            logger.warn('***************************************')
-            logger.warn('Warnings generated stopping cluster')
-            logger.warn('***************************************')
-        if retcode > 1:
-            logger.fatal('***************************************')
-            logger.fatal('gpstop failed with error code %d' % retcode)
-            logger.fatal('***************************************')
-            raise Exception('gpstop failed')
-
-    except OSError, e:
-        logger.fatal(str(e))
-        raise Exception('gpstop failed')
-
-
-#-----------------------------------------------------------                                                                             
-def Startup(env, utility=False):
-    '''                                                                                                                                   
-    Starts up the specified database                                                                                                      
-    '''
-
-    try:
-        if utility:
-           cmd = 'gpstart -a -m'
-           env['GPSTART_INTERNAL_MASTER_ONLY'] = '1'
-        else:
-           cmd = 'gpstart -a'
-
-        logger.debug("Starting cluster where cmd = %s,  with env = %s" % (str(cmd), str(env)))
-        pid = subprocess.Popen( cmd
-                              , preexec_fn = os.setpgrp
-                              , env = env
-                              , shell = True
-                              , stdout = subprocess.PIPE
-                              , stderr = subprocess.PIPE
-                              , close_fds = True
-                              )
-                                                                            
-        retcode = None
-        
-        """
-        while retcode == None:
-            retcode = pid.wait()
-        """
-        stdoutdata, stderrdata = pid.communicate()
-        logger.debug("gpstart for master output = \n" + stdoutdata)
-        if len(stderrdata) > 0:
-           logger.debug("gpstart for master error  = " + stderrdata)
-        retcode = pid.returncode
-        
-        if retcode < 0:
-            raise Exception("Startup terminated by signal");
-
-        if retcode == 1:
-            logger.warn('***************************************')
-            logger.warn('Warnings generated starting cluster')
-            logger.warn('***************************************')
-        if retcode > 1:
-            logger.fatal('***************************************')
-            logger.fatal('Startup failed with error code %d' % retcode)
-            logger.fatal('***************************************')
-            raise Exception('Startup failed')
-
-    except OSError, e:
-        logger.fatal(str(e))
-        raise Exception('Startup failed')
-
-
-#-------------------------------------------------------------------------------
-def StartupPrimaries(gparray):
-     """ Startup all the primary segments in the cluster """
-
-     logging.debug("Attempting to start primaries.")
-
-     """ get all the primary segments """
-     allSegDbList = gparray.getSegDbList()
-
-     if len(allSegDbList) / 2 > 64:
-        maxThreads = 64
-     else:
-        maxThreads = len(allSegDbList) / 2
-     primarySegCmdList = []
-
-     try:
-         pool = WorkerPool(numWorkers = maxThreads);
-
-         """ get all the primary segments """
-         allSegDbList = gparray.getSegDbList()
-         for db in allSegDbList:
-             if db.isSegmentPrimary() == True:
-                segStartCmd = SegmentStart( name = "start command primary segment %s" + str(db.getSegmentDbId())
-                                          , gpdb = db
-                                          , numContentsInCluster = 123    # unused here, so just pass any number
-                                          , mirroringMode = "mirrorless"
-                                          , utilityMode = True
-                                          , ctxt = REMOTE
-                                          , remoteHost = db.getSegmentAddress()
-                                          , noWait = False
-                                          , timeout = 60
-                                          )
-                logging.debug("attempting to start primary with: " + str(segStartCmd))
-                pool.addCommand(segStartCmd)
-
-         # Wait for the segments to finish
-         try:
-             pool.join()
-         except:
-             pool.haltWork()
-             pool.joinWorkers()
-            
-         failure = False
-         results = []
-         for cmd in pool.getCompletedItems():
-             r = cmd.get_results()
-             if not cmd.was_successful():
-                 logging.error("Unable to start segment: " + str(r))
-                 failure = True
-
-         if failure:
-             raise Exception("There was an issue starting the primary segments.")
-     except Exception, e:
-         ShutdownPrimaries(gparray)
-         raise Exception("Could not start primary segments: " + str(e))
-
-
-#-------------------------------------------------------------------------------                                                                     
-def ShutdownPrimaries(gparray):
-     """ Shutdown all the primary segments in the cluster """
-
-     logging.debug("Attempting to stop primaries.")
-
-     """ get all the primary segments """
-     allSegDbList = gparray.getSegDbList()
-
-     if len(allSegDbList) / 2 > 64:
-        maxThreads = 64
-     else:
-        maxThreads = len(allSegDbList) / 2
-     primarySegCmdList = []
-
-     try:
-         pool = WorkerPool(numWorkers = maxThreads);
-
-         """ get all the primary segments """
-         allSegDbList = gparray.getSegDbList()
-         for db in allSegDbList:
-             if db.isSegmentPrimary() == True:
-                segStopCmd = SegmentStop( name = "stop command primary segment %s" + str(db.getSegmentDbId())
-                                        , dataDir = db.getSegmentDataDirectory()
-                                        , mode = "fast"
-                                        , nowait = False
-                                        , ctxt = REMOTE
-                                        , remoteHost = db.getSegmentAddress()
-                                        , timeout = 60
-                                        )
-                logging.debug("attempting to stop primary with: " + str(segStopCmd))
-                pool.addCommand(segStopCmd)
-
-         # Wait for the segments to finish                                                                                                                
-         try:
-             pool.join()
-         except:
-             pool.haltWork()
-             pool.joinWorkers()
-
-         failure = False
-         results = []
-         for cmd in pool.getCompletedItems():
-             r = cmd.get_results()
-             if not cmd.was_successful():
-                 logging.error("Unable to stop segment: " + str(r))
-                 failure = True
-
-         if failure:
-             raise Exception("There was an issue stopping the primary segments.")
-     except Exception, e:
-         raise Exception("Could not stop primary segments: " + str(e))
-
-
-
-#-------------------------------------------------------------------------------
-def SetupEnv(gphome):
-        '''                                                                                                                                   
-        Sets up environment variables for Greenplum Administration                                                                            
-        '''
-        
-        if len(gphome) > 2 and gphome[0] == '"' and gphome[len(gphome) - 1] == '"':
-           gphome = gphome[1:len(gphome) - 1]
-        
-        thePath = '/usr/kerberos/bin:/usr/sfw/bin:/opt/sfw/bin'
-        thePath += ':/usr/local/bin:/bin:/usr/bin:/sbin:/usr/sbin:/usr/ucb'
-        thePath += ':/sw/bin'
-        user = os.environ.get('USER') or os.environ.get('LOGNAME')
-        home  = os.environ.get('HOME')
-        lpath = os.environ.get('LD_LIBRARY_PATH')
-        dypath = os.environ.get('DYLD_LIBRARY_PATH')
-
-        # Add $GPHOME/bin to the path for this environment                                                                                    
-        path = '%s/bin:%s/ext/python/bin:%s' % (gphome, gphome, thePath)
-
-        if lpath:
-            lpath = '%s/lib:%s/ext/python/lib:%s' % (gphome, gphome, lpath)
-        else:
-            lpath = '%s/lib:%s/ext/python/lib' % (gphome, gphome)
-        if dypath:
-            dypath = '%s/lib:%s/ext/python/lib:%s' % (gphome, gphome, dypath)
-        else:
-            dypath = '%s/lib:%s/ext/python/lib' % (gphome, gphome)
-
-        env = {}
-        env['HOME']    = home
-        env['USER']    = user
-        env['LOGNAME'] = user
-        env['GPHOME']  = gphome
-        env['PATH']    = path
-        env['LD_LIBRARY_PATH'] = lpath
-        env['DYLD_LIBRARY_PATH'] = dypath
-        env['PYTHONPATH'] = os.path.join(gphome, 'lib', 'python')
-        env['PYTHONHOME'] = os.path.join(gphome, 'ext', 'python')
-        env['MASTER_DATA_DIRECTORY'] = get_masterdatadir()
-        return env
-
-
-#-------------------------------------------------------------------------------
-def findRelFileDotNodes(relFileNode, sortedList):
-    """
-      This function will take a relfilenode without a dot '.' suffix, and find all of its
-      dot suffix files in a sorted list (sortedList). Just the suffixes are returned.
-      relFileNode and sortedList are assumed to contain full path lists.
-    """
-    retValue = []
-    relFileNodeWithDot = relFileNode + "."
-    index = bisect.bisect_left(sortedList, relFileNode)
-    index = index + 1
-
-    while index < len(sortedList) and sortedList[index].startswith(relFileNodeWithDot):
-       fullName = sortedList[index]
-       dotIndex = len(relFileNodeWithDot)
-       suffix = fullName[dotIndex:]
-       retValue.append(suffix)
-       index = index + 1
-    
-    return retValue
-
-
-#-------------------------------------------------------------------------------
-def sshBusy(cmd):
-    """ 
-      This function will check the results of a Command to see if ssh was too busy.
-      It will return False if the command completed, successfully or not, 
-      and a retry is not possible or necessary. 
-    """
-    retValue = False
-    results = cmd.get_results()
-    resultStr = results.printResult()
-    if results.rc != 0:
-       if resultStr.find("ssh_exchange_identification: Connection closed by remote host") != -1:
-          retValue = True
-       else:
-          retValue = False
-    else:
-       retValue = False
-    return retValue
-
-
-
-#-------------------------------------------------------------------------------
-def runAndCheckCommandComplete(cmd):
-    """ 
-      This function will run a Command and return False if ssh was too busy.
-      It will return True if the command completed, successfully or not, 
-      if ssh wasn't busy. 
-    """
-    retValue = True
-    cmd.run(validateAfter = False)
-    if sshBusy(cmd) == True:
-       """ Couldn't make the connection. put in a delay, and return"""
-       self.logger.debug("gpupgrademirror ssh is busy... need to retry the command: " + str(cmd))
-       time.sleep(1)
-       retValue = False
-    else:
-       retValue = True
-    return retValue
-
-
-#-------------------------------------------------------------------------------
-                                                                
-def parseargs():
-    parser = OptParser( option_class = OptChecker
-                      , description  = ' '.join(DESCRIPTION.split())
-                      , version      = '%prog version $Revision: #12 $'
-                      )
-    parser.setHelp(_help)
-    parser.set_usage('%prog ' + _usage)
-    parser.remove_option('-h')
-
-    parser.add_option('-r', '--rollback', action='store_true',
-                      help='rollback failed expansion setup.', default=False)
-    parser.add_option('-c', '--continue-upgrade', action='store_true',
-                      help='continue expansion.', default=False)
-    parser.add_option('-m', '--mode', default='S',
-                      help='Valid values are S for safe mode and U for un-safe mode')
-    parser.add_option('-P', '--phase2', action='store_true',
-                      help='phase 2 for upgrade of mirrors.', default=False)
-    parser.add_option('-v','--verbose', action='store_true',
-                      help='debug output.', default=False)
-    parser.add_option('-h', '-?', '--help', action='help',
-                        help='show this help message and exit.', default=False)
-    parser.add_option('-i', '--ids', default='',
-                      help='comma separated list of dbids for use in upgrade')
-    parser.add_option('-g', '--gphome', default=get_gphome(),
-                      help='location of gphome for 3.3.x system')
-    parser.add_option('-t', '--temp-dir', default='',
-                      help='location for temp upgrade mirror data')
-    parser.add_option('-s', '--speed-network', default='',
-                      help='speed of NIC on he network')
-    parser.add_option('--usage', action="briefhelp")
-    
-
-    """
-     Parse the command line arguments
-    """
-    (options, args) = parser.parse_args()
-
-    if len(args) > 0:
-        logger.error('Unknown argument %s' % args[0])
-        parser.exit()
-
-    """
-     Make a directory to hold information about this run. We will need this information in the
-     event we run again in rollback or continue mode. We use temporary directory gpmigrator create
-     under the master data directory, or the one it creates under the segment data directory (i.e.
-     the directory passed in via the -t option.  
-    """
-    try:
-        if len(options.temp_dir) > 0:
-           options.info_data_directory = options.temp_dir
-        else:
-           mDir = os.path.split(get_masterdatadir())
-           dirPrefix = mDir[0]
-           if len(dirPrefix) == 1 and str(dirPrefix) == str('/'):
-              """ Special case where the directory is '/'. """
-              dirPrefix = ""
-           dirSuffix = mDir[1]
-           options.info_data_directory = dirPrefix + "/" + GPMIGRATOR_TEMP_DIR + "/" + dirSuffix + INFO_DIR 
-    except GpError, msg:
-        logger.error(msg)
-        parser.exit()
-
-    if not os.path.exists(options.info_data_directory):
-       MakeDirectoryWithMode.local( name = 'gpupgrademirror make info dir: %s' % options.info_data_directory
-                                  , directory = options.info_data_directory
-                                  , mode = 700
-                                  )
-    return options, args
-
-#-------------------------------------------------------------------------------
-def sig_handler(sig, arg):
-    print "Handling signal..."
-    signal.signal(signal.SIGTERM, signal.SIG_DFL)
-    signal.signal(signal.SIGHUP, signal.SIG_DFL)
-
-    # raise sig
-    os.kill(os.getpid(), sig)
-
-
-#-------------------------------------------------------------------------------
-def create_pid_file(info_data_directory):
-    """Creates gpupgradmirror pid file"""
-    try:
-        fp = open(info_data_directory + '/' + PID_FILE, 'w')
-        fp.write(str(os.getpid()))
-    except IOError:
-        raise
-    finally:
-        if fp: fp.close()
-
-
-#-------------------------------------------------------------------------------
-def remove_pid_file(info_data_directory):
-    """Removes upgrademirror pid file"""
-    try:
-        os.unlink(info_data_directory + '/' + PID_FILE)
-    except:
-        pass
-
-
-#-------------------------------------------------------------------------------
-def is_gpupgrademirror_running(info_data_directory):
-    """Checks if there is another instance of gpupgrademirror running"""
-    is_running = False
-    try:
-        fp = open(info_data_directory + '/' + PID_FILE, 'r')
-        pid = int(fp.readline().strip())
-        fp.close()
-        is_running = check_pid(pid)
-    except IOError:
-        pass
-    except Exception, msg:
-        raise
-
-    return is_running
-
-
-#-------------------------------------------------------------------------------
-#-------------------------------------------------------------------------------
-class InvalidStatusError(Exception): pass
-
-
-#-------------------------------------------------------------------------------
-#-------------------------------------------------------------------------------
-class ValidationError(Exception): pass
-
-
-#-------------------------------------------------------------------------------
-#-------------------------------------------------------------------------------
-class GPHostCache:
-    """ This class represents the information stored in the .gphostcache file """
-    
-    def __init__(self, url):
-        self.hostcache = {}
-        self.readfile()
-        
-    #-------------------------------------------------------------------------------
-    def __str__(self):   
-       tempStr = "self.hostcache = " + str(self.hostcache) + '\n'
-       return tempStr
-        
-    #-------------------------------------------------------------------------------
-    def readfile(self):
-        if len(options.ids) == 0:
-           FILEDIR   = os.path.expanduser("~")
-        else:
-           FILEDIR = options.info_data_directory
-        self.FILENAME  = ".gphostcache"
-        self.CACHEFILE = FILEDIR + "/" + self.FILENAME
-
-        try:
-            file = None
-            file = open(self.CACHEFILE, 'r')
-                
-            for line in file:
-                (interface,hostname) = line.split(':')
-                self.hostcache[interface.strip()] = hostname.strip()
-            file.close()             
-        except IOError, ioe:
-            logger.error("Can not read host cache file %s. Exception: %s" % (self.CACHEFILE, str(ioe)))
-            raise Exception("Unable to read host cache file: %s" % self.CACHEFILE)                  
-        finally:
-            if file != None:
-               file.close()        
-    
-    #-------------------------------------------------------------------------------
-    def getInterfaces(self, hostName):
-        retList = []
-        
-        for interface in self.hostcache:
-            if self.hostcache[interface] == hostName:
-               retList.append(interface)
-        return retList
-    
-    #-------------------------------------------------------------------------------
-    def getHost(self, interface):
-        return self.hostcache[interface]
-    
-    #-------------------------------------------------------------------------------
-    def getHostList(self):
-        retList = []
-        
-        for (interface, host) in self.hostcache:
-            if host in retList:
-                continue
-            else:
-                retList.append(host)
-
-
-#-------------------------------------------------------------------------------
-#-------------------------------------------------------------------------------
-class RelFileNodes():
-    """ This class represents a complete set of tables and dependencies for a given segment. """
-    """ WARNING: This class isn't currently used, so it is not tested. Beware...."""
-
-    def __init__(self, logger, mirrorInfo, database_list):
-        self.info_data_directory = mirrorInfo.mirror_dbid
-        self.mirrorInfo = mirrorInfo
-        self.databaseList = database_list
-        self.tableRelFileList = []
-        
-    #-------------------------------------------------------------------------------    
-    def __str__(self): 
-        retValue = ''
-         
-        for tableFileList in self.tableRelFileList:
-            tempStr = ':'.join(tableFileList)
-            retValue = retValue + tempStr + '\n'
-        return retValue
-        
-#   #-------------------------------------------------------------------------------            
-    def __repr__(self):
-        return self.__str__()
-        
-    #-------------------------------------------------------------------------------
-    def setup(self):
-        
-        """ Connect to each database on the segment, and get its list of tables and table dependencies. """
-        for db in self.databaseList:
-           if str(db.databaseName) == str("template0"):
-              continue
-           segmentURL = dbconn.DbURL( hostname = self.mirrorInfo.primary_host
-                                    , port = self.mirrorInfo.primary_host_port
-                                    , dbname = db.databaseName 
-                                    )
-           conn   = dbconn.connect( dburl   = segmentURL
-                                  , utility = True
-                                  )
-           
-           tablesCursor = dbconn.execSQL(conn, TableRelFileNodesRow.query())
-           aTable = []
-           for row in tablesCursor:
-               aTable = []
-               """ KAS need to added code to look for *.1, *.2 files for this table """
-               tableRelRow = TableRelFileNodesRow(row)
-               aTable.append(str(db.databaseDirectory) + '/' + str(tableRelRow.table_file))
-               if tableRelRow.toast_file != None:
-                  aTable.append(str(db.databaseDirectory) + '/' + str(tableRelRow.toast_file))
-               if tableRelRow.toast_index_file != None:
-                  aTable.append(str(db.databaseDirectory) + '/' + str(tableRelRow.toast_index_file))
-               if tableRelRow.ao_file != None:            
-                  aTable.append(str(db.databaseDirectory) + '/' + str(tableRelRow.ao_file))
-               if tableRelRow.ao_index_file != None:
-                  aTable.append(str(db.databaseDirectory) + '/' + str(tableRelRow.ao_index_file))
-               indexesCursor = dbconn.execSQL(conn, TableIndexRow.query(str(tableRelRow.table_file_oid)))
-               """ KAS note to self. Do I need to worry about *.1, *.2... files? """
-               for indexRow in indexesCursor:
-                   indexEntry = TableIndexRow(indexRow)
-                   aTable.append(str(db.databaseDirectory) + '/' + str(indexEntry.relfilenode))
-               self.tableRelFileList.append(aTable)
-           conn.close()
-           
-   #-------------------------------------------------------------------------------
-    def _read_rel_file_node_file(self):
-        """Reads in an existing relfile node file"""
-        self.logger.debug("Trying to read in a pre-existing relfile node file" + str(self._rel_file_node_filename))
-        try:
-            self._fp = open(self._rel_file_node_filename, 'r')
-            self._fp.seek(0)
-
-            for line in self._fp:
-                tableFileList = line.split(':')
-                tableRelFileList.append(tableFileList)
-            self._fp.close()
-                
-        except IOErrormm, e:
-           logger.error('Unable to read relfile node file: ' + str(e))
-           logger.error('gpupgradmirror exiting')
-           exit("The gpupgrademirror log information can be found in " + str(get_logfile()) + "\n")
-
-   #-------------------------------------------------------------------------------
-    def create_rel_file_node_file(self):
-        """Creates a new gpupgrademirror relfile node file"""
-        try:                
-            self._fp = open(self._rel_file_node_filename, 'w')
-        except IOError, e:
-            logging.error('Unable to create relfile node file %s.' % self._rel_file_node_filename)
-            raise e
-
-   #-------------------------------------------------------------------------------
-    def remove_rel_file_node_file(self):
-        """Closes and removes the gpupgrademirror relfile node file"""
-        if self._fp:
-            self._fp.close()
-            self._fp = None
-        if os.path.exists(self._rel_file_node_filename):
-            os.unlink(self._rel_file_node_filename)
-
-
-#-------------------------------------------------------------------------------
-#-------------------------------------------------------------------------------
-class TableRelFileNodesRow:
-
-    def __init__(self, row):
-        self.table_file_oid   = row[0]
-        self.table_file       = row[1]
-        self.toast_file       = row[2]
-        self.toast_index_file = row[3]
-        self.ao_file          = row[4]
-        self.ao_index_file    = row[5]
-        
-    #-------------------------------------------------------------------------------
-    @staticmethod
-    def query():    
-        exeSql = """SELECT 
-            t1.oid         AS table_file_oid 
-          , t1.relfilenode AS table_file
-          , t2.relfilenode AS toast_file
-          , t3.relfilenode AS toast_index_file
-          ,    t4.relfilenode AS ao_file
-          ,    t5.relfilenode AS ao_index_file
-           FROM    (((pg_class AS t1 left outer join pg_class AS t2 ON t2.oid = t1.reltoastrelid)
-              left outer join  pg_class AS t3 ON t3.oid = t2.reltoastidxid)
-                  left outer join pg_class AS    t4 ON t4.oid = t1.relaosegrelid)
-                    left outer join pg_class AS t5 ON    t5.oid = t4.relaosegidxid
-           WHERE   t1.relkind = 'r' or t1.relkind = 'x'
-           ORDER BY t1.relfilenode
-        """
-
-        return exeSql
-
-
-#-------------------------------------------------------------------------------
-#-------------------------------------------------------------------------------
-
-class TableIndexRow:
-
-    def __init__(self, row):
-        self.relfilenode       = row[0]
-
-        
-#-------------------------------------------------------------------------------
-    @staticmethod
-    def query(table_oid):
-        exeSql = """ SELECT t2.relfilenode
-                     FROM   pg_catalog.pg_index AS t1
-                             INNER JOIN
-                            pg_catalog.pg_class AS t2
-                     ON     t1.indexrelid = t2.oid
-                     WHERE  t1.indrelid = """ + str(table_oid)
-
-        return exeSql
-
-
-#-------------------------------------------------------------------------------
-#-------------------------------------------------------------------------------
-class MirrorNICRow:
-    
-    def __init__(self, row):
-        self.nic = row[0]
-        self.dbid = row[1]
-
-    #-------------------------------------------------------------------------------
-    @staticmethod
-    def query():
-        if options.phase2 == False:
-           exeStr = '''  SELECT    hostname as nicaddress, dbid
-                         FROM      pg_catalog.gp_configuration
-                         WHERE     isprimary = 'f'
-                         ORDER BY  hostname '''
-        else:
-           raise Exception('Internal Error. MirrorNICRow.qurey only for 3.x systems.')           
-        return exeStr
-
-
-#-------------------------------------------------------------------------------
-#-------------------------------------------------------------------------------
-class MirrorHostsRow:
-    
-    def __init__(self, row):
-        self.hostname = row[0]
-        self.dbid     = row[1]
-
-    #-------------------------------------------------------------------------------
-    @staticmethod
-    def query():
-        if options.phase2 == False:
-           raise Exception('Internal Error. MirrorHostsRow.qurey only for 3.3.x systems.')
-        else:
-           exeStr = ''' SELECT      hostname, dbid 
-                        FROM        pg_catalog.gp_segment_configuration
-                        WHERE       role = 'm'
-                        ORDER BY    hostname '''
-        return exeStr
-
-
-#-------------------------------------------------------------------------------
-#-------------------------------------------------------------------------------
-class MirrorHostInfoRow:
-    
-    def __init__(self, row):
-        self.mirror_host            = row[0]
-        self.mirror_host_port       = row[1]
-        self.primary_host           = row[2]
-        self.primary_host_port      = row[3]
-        self.primary_data_directory = row[4]
-        self.mirror_data_directory  = row[5]
-        self.mirror_dbid            = row[6]
-        self.primary_dbid           = row[7]
-        if options.phase2 == True:
-           self.nic_address = row[8]
-
-    #-------------------------------------------------------------------------------
-    def __repr__(self):
-        return self.__str__()
-
-    #-------------------------------------------------------------------------------
-    def __str__(self):  
-        tempStr = 'mirror_host='            + str(self.mirror_host)            + '\n' \
-        +         'mirror_host_port='       + str(self.mirror_host_port)       + '\n' \
-        +         'primary_host='           + str(self.primary_host)           + '\n' \
-        +         'primary_host_port='      + str(self.primary_host_port)      + '\n' \
-        +         'primary_data_directory=' + str(self.primary_data_directory) + '\n' \
-        +         'mirror_data_directory='  + str(self.mirror_data_directory)  + '\n' \
-        +         'mirror_dbid='            + str(self.mirror_dbid)            + '\n' \
-        +         'primary_dbid='           + str(self.primary_dbid)           + '\n'
-        if options.phase2 == True:
-           tempStr = tempStr + 'nic_address='   + str(self.nic_address)        + '\n' 
-        return tempStr
-
-    #-------------------------------------------------------------------------------
-    @staticmethod
-    def readListFromFile(nvFileName):
-        """ Static method to convert file with information in the __str__ method format """
-        """ for this object into a list of rows. The rows are lists of column values.                                      """
-        retList = []
-        
-        logger.debug("Trying to read in a pre-existing gpupgrademirror MirrorHostInfoRow file. " + str(nvFileName))
-        
-        try:
-            fp = open(nvFileName,'r')
-            fp.seek(0)
-
-            if options.phase2 == True:
-              colsPerRow = 9
-            else:
-              colsPerRow = 8
-            index = 0
-            row = []
-            for line in fp:
-                (name, value) = line.strip().rsplit('=', 1)
-                row.append(value)
-                index = index + 1
-                if index == colsPerRow:     
-                   retList.append(MirrorHostInfoRow(row))
-                   row = []
-                   index = 0
-            return retList
-        except IOError:
-            raise
-        finally:
-            try:
-                if fp != None:
-                   fp.close()
-            except:
-                pass
-            
-
-    #-------------------------------------------------------------------------------
-    @staticmethod
-    def writeListToFile(nvFileName, listOfRows):
-        """ Static method to write out a list of rows in the __str__ method format """
-
-        logger.debug("Trying to write a list of MirrorHostInfoRow to a file. " + str(nvFileName))
-       
-        try:
-           fp = open(nvFileName, 'w')
-           fp.seek(0)
-           for row in listOfRows:
-               fp.write(str(row))
-           fp.flush()
-           fp.close()
-        except IOError:
-           raise
-
-    #-------------------------------------------------------------------------------
-    @staticmethod
-    def query():     
-        if options.phase2 == False:
-           exeStr =  '''SELECT         t2.hostname as mirror_host
-                                     , t2.port     as mirror_host_port
-                                     , t1.hostname as primary_host
-                                     , t1.port     as primary_host_port
-                                     , t1.datadir  as primary_data_directory
-                                     , t2.datadir  as mirror_data_directory
-                                     , t2.dbid     as mirror_dbid
-                                     , t1.dbid     as primary_dbid
-                                FROM   pg_catalog.gp_configuration as t1 
-                                        INNER JOIN  
-                                       pg_catalog.gp_configuration as t2 
-                                ON     t1.content = t2.content 
-                                   AND t2.hostname = '%s'
-                                   AND t1.content >= 0 
-                                   AND t1.isprimary = 't' 
-                                   AND t2.isprimary = 'f' 
-                                ORDER BY  primary_host '''
-        else: 
-           exeStr = """SELECT          t2.hostname as mirror_host
-                                     , t2.port     as mirror_host_port
-                                     , t1.hostname as primary_host
-                                     , t1.port     as primary_host_port
-                                     , t3.fselocation  as primary_data_directory
-                                     , t4.fselocation  as mirror_data_directory
-                                     , t2.dbid     as mirror_dbid
-                                     , t1.dbid     as primary_dbid
-                                     , t2.address as nic_address
-                            FROM   ((pg_catalog.gp_segment_configuration as t1
-                                      INNER JOIN
-                                    pg_catalog.gp_segment_configuration as t2
-                                      ON     t1.content = t2.content
-                                         AND t2.hostname = '%s'
-                                         AND t1.content >= 0
-                                         AND t1.role = 'p'
-                                         AND t2.role = 'm'
-                                    ) INNER JOIN pg_catalog.pg_filespace_entry as t3
-                                          ON   t3.fsedbid = t1.dbid
-                                   ) INNER JOIN pg_catalog.pg_filespace_entry as t4
-                                      ON t4.fsedbid = t2.dbid
-                            ORDER BY  primary_host"""
-            
-        return exeStr
-
-
-#-------------------------------------------------------------------------------
-#-------------------------------------------------------------------------------
-class DatabaseRow:
-    
-    def __init__(self, row):
-        self.databaseDirectory = row[0]
-        self.databaseName = row[1]
-
-    @staticmethod
-    def query():
-        return '''SELECT oid, datname
-                  FROM pg_database
-                  ORDER BY oid '''
-
-
-#-------------------------------------------------------------------------------
-#-------------------------------------------------------------------------------
-class InformationSchemaTablesRow:
-    
-    def __init__(self, row):
-        self.filename = row[0]
-
-    @staticmethod
-    def query():
-        return '''SELECT  c.relfilenode
-                  FROM    pg_class c JOIN pg_namespace n ON (c.relnamespace = n.oid)
-                  WHERE  n.nspname = 'information_schema' AND (relkind = 'r' OR relkind = 'i')'''
-
-
-#-------------------------------------------------------------------------------
-#-------------------------------------------------------------------------------
-class NonCatalogToastTablesRow:
-    
-    def __init__(self, row):
-        self.filename = row[0]
-
-    @staticmethod
-    def query():
-        return '''SELECT relfilenode
-                  FROM pg_class
-                  WHERE relnamespace  =  11
-                     AND relisshared   =  'f'
-                     AND (relkind  = 'r' OR relkind = 'i')
-                     AND relstorage    =  'h'
-                     AND reltoastrelid = 0
-                  ORDER BY relfilenode '''
-
-
-#-------------------------------------------------------------------------------
-#-------------------------------------------------------------------------------
-class CatalogToastTablesRow:
-    
-    def __init__(self, row):
-        self.filename = row[0]
-        self.toastfilename = row[1]
-        self.toastfileindex = row[2]
-        
-    @staticmethod
-    def query():
-        return """SELECT  t1.relfilenode, t2.relfilenode as toast_filenode, t3.relfilenode as toast_filenode_index
-                  FROM  pg_class as t1, pg_class as t2, pg_class as t3
-                  WHERE   t1.relnamespace  = 11
-                    AND   t1.relisshared   = 'f'
-                    AND   t1.reltoastrelid = t2.oid
-                    AND   t2.reltoastidxid = t3.oid
-                  ORDER BY t1.relfilenode"""
-    
-
-#-------------------------------------------------------------------------------
-#-------------------------------------------------------------------------------
-class Phase1and2MirrorInfo:
-    """
-    This class contains information on all the mirror nodes in the cluster.
-    The way it gathers information depends on if we are in phase 1 or phase 2.
-    
-    If this process is a minion (i.e. it is a sub process of the overall gpupgrademirror),
-    it will just get those mirrors have have dbids in the options.ids list.
-    """
-
-    def __init__(self, url = None):
-        self.dbUrl = url
-        self.mirrorInfoFileUnqualified = 'gpupgrademirror.mirrorinfo'
-        self.mirrorInfoFile = options.info_data_directory + '/' + self.mirrorInfoFileUnqualified
-        self.mirrorNodeList = []
-        self.allMirrorInfoList = []
-        self.gpHostCacheInfo = GPHostCache(url = url)
-        conn = None
-    
-        if (options.rollback == True or options.continue_upgrade == True) and options.phase2 == False:
-           """ We are in a rollback or continue state in Phase 1.            """
-           """ Retrieve configuration information from previously saved file """
-           self.allMirrorInfoList = MirrorHostInfoRow.readListFromFile(Phase1and2MirrorInfo.getMirrorInfoFile())
-
-           """ for each NIC in the list of mirror NICs, make a list of distinct mirror nodes """
-           for mirrorInfo in self.allMirrorInfoList:
-               mirrorHostName = self.gpHostCacheInfo.getHost(mirrorInfo.mirror_host)
-               if mirrorHostName in self.mirrorNodeList:
-                  pass
-               elif len(options.ids) == 0 or mirrorInfo.mirror_dbid in options.ids:
-                  self.mirrorNodeList.append(mirrorHostName)
-        else:
-           """ We are in normal Phase1 or normal Phase 2 (no rollback or continue) """
-           conn   = dbconn.connect( dburl   = dbUrl
-                                  , utility = True
-                                  )
-           
-           ''' Get a list of mirror nodes '''
-           if options.phase2 == True:
-              mirrorNodeCursor = dbconn.execSQL(conn, MirrorHostsRow.query())
-              for row in mirrorNodeCursor:
-                  mirror = MirrorHostsRow(row)
-                  if len(options.ids) == 0 or str(mirror.dbid) in options.ids:
-                     if mirror.hostname not in self.mirrorNodeList:
-                        self.mirrorNodeList.append(mirror.hostname)
-
-                        ''' Get info on mirror segments for this host'''
-                        mirrorSegInfoCursor = dbconn.execSQL(conn, MirrorHostInfoRow.query() % mirror.hostname)
-                        for mirrorInfoRow in mirrorSegInfoCursor:
-                            mirrorInfo = MirrorHostInfoRow(mirrorInfoRow)
-                            if len(options.ids) == 0 or str(mirrorInfo.mirror_dbid) in options.ids:
-                               self.allMirrorInfoList.append(mirrorInfo)
-           else:
-              ''' We are in phase1              '''
-              ''' Get a list of all mirror NICs '''
-              mirrorNicCursor = dbconn.execSQL(conn, MirrorNICRow.query())
-           
-              ''' Make a list of distinct nodes associated with the mirror NICs '''
-              for row in mirrorNicCursor:
-                  nicRow = MirrorNICRow(row)
-                  host = self.gpHostCacheInfo.getHost(nicRow.nic)
-                  if host in self.mirrorNodeList:
-                     pass
-                  else:
-                     if len(options.ids) == 0 or str(nicRow.dbid) in options.ids:
-                        self.mirrorNodeList.append(host)
-           
-              for nodeName in self.mirrorNodeList:
-                  ''' for each node, use its NICs to find its segments '''
-                  nicList = self.gpHostCacheInfo.getInterfaces(nodeName)
-                  ''' Get info on mirror segments for this host'''
-                  for nic in nicList:
-                     mirrorSegInfoCursor = dbconn.execSQL(conn, MirrorHostInfoRow.query() % nic)
-                     for mirrorInfoRow in mirrorSegInfoCursor:
-                         mirrorInfo = MirrorHostInfoRow(mirrorInfoRow)
-                         if len(options.ids) == 0 or str(mirrorInfo.mirror_dbid) in options.ids:
-                            self.allMirrorInfoList.append(mirrorInfo)
-
-           if conn != None:
-              conn.close()  
-            
-           MirrorHostInfoRow.writeListToFile(Phase1and2MirrorInfo.getMirrorInfoFile(), self.allMirrorInfoList)   
-
-    #-------------------------------------------------------------------------------
-    def __str__(self):    
-        tempStr = '  dbUrl                  = ' + str(self.dbUrl)             + '\n' \
-        +         '  mirrorInfoFile         = ' + str(self.mirrorInfoFile)    + '\n' \
-        +         '  self.mirrorNodeList    = \n' + str(self.mirrorNodeList)  + '\n' \
-        +         '  self.allMirrorInfoList = ' + str(self.allMirrorInfoList) + '\n' \
-        +         '  self.gpHostCacheInfo   = ' + str(self.gpHostCacheInfo)   + '\n'
-        return tempStr 
-        
-    #-------------------------------------------------------------------------------
-    def getAllMirrorInfoList(self):
-        return self.allMirrorInfoList
-        
-    #-------------------------------------------------------------------------------
-    def getMirrorNodeList(self):
-        return self.mirrorNodeList
-    
-    
-    #-------------------------------------------------------------------------------
-    def getMirrorInfo(self, mirrorName):
-        retList = []
-        
-        if options.phase2 == True:
-           for mirrorSeg in self.allMirrorInfoList:
-               if mirrorSeg.mirror_host == mirrorName:
-                   retList.append(mirrorSeg)
-        else:
-           nicNameList = self.gpHostCacheInfo.getInterfaces(mirrorName)
-           for nicName in nicNameList:
-               for mirrorSeg in self.allMirrorInfoList:
-                   if nicName == mirrorSeg.mirror_host:
-                       retList.append(mirrorSeg)
-        
-        return retList
-
-    #-------------------------------------------------------------------------------
-    def getMirrorInfoFromMirrordbid(self, mirrordbid):
-        retValue = None
-    
-        for mirrorSeg in self.allMirrorInfoList:
-            if str(mirrorSeg.mirror_dbid) == str(mirrordbid):
-               retValue = mirrorSeg
-               break
-        return retValue
-
-    #-------------------------------------------------------------------------------
-    def getInterfaceList(self, node_name):
-        retList = []
-        
-        if options.phase2 == True:
-           for mirrorSeg in self.allMirrorInfoList:
-               if mirrorSeg.mirror_host == node_name:
-                  retList.append(mirrorSeg.nic_address)
-        else:
-           allHostCacheInterfaces = self.gpHostCacheInfo.getInterfaces(node_name)
-           """ The host cache contains entries that are not actual nics, so """
-           """ we must only return the real nics that are referenced from   """
-           """ from the database cluster.                                   """                        
-           for interface in allHostCacheInterfaces:
-               for seg in self.allMirrorInfoList:
-                   if seg.mirror_host not in retList and interface == seg.mirror_host:
-                       retList.append(interface)
-           
-        return retList
-
-    #-------------------------------------------------------------------------------
-    @staticmethod
-    def getMirrorInfoFile():
-        return options.info_data_directory + '/' + 'gpupgrademirror.mirrorinfo'
-
-
-    #-------------------------------------------------------------------------------
-    @staticmethod
-    def removeMirrorInfoFile():
-        try:
-           os.remove(Phase1and2MirrorInfo.getMirrorInfoFile())
-        except:
-           pass
-           
-
-
-#-------------------------------------------------------------------------
-#-------------------------------------------------------------------------                    
-class NetworkSpeed(Command):
-    """
-    This class will attempt to get the network speed on the cluster. 
-    It assumes the the 2 nodes being tested are different nodes, and 
-    that no other traffic is present between the two nodes.
-    """
-    
-    def __init__(self,name, host1, host2, tempDir1, tempDir2, ctxt=LOCAL, remoteHost=None):        
-        self.host1 = host1
-        self.host2 = host2
-        self.tempDir1 = tempDir1
-        self.tempDir2 = tempDir2
-        self.remoteHost = remoteHost
-        cmdStr="gpcheckperf -h %s -h %s -r N -d %s -d %s" % (host1, host2, tempDir1, tempDir2)
-        Command.__init__(self,name,cmdStr,ctxt,remoteHost)
-        
-    def getSpeed(self):
-        """ 
-        Return the network speed in megabytes per second.
-        Don't ever fail, just say it is 0 if something goes wrong.
-        """
-        retValue = 0
-        results = ""
-        try:
-           results = self.get_results()
-           resultStr = results.printResult()
-           index = resultStr.find("median = ")
-           substring = resultStr[index:]
-           substringList = substring.split(" ")
-           retValue = int(float(substringList[2]))
-        except Exception, e:
-           logger.warning("There was a problem getting network speed: results = %s. exception = %s" % (str(results), str(e)))
-           pass
-        return retValue
-        
-
-#-------------------------------------------------------------------------
-#-------------------------------------------------------------------------                    
-class MakeDirectoryWithMode(Command):
-    
-    def __init__(self,name,directory,ctxt=LOCAL,remoteHost=None,mode=None):        
-        self.directory=directory
-        if mode == None:
-            pMode = ''
-        else:
-            pMode = "-m " + str(mode)
-        cmdStr="%s -p %s %s" % (findCmdInPath('mkdir'),pMode,directory)
-        Command.__init__(self,name,cmdStr,ctxt,remoteHost)
-    
-    @staticmethod
-    def local(name,directory,mode=None):    
-        mkdirCmd=MakeDirectoryWithMode(name,directory,mode = mode)
-        mkdirCmd.run(validateAfter=True)
-            
-    @staticmethod
-    def remote(name,remote_host,directory,mode=None):
-        mkdirCmd=MakeDirectoryWithMode(name,directory,ctxt=REMOTE,remoteHost=remote_host,mode=mode)
-        mkdirCmd.run(validateAfter=True)
-
-
-#-------------------------------------------------------------------------
-#-------------------------------------------------------------------------                    
-class CreateDirIfNecessaryWithMode(Command):
-    def __init__(self,name,directory,ctxt=LOCAL,remoteHost=None,mode=None):
-        if mode == None:
-           cmdStr="""python -c "import os; os.mkdir('%s')" """ % (directory)
-        else:
-           if len(str(mode)) != 4:
-               raise Exception("Internal error: os.mkdir mode parameter must have length of 4")
-           pythonPg = """\"\"\"import os
-if os.path.exists('%s') == False:
-  os.mkdir('%s', %s)\"\"\"
-""" % (directory, directory, str(mode))
-           cmdStr = """python -c %s""" % pythonPg
-        Command.__init__(self,name,cmdStr,ctxt,remoteHost)
-        
-    @staticmethod
-    def remote(name,remote_host,directory,mode=None):
-        cmd=CreateDirIfNecessaryWithMode(name,directory,ctxt=REMOTE,remoteHost=remote_host,mode=mode)
-        cmd.run(validateAfter=False)
-                
-
-#-------------------------------------------------------------------------
-#-------------------------------------------------------------------------
-class CopyToLocal(Command):
-    def __init__(self, name, srcHost, srcDirectory, dstDirectory, ctxt=LOCAL):
-        self.srcDirectory=srcDirectory
-        self.srcHost=srcHost
-        self.dstDirectory=dstDirectory
-        cmdStr="%s -r %s:%s %s" % (findCmdInPath('scp'), srcHost, srcDirectory, dstDirectory)
-        Command.__init__(self, name, cmdStr, ctxt=LOCAL)
-
-
-#-------------------------------------------------------------------------
-#-------------------------------------------------------------------------                    
-class RemoteCopyPreserve(Command):
-    def __init__(self,name,srcDirectory,dstHost,dstDirectory,ctxt=LOCAL,remoteHost=None):
-        self.srcDirectory=srcDirectory
-        self.dstHost=dstHost
-        self.dstDirectory=dstDirectory        
-        cmdStr="%s -rp %s %s:%s" % (findCmdInPath('scp'),srcDirectory,dstHost,dstDirectory)
-        Command.__init__(self,name,cmdStr,ctxt,remoteHost)
-
-
-#-------------------------------------------------------------------------
-#-------------------------------------------------------------------------                    
-class GPumDiskFree(Command):
-    """ This method checks for available space in a directory   """
-    """ The same method, without the GPum prefix is in gppylib, """
-    """ but it doesn't work correctly in 3.3.x.                 """
-
-    def __init__(self,name,directory,ctxt=LOCAL,remoteHost=None):
-        self.directory=directory
-        dfCommand = SYSTEM.getDiskFreeCmd()
-        if options.phase2 == False and isinstance(SYSTEM, SolarisPlatform):
-           """ This is a work around for a bug in 3.3.x (see MPP-6647) """
-           dfCommand = dfCommand + 'k'
-        if options.phase2 == False and isinstance(SYSTEM, LinuxPlatform):
-           """ This is a work around for a bug in 3.3.x (see MPP-6647) """
-           dfCommand = dfCommand + 'Pk'
-        cmdStr="%s %s" % (dfCommand,directory)
-        Command.__init__(self,name,cmdStr,ctxt,remoteHost)
-
-    @staticmethod
-    def get_size(name,remote_host,directory):
-        dfCmd=GPumDiskFree( name = name
-                      , directory = directory
-                      , ctxt = REMOTE
-                      , remoteHost = remote_host
-                      )
-        dfCmd.run(validateAfter=True)
-        return dfCmd.get_bytes_free()
-
-
-    def get_bytes_free(self):
-        '''expected output of the form:                                                                                                          
-           Filesystem   512-blocks      Used Available Capacity  Mounted on                                                                      
-           /dev/disk0s2  194699744 158681544  35506200    82%    /                                                                               
-        '''
-        rawIn=self.results.stdout.split('\n')[1]
-        bytesFree=int(rawIn.split()[3])*1024
-        return bytesFree
-
-
-#-------------------------------------------------------------------------
-#-------------------------------------------------------------------------                                                       
-class CreateHardLink(Command):
-    """ This class create a hard link or links. """
-
-    def __init__(self, name, srcFile, hardLink, ctxt = LOCAL, remoteHost = None, argsFile = None):
-        if argsFile == None:
-           self.srcFile = srcFile
-           self.hardLink = hardLink
-           cmdStr="%s %s %s" % (findCmdInPath('ln'), srcFile, hardLink)
-           Command.__init__(self,name,cmdStr,ctxt,remoteHost)
-        else:
-           cmdStr="%s %s | %s -n 2 %s" % (findCmdInPath("cat"), argsFile, findCmdInPath("xargs"), findCmdInPath('ln'))
-           Command.__init__(self,name,cmdStr,ctxt,remoteHost)
-           
-
-#-------------------------------------------------------------------------                                                                                            
-#-------------------------------------------------------------------------                                                                                            
-class GPumMoveDirectory(Command):
-    """ This class moves a local directory."""
-
-    def __init__(self,name,srcDirectory,dstDirectory,ctxt=LOCAL,remoteHost=None):
-        self.srcDirectory=srcDirectory
-        self.dstDirectory=dstDirectory
-        cmdStr="%s -f %s %s" % (findCmdInPath('mv'),srcDirectory,dstDirectory)
-        Command.__init__(self,name,cmdStr,ctxt,remoteHost)
-
-
-#-------------------------------------------------------------------------
-#-------------------------------------------------------------------------                                                       
-class GPumMoveDirectoryContents(Command):
-    """ This class moves the contents of a local directory."""
-
-    def __init__(self,name,srcDirectory,dstDirectory,ctxt=LOCAL,remoteHost=None):
-        self.srcDirectory = srcDirectory
-        self.srcDirectoryFiles = self.srcDirectory + "." + 'dirfilelist' 
-        self.dstDirectory = dstDirectory
-        ls = findCmdInPath("ls")
-        cat = findCmdInPath("cat")
-        xargs = findCmdInPath("xargs")
-        mv = findCmdInPath("mv")
-        cmdStr = "%s -1 %s > %s" % (ls, self.srcDirectory, self.srcDirectoryFiles)
-        cmdStr = cmdStr + ";%s %s" % (cat, self.srcDirectoryFiles)
-        cmdStr = cmdStr + " | %s -I xxx %s %s/xxx %s" % (xargs, mv, self.srcDirectory, self.dstDirectory)
-        cmdStr = cmdStr + "; rm %s" % (self.srcDirectoryFiles)
-        """
-        ls -1 temp1 > temp1.list;cat temp1.list | xargs -I xxx mv temp1/xxx temp2
-        cmdStr = "%s -1 %s > %s;%s %s | %s -I xxx %s %s/xxx %s"
-        cmdStr="%s -f %s %s" % (findCmdInPath('mv'),srcDirectory,dstDirectory)
-        """
-        Command.__init__(self,name,cmdStr,ctxt,remoteHost)
-        
-        
-#-------------------------------------------------------------------------
-#-------------------------------------------------------------------------                                                       
-class RemoveFileDirectoryFromList(Command):
-    """
-    This class will remove a list of files from a given locaiton.
-    """
-    def __init__(self,name,fileListLocation,ctxt=LOCAL,remoteHost=None):
-        self.fileListLocation = fileListLocation
-        self.ctxt = ctxt
-        self.remoteHost = remoteHost
-        cmdStr="%s %s | %s %s -rf" % \
-                    ( findCmdInPath("cat")
-                    , fileListLocation
-                    , findCmdInPath("xargs")
-                    , findCmdInPath('rm')
-                    )
-        Command.__init__(self,name,cmdStr,ctxt,remoteHost)
-
-    def setupListFile(self, tempDir, theList):
-        fd, fdName = tempfile.mkstemp(prefix='tmpRFDFL', dir=tempDir)
-
-        fd = open(fdName, "w")
-        for row in theList:
-            fd.write(row +"\n")
-        fd.close()
-        rmCmd = RemoteCopy( name = "gpupgrademirror copy RFDFL: %s to %s:%s" % (fdName, self.remoteHost, self.fileListLocation)
-                          , srcDirectory = fdName
-                          , dstHost = self.remoteHost
-                          , dstDirectory = self.fileListLocation
-                          , ctxt = LOCAL
-                          , remoteHost = None
-                          )
-        rmCmd.run(validateAfter=True)
-        os.unlink(fdName)
-                
-#-------------------------------------------------------------------------
-#------------------------------------------------------------------------- 
-class CreateTarFromList(Command):
-    """
-    This class will create a tar file from a list of files.
-    WARNING. This class is not used and is untested.
-    """
-    def __init__(self, name, fileListLocation, dstTarFile, srcDirectory, ctxt=LOCAL, remoteHost=None):
-        self.fileListLocation = fileListLocation
-        self.dstTarFile = dstTarFile
-        self.srcDirectory = srcDirectory
-        self.ctxt = ctxt
-        self.remoteHost = remoteHost
-        tarCmd = SYSTEM.getTarCmd()
-        cmdStr="%s %s | %s %s rvPpf %s -C %s  " % \
-                       ( findCmdInPath("cat")
-                       , fileListLocation
-                       , findCmdInPath('xargs')
-                       , tarCmd
-                       , self.dstTarFile
-                       , srcDirectory
-                       )
-        Command.__init__(self,name,cmdStr,ctxt,remoteHost)
-
-    def setupListFile(self, tempDir, theList):
-        fd, fdName = tempfile.mkstemp(prefix='tmpCTFL', dir=tempDir)
-
-        fd = open(fdName, "w")
-        for row in theList:
-            fd.write(row +"\n")
-        fd.close()
-        rmCmd = RemoteCopy( name = "gpupgrademirror copy CTFL: %s to %s:%s" % (fdName, self.remoteHost, self.fileListLocation)
-                          , srcDirectory = fdName
-                          , dstHost = self.remoteHost
-                          , dstDirectory = self.fileListLocation
-                          , ctxt = LOCAL
-                          , remoteHost = None
-                          )
-        rmCmd.run(validateAfter=True)
-        os.unlink(fdName)                
-
-
-#-------------------------------------------------------------------------
-#-------------------------------------------------------------------------                                                       
-class FileDirectoryList(Command):
-    """ This class gets list of files based on a pattern used by ls. """
-
-    def __init__(self, name, filePattern, ctxt=LOCAL, remoteHost=None):
-        self.filePattern = filePattern
-        cmdStr="%s -1 %s" % (findCmdInPath('ls'), filePattern)
-        Command.__init__(self,name,cmdStr,ctxt,remoteHost)
-
-    @staticmethod
-    def get_list(name, remote_host, file_pattern):
-        lsCmd = FileDirectoryList(name, filePattern = file_pattern, ctxt = REMOTE, remoteHost = remote_host)
-        lsCmd.run(validateAfter=False)
-        return lsCmd.get_result_list()
-    
-    def get_result_list(self):
-        files = self.results.stdout.split('\n')
-        if files != None and len(files) > 0 and files[-1] == '':
-           ''' Remove any trailing empty string '''
-           files.pop()
-        return files
-
-
-#-------------------------------------------------------------------------
-#-------------------------------------------------------------------------                                                       
-class FilesInDir(Command):
-    """ This class gets a list of files in a directory """
-
-    def __init__(self, name, filePattern, ctxt=LOCAL, remoteHost=None, remoteTempFile=None):
-        self.filePattern = filePattern
-        self.remoteHost = remoteHost
-        self.remoteTempFile = remoteTempFile
-        if remoteTempFile != None:
-           remoteTempFilePart = " > %s" % remoteTempFile
-        else:
-           remoteTempFilePart = ""
-        find  = findCmdInPath('find')
-        cmdStr= "%s %s -type f %s" % (find, filePattern, remoteTempFilePart)
-        self.cmdStr = cmdStr
-        Command.__init__(self,name,cmdStr,ctxt,remoteHost)
-    
-    
-    def get_result_list(self, localTempFile = None):
-        retList = []
-
-        if localTempFile != None:
-           rcCmd = CopyToLocal( name = "gpupgrademirror get result list copy: %s:%s to %s" % (self.remoteHost, self.remoteTempFile, localTempFile)
-                           , srcHost = self.remoteHost
-                           , srcDirectory = self.remoteTempFile
-                           , dstDirectory = localTempFile
-                           )
-           rcCmd.run(validateAfter = True)
-           rmCmd = RemoveFiles( name = 'gpupgrademirror remove remote temp file: %s:%s' % (self.remoteHost, self.remoteTempFile)
-                              , directory = self.remoteTempFile
-                              , ctxt = REMOTE
-                              , remoteHost = self.remoteHost
-                              )
-           rmCmd.run(validateAfter = True)
-
-           fd = open(localTempFile, "r")
-           fd.seek(0)
-           fileList = []
-           for line in fd:
-               line = line.rstrip('\n')
-               fileList.append(line)
-           fd.close()
-        else:
-           fileList = self.results.stdout.split('\n')
-
-        if fileList != None and len(fileList) > 0 and fileList[-1] == '':
-           ''' Remove any trailing empty string '''
-           fileList.pop()
-        retList = fileList
- 
-        return retList
-
-
-#-------------------------------------------------------------------------
-#-------------------------------------------------------------------------                                                       
-class FileAndSize(Command):
-    """ 
-     This class gets a list of files in a directory and their sizes in k-bytes 
-     The results are sorted by file size. If "zeroSize is true" only files that
-     have 0 bytes are returned.
-    """
-
-    def __init__(self, name, filePattern, ctxt=LOCAL, remoteHost=None, zeroSize=False, remoteTempFile=None):
-        self.filePattern = filePattern
-        self.remoteHost = remoteHost
-        self.remoteTempFile = remoteTempFile
-        if remoteTempFile != None:
-           remoteTempFilePart = " > %s" % remoteTempFile
-        else:
-           remoteTempFilePart = ""
-        if zeroSize == True:
-           sizeArg = "-size 0c"
-        else:
-           sizeArg = ''
-        find  = findCmdInPath('find')
-        xargs = findCmdInPath('xargs')
-        du    = findCmdInPath('du') 
-        sort  = findCmdInPath('sort')
-        cmdStr= "%s %s -type f %s | %s %s -k | %s -n %s" % (find, filePattern, sizeArg, xargs, du, sort, remoteTempFilePart)
-        self.cmdStr = cmdStr
-        Command.__init__(self,name,cmdStr,ctxt,remoteHost)
-
-    
-    def get_result_list(self, localTempFile = None):
-        retList = []
-
-        if localTempFile != None:
-           rcCmd = CopyToLocal( name = "gpupgrademirror get result list copy: %s:%s to %s" % (self.remoteHost, self.remoteTempFile, localTempFile)
-                           , srcHost = self.remoteHost
-                           , srcDirectory = self.remoteTempFile
-                           , dstDirectory = localTempFile
-                           )
-           rcCmd.run(validateAfter = True)
-           rmCmd = RemoveFiles( name = 'gpupgrademirror remove remote temp file: %s:%s' % (self.remoteHost, self.remoteTempFile)
-                              , directory = self.remoteTempFile
-                              , ctxt = REMOTE
-                              , remoteHost = self.remoteHost
-                              )
-           rmCmd.run(validateAfter = True)
-
-           fd = open(localTempFile, "r")
-           fd.seek(0)
-           fileList = []
-           for line in fd:
-               fileList.append(line)
-           fd.close()
-        else:
-           fileList = self.results.stdout.split('\n')
-
-        if fileList != None and len(fileList) > 0 and fileList[-1] == '':
-           ''' Remove any trailing empty string '''
-           fileList.pop()
-        for file in fileList:
-            sizef, filef = file.split()
-            sizef = sizef.strip()
-            filef = filef.strip()
-            ###print "sizef = " + str(sizef)
-            ###print "filef = >>>" + str(filef) + "<<<"
-            retList.append([sizef, filef])
-        return retList
-
-    
-#-------------------------------------------------------------------------
-#-------------------------------------------------------------------------                                                       
-class DirectoryList(Command):
-    """ This method gets a list of all directories and any sub-directories in a directory  """
-
-    def __init__(self, name, dirLocation, ctxt=LOCAL, remoteHost=None):
-        self.dirLocation = dirLocation
-        find  = findCmdInPath('find')
-        sort  = findCmdInPath('sort')
-        cmdStr= "%s %s -type d | %s " % (find, dirLocation, sort)         
-        Command.__init__(self,name,cmdStr,ctxt,remoteHost)
-
-    @staticmethod
-    def get_list(name, remote_host, dir_location):
-        theCmd = DirectoryList(name, dirLocation = dir_location, ctxt = REMOTE, remoteHost = remote_host)
-        theCmd.run(validateAfter=False)
-        return theCmd.get_result_list()
-    
-    def get_result_list(self):
-        dirList = self.results.stdout.split('\n')
-        if dirList != None and len(dirList) > 0 and dirList[-1] == '':
-           ''' Remove any trailing empty string '''
-           dirList.pop()
-        if dirList != None and len(dirList) > 0:
-            ''' The first element is the dirLcation itself. Remove it'''
-            dirList.pop(0)
-        return dirList 
-    
-#-------------------------------------------------------------------------
-#-------------------------------------------------------------------------    
-class SpecialFileHandling():
-    """
-    This is a specialized class for dealing with Phase 2 special files that must be
-    copied from the primary to the mirror. It creats hard links from a directory to
-    files that must be copied, and then copies the entire directory contents over.
-    It assumes all the files exist under the same directory (i.e. base).
-    """
-    def __init__(self, sourceList, seg):
-        self.sourceList = sourceList
-        self.seg = seg
-        self.fullPathLinkDir = seg.primaryTempDir + "/speciallinkdir"
-
-        
-    def createLinks(self):
-        """ 
-        Create a file containing a list of ln parameters locally, and the copy it 
-        to the primary node where we will run ln and create the links.
-        """
-        linkFile = "gpupgrademirror%s.linkspecialfile" % str(self.seg.primary_dbid)
-        sourceFullPathLinkFile = options.info_data_directory + "/" + linkFile
-        targetFullPathLinkFile = self.seg.primaryTempDir + "/" + linkFile
-        linkFP = open(sourceFullPathLinkFile, 'w')
-        linkFP.seek(0)
-        for sfile in self.sourceList:
-            fullPathSrcFile = self.seg.primary_data_directory + "/" + sfile
-            fullPathTargetFile = self.fullPathLinkDir + "/" + sfile
-            linkFP.write("%s %s\n" % (fullPathSrcFile, fullPathTargetFile))
-        linkFP.flush()
-        linkFP.close()
-
-        """ Copy the link file to the primary temp dir """
-        cmdComplete = False
-        while cmdComplete == False:
-          linkCopy = RemoteCopyPreserve( name = "gpupgrademirror copy ln file for primary: %s:%s" % (self.seg.primary_host, self.fullPathLinkDir)
-                                       , srcDirectory = sourceFullPathLinkFile
-                                       , dstHost = self.seg.primary_host
-                                       , dstDirectory = targetFullPathLinkFile
-                                       , ctxt = LOCAL
-                                       )
-          cmdComplete = runAndCheckCommandComplete(linkCopy)
-        linkCopy.validate()
-
-        """ Create a directory to store all our links to the actual files """
-        cmdComplete = False
-        while cmdComplete == False:
-           rmCmd = RemoveFiles( name = 'gpupgrademirror special file link dir: %s:%s' % (self.seg.primary_host, self.fullPathLinkDir)
-                             , directory = self.fullPathLinkDir
-                             , ctxt = REMOTE
-                             , remoteHost = self.seg.primary_host
-                             )
-           cmdComplete = runAndCheckCommandComplete(rmCmd)
-        rmCmd.validate()
-        
-        cmdComplete = False
-        while cmdComplete == False:
-           rmCmd = MakeDirectoryWithMode( name = 'gpupgrademirror special file link dir: %s:%s' % (self.seg.primary_host, self.fullPathLinkDir)
-                             , directory = self.fullPathLinkDir
-                             , ctxt = REMOTE
-                             , remoteHost = self.seg.primary_host
-                             , mode = 700
-                             )
-           cmdComplete = runAndCheckCommandComplete(rmCmd)
-        rmCmd.validate()
-
-        """ Create any subdirectories for our links """
-        subDirList = []
-        for candidate in self.sourceList:
-            candidateList = candidate.split("/")
-            if len(candidateList) > 1:
-               dirPart = "/".join(candidateList[:-1])
-               if dirPart not in subDirList:
-                  subDirList.append(dirPart)
-        for subDir in subDirList:
-            fullPathSubDir = self.fullPathLinkDir + "/" + subDir
-            cmdComplete = False
-            while cmdComplete == False:
-               rmCmd = MakeDirectoryWithMode( name = 'gpupgrademirror special file link dir: %s:%s' % (self.seg.primary_host, fullPathSubDir)
-                             , directory = fullPathSubDir
-                             , ctxt = REMOTE
-                             , remoteHost = self.seg.primary_host
-                             , mode = 700
-                             )
-               cmdComplete = runAndCheckCommandComplete(rmCmd)
-            rmCmd.validate()
-
-        """ Create the links """               
-        cmdComplete = False
-        while cmdComplete == False:
-            link = CreateHardLink( name = "gpupgrademirror creating links for special files %s:%s." % (self.seg.primary_host, str(self.seg.primary_data_directory))
-                                    , srcFile = None
-                                    , hardLink = None
-                                    , ctxt = REMOTE
-                                    , remoteHost = self.seg.primary_host
-                                    , argsFile = targetFullPathLinkFile
-                                    )
-            cmdComplete = runAndCheckCommandComplete(link)
-        link.validate()
-
-        cmdComplete = False
-        while cmdComplete == False:
-            rmCmd = RemoveFiles( name = 'gpupgrademirror remove links file: %s' % (targetFullPathLinkFile)
-                                  , directory = targetFullPathLinkFile
-                                  , ctxt = REMOTE
-                                  , remoteHost = self.seg.primary_host
-                                  )
-            cmdComplete = runAndCheckCommandComplete(rmCmd)
-        rmCmd.validate()              
-        
-    
-#-------------------------------------------------------------------------
-#-------------------------------------------------------------------------    
-class SegHardLinks(Command):
-    """ 
-      This class will generate groups of hard links to files in a segment directory.
-      The overall design is to create a new temporary directory "gpupgrademirrorhardbase", which
-      will contain a many to one relationship between its sub-directories and
-      the real database directories (e.g. gpupgrademirrorhardbase/1.1, gpupgrademirrorhardbase/1.2 both contian
-      links that point to base/1 files).
-      
-      We will special case any file that has zero length. This is because pysync did not
-      handle zero based files correctly at the time this class was written.
-    """
-    
-    """ These two values determin the upper limit as to the number and size of files copied with each copy. """
-    _max_files_per_pysync = 500
-    _max_size_kb_per_pysync  = 10000000
-
-    def __init__(self, mirror_info, logger):
-        logger.debug("in SegHardLinks__init__() %s:%s primary seg id: %s" % (mirror_info.primary_host, mirror_info.primary_data_directory, str(mirror_info.primary_dbid)))
-        self.logger = logger
-        self.theMirror = mirror_info
-        self.hardBaseDir = self.theMirror.primary_data_directory + '/gpupgrademirrorhardbase'
-        
-    #-------------------------------------------------------------------------------
-    def createLinks(self):
-        self.logger.debug('in SegHardLinks.createLinks(). Primary seg id %s' % str(self.theMirror.primary_dbid))
-        cmdComplete = False
-        while cmdComplete == False:
-           baseDir = DirectoryList( name = "gpupgrademirror get directory list under base for primary seg: " + str(self.theMirror.primary_dbid)
-                                  , dirLocation = self.theMirror.primary_data_directory + '/base' 
-                                  , ctxt = REMOTE
-                                  , remoteHost = self.theMirror.primary_host
-                                  )
-           cmdComplete = runAndCheckCommandComplete(baseDir)
-        baseDir.validate()   
-        dirResultList = baseDir.get_result_list()
-        
-        """ Remove any reference to a dir called pgsql_tmp, which may exist in some database directories. """
-        for aDir in dirResultList:
-           dirSuffixIndex = aDir.rfind("/")
-           candidateDir = aDir[dirSuffixIndex + 1:].rstrip()
-           if candidateDir == "pgsql_tmp":
-              dirResultList.remove(aDir)
-
-        
-        """ Create a directory to store all our links to the actual files """
-        cmdComplete = False
-        while cmdComplete == False:
-           rmCmd = MakeDirectoryWithMode( name = 'gpupgrademirror make soft base: %s' % self.hardBaseDir
-                             , directory = self.hardBaseDir
-                             , ctxt = REMOTE
-                             , remoteHost = self.theMirror.primary_host
-                             , mode = 700
-                             )
-           cmdComplete = runAndCheckCommandComplete(rmCmd)
-        rmCmd.validate()
-        
-        for dir in dirResultList:
-            """ For each database directory """
-            self.logger.debug("working on dir %s:%s" % (self.theMirror.primary_host, str(dir)))
-            
-            """ Get a list of all files in the dir and their sizes in k-bytes """
-            cmdComplete = False
-            while cmdComplete == False:
-               dirFileSizeAll = FileAndSize( name = 'gpupgradmirror get list of files and size in database dir'
-                                     , filePattern = str(dir)
-                                     , ctxt = REMOTE
-                                     , remoteHost = self.theMirror.primary_host
-                                     , remoteTempFile = self.theMirror.primary_data_directory + '/gpupgrademirrorhardbase/tempfileandsize'
-                                     )
-               cmdComplete = runAndCheckCommandComplete(dirFileSizeAll)
-            dirFileSizeAll.validate()
-            dbFileListAll = dirFileSizeAll.get_result_list(localTempFile = options.info_data_directory + "/tempfileandsize" + str(self.theMirror.primary_dbid))
-            
-            """ Get a sub-list of files that have zero length """ 
-            cmdComplete = False
-            while cmdComplete == False:
-               dirFileSizeZero = FileAndSize( name = 'gpupgradmirror get list of files and size in database dir'
-                                     , filePattern = str(dir)
-                                     , ctxt = REMOTE
-                                     , remoteHost = self.theMirror.primary_host
-                                     , zeroSize = True
-                                     , remoteTempFile = self.theMirror.primary_data_directory + '/gpupgrademirrorhardbase/tempfileandsize'
-                                     )
-               cmdComplete = runAndCheckCommandComplete(dirFileSizeZero)
-            dirFileSizeZero.validate()           
-            dbFileListZero = dirFileSizeZero.get_result_list(localTempFile = options.info_data_directory + "/tempfileandsize" + str(self.theMirror.primary_dbid))
-            dbFileListZeroLength = len(dbFileListZero) 
-               
-            dbFileListZeroName = []
-            for iii in range(len(dbFileListZero)):
-                dbFileListZeroName.append(dbFileListZero[iii][1])
-            
-            """ Divide the two lists into distinct sub-lists """
-            dbFileNonZeroList = []
-            for candidate in dbFileListAll:
-                if candidate[0] == str(0):
-                   """ size is less than 1k """
-                   if candidate[1] not in dbFileListZeroName:
-                      dbFileNonZeroList.append(candidate)
-                else:
-                   dbFileNonZeroList.append(candidate)
-
-            """ Recombine the lists, putting the zero length file list on the front """
-            dbFileList = dbFileListZero + dbFileNonZeroList
-
-            newDirSuffix = 0
-            fileIndex    = 0
-            dirIndex = dir.rfind('/')
-            theDir = dir[dirIndex + 1:]
-            while fileIndex < len(dbFileList):
-               """ For each file in the database directory """
-               dirPlusSuffix = self.hardBaseDir + "/" + str(theDir) + "." + str(newDirSuffix)
-               
-               cmdComplete = False
-               while cmdComplete == False:
-                  rmCmd = MakeDirectoryWithMode( name = 'gpupgrademirror make soft base sub-database dir: %s' % dirPlusSuffix
-                                    , directory = dirPlusSuffix
-                                    , ctxt = REMOTE
-                                    , remoteHost = self.theMirror.primary_host
-                                    , mode = 700
-                                    )
-                  cmdComplete = runAndCheckCommandComplete(rmCmd)
-               rmCmd.validate()               
-               numFiles    = 0
-               currentSize = 0
-               srcFileList = []
-               hardLinkList = []
-               while  (newDirSuffix == 0 and fileIndex < dbFileListZeroLength) \
-                      or \
-                      (   newDirSuffix != 0 and fileIndex < len(dbFileList) \
-                          and numFiles < self._max_files_per_pysync \
-                          and currentSize < self._max_size_kb_per_pysync \
-                      ):
-                  """ Make groups of the files and create links to the grouped files """
-                  sizeSourceFile     = dbFileList[fileIndex][0]
-                  fullPathSourceFile = dbFileList[fileIndex][1]
-                  subfileIndex = fullPathSourceFile.rfind('/')
-                  theFile = fullPathSourceFile[subfileIndex + 1:]
-                  sLink = dirPlusSuffix + "/" + theFile
-                  srcFileList.append(fullPathSourceFile)
-                  hardLinkList.append(sLink)
-                  numFiles = numFiles + 1
-                  currentSize = currentSize + int(sizeSourceFile)
-                  fileIndex = fileIndex + 1
-                  if newDirSuffix == 0 and fileIndex == dbFileListZeroLength:
-                     zeroFilesFlag = False
-               
-               """ 
-                 Create a file containing a list of ln parameters locally, and the copy it 
-                 to the primary node where we will run ln and create the links.
-               """
-               linkFile = "gpupgrademirror%s.linkfile" % str(self.theMirror.primary_dbid)       
-               sourceFullPathLinkFile = options.info_data_directory + "/" + linkFile
-               targetFullPathLinkFile = self.hardBaseDir + "/" + linkFile
-               linkFP = open(sourceFullPathLinkFile, 'w')
-               linkFP.seek(0)
-               for lnIndex in range(len(srcFileList)):
-                   linkFP.write("%s %s\n" % (srcFileList[lnIndex], hardLinkList[lnIndex]))
-               linkFP.flush()
-               linkFP.close()
-               cmdComplete = False
-               while cmdComplete == False:
-                  linkCopy = RemoteCopyPreserve( name = "gpupgrademirror copy ln file for primary dbid: " + str(self.theMirror.primary_dbid)
-                                       , srcDirectory = sourceFullPathLinkFile
-                                       , dstHost = self.theMirror.primary_host
-                                       , dstDirectory = targetFullPathLinkFile
-                                       , ctxt = LOCAL
-                                       )
-                  cmdComplete = runAndCheckCommandComplete(linkCopy)
-               linkCopy.validate()
-               
-               cmdComplete = False
-               while cmdComplete == False:
-                  link = CreateHardLink( name = "gpupgrademirror creating links for dbid %s directory %s ." % (self.theMirror.primary_dbid, str(newDirSuffix))
-                                    , srcFile = srcFileList
-                                    , hardLink = hardLinkList
-                                    , ctxt = REMOTE
-                                    , remoteHost = self.theMirror.primary_host
-                                    , argsFile = targetFullPathLinkFile
-                                    )
-                  cmdComplete = runAndCheckCommandComplete(link)
-               link.validate()
-
-               cmdComplete = False
-               while cmdComplete == False:
-                  rmCmd = RemoveFiles( name = 'gpupgrademirror remove links file: %s' % (targetFullPathLinkFile)
-                                  , directory = targetFullPathLinkFile
-                                  , ctxt = REMOTE
-                                  , remoteHost = self.theMirror.primary_host
-                                  )
-                  cmdComplete = runAndCheckCommandComplete(rmCmd)
-               rmCmd.validate()              
-               newDirSuffix = newDirSuffix + 1
-        
-    #-------------------------------------------------------------------------------    
-    def removeLinks(self):
-        self.logger.debug("in SegHardLinks.removeLinks %s:%s" % (self.theMirror.primary_host, self.hardBaseDir))
-        
-        cmdComplete = False
-        while cmdComplete == False:
-           cmd = RemoveFiles( name = 'gpupgrademirror remove gpupgrademirrorhardbase dir'
-                         , remoteHost = self.theMirror.primary_host 
-                         , directory = self.hardBaseDir
-                         , ctxt = REMOTE
-                         )
-           cmdComplete = runAndCheckCommandComplete(cmd)
-        cmd.validate()
- 
-    #-------------------------------------------------------------------------------    
-    def getHardDirList(self):
-        """ 
-          Returns a list of the fully qualified link dirs
-          (e.g. [ '/db1/seg1/g0/base/2.1' , '/db1/seg1/g0/base/2.2' ,...] )
-        """
-        retList = [] 
-        cmdComplete = False
-        while cmdComplete == False:
-           baseDir = DirectoryList( name = "gpupgrademirror get directory list under base for primary seg: " + str(self.theMirror.primary_dbid)
-                               , dirLocation = self.hardBaseDir 
-                               , ctxt = REMOTE
-                               , remoteHost = self.theMirror.primary_host
-                               )
-           cmdComplete = runAndCheckCommandComplete(baseDir)
-        baseDir.validate()
-        retList = baseDir.get_result_list()
-        return retList
- 
-    #-------------------------------------------------------------------------------    
-    @staticmethod
-    def getUnqualifiedLinkDir(qualifiedDir):
-        retValue = ''        
-        startIndex = qualifiedDir.rfind("/")
-        retValue = qualifiedDir[startIndex +1:]
-        return retValue
- 
-     #-------------------------------------------------------------------------------    
-    @staticmethod
-    def getUnqualifiedRealDir(qualifiedDir):
-        retValue = ''       
-        startIndex = qualifiedDir.rfind("/")
-        endIndex = qualifiedDir.rfind(".")
-        retValue = qualifiedDir[startIndex +1:endIndex]       
-        return retValue
- 
-#-------------------------------------------------------------------------
-#-------------------------------------------------------------------------
-class PySyncPlus(PySync):
-    """
-    This class is really just PySync but it records all the parameters passed in.
-    """
-
-    def __init__(self,name,srcDir,dstHost,dstDir,ctxt=LOCAL,remoteHost=None, options=None):
-        self.namePlus = name
-        self.srcDirPlus = srcDir
-        self.dstHostPlus = dstHost
-        self.dstDirPlus = dstDir
-        self.ctxtPlus = ctxt
-        self.remoteHostPlus = remoteHost
-        self.optionsPlus = options
-        PySync.__init__( self
-                       , name = name
-                       , srcDir = srcDir
-                       , dstHost = dstHost
-                       , dstDir = dstDir
-                       , ctxt = ctxt
-                       , remoteHost = remoteHost
-                       , options = options 
-                       )
-        self.destinationHost = dstHost
-
-
-#---------

<TRUNCATED>


Mime
View raw message