hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r...@apache.org
Subject incubator-hawq git commit: HAWQ-351. Add movefilespace option to hawq filespace
Date Wed, 27 Jan 2016 02:53:05 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 095368465 -> 71ca7d577


HAWQ-351. Add movefilespace option to hawq filespace


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/71ca7d57
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/71ca7d57
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/71ca7d57

Branch: refs/heads/master
Commit: 71ca7d5777770966d819decf3ea6b771ce9f1f3f
Parents: 0953684
Author: rlei <rlei@pivotal.io>
Authored: Tue Jan 19 11:36:43 2016 +0800
Committer: rlei <rlei@pivotal.io>
Committed: Tue Jan 26 17:42:49 2016 +0800

----------------------------------------------------------------------
 tools/bin/gppylib/operations/filespace.py | 126 +++++++------------------
 tools/bin/hawqfilespace                   | 113 ++++++++++++++++++++--
 2 files changed, 139 insertions(+), 100 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/71ca7d57/tools/bin/gppylib/operations/filespace.py
----------------------------------------------------------------------
diff --git a/tools/bin/gppylib/operations/filespace.py b/tools/bin/gppylib/operations/filespace.py
index eff33dc..c4c0d3b 100644
--- a/tools/bin/gppylib/operations/filespace.py
+++ b/tools/bin/gppylib/operations/filespace.py
@@ -33,6 +33,8 @@ from gppylib.operations import Operation
 from gppylib.operations.utils import RemoteOperation, ParallelOperation
 from gppylib.operations.unix import CheckFile, CheckDir, ListFiles
 from gppylib.gparray import GpArray
+from hawqpylib.hawqarray import HAWQArray
+from hawqpylib.hawqlib import local_ssh, HawqXMLParser, parse_hosts_file
 
 logger = gplog.get_default_logger()
 
@@ -940,28 +942,28 @@ class MoveFileSpaceLocation(Operation):
         self.dburl = dbconn.DbURL(username=self.user, password=self.pswd)
     
     def start_master_only(self):
-        logger.info('Starting Database in master only mode')
-        cmd = GpStart('Start Database in master only mode', masterOnly=True, upgrade=True)
-        cmd.run()
-        if cmd.get_results().rc != 0:
-            logger.error('Failed to start Greenplum Database in master only mode.')
-            cmd.validate()
+        logger.info('Starting Database in upgrade mode')
+        cmd = 'hawq start master -U upgrade'
+        result = local_ssh(cmd, logger)
+        if result != 0:
+            logger.error("Failed to start HAWQ Database in upgrade mode")
+        return result
 
     def stop_master_only(self):
-        logger.info('Stopping Database in master only mode')
-        cmd = GpStop('Stop Database in master only mode', masterOnly=True)
-        cmd.run()
-        if cmd.get_results().rc != 0:
-            logger.error('Failed to stop Greenplum Database in master only mode.')
-            cmd.validate()
+        logger.info('Stopping HAWQ master')
+        cmd = 'hawq stop master -a'
+        result = local_ssh(cmd, logger)
+        if result != 0:
+            logger.error("Failed to stop HAWQ master")
+        return result
 
     def stop_database(self):
-        logger.info('Stopping Greenplum Database')
-        cmd = GpStop('Stop Greenplum Databse')
-        cmd.run()
-        if cmd.get_results().rc != 0:
-            logger.error('Failed to stop Greenplum Database.')
-            cmd.validate()
+        logger.info('Stopping HAWQ cluster')
+        cmd = 'hawq stop cluster -a'
+        result = local_ssh(cmd, logger)
+        if result != 0:
+            logger.error("Failed to stop HAWQ master")
+        return result
 
     def check_database_stopped(self):
         try:
@@ -997,102 +999,40 @@ class MoveFileSpaceLocation(Operation):
             self.fsoid =  filespace[0]
             filespaceRow.close()
             
-            #get segments dbid
-            segmentsDbidRows = dbconn.execSQL(conn, '''
-                            SELECT dbid, content
-                            FROM gp_segment_configuration
-                            WHERE content != -1
-                            ''' )
-            
-            if segmentsDbidRows.rowcount == 0:
-                logger.error('No segment is configured')
-                return None
-            
-            segconf = {}
-            for row in segmentsDbidRows:
-                segconf[row[0]] = row[1]
-                
-            segdbids = segconf.keys()
-                        
-            #get segments prefix and verify consistency of filespace entries
-            entryRows = dbconn.execSQL(conn, '''
-                            SELECT fselocation, fsedbid
-                            FROM pg_filespace_entry 
-                            WHERE fsefsoid = %d and fsedbid in (%s);
-                        ''' % (self.fsoid, ','.join(str(x) for x in segdbids)))
-
-            segmentLocation = []
-            
-            if entryRows.rowcount == 0:
-                logger.error('Cannot get filespace entry for filespace "%s"' % self.fsname)
-                return None
-            
-            targetDbids = []
-            for row in entryRows:
-                segmentLocation.append(row[0]);
-                targetDbids.append(row[1])
-            
-            entryRows.close()
-
-            prefixs = [os.path.split(loc)[1] for loc in segmentLocation]
-            self.prefix = os.path.commonprefix(prefixs)
-            
-            if len(self.prefix) == 0:
-                logger.error('Cannot get segments prefix for filespace "%s"' % self.fsname)
-                return None
-            
-            schemes = [urlparse.urlparse(loc).scheme for loc in segmentLocation]
-            
-            if len(set(schemes)) != 1:
-                logger.warn('The scheme of segment are not consistent')
-                
-            targetScheme = urlparse.urlparse(self.location).scheme
-            
-            if len(targetScheme) == 0:
-                logger.error('Target location: "%s" is not a valid url' % self.location)
-                return None
-            
-            if targetScheme not in set(schemes):
-                logger.error("Target location's scheme: \"%s\" does not match file system
requirement", targetScheme)
-                return None
-            
             #remove last slash
             if self.location[-1] is '/':
                 self.location = self.location[:-1]
                 
             queries = []
             #generate sql for pg_filespace_entry
-            for dbid in targetDbids:
-                uri = '%s/%s%d' % (self.location, self.prefix, segconf[dbid])
-                queries.append('''
-                        UPDATE pg_filespace_entry 
-                        SET fselocation = '%s' 
-                        WHERE fsefsoid = %d 
-                        AND fsedbid = %d;
-                        ''' %(uri, self.fsoid, dbid))
+            uri = self.location
+            queries.append('''
+                    UPDATE pg_filespace_entry 
+                    SET fselocation = '%s' 
+                    WHERE fsefsoid = %d; 
+                    ''' %(uri, self.fsoid))
             
             #get gp_persistent_filespace
             rows = dbconn.execSQL(conn, '''
-                        SELECT contentid, location_1
+                        SELECT location
                         FROM gp_persistent_filespace_node 
-                        WHERE filespace_oid = %d and contentid <> -1
+                        WHERE filespace_oid = %d;
                     ''' % self.fsoid)
             
             #generate sql for gp_persistent_filespace
             for row in rows:
-                uri = '%s/%s%d' % (self.location, self.prefix, row[0])
+                uri = '%s' % (self.location)
                 
-                paddinglen = len(row[1])
+                paddinglen = len(row[0])
                 if len(uri) > paddinglen:
                     logger.error('target location "%s" is too long' % self.location)
                     return None
                 
                 queries.append('''
                         UPDATE gp_persistent_filespace_node 
-                        SET location_1 = '%s' 
-                        WHERE filespace_oid = %d 
-                        AND contentid = %d;
-                        ''' %(uri.ljust(paddinglen), self.fsoid, row[0]))
+                        SET location = '%s' 
+                        WHERE filespace_oid = %d; 
+                        ''' %(uri.ljust(paddinglen), self.fsoid))
             
             return queries
         

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/71ca7d57/tools/bin/hawqfilespace
----------------------------------------------------------------------
diff --git a/tools/bin/hawqfilespace b/tools/bin/hawqfilespace
index ab9dc9c..ed57002 100755
--- a/tools/bin/hawqfilespace
+++ b/tools/bin/hawqfilespace
@@ -35,6 +35,10 @@ Execution Options:
   -c | --config FILE         configuration file
   -o | --output DIRECTORY    output directory for config file
 
+Move Options:
+  --movefilespace FILESPACE | default move filespace to new location on distributed file
system
+  --location <dfslocation>    new location which the filespace will be moved to
+
 '''
 # ============================================================================
 __version__ = '$Revision$'
@@ -68,6 +72,7 @@ try:
     from getpass import getpass
     from gppylib.parseutils import line_reader, parse_fspacename, parse_dfs_url, parse_fspacesys,
parse_fspacereplica, parse_gpfilespace_line, \
         canonicalize_address
+    from gppylib.operations.filespace import MoveFilespaceError, MoveFileSpaceLocation, CheckSuperUser
 
 except ImportError, e:
     sys.exit('Error: unable to import module: ' + str(e))
@@ -146,6 +151,8 @@ def ParseOptions():
                       action='store_false', help=optparse.SUPPRESS_HELP)
     parser.add_option('-q', '--quiet',
                       action='store_true', help=optparse.SUPPRESS_HELP)
+    parser.add_option('--movefilespace')
+    parser.add_option('--location')
 
     (options, args) = parser.parse_args()
 
@@ -215,6 +222,21 @@ def getstring(prompt, question=False, default=""):
         sys.exit(1)
 
 # ============================================================================
+# ============================================================================
+def getbool(prompt, question, default):
+    '''
+    getbool(prompt, question, default)
+
+    Prompts the user for input (yes|no), returns True/False
+    '''
+    while True:
+        value = getstring(prompt, question, default).lower()
+        if value in ("y", "ye", "yes"):
+            return True
+        elif value in ("n", "no"):
+            return False
+
+
 def getdir(prompt, hosts=[], primary=None, shared=False, fsysn=None, db=None):
     '''
     getdir(prompt, hosts)
@@ -694,12 +716,89 @@ if __name__ == '__main__':
     else:
         opt.password = None
 
-    gpf = GPFilespace(opt)
+
     try:
-        gpf.Setup()
-        if opt.config == None:
-            gpf.generate_config()
+        if opt.movefilespace:
+            logger.debug("Start filespace move")
+            logger.info("""
+A tablespace requires a file system location to store its database
+files. A filespace is a collection of file system locations for all components
+in a HAWQ system.
+Once a filespace is created, it can be used by one or more tablespaces.
+
+""")
+            if opt.movefilespace == 'default':
+                opt.movefilespace = "dfs_system"
+
+            if opt.location is None or len(opt.location) == 0:
+                usage()
+                logger.error("Target location is not set");
+                sys.exit(1)
+
+            logger.warn('PLEASE STOP DATABASE AND BACKUP MASTER DATA DIRECTORY IN CASE OF
LOSING DATA!')
+
+            question = "I understand the risk and already backuped master data directory?
 (Y|N)"
+            if not getbool("> ", question, "N"):
+                raise KeyboardInterrupt()
+
+            MoveFileSpaceLocation(opt.movefilespace, opt.location, opt.username, opt.password).run()
+
+            logger.info("Move filespace successfully.")
+            logger.info("IF STANDBY MASTER IS CONFIGURED, PLEASE REMOVE IT AND INITIALIZE
A NEW ONE!")
         else:
-            gpf.execute_config()
-    finally:
-        gpf.cleanup()
+            gpf = GPFilespace(opt)
+            try:
+                gpf.Setup()
+                if opt.config == None:
+                    gpf.generate_config()
+                else:
+                    gpf.execute_config()
+            finally:
+                gpf.cleanup()
+
+    except MoveFilespaceError, e:
+        logger.error(e)
+        sys.exit(1)
+    # An expected error
+    except GPError, e:
+        try:
+            # Cast to unicode using UTF-8 to work around issues with the
+            # logging module:
+            #    logger.error("퀙퀠퀩퀘ㄲㄸㅉㅃ12ㅁavb")  => UnicodeDecodeError
+            err = unicode(str(e), 'UTF-8')
+            logger.error(err)
+            logger.debug("exit(1)")
+        except BaseException, e:
+            sys.stderr.write("[Error] %s\n" % str(e))
+            sys.stderr.write(traceback.format_exc())
+            sys.stderr.flush()
+        sys.exit(1)
+
+    # User cancelation
+    except KeyboardInterrupt, e:
+        try:
+            logger.info('[Cancelled]')
+        except BaseException, e:
+            sys.stderr.write("error %s\n" % str(e))
+        sys.exit(1)
+
+    # Let SystemExit exceptions through
+    except SystemExit, e:
+        try:
+            logger.debug('exit(%s)' % str(e))
+        except BaseException, e:
+            sys.stderr.write("error %s\n" % str(e))
+        raise e
+
+    # Catch anything else - shouldn't ever occur
+    except BaseException, e:
+        try:
+            err = '[%s]: "%s"' % (type(e).__name__,  unicode(str(e), 'UTF-8'))
+            logger.error(err)
+            logger.debug(traceback.format_exc())
+            logger.debug("exit(1)")
+
+            sys.stderr.write('[Error] See "%s" for details\n' % get_logfile())
+        except BaseException, e:
+            sys.stderr.write("error %s\n" % str(e))
+        sys.exit(1)


Mime
View raw message