hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From h...@apache.org
Subject [1/2] incubator-hawq git commit: HAWQ-247. Remove deprecated GUCs related to FTS
Date Mon, 14 Dec 2015 02:19:36 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 1500475b3 -> c90ba7cf7


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c90ba7cf/tools/bin/gppylib/system/faultProberImplGpdb.py
----------------------------------------------------------------------
diff --git a/tools/bin/gppylib/system/faultProberImplGpdb.py b/tools/bin/gppylib/system/faultProberImplGpdb.py
deleted file mode 100644
index a819b4f..0000000
--- a/tools/bin/gppylib/system/faultProberImplGpdb.py
+++ /dev/null
@@ -1,93 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-# 
-#   http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-
-Note that this does NOT pay attention to the current pause state in the db.  It can't do
that right now
-  anyway because gp_fts_probe_pause does not reflect the shared memory variable?
-
-"""
-import os
-import time
-import re
-
-from gppylib.gplog import *
-from gppylib.utils import checkNotNone
-from gppylib.system.faultProberInterface import GpFaultProber
-from gppylib.db import dbconn, catalog
-
-logger = get_default_logger()
-
-class GpFaultProberImplGpdb(GpFaultProber):
-    def __init__(self):
-        self.__isPaused = False
-        self.__masterDbUrl = None
-        self.__conn = None
-
-    #
-    # returns self
-    #
-    def initializeProber( self, masterPort ) :
-        self.__masterDbUrl = dbconn.DbURL(port=masterPort, dbname='template1')
-        return self
-
-    def pauseFaultProber(self):
-        assert not self.__isPaused
-        assert self.__masterDbUrl is not None # must be initialized
-        assert self.__conn is None
-
-        logger.debug("Pausing fault prober")
-        
-        self.__conn = dbconn.connect(self.__masterDbUrl, True) # use utility mode so we don't
do any segment connection stuff
-        dbconn.execSQL( self.__conn, "set gp_fts_probe_pause = on")
-
-        self.__isPaused = True
-
-    def unpauseFaultProber(self):
-        assert self.__isPaused
-        assert self.__masterDbUrl is not None  # must be initialized
-        assert self.__conn is not None
-
-        logger.debug("Unpausing fault prober")
-
-        self.__conn.close() # closing connection will rollback GUC setting and so unpause
prober
-        self.__conn = None
-        self.__isPaused = False
-
-    def isFaultProberPaused(self):
-        return self.__isPaused
-
-    def getFaultProberInterval(self):
-        probe_interval_re = re.compile(r'(?P<val>\d+)(?P<unit>[a-zA-Z]*)')
-        probe_interval_secs = 60
-        
-        conn = None
-        
-        try:
-            conn = dbconn.connect(self.__masterDbUrl, True)
-            fts_probe_interval_value = catalog.getSessionGUC(conn, 'gp_fts_probe_interval')
-            m = probe_interval_re.match(fts_probe_interval_value)
-            if m.group('unit') == 'min':
-                probe_interval_secs = int(m.group('val')) * 60
-            else:
-                probe_interval_secs = int(m.group('val'))
-        except:
-            raise
-        finally:
-            if conn:
-                conn.close()
-                
-        return probe_interval_secs

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c90ba7cf/tools/bin/lib/gp_bash_functions.sh
----------------------------------------------------------------------
diff --git a/tools/bin/lib/gp_bash_functions.sh b/tools/bin/lib/gp_bash_functions.sh
index 4c000bf..6443245 100755
--- a/tools/bin/lib/gp_bash_functions.sh
+++ b/tools/bin/lib/gp_bash_functions.sh
@@ -1924,42 +1924,6 @@ CHK_START_ERROR_TXT () {
 	LOG_MSG "[INFO]:-End Function $FUNCNAME"
 }
 
-# Make a dbid file at a particular host. The dbid file is used by gpstart
-# to tell the process in question which segment/master it is.
-# Arguments:
-#   1 - DBID
-#   2 - host name
-#   3 - path to data directory
-MAKE_DBID_FILE() {
-	DBID=$1; shift
-	HOST=$1; shift
-	DATADIR=$1; shift
-
-	FILEPATH=$DATADIR/gp_dbid
-
-	if [ "$FILEPATH" = "/gp_dbid" ]; then # DATADIR is empty
-		ERROR_EXIT "[FATAL]:-Internal error -- expected non-empty data directory" 2
-	fi
-
-	$TRUSTED_SHELL $HOST \
-	  "$ECHO \"# Greenplum Database identifier for this master/segment.
-# Do not change the contents of this file.
-dbid = $DBID\" > $FILEPATH &&  chmod 400 $FILEPATH"
-}
-
-MAKE_CONTENTNUM_FILE() {
-	CONTENT_NUM=$1; shift
-	DATADIR=$1; shift
-
-	FILEPATH=$DATADIR/gp_contentnum
-	if [ "$FILEPATH" = "/gp_contentnum" ]; then # DATADIR is empty
-		ERROR_EXIT "[FATAL]:-Internal error -- expected non-empty data directory" 2
-	fi
-
-	$ECHO "# Greenplum Database content num for master/standby.
-# Do not change the contents of this file.
-content_num = $CONTENT_NUM" > "$FILEPATH" && chmod 600 "$FILEPATH"
-}
 
 UPDATE_MPP () {
 	LOG_MSG "[INFO][$INST_COUNT]:-Start Function $FUNCNAME"

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c90ba7cf/tools/bin/lib/gpconfigurenewsegment
----------------------------------------------------------------------
diff --git a/tools/bin/lib/gpconfigurenewsegment b/tools/bin/lib/gpconfigurenewsegment
deleted file mode 100755
index b6612c7..0000000
--- a/tools/bin/lib/gpconfigurenewsegment
+++ /dev/null
@@ -1,404 +0,0 @@
-#!/usr/bin/env python
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-# 
-#   http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import sys
-import os
-from optparse import Option, OptionGroup, OptionParser, OptionValueError, SUPPRESS_USAGE
-
-from gppylib.gpparseopts import OptParser, OptChecker
-from gppylib.commands.gp import ModifyPostgresqlConfSetting, SegmentStart, SegmentStop
-from gppylib.db import dbconn
-from gppylib.db import catalog
-from gppylib.commands import unix
-from gppylib.commands.pg import DbStatus
-from gppylib import gplog
-from gppylib.commands.base import *
-from gppylib.gparray import GpDB
-from gppylib.utils import canStringBeParsedAsInt
-from gppylib.gp_dbid import writeGpDbidFile
-from time import sleep
-from gppylib.operations.buildMirrorSegments import gDatabaseDirectories, gDatabaseFiles
-from gppylib.operations.filespace import update_temporary_directories
-
-description = (""" 
-
-Configure segment directories for installation into a pre-existing GPDB array.
-
-Used by at least gpexpand, gprecoverseg, and gpaddmirrors
-
-""")
-
-DEFAULT_BATCH_SIZE=8
-EXECNAME = os.path.split(__file__)[-1]
-
-DESTINATION_FILE_SPACES_DIRECTORY = "fs_directory"
-
-class ValidationException(Exception):
-
-    def __init__(self, msg):
-        self.__msg = msg
-        Exception.__init__(self, msg)
-
-    def getMessage(self):
-        return self.__msg
-
-class ConfExpSegCmd(Command):
-    def __init__(self, name, cmdstr, datadir, port, dbid, content, newseg, tarfile, useLighterDirectoryValidation,
isPrimary,
-                    filespaceOids, filespaceDirs, verbose, validationOnly, writeGpDbidFileOnly):
-        """
-        @param useLighterDirectoryValidation if True then we don't require an empty directory;
we just require that
-                                             database stuff is not there.
-        """
-        self.datadir = datadir
-        self.port = int(port)
-        self.dbid = dbid
-        self.content = content
-        self.newseg = newseg
-        self.tarfile = tarfile
-        self.verbose = verbose
-        self.useLighterDirectoryValidation = useLighterDirectoryValidation
-        self.filespaceOids = filespaceOids
-        self.filespaceDirs = filespaceDirs
-        self.isPrimary = isPrimary
-
-        #
-        # validationOnly if True then we validate directories and other simple things only;
we don't
-        #                actually configure the segment
-        #
-        self.validationOnly = validationOnly
-
-        self.writeGpDbidFileOnly = writeGpDbidFileOnly
-        Command.__init__(self, name, cmdstr)
-
-
-    def __str__(self):
-        if self.results:
-            return "%s cmdStr='%s'  had result: %s" % (self.name, self.cmdStr, self.results)
-        else:
-            return "%s cmdStr='%s'" % (self.name, self.cmdStr)
-
-    def validatePath(self, path, isSystemFilespaceDir):
-        """
-        Raises ValidationException when a validation problem is detected
-        """
-
-        # This is special case, we need some method to check this director.
-        if path.find('://') != -1:
-            return
-
-        if not os.path.exists(os.path.dirname(path)):
-            raise ValidationException("Parent directory for %s directory '%s' does not exist"
%
-                    ( "system data" if isSystemFilespaceDir else "filespace", path) )
-
-        if not os.path.exists(path):
-            #
-            # dir doesn't exist is okay -- other scripts, or filerep mirroring code, will
create it
-            #
-            return
-
-        if self.useLighterDirectoryValidation:
-
-            if isSystemFilespaceDir:
-                # validate that we don't contain database directories or files
-                for toCheck in gDatabaseDirectories:
-                    if toCheck != "pg_log": # it's okay to have pg_log -- to save old logs
around to look at
-                        if os.path.exists(os.path.join(path, toCheck)):
-                            raise ValidationException("Segment directory '%s' contains directory
%s but should not!" %
-                                        (path, toCheck))
-                for toCheck in gDatabaseFiles:
-                    if os.path.exists(os.path.join(path, toCheck)):
-                        raise ValidationException("Segment directory '%s' contains file %s
but should not!" %
-                                    (path, toCheck))
-            else:
-
-                for name in os.listdir( path ):
-                    if name[0] in "0123456789": # the database files here will have  
-                        raise ValidationException("Filespace directory contains invalid file(s):
'%s'" % path)
-        else:
-            # it better be empty
-            if len(os.listdir(path)) != 0:
-                raise ValidationException("Segment directory '%s' exists but is not empty!"
% path)
-
-    def makeOrUpdatePathAsNeeded(self, path):
-        if os.path.exists(path):
-            os.chmod(path, 0700)
-        else:
-            os.mkdir(path, 0700)
-            
-    def fixupFilespaces(self):
-        """
-        This method will take file spaces stored in the system data directory and put them
in their proper locations.
-        """
-        if self.filespaceOids == None:
-            return
-        try:
-            logger.info("copy filespaces to their locations")
-           # only on primary do we create it -- otherwise, mirroring code creates it on replay
-            filespaceDataDir = self.datadir + "/" + DESTINATION_FILE_SPACES_DIRECTORY
-            i = 0
-            for i in range(len(self.filespaceOids)):
-                sourceDir = filespaceDataDir + "/" + str(self.filespaceOids[i])
-                targetDir = self.filespaceDirs[i]
-                cpCmd = unix.LocalDirCopy("copy filespace from %s to %s" % (sourceDir, targetDir),
sourceDir, targetDir)
-                cpCmd.run(validateAfter=True)
-            unix.RemoveFiles.local("remove filespace template after copy to proper location",
filespaceDataDir)
-        except Exception, e:
-            self.set_results(CommandResult(1, '', e, True, False))
-            raise
-
-    def run(self):
-        try:
-            if self.writeGpDbidFileOnly:
-                logger.info("create gp_dbid file for segment")
-                writeGpDbidFile(self.datadir, self.dbid, logger=gplog.get_logger_if_verbose())
-            else:
-                if self.newseg:
-                    # make directories, extract template update postgresql.conf
-                    logger.info("Validate data directories for new segment")
-                    try:
-                        self.validatePath(self.datadir, True)
-                    except ValidationException, e:
-                        msg = "for segment with port %s: %s" %  (self.port, e.getMessage())
-                        self.set_results(CommandResult(1, '', msg, True, False))
-                        raise
-                    except Exception, e:
-                        self.set_results(CommandResult(1, '', e, True, False))
-                        raise
-
-                    if self.validationOnly:
-                        self.set_results(CommandResult(0, '', '', True, False))
-                        return
-
-                    logger.info("Create or update data directories for new segment")
-                    try:
-                        self.makeOrUpdatePathAsNeeded(self.datadir)
-                    except Exception, e:
-                        self.set_results(CommandResult(1,'',e,True,False))
-                        raise
-
-                    logger.info("extract tar file to new segment")
-                    extractTarCmd = unix.ExtractTar('extract tar file to new segment', self.tarfile,
self.datadir )
-                    try:
-                        logger.debug('Extracting tar file %s to %s' % (self.tarfile, self.datadir))
-                        extractTarCmd.run(validateAfter=True)
-                    except:
-                        self.set_results(extractTarCmd.get_results())
-                        logger.error(extractTarCmd.get_results())
-                        raise
-
-                    logger.info("create gp_dbid file for segment")
-                    writeGpDbidFile(self.datadir, self.dbid, logger=gplog.get_logger_if_verbose())
-
-                    logger.info('create temporary direcotries')
-                    update_temporary_directories(self.datadir, self.content)
-
-                    logger.info("Updating %s/postgresql.conf" % self.datadir)
-                    modifyPostgresqlConfCmd = ModifyPostgresqlConfSetting('Updating %s/postgresql.conf'
% self.datadir,
-                                                                          self.datadir +
'/postgresql.conf',
-                                                                          'port', self.port,
optType='number')
-                    try:
-                        modifyPostgresqlConfCmd.run(validateAfter=True)
-                    except:
-                        self.set_results(modifyPostgresqlConfCmd.get_results())
-                        raise
-
-                # We might need to stop the segment if the last setup failed past this point
-                if os.path.exists('%s/postmaster.pid' % self.datadir):
-                    logger.info('%s/postmaster.pid exists.  Stopping segment' % self.datadir)
-                    stopCmd = SegmentStop('Stop new segment', self.datadir)
-                    try:
-                        stopCmd.run(validateAfter=True)
-                    except:
-                        pass
-                    
-        except Exception, e:
-            self.set_results(CommandResult(1, '', e, True, False))
-            if self.verbose:
-                logger.exception(e)
-            return
-        else:
-            self.set_results(CommandResult(0, '', '', True, False))
-
-    def set_results(self, results):
-        self.results = results
-        
-    def get_results(self):
-        return self.results
-
-    def cancel(self):
-        self.exec_context.cancel()    
-    
-    def interrupt(self):
-        self.exec_context.interrupt()
-    
-    def was_successful(self):
-        if self.results is None:
-            return False
-        else:
-            return self.results.wasSuccessful()
-    
-        
-    def validate(self):
-        """Plain vanilla validation which expects a 0 return code."""        
-        if self.results.rc != 0:
-            raise ExecutionError("non-zero rc: %d" % self.results.rc, self)  
-
-def getOidDirLists(oidDirs):
-    """ break up list in <oid>:<dir> format into list of oid list of dir """
-    oidList = []
-    dirList = []
-    i = 0
-    while i < len(oidDirs):
-        oidList.append(oidDirs[i])
-        i = i + 1
-        dirList.append(oidDirs[i])
-        i = i + 1
-    return oidList, dirList
-        
-
-def parseargs():
-    parser = OptParser(option_class=OptChecker,
-                description=' '.join(description.split()),
-                version='%prog version $Revision: $')
-    parser.set_usage('%prog is a utility script used by gpexpand, gprecoverseg, and gpaddmirrors
and is not intended to be run separately.')
-    parser.remove_option('-h')
-    
-    parser.add_option('-v','--verbose', action='store_true', help='debug output.')
-    parser.add_option('-c', '--confinfo', type='string')                        
-    parser.add_option('-t', '--tarfile', type='string')
-    parser.add_option('-n', '--newsegments', action='store_true')
-    parser.add_option('-B', '--batch-size', type='int', default=DEFAULT_BATCH_SIZE, metavar='<batch_size>')
-    parser.add_option("-V", "--validation-only", dest="validationOnly", action='store_true',
default=False)
-    parser.add_option("-W", "--write-gpid-file-only", dest="writeGpidFileOnly", action='store_true',
default=False)
-
-    parser.set_defaults(verbose=False, filters=[], slice=(None, None))
-    
-    # Parse the command line arguments
-    (options, args) = parser.parse_args()
-
-    if not options.confinfo:
-        raise Exception('Missing --confinfo argument.')
-
-    if options.batch_size <= 0:
-        logger.warn('batch_size was less than zero.  Setting to 1.')
-        batch_size = 1
-
-    if options.newsegments and not options.tarfile:
-        raise Exception('-n option requires -t option to be specified')
-
-    if options.validationOnly and options.writeGpidFileOnly:
-        raise Exception('Only one of --validation-only and --write-gpid-file-only can be
specified')
-
-    seg_info = []
-    conf_lines = options.confinfo.split(',')
-    for line in conf_lines:
-        conf_vals = line.split('|')
-        if len(conf_vals) < 5:
-            raise Exception('Invalid configuration value: %s' % conf_vals)
-        if conf_vals[0] == '':
-            raise Exception('Missing data directory in: %s' % conf_vals)
-        if not canStringBeParsedAsInt(conf_vals[1]) or conf_vals[1] < 1024:
-            raise Exception('Invalid port in: %s' % conf_vals)
-        if conf_vals[2] != 'true' and conf_vals[2] != 'false':
-            raise Exception('Invalid isPrimary option in: %s' % conf_vals)
-        if conf_vals[3] != 'true' and conf_vals[3] != 'false':
-            raise Exception('Invalid directory validation option in: %s' % conf_vals)
-        if not canStringBeParsedAsInt(conf_vals[4]):
-            raise Exception('Invalid dbid option in: %s' % conf_vals)
-        seg_info.append(conf_vals)
-
-    seg_info_len = len(seg_info)
-    if seg_info_len == 0:
-        raise Exception('No segment configuration values found in --confinfo argument')
-    elif seg_info_len < options.batch_size:
-        # no need to have more threads than segments
-        options.batch_size = seg_info_len
-
-    return options, args, seg_info
-
-
-try:
-    logger = gplog.setup_tool_logging(EXECNAME, unix.getLocalHostname(), unix.getUserName())
-
-    (options, args, seg_info) = parseargs()
-    if options.verbose:
-        gplog.enable_verbose_logging()
-        
-    logger.info("Starting gpconfigurenewsegment with args: %s" % ' '.join(sys.argv[1:]))
-    
-    pool = WorkerPool(numWorkers=options.batch_size)
-    
-    for seg in seg_info:
-        dataDir = seg[0]
-        port = seg[1]
-        isPrimary = seg[2] == "true"
-        directoryValidationLevel = seg[3] == "true"
-        dbid = int(seg[4])
-        content = int(seg[5])
-        #if len(seg) >= 7:
-        #    filespaceOids, filespaceDirs = getOidDirLists(seg[6].split('+'))
-        #else:
-        #    filespaceOids, filespaceDirs = [], []
-        filespaceOids, filespaceDirs = [], []
-
-        cmd = ConfExpSegCmd( name = 'Configure segment directory'
-                           , cmdstr = ' '.join(sys.argv)
-                           , datadir = dataDir
-                           , port = port
-                           , dbid = dbid
-                           , content = content
-                           , newseg = options.newsegments
-                           , tarfile = options.tarfile
-                           , useLighterDirectoryValidation = directoryValidationLevel
-                           , isPrimary = isPrimary
-                           , filespaceOids = filespaceOids
-                           , filespaceDirs = filespaceDirs
-                           , verbose = options.verbose
-                           , validationOnly = options.validationOnly
-                           , writeGpDbidFileOnly = options.writeGpidFileOnly
-                           )
-        pool.addCommand(cmd)
-
-    pool.join()
-
-    if options.validationOnly:
-        errors = []
-        for item in pool.getCompletedItems():
-            if item.get_results().rc != 0:
-                errors.append(str(item.get_results().stderr).replace("\n", " "))
-
-        if errors:
-            print >> sys.stderr, "\n".join(errors)
-            sys.exit(1)
-        else: sys.exit(0)
-    else:
-        try:
-            pool.check_results()
-        except Exception, e:
-            if options.verbose:
-                logger.exception(e)
-            logger.error(e)
-            print >> sys.stderr, e
-            sys.exit(1)
-
-    sys.exit(0)
-
-except Exception, msg:
-    logger.error(msg)
-    print >> sys.stderr, msg
-    sys.exit(1)


Mime
View raw message