hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject incubator-hawq git commit: HAWQ-1480 - Added feature for packing a core file
Date Tue, 13 Jun 2017 02:07:34 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master b7f9f36f5 -> ca53f37df


HAWQ-1480 - Added feature for packing a core file


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ca53f37d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ca53f37d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ca53f37d

Branch: refs/heads/master
Commit: ca53f37df70a383fc7f107ac75de1015d9e22467
Parents: b7f9f36
Author: Shubham Sharma <shubham.uconn@gmail.com>
Authored: Mon Jun 5 21:32:15 2017 -0700
Committer: Wen Lin <wlin@pivotal.io>
Committed: Tue Jun 13 10:05:52 2017 +0800

----------------------------------------------------------------------
 tools/sbin/packcore | 261 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 261 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ca53f37d/tools/sbin/packcore
----------------------------------------------------------------------
diff --git a/tools/sbin/packcore b/tools/sbin/packcore
new file mode 100755
index 0000000..6c04672
--- /dev/null
+++ b/tools/sbin/packcore
@@ -0,0 +1,261 @@
+#!/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Merged from https://github.com/greenplum-db/gpdb/blob/master/gpMgmt/sbin/packcore
+
+'''
+USAGE:  ./packcore -b|--binary BINARY COREFILE
+        where BINARY is the path to postgres binary
+        where COREFILE is the core file
+'''
+
+import glob
+import os
+import re
+import string
+import shutil
+import stat
+import sys
+from optparse import OptionParser
+from subprocess import Popen, PIPE
+
+
+def _getPlatformInfo():
+    if which('lsb_release') is None:
+        for file in glob.glob('/etc/*release'):
+            shutil.copy(file, '.')
+    else:
+        Popen('/usr/bin/lsb_release -a > ./lsb_release.out', shell=True)
+
+    Popen('uname -r > uname.out', shell=True)
+
+
+def _getFileInfo(coreFile):
+    cmd = Popen('/usr/bin/file ' + coreFile, shell=True, stdout=PIPE)
+    return cmd.communicate()[0]
+
+
+def _isCore(fileCmdOutput):
+    if fileCmdOutput.find('LSB core file') is -1:
+        return False
+    return True
+
+
+def _findBinary(fileCmdOutput):
+    start = string.find(fileCmdOutput, "'") + 1
+    end = string.find(fileCmdOutput, "'", start)
+    cmd = fileCmdOutput[start:end].split()[0].translate(None, string.punctuation)
+    if os.path.isabs(cmd):
+        return cmd
+    return which(cmd)
+
+
+def _getLibraryListWithLDD(binary):
+    # We manually seed this with few libraries that are missed
+    # This may not be needed for all processes, but will round out the
+    # postgres binary debugging
+    # TODO: Identify methods to distinguish a 32 vs. 64 bit executable
+    libraries = ['/lib64/libgcc_s.so.1', '/lib64/libnss_files.so.2', '/lib/libgcc_s.so.1',
'/lib/libnss_files.so.2']
+    ldd_output = Popen('ldd `which postgres`', shell=True, stdout=PIPE)
+    for line in ldd_output.stdout:
+        match = re.search(r'(\S+) \(0x', line)
+        if match and match.group(1):
+            libraries.append(match.group(1).strip())
+    return libraries
+
+
+def _getLibraryListWithGDB(coreFile, binary):
+    gdb = which('gdb')
+    if gdb is None:
+        return False
+
+    libraries = []
+    # fix for issues with PYTHONPATH and PYTHONHOME
+    environ = os.environ.copy()
+    for key in ('PYTHONHOME', 'PYTHONPATH', 'LD_LIBRARY_PATH'):
+        if key in environ:
+            del environ[key]
+    cmd = Popen(gdb + ' --eval-command="quit" ' + binary + ' -c ' + coreFile, shell=True,
stdout=PIPE, stderr=PIPE)
+    result = cmd.communicate()[0]
+
+    for line in result.splitlines():
+        if line.find('Reading symbols') is 0:
+            end = line.find('...')
+            libraries.append(line[21:end])
+
+    return libraries
+
+
+def _copyFilePath(src, dst):
+    srcDir = os.path.dirname(src)
+    if srcDir.find('/') is 0:
+        srcDir = srcDir[1:]
+    dstDir = os.path.join(dst, srcDir)
+    if not os.path.exists(dstDir):
+        os.makedirs(dstDir)
+    shutil.copy(src, dstDir)
+
+
+def _generateGDBScript(b, c):
+    with open('runGDB.sh', 'w') as f1:
+        print >>f1, '#!/bin/bash'
+        print >>f1, 'unset PYTHONHOME'
+        print >>f1, 'unset PYTHONPATH'
+        print >>f1, 'curDIR=`pwd`'
+        print >>f1, '/usr/bin/gdb --eval-command="set sysroot $curDIR" --eval-command="core
%s" %s' % (c, b)
+    os.chmod('runGDB.sh', 0755)
+
+
+# This is taken from Python 3.3:
+def which(cmd, mode=os.F_OK | os.X_OK, path=None):
+    """Given a command, mode, and a PATH string, return the path which
+    conforms to the given mode on the PATH, or None if there is no such
+    file.
+
+    `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
+    of os.environ.get("PATH"), or can be overridden with a custom search
+    path.
+
+    """
+    # Check that a given file can be accessed with the correct mode.
+    # Additionally check that `file` is not a directory, as on Windows
+    # directories pass the os.access check.
+    def _access_check(fn, mode):
+        return (os.path.exists(fn) and os.access(fn, mode)
+                and not os.path.isdir(fn))
+
+    # If we're given a path with a directory part, look it up directly rather
+    # than referring to PATH directories. This includes checking relative to the
+    # current directory, e.g. ./script
+    if os.path.dirname(cmd):
+        if _access_check(cmd, mode):
+            return cmd
+        return None
+
+    if path is None:
+        path = os.environ.get("PATH", os.defpath)
+        if not path:
+            return None
+        path = path.split(os.pathsep)
+
+    if sys.platform == "win32":
+        # The current directory takes precedence on Windows.
+        if not os.curdir in path:
+            path.insert(0, os.curdir)
+
+        # PATHEXT is necessary to check on Windows.
+        pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
+        # See if the given file matches any of the expected path extensions.
+        # This will allow us to short circuit when given "python.exe".
+        # If it does match, only test that one, otherwise we have to try
+        # others.
+        if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
+            files = [cmd]
+        else:
+            files = [cmd + ext for ext in pathext]
+    else:
+            # On other platforms you don't have things like PATHEXT to tell you
+            # what file suffixes are executable, so just pass on cmd as-is.
+        files = [cmd]
+
+    seen = set()
+    for dir in path:
+        normdir = os.path.normcase(dir)
+        if not normdir in seen:
+            seen.add(normdir)
+            for thefile in files:
+                name = os.path.join(dir, thefile)
+                if _access_check(name, mode):
+                    return name
+    return None
+
+
+def packCoreFile(coreFile, binary):
+    packDir = './packcore-' + os.path.basename(coreFile)
+    oldDir = os.getcwd()
+    try:
+        os.mkdir(packDir)
+        os.chdir(packDir)
+        shutil.copy(coreFile, '.')
+        _getPlatformInfo()
+        shutil.copy(binary, '.')
+
+        libraries = _getLibraryListWithGDB(coreFile, binary)
+
+        if libraries is False:
+            libraries = _getLibraryListWithLDD(binary)
+
+        for lib in libraries:
+            try:
+                _copyFilePath(lib, '.')
+            except IOError:
+                continue
+
+        _generateGDBScript(os.path.basename(binary), os.path.basename(coreFile))
+        os.chdir(oldDir)
+        cmd = Popen('tar zcf packcore-' + os.path.basename(coreFile) + '.tgz ' + packDir,
shell=True)
+        cmd.wait()
+    finally:
+        os.chdir(oldDir)
+        Popen('rm -rf ' + packDir, shell=True)
+
+
+def parseArgs():
+    u = '''%prog [options] core_file
+This will create an archive with the core file and all required
+libraries for analysis.  The preference is to use GDB so that we can
+resolve dependencies for extensions.'''
+
+    parser = OptionParser(version='%prog: 0.4beta', usage=u)
+    parser.add_option('-b', '--binary', action='store', type='string', dest='binary', metavar='PROGRAMME',
help='The full path to the binary that created the core file.  Used when packcore cannot determine
the source binary')
+    (option, args) = parser.parse_args()
+    if len(args) != 1:
+        parser.error('Please specify a core file')
+        sys.exit(1)
+    return (option, args)
+
+
+def main():
+    # Check python vesion
+    if sys.hexversion < 0x020600f0:
+        sys.stderr.write('packcore requires a minimum python version of 2.6.  Current version
is:\n' + sys.version)
+        sys.exit(1)
+
+    (options, args) = parseArgs()
+
+    coreFile = os.path.abspath(args[0])
+    fileCmd = _getFileInfo(coreFile)
+    if not _isCore(fileCmd):
+        sys.stderr.write(args[0] + ' is not a valid core file\n')
+        sys.exit(1)
+
+    if options.binary:
+        binary = which(options.binary)
+    else:
+        binary = _findBinary(fileCmd)
+
+    if not binary:
+        sys.stderr.write("Unable to find full path to binary for core file\n")
+        sys.exit(1)
+
+    packCoreFile(coreFile, binary)
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()


Mime
View raw message