hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r...@apache.org
Subject [28/28] incubator-hawq git commit: HAWQ-837. Add python modules into HAWQ code
Date Tue, 21 Jun 2016 02:41:44 GMT
HAWQ-837. Add python modules into HAWQ code

Added figleaf, pg8000, simplejson, pyyaml, lockfile, psi, pychecker, unittest2
Paramiko is not included because of License concern.
Pycrypto is not included because not in use anymore.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/80e25b46
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/80e25b46
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/80e25b46

Branch: refs/heads/master
Commit: 80e25b468cecb12fd58d28b5c08494ead7dbba81
Parents: 84dd4d4
Author: rlei <rlei@pivotal.io>
Authored: Mon Jun 20 16:49:13 2016 +0800
Committer: rlei <rlei@pivotal.io>
Committed: Tue Jun 21 10:15:46 2016 +0800

----------------------------------------------------------------------
 tools/bin/Makefile                              |   72 +-
 tools/bin/ext/__init__.py                       |  306 +++
 tools/bin/ext/figleaf/__init__.py               |  325 +++
 tools/bin/ext/figleaf/_lib.py                   |   22 +
 tools/bin/ext/figleaf/annotate.py               |  241 +++
 tools/bin/ext/figleaf/annotate_cover.py         |  159 ++
 tools/bin/ext/figleaf/annotate_html.py          |  292 +++
 tools/bin/ext/figleaf/annotate_sections.py      |   95 +
 tools/bin/ext/figleaf/figleaf2html              |    7 +
 tools/bin/ext/figleaf/internals.py              |  257 +++
 tools/bin/ext/figleaf/nose_sections.py          |  133 ++
 tools/bin/ext/pg8000/__init__.py                |   37 +
 tools/bin/ext/pg8000/dbapi.py                   |  621 ++++++
 tools/bin/ext/pg8000/errors.py                  |  115 +
 tools/bin/ext/pg8000/interface.py               |  542 +++++
 tools/bin/ext/pg8000/protocol.py                | 1340 ++++++++++++
 tools/bin/ext/pg8000/types.py                   |  687 ++++++
 tools/bin/ext/pg8000/util.py                    |   20 +
 tools/bin/ext/simplejson/__init__.py            |  303 +++
 tools/bin/ext/simplejson/_speedups.c            |  234 +++
 tools/bin/ext/simplejson/decoder.py             |  289 +++
 tools/bin/ext/simplejson/encoder.py             |  387 ++++
 tools/bin/ext/simplejson/jsonfilter.py          |   56 +
 tools/bin/ext/simplejson/scanner.py             |   79 +
 tools/bin/ext/simplejson/tests/__init__.py      |    0
 tools/bin/ext/simplejson/tests/test_attacks.py  |   22 +
 tools/bin/ext/simplejson/tests/test_dump.py     |   26 +
 tools/bin/ext/simplejson/tests/test_fail.py     |   86 +
 tools/bin/ext/simplejson/tests/test_float.py    |   20 +
 tools/bin/ext/simplejson/tests/test_indent.py   |   57 +
 tools/bin/ext/simplejson/tests/test_pass1.py    |   88 +
 tools/bin/ext/simplejson/tests/test_pass2.py    |   27 +
 tools/bin/ext/simplejson/tests/test_pass3.py    |   32 +
 .../bin/ext/simplejson/tests/test_recursion.py  |   78 +
 .../bin/ext/simplejson/tests/test_separators.py |   57 +
 tools/bin/ext/simplejson/tests/test_unicode.py  |   32 +
 tools/bin/ext/yaml/__init__.py                  |  306 +++
 tools/bin/ext/yaml/composer.py                  |  134 ++
 tools/bin/ext/yaml/constructor.py               |  691 ++++++
 tools/bin/ext/yaml/cyaml.py                     |  101 +
 tools/bin/ext/yaml/dumper.py                    |   78 +
 tools/bin/ext/yaml/emitter.py                   | 1179 +++++++++++
 tools/bin/ext/yaml/error.py                     |   91 +
 tools/bin/ext/yaml/events.py                    |  102 +
 tools/bin/ext/yaml/loader.py                    |   56 +
 tools/bin/ext/yaml/nodes.py                     |   65 +
 tools/bin/ext/yaml/parser.py                    |  602 ++++++
 tools/bin/ext/yaml/reader.py                    |  241 +++
 tools/bin/ext/yaml/representer.py               |  504 +++++
 tools/bin/ext/yaml/resolver.py                  |  239 +++
 tools/bin/ext/yaml/scanner.py                   | 1472 +++++++++++++
 tools/bin/ext/yaml/serializer.py                |  127 ++
 tools/bin/ext/yaml/tokens.py                    |  120 ++
 tools/bin/pythonSrc/PSI-0.3b2_gp/DESIGN         |   95 +
 .../bin/pythonSrc/PSI-0.3b2_gp/GREENPLUM_README |    1 +
 tools/bin/pythonSrc/PSI-0.3b2_gp/LICENSE        |   26 +
 tools/bin/pythonSrc/PSI-0.3b2_gp/MANIFEST       |   61 +
 tools/bin/pythonSrc/PSI-0.3b2_gp/PKG-INFO       |  129 ++
 tools/bin/pythonSrc/PSI-0.3b2_gp/README         |  281 +++
 tools/bin/pythonSrc/PSI-0.3b2_gp/TODO           |   72 +
 .../PSI-0.3b2_gp/examples/process_details.py    |  165 ++
 tools/bin/pythonSrc/PSI-0.3b2_gp/include/arch.h |   89 +
 .../PSI-0.3b2_gp/include/linux_utils.h          |   46 +
 .../bin/pythonSrc/PSI-0.3b2_gp/include/mount.h  |  126 ++
 .../PSI-0.3b2_gp/include/posix_mount.h          |   42 +
 .../PSI-0.3b2_gp/include/posix_utils.h          |   63 +
 .../pythonSrc/PSI-0.3b2_gp/include/process.h    |  217 ++
 .../PSI-0.3b2_gp/include/procfs_utils.h         |   91 +
 tools/bin/pythonSrc/PSI-0.3b2_gp/include/psi.h  |  284 +++
 .../pythonSrc/PSI-0.3b2_gp/include/psifuncs.h   |   85 +
 .../bin/pythonSrc/PSI-0.3b2_gp/misc/mem_test.py |   72 +
 .../bin/pythonSrc/PSI-0.3b2_gp/misc/mktests.sh  |   38 +
 .../PSI-0.3b2_gp/misc/valgrind-python.supp      |  391 ++++
 .../bin/pythonSrc/PSI-0.3b2_gp/psi/__init__.py  |   55 +
 .../bin/pythonSrc/PSI-0.3b2_gp/psi/_version.py  |   44 +
 tools/bin/pythonSrc/PSI-0.3b2_gp/setup.py       |  570 +++++
 .../bin/pythonSrc/PSI-0.3b2_gp/src/_psimodule.c |  389 ++++
 tools/bin/pythonSrc/PSI-0.3b2_gp/src/arch.c     |  561 +++++
 .../pythonSrc/PSI-0.3b2_gp/src/arch/aix_mount.c |  189 ++
 .../PSI-0.3b2_gp/src/arch/aix_process.c         |  597 ++++++
 .../pythonSrc/PSI-0.3b2_gp/src/arch/aix_psi.c   |   89 +
 .../PSI-0.3b2_gp/src/arch/darwin_mount.c        |  170 ++
 .../PSI-0.3b2_gp/src/arch/darwin_process.c      |  562 +++++
 .../PSI-0.3b2_gp/src/arch/darwin_processtable.c |  131 ++
 .../PSI-0.3b2_gp/src/arch/darwin_psi.c          |   93 +
 .../PSI-0.3b2_gp/src/arch/getloadavg.c          |   67 +
 .../PSI-0.3b2_gp/src/arch/linux_mount.c         |  156 ++
 .../PSI-0.3b2_gp/src/arch/linux_process.c       |  771 +++++++
 .../pythonSrc/PSI-0.3b2_gp/src/arch/linux_psi.c |   77 +
 .../PSI-0.3b2_gp/src/arch/linux_utils.c         |   83 +
 .../PSI-0.3b2_gp/src/arch/posix_arch.c          |   75 +
 .../PSI-0.3b2_gp/src/arch/posix_mount.c         |   67 +
 .../PSI-0.3b2_gp/src/arch/posix_utils.c         |  167 ++
 .../PSI-0.3b2_gp/src/arch/procfs_processtable.c |  158 ++
 .../PSI-0.3b2_gp/src/arch/procfs_utils.c        |  131 ++
 .../pythonSrc/PSI-0.3b2_gp/src/arch/sargs64.c   |  202 ++
 .../PSI-0.3b2_gp/src/arch/sunos_mount.c         |  171 ++
 .../PSI-0.3b2_gp/src/arch/sunos_process.c       | 1021 +++++++++
 .../pythonSrc/PSI-0.3b2_gp/src/arch/sunos_psi.c |   66 +
 .../bin/pythonSrc/PSI-0.3b2_gp/src/archmodule.c |  210 ++
 tools/bin/pythonSrc/PSI-0.3b2_gp/src/mount.c    |  502 +++++
 .../pythonSrc/PSI-0.3b2_gp/src/mountmodule.c    |  229 ++
 tools/bin/pythonSrc/PSI-0.3b2_gp/src/process.c  |  981 +++++++++
 .../pythonSrc/PSI-0.3b2_gp/src/processmodule.c  |  208 ++
 .../pythonSrc/PSI-0.3b2_gp/src/processtable.c   |  240 +++
 tools/bin/pythonSrc/PSI-0.3b2_gp/src/timespec.c |  852 ++++++++
 tools/bin/pythonSrc/PSI-0.3b2_gp/src/util.c     |  604 ++++++
 .../pythonSrc/PSI-0.3b2_gp/tests/_psi_test.py   |  145 ++
 tools/bin/pythonSrc/PSI-0.3b2_gp/tests/aixapp.c |   34 +
 tools/bin/pythonSrc/PSI-0.3b2_gp/tests/app.c    |   19 +
 .../pythonSrc/PSI-0.3b2_gp/tests/apphelper.py   |  225 ++
 .../pythonSrc/PSI-0.3b2_gp/tests/arch_test.py   |   68 +
 .../pythonSrc/PSI-0.3b2_gp/tests/mount_test.py  |  169 ++
 .../PSI-0.3b2_gp/tests/process_test.py          |  673 ++++++
 .../PSI-0.3b2_gp/tests/processtable_test.py     |   90 +
 .../PSI-0.3b2_gp/tests/timespec_test.py         |  233 +++
 tools/bin/pythonSrc/lockfile-0.9.1/2.4.diff     |   99 +
 tools/bin/pythonSrc/lockfile-0.9.1/ACKS         |    6 +
 tools/bin/pythonSrc/lockfile-0.9.1/LICENSE      |   21 +
 tools/bin/pythonSrc/lockfile-0.9.1/MANIFEST     |   19 +
 tools/bin/pythonSrc/lockfile-0.9.1/PKG-INFO     |   51 +
 tools/bin/pythonSrc/lockfile-0.9.1/README       |   27 +
 .../bin/pythonSrc/lockfile-0.9.1/RELEASE-NOTES  |   50 +
 tools/bin/pythonSrc/lockfile-0.9.1/doc/Makefile |   73 +
 tools/bin/pythonSrc/lockfile-0.9.1/doc/conf.py  |  179 ++
 .../pythonSrc/lockfile-0.9.1/doc/glossary.rst   |   15 +
 .../bin/pythonSrc/lockfile-0.9.1/doc/index.rst  |   22 +
 .../pythonSrc/lockfile-0.9.1/doc/lockfile.rst   |  257 +++
 .../lockfile-0.9.1/lockfile/__init__.py         |  286 +++
 .../lockfile-0.9.1/lockfile/linklockfile.py     |   69 +
 .../lockfile-0.9.1/lockfile/mkdirlockfile.py    |   79 +
 .../lockfile-0.9.1/lockfile/pidlockfile.py      |  189 ++
 .../lockfile-0.9.1/lockfile/sqlitelockfile.py   |  146 ++
 tools/bin/pythonSrc/lockfile-0.9.1/setup.py     |   32 +
 .../lockfile-0.9.1/test/compliancetest.py       |  222 ++
 .../lockfile-0.9.1/test/test_lockfile.py        |   30 +
 tools/bin/pythonSrc/pychecker-0.8.18/COPYRIGHT  |   31 +
 tools/bin/pythonSrc/pychecker-0.8.18/ChangeLog  |  349 ++++
 tools/bin/pythonSrc/pychecker-0.8.18/KNOWN_BUGS |  100 +
 .../bin/pythonSrc/pychecker-0.8.18/MAINTAINERS  |   81 +
 .../bin/pythonSrc/pychecker-0.8.18/MANIFEST.in  |    2 +
 tools/bin/pythonSrc/pychecker-0.8.18/NEWS       |  406 ++++
 tools/bin/pythonSrc/pychecker-0.8.18/README     |  152 ++
 tools/bin/pythonSrc/pychecker-0.8.18/TODO       |  101 +
 tools/bin/pythonSrc/pychecker-0.8.18/VERSION    |    1 +
 .../pychecker-0.8.18/pychecker/CodeChecks.py    | 1969 ++++++++++++++++++
 .../pychecker-0.8.18/pychecker/Config.py        |  475 +++++
 .../pythonSrc/pychecker-0.8.18/pychecker/OP.py  |  131 ++
 .../pychecker-0.8.18/pychecker/OptionTypes.py   |  117 ++
 .../pychecker-0.8.18/pychecker/Options.ad       |    9 +
 .../pychecker-0.8.18/pychecker/Stack.py         |  115 +
 .../pychecker-0.8.18/pychecker/Warning.py       |   50 +
 .../pychecker-0.8.18/pychecker/__init__.py      |   17 +
 .../pychecker-0.8.18/pychecker/checker.py       |  961 +++++++++
 .../pychecker-0.8.18/pychecker/function.py      |  159 ++
 .../pychecker-0.8.18/pychecker/msgs.py          |  175 ++
 .../pychecker-0.8.18/pychecker/options.py       |  275 +++
 .../pychecker-0.8.18/pychecker/pcmodules.py     |   19 +
 .../pychecker-0.8.18/pychecker/printer.py       |   47 +
 .../pychecker-0.8.18/pychecker/python.py        |  427 ++++
 .../pychecker-0.8.18/pychecker/utils.py         |  102 +
 .../pychecker-0.8.18/pychecker/warn.py          |  778 +++++++
 .../pychecker-0.8.18/pychecker2/Check.py        |   96 +
 .../pychecker-0.8.18/pychecker2/ClassChecks.py  |  364 ++++
 .../pychecker2/ConditionalChecks.py             |   44 +
 .../pychecker-0.8.18/pychecker2/File.py         |   39 +
 .../pychecker2/FormatStringChecks.py            |  183 ++
 .../pychecker-0.8.18/pychecker2/ImportChecks.py |   93 +
 .../pychecker-0.8.18/pychecker2/NOTES.txt       |  131 ++
 .../pychecker-0.8.18/pychecker2/OpChecks.py     |   68 +
 .../pychecker-0.8.18/pychecker2/Options.py      |  106 +
 .../pychecker-0.8.18/pychecker2/ParseChecks.py  |   84 +
 .../pychecker2/ReachableChecks.py               |   91 +
 .../pychecker-0.8.18/pychecker2/ReturnChecks.py |   43 +
 .../pychecker-0.8.18/pychecker2/ScopeChecks.py  |   28 +
 .../pychecker-0.8.18/pychecker2/TestSupport.py  |   57 +
 .../pychecker2/VariableChecks.py                |  354 ++++
 .../pychecker-0.8.18/pychecker2/Warning.py      |   13 +
 .../pychecker-0.8.18/pychecker2/__init__.py     |    1 +
 .../pychecker-0.8.18/pychecker2/main.py         |  115 +
 .../pychecker2/scripts/check.sh                 |    4 +
 .../pychecker2/scripts/clean.sh                 |    5 +
 .../pychecker-0.8.18/pychecker2/scripts/lib.sh  |    2 +
 .../pychecker2/scripts/pychecker2.sh            |    6 +
 .../pychecker-0.8.18/pychecker2/symbols.py      |  372 ++++
 .../pychecker-0.8.18/pychecker2/test.py         |   55 +
 .../pychecker2/tests/__init__.py                |    0
 .../pychecker2/tests/badparse.py                |    1 +
 .../pychecker-0.8.18/pychecker2/tests/class.py  |  152 ++
 .../pychecker-0.8.18/pychecker2/tests/except.py |   11 +
 .../pychecker2/tests/expected/incremental       |   49 +
 .../pychecker2/tests/expected/no-importError    |   48 +
 .../pychecker2/tests/expected/no-operator       |   47 +
 .../pychecker2/tests/expected/no-operatorPlus   |   48 +
 .../pychecker2/tests/expected/no-redefinedScope |   47 +
 .../pychecker2/tests/expected/no-selfName       |   48 +
 .../tests/expected/no-shadowIdentifier          |   42 +
 .../pychecker2/tests/expected/no-syntaxErrors   |   45 +
 .../pychecker2/tests/expected/no-unknown        |   43 +
 .../pychecker2/tests/expected/no-unpackedUsed   |   52 +
 .../pychecker2/tests/expected/no-unused         |   36 +
 .../pychecker2/tests/expected/normal            |   47 +
 .../pychecker2/tests/expected/options           |   33 +
 .../pychecker2/tests/expected/reportUnusedSelf  |   55 +
 .../pychecker2/tests/expected/shadowBuiltins    |   50 +
 .../pychecker2/tests/expected/verbose           |   69 +
 .../pychecker-0.8.18/pychecker2/tests/format.py |   48 +
 .../pychecker-0.8.18/pychecker2/tests/global.py |   14 +
 .../pychecker-0.8.18/pychecker2/tests/import.py |   19 +
 .../pychecker-0.8.18/pychecker2/tests/nested.py |    8 +
 .../pychecker-0.8.18/pychecker2/tests/op.py     |   24 +
 .../pychecker-0.8.18/pychecker2/tests/self.py   |   31 +
 .../pychecker-0.8.18/pychecker2/tests/shadow.py |   59 +
 .../pychecker2/tests/unreachable.py             |   58 +
 .../pychecker-0.8.18/pychecker2/tests/unused.py |   31 +
 .../pychecker2/tests/unusedImport.py            |    4 +
 .../pychecker2/utest/__init__.py                |    1 +
 .../pychecker-0.8.18/pychecker2/utest/class.py  |  232 +++
 .../pychecker2/utest/conditional.py             |   27 +
 .../pychecker-0.8.18/pychecker2/utest/data.py   |    8 +
 .../pychecker-0.8.18/pychecker2/utest/format.py |   78 +
 .../pychecker-0.8.18/pychecker2/utest/import.py |   33 +
 .../pychecker-0.8.18/pychecker2/utest/ops.py    |   43 +
 .../pychecker2/utest/parseerror.py              |   24 +
 .../pychecker2/utest/reachable.py               |   73 +
 .../pychecker2/utest/returns.py                 |   21 +
 .../pychecker-0.8.18/pychecker2/utest/scopes.py |   47 +
 .../pychecker-0.8.18/pychecker2/utest/self.py   |   45 +
 .../pychecker-0.8.18/pychecker2/utest/shadow.py |   21 +
 .../pychecker2/utest/unknown.py                 |   27 +
 .../pychecker-0.8.18/pychecker2/utest/unused.py |   96 +
 .../pychecker-0.8.18/pychecker2/utest/usage.py  |    7 +
 .../pychecker2/utest/warnings.py                |   23 +
 .../pychecker-0.8.18/pychecker2/util.py         |   73 +
 tools/bin/pythonSrc/pychecker-0.8.18/pycheckrc  |  225 ++
 tools/bin/pythonSrc/pychecker-0.8.18/setup.cfg  |    8 +
 tools/bin/pythonSrc/pychecker-0.8.18/setup.py   |  281 +++
 .../pythonSrc/pychecker-0.8.18/test/common.py   |   88 +
 .../test/expected/test_getmodule                |    8 +
 .../test/expected/test_zope_interface           |    5 +
 .../test/input/getmodule/A/C.py                 |    4 +
 .../test/input/getmodule/A/__init__.py          |    0
 .../test/input/getmodule/B/C.py                 |    4 +
 .../test/input/getmodule/B/__init__.py          |    0
 .../test/input/getmodule/__init__.py            |    1 +
 .../test/input/test_getmodule.py                |    5 +
 .../test/input/test_zope_interface.py           |    4 +
 .../bin/pythonSrc/pychecker-0.8.18/test/main.py |   13 +
 .../pychecker-0.8.18/test/test_module.py        |   22 +
 .../pychecker-0.8.18/test/test_stdlib.py        |   25 +
 .../pythonSrc/pychecker-0.8.18/test_check.sh    |   69 +
 .../pychecker-0.8.18/test_expected/test1        |   35 +
 .../pychecker-0.8.18/test_expected/test1-2.3    |   33 +
 .../pychecker-0.8.18/test_expected/test1-2.4    |   33 +
 .../pychecker-0.8.18/test_expected/test1-2.5    |   33 +
 .../pychecker-0.8.18/test_expected/test1-2.6    |   33 +
 .../pychecker-0.8.18/test_expected/test10       |    5 +
 .../pychecker-0.8.18/test_expected/test100      |    5 +
 .../pychecker-0.8.18/test_expected/test101      |    5 +
 .../pychecker-0.8.18/test_expected/test102      |    5 +
 .../pychecker-0.8.18/test_expected/test11       |    5 +
 .../pychecker-0.8.18/test_expected/test12       |    5 +
 .../pychecker-0.8.18/test_expected/test13       |    6 +
 .../pychecker-0.8.18/test_expected/test14       |    9 +
 .../pychecker-0.8.18/test_expected/test15       |    8 +
 .../pychecker-0.8.18/test_expected/test16       |    5 +
 .../pychecker-0.8.18/test_expected/test17       |    8 +
 .../pychecker-0.8.18/test_expected/test17-2.4   |    7 +
 .../pychecker-0.8.18/test_expected/test18       |   16 +
 .../pychecker-0.8.18/test_expected/test19       |    6 +
 .../pychecker-0.8.18/test_expected/test2        |    8 +
 .../pychecker-0.8.18/test_expected/test20       |    7 +
 .../pychecker-0.8.18/test_expected/test20-2.4   |    6 +
 .../pychecker-0.8.18/test_expected/test20-2.5   |    6 +
 .../pychecker-0.8.18/test_expected/test20-2.6   |    6 +
 .../pychecker-0.8.18/test_expected/test21       |    7 +
 .../pychecker-0.8.18/test_expected/test22       |   33 +
 .../pychecker-0.8.18/test_expected/test23       |    8 +
 .../pychecker-0.8.18/test_expected/test24       |   10 +
 .../pychecker-0.8.18/test_expected/test25       |    8 +
 .../pychecker-0.8.18/test_expected/test26       |    5 +
 .../pychecker-0.8.18/test_expected/test27       |    7 +
 .../pychecker-0.8.18/test_expected/test27-2.2   |    5 +
 .../pychecker-0.8.18/test_expected/test27-2.3   |    5 +
 .../pychecker-0.8.18/test_expected/test27-2.4   |    5 +
 .../pychecker-0.8.18/test_expected/test27-2.5   |    5 +
 .../pychecker-0.8.18/test_expected/test27-2.6   |    5 +
 .../pychecker-0.8.18/test_expected/test28       |    5 +
 .../pychecker-0.8.18/test_expected/test29       |    5 +
 .../pychecker-0.8.18/test_expected/test3        |    8 +
 .../pychecker-0.8.18/test_expected/test3-2.2    |    8 +
 .../pychecker-0.8.18/test_expected/test3-2.3    |    9 +
 .../pychecker-0.8.18/test_expected/test3-2.4    |    8 +
 .../pychecker-0.8.18/test_expected/test3-2.5    |    8 +
 .../pychecker-0.8.18/test_expected/test3-2.6    |    8 +
 .../pychecker-0.8.18/test_expected/test30       |    5 +
 .../pychecker-0.8.18/test_expected/test31       |   10 +
 .../pychecker-0.8.18/test_expected/test32       |    6 +
 .../pychecker-0.8.18/test_expected/test33       |    5 +
 .../pychecker-0.8.18/test_expected/test34       |   14 +
 .../pychecker-0.8.18/test_expected/test34-2.2   |   17 +
 .../pychecker-0.8.18/test_expected/test34-2.3   |   17 +
 .../pychecker-0.8.18/test_expected/test34-2.4   |   14 +
 .../pychecker-0.8.18/test_expected/test34-2.5   |   14 +
 .../pychecker-0.8.18/test_expected/test34-2.6   |   14 +
 .../pychecker-0.8.18/test_expected/test35       |    6 +
 .../pychecker-0.8.18/test_expected/test36       |    7 +
 .../pychecker-0.8.18/test_expected/test37       |    5 +
 .../pychecker-0.8.18/test_expected/test38       |   22 +
 .../pychecker-0.8.18/test_expected/test39       |   14 +
 .../pychecker-0.8.18/test_expected/test39-2.2   |   14 +
 .../pychecker-0.8.18/test_expected/test39-2.3   |   14 +
 .../pychecker-0.8.18/test_expected/test39-2.4   |   14 +
 .../pychecker-0.8.18/test_expected/test39-2.5   |   14 +
 .../pychecker-0.8.18/test_expected/test39-2.6   |   14 +
 .../pychecker-0.8.18/test_expected/test4        |    6 +
 .../pychecker-0.8.18/test_expected/test40       |   15 +
 .../pychecker-0.8.18/test_expected/test41       |    7 +
 .../pychecker-0.8.18/test_expected/test42       |    5 +
 .../pychecker-0.8.18/test_expected/test43       |    5 +
 .../pychecker-0.8.18/test_expected/test44       |    5 +
 .../pychecker-0.8.18/test_expected/test45       |    6 +
 .../pychecker-0.8.18/test_expected/test46       |   10 +
 .../pychecker-0.8.18/test_expected/test47       |    5 +
 .../pychecker-0.8.18/test_expected/test48       |   40 +
 .../pychecker-0.8.18/test_expected/test49       |    8 +
 .../pychecker-0.8.18/test_expected/test5        |    5 +
 .../pychecker-0.8.18/test_expected/test50       |    8 +
 .../pychecker-0.8.18/test_expected/test51       |    6 +
 .../pychecker-0.8.18/test_expected/test52       |    5 +
 .../pychecker-0.8.18/test_expected/test53       |    9 +
 .../pychecker-0.8.18/test_expected/test54       |    9 +
 .../pychecker-0.8.18/test_expected/test55       |    9 +
 .../pychecker-0.8.18/test_expected/test56       |    5 +
 .../pychecker-0.8.18/test_expected/test57       |    5 +
 .../pychecker-0.8.18/test_expected/test58       |   10 +
 .../pychecker-0.8.18/test_expected/test58-2.2   |   14 +
 .../pychecker-0.8.18/test_expected/test58-2.3   |    8 +
 .../pychecker-0.8.18/test_expected/test58-2.4   |    8 +
 .../pychecker-0.8.18/test_expected/test58-2.5   |    8 +
 .../pychecker-0.8.18/test_expected/test58-2.6   |    8 +
 .../pychecker-0.8.18/test_expected/test59       |    9 +
 .../pychecker-0.8.18/test_expected/test6        |    5 +
 .../pychecker-0.8.18/test_expected/test60       |    6 +
 .../pychecker-0.8.18/test_expected/test61       |   12 +
 .../pychecker-0.8.18/test_expected/test62       |    5 +
 .../pychecker-0.8.18/test_expected/test63       |    5 +
 .../pychecker-0.8.18/test_expected/test64       |    9 +
 .../pychecker-0.8.18/test_expected/test65       |    5 +
 .../pychecker-0.8.18/test_expected/test66       |    5 +
 .../pychecker-0.8.18/test_expected/test67       |    5 +
 .../pychecker-0.8.18/test_expected/test68       |    9 +
 .../pychecker-0.8.18/test_expected/test68-2.4   |    5 +
 .../pychecker-0.8.18/test_expected/test68-2.5   |    5 +
 .../pychecker-0.8.18/test_expected/test68-2.6   |    5 +
 .../pychecker-0.8.18/test_expected/test69       |    5 +
 .../pychecker-0.8.18/test_expected/test7        |    5 +
 .../pychecker-0.8.18/test_expected/test70       |   12 +
 .../pychecker-0.8.18/test_expected/test71       |   10 +
 .../pychecker-0.8.18/test_expected/test71-2.4   |    8 +
 .../pychecker-0.8.18/test_expected/test71-2.5   |    8 +
 .../pychecker-0.8.18/test_expected/test71-2.6   |    8 +
 .../pychecker-0.8.18/test_expected/test72       |    8 +
 .../pychecker-0.8.18/test_expected/test73       |    9 +
 .../pychecker-0.8.18/test_expected/test74       |   11 +
 .../pychecker-0.8.18/test_expected/test74-2.4   |   10 +
 .../pychecker-0.8.18/test_expected/test74-2.5   |   10 +
 .../pychecker-0.8.18/test_expected/test74-2.6   |   10 +
 .../pychecker-0.8.18/test_expected/test75       |    6 +
 .../pychecker-0.8.18/test_expected/test75-2.2   |    9 +
 .../pychecker-0.8.18/test_expected/test75-2.3   |    9 +
 .../pychecker-0.8.18/test_expected/test75-2.4   |    9 +
 .../pychecker-0.8.18/test_expected/test75-2.5   |    9 +
 .../pychecker-0.8.18/test_expected/test75-2.6   |    9 +
 .../pychecker-0.8.18/test_expected/test76       |    6 +
 .../pychecker-0.8.18/test_expected/test77       |   11 +
 .../pychecker-0.8.18/test_expected/test78       |    5 +
 .../pychecker-0.8.18/test_expected/test79       |    6 +
 .../pychecker-0.8.18/test_expected/test8        |    8 +
 .../pychecker-0.8.18/test_expected/test80       |    6 +
 .../pychecker-0.8.18/test_expected/test80-2.2   |    6 +
 .../pychecker-0.8.18/test_expected/test80-2.3   |    6 +
 .../pychecker-0.8.18/test_expected/test80-2.4   |    6 +
 .../pychecker-0.8.18/test_expected/test80-2.5   |    6 +
 .../pychecker-0.8.18/test_expected/test80-2.6   |    6 +
 .../pychecker-0.8.18/test_expected/test81       |    5 +
 .../pychecker-0.8.18/test_expected/test82       |    5 +
 .../pychecker-0.8.18/test_expected/test83       |    9 +
 .../pychecker-0.8.18/test_expected/test84       |    5 +
 .../pychecker-0.8.18/test_expected/test85       |    6 +
 .../pychecker-0.8.18/test_expected/test86       |   13 +
 .../pychecker-0.8.18/test_expected/test87       |   25 +
 .../pychecker-0.8.18/test_expected/test88       |    9 +
 .../pychecker-0.8.18/test_expected/test88-2.4   |   11 +
 .../pychecker-0.8.18/test_expected/test89       |   11 +
 .../pychecker-0.8.18/test_expected/test89-2.2   |    5 +
 .../pychecker-0.8.18/test_expected/test89-2.3   |    5 +
 .../pychecker-0.8.18/test_expected/test89-2.4   |    5 +
 .../pychecker-0.8.18/test_expected/test89-2.5   |    5 +
 .../pychecker-0.8.18/test_expected/test89-2.6   |    5 +
 .../pychecker-0.8.18/test_expected/test9        |    6 +
 .../pychecker-0.8.18/test_expected/test90       |   16 +
 .../pychecker-0.8.18/test_expected/test92       |    5 +
 .../pychecker-0.8.18/test_expected/test93       |    6 +
 .../pychecker-0.8.18/test_expected/test94       |   16 +
 .../pychecker-0.8.18/test_expected/test95       |    5 +
 .../pychecker-0.8.18/test_expected/test96       |    6 +
 .../pychecker-0.8.18/test_expected/test97       |    5 +
 .../pychecker-0.8.18/test_expected/test98       |    8 +
 .../pychecker-0.8.18/test_expected/test99       |    5 +
 .../pychecker-0.8.18/test_input/__init__.py     |    3 +
 .../pychecker-0.8.18/test_input/import24.py     |   14 +
 .../pychecker-0.8.18/test_input/import37.py     |    7 +
 .../pychecker-0.8.18/test_input/import44.py     |   12 +
 .../pychecker-0.8.18/test_input/import47.py     |    7 +
 .../pychecker-0.8.18/test_input/import56a.py    |    9 +
 .../pychecker-0.8.18/test_input/import56b.py    |    6 +
 .../pychecker-0.8.18/test_input/import69.py     |    8 +
 .../pychecker-0.8.18/test_input/import73.py     |    9 +
 .../pychecker-0.8.18/test_input/import98.py     |   11 +
 .../pychecker-0.8.18/test_input/pycheckrc       |    7 +
 .../pychecker-0.8.18/test_input/test1.py        |   75 +
 .../pychecker-0.8.18/test_input/test10.py       |    8 +
 .../pychecker-0.8.18/test_input/test100.py      |   15 +
 .../pychecker-0.8.18/test_input/test101.py      |   33 +
 .../pychecker-0.8.18/test_input/test102.py      |   16 +
 .../pychecker-0.8.18/test_input/test11.py       |   20 +
 .../pychecker-0.8.18/test_input/test12.py       |    8 +
 .../pychecker-0.8.18/test_input/test13.py       |   30 +
 .../pychecker-0.8.18/test_input/test14.py       |   24 +
 .../pychecker-0.8.18/test_input/test15.py       |   39 +
 .../pychecker-0.8.18/test_input/test16.py       |   19 +
 .../pychecker-0.8.18/test_input/test17.py       |   35 +
 .../pychecker-0.8.18/test_input/test18.py       |   44 +
 .../pychecker-0.8.18/test_input/test19.py       |   10 +
 .../pychecker-0.8.18/test_input/test2.py        |   20 +
 .../pychecker-0.8.18/test_input/test20.py       |   41 +
 .../pychecker-0.8.18/test_input/test21.py       |   57 +
 .../pychecker-0.8.18/test_input/test22.py       |   86 +
 .../pychecker-0.8.18/test_input/test23.py       |   48 +
 .../pychecker-0.8.18/test_input/test24.py       |   12 +
 .../pychecker-0.8.18/test_input/test25.py       |   16 +
 .../pychecker-0.8.18/test_input/test26.py       |   11 +
 .../pychecker-0.8.18/test_input/test27.py       |   15 +
 .../pychecker-0.8.18/test_input/test28.py       |    8 +
 .../pychecker-0.8.18/test_input/test29.py       |   14 +
 .../pychecker-0.8.18/test_input/test3.py        |   14 +
 .../pychecker-0.8.18/test_input/test30.py       |    8 +
 .../pychecker-0.8.18/test_input/test31.py       |   38 +
 .../pychecker-0.8.18/test_input/test32.py       |   16 +
 .../pychecker-0.8.18/test_input/test33.py       |   10 +
 .../pychecker-0.8.18/test_input/test34.py       |   27 +
 .../pychecker-0.8.18/test_input/test35.py       |   13 +
 .../pychecker-0.8.18/test_input/test36.py       |   40 +
 .../pychecker-0.8.18/test_input/test37.py       |    7 +
 .../pychecker-0.8.18/test_input/test38.py       |   54 +
 .../pychecker-0.8.18/test_input/test39.py       |   44 +
 .../pychecker-0.8.18/test_input/test4.py        |   32 +
 .../pychecker-0.8.18/test_input/test40.py       |   34 +
 .../pychecker-0.8.18/test_input/test41.py       |   18 +
 .../pychecker-0.8.18/test_input/test42.py       |   16 +
 .../pychecker-0.8.18/test_input/test43.py       |   11 +
 .../pychecker-0.8.18/test_input/test44.py       |    9 +
 .../pychecker-0.8.18/test_input/test45.py       |   23 +
 .../pychecker-0.8.18/test_input/test46.py       |    7 +
 .../pychecker-0.8.18/test_input/test47.py       |    7 +
 .../pychecker-0.8.18/test_input/test48.py       |   84 +
 .../pychecker-0.8.18/test_input/test49.py       |   37 +
 .../pychecker-0.8.18/test_input/test5.py        |   20 +
 .../pychecker-0.8.18/test_input/test50.py       |   21 +
 .../pychecker-0.8.18/test_input/test51.py       |   19 +
 .../pychecker-0.8.18/test_input/test52.py       |   54 +
 .../pychecker-0.8.18/test_input/test53.py       |   28 +
 .../pychecker-0.8.18/test_input/test54.py       |   20 +
 .../pychecker-0.8.18/test_input/test55.py       |   47 +
 .../pychecker-0.8.18/test_input/test56.py       |    7 +
 .../pychecker-0.8.18/test_input/test57.py       |   11 +
 .../pychecker-0.8.18/test_input/test58.py       |   11 +
 .../pychecker-0.8.18/test_input/test59.py       |   31 +
 .../pychecker-0.8.18/test_input/test6.py        |   31 +
 .../pychecker-0.8.18/test_input/test60.py       |   22 +
 .../pychecker-0.8.18/test_input/test61.py       |   28 +
 .../pychecker-0.8.18/test_input/test62.py       |    4 +
 .../pychecker-0.8.18/test_input/test63.py       |   15 +
 .../pychecker-0.8.18/test_input/test64.py       |   17 +
 .../pychecker-0.8.18/test_input/test65.py       |   21 +
 .../pychecker-0.8.18/test_input/test66.py       |    7 +
 .../pychecker-0.8.18/test_input/test67.py       |   16 +
 .../pychecker-0.8.18/test_input/test68.py       |  170 ++
 .../pychecker-0.8.18/test_input/test69.py       |   22 +
 .../pychecker-0.8.18/test_input/test7.py        |   18 +
 .../pychecker-0.8.18/test_input/test70.py       |   60 +
 .../pychecker-0.8.18/test_input/test71.py       |   50 +
 .../pychecker-0.8.18/test_input/test72.py       |   29 +
 .../pychecker-0.8.18/test_input/test73.py       |   37 +
 .../pychecker-0.8.18/test_input/test74.py       |   41 +
 .../pychecker-0.8.18/test_input/test75.py       |   87 +
 .../pychecker-0.8.18/test_input/test76.py       |   12 +
 .../pychecker-0.8.18/test_input/test77.py       |    9 +
 .../pychecker-0.8.18/test_input/test78.py       |   19 +
 .../pychecker-0.8.18/test_input/test79.py       |   33 +
 .../pychecker-0.8.18/test_input/test8.py        |    4 +
 .../pychecker-0.8.18/test_input/test80.py       |   25 +
 .../pychecker-0.8.18/test_input/test81.py       |   30 +
 .../pychecker-0.8.18/test_input/test82.py       |   18 +
 .../pychecker-0.8.18/test_input/test83.py       |   43 +
 .../pychecker-0.8.18/test_input/test84.py       |   10 +
 .../pychecker-0.8.18/test_input/test85.py       |   22 +
 .../pychecker-0.8.18/test_input/test86.py       |   96 +
 .../pychecker-0.8.18/test_input/test87.py       |   56 +
 .../pychecker-0.8.18/test_input/test88.py       |   23 +
 .../pychecker-0.8.18/test_input/test89.py       |    8 +
 .../pychecker-0.8.18/test_input/test9.py        |   17 +
 .../pychecker-0.8.18/test_input/test90.py       |   22 +
 .../pychecker-0.8.18/test_input/test92.py       |    6 +
 .../pychecker-0.8.18/test_input/test93.py       |   11 +
 .../pychecker-0.8.18/test_input/test94.py       |   54 +
 .../pychecker-0.8.18/test_input/test95.py       |    5 +
 .../pychecker-0.8.18/test_input/test96.py       |   15 +
 .../pychecker-0.8.18/test_input/test97.py       |   20 +
 .../pychecker-0.8.18/test_input/test98.py       |   33 +
 .../pychecker-0.8.18/test_input/test99.py       |   22 +
 tools/bin/pythonSrc/unittest2-0.5.1/PKG-INFO    |  321 +++
 tools/bin/pythonSrc/unittest2-0.5.1/README.txt  |  296 +++
 tools/bin/pythonSrc/unittest2-0.5.1/setup.cfg   |    8 +
 tools/bin/pythonSrc/unittest2-0.5.1/setup.py    |   85 +
 tools/bin/pythonSrc/unittest2-0.5.1/unit2       |    7 +
 tools/bin/pythonSrc/unittest2-0.5.1/unit2.py    |    7 +
 .../unittest2-0.5.1/unittest2.egg-info/PKG-INFO |  321 +++
 .../unittest2.egg-info/SOURCES.txt              |   39 +
 .../unittest2.egg-info/dependency_links.txt     |    1 +
 .../unittest2.egg-info/entry_points.txt         |    4 +
 .../unittest2.egg-info/top_level.txt            |    1 +
 .../unittest2-0.5.1/unittest2/__init__.py       |   68 +
 .../unittest2-0.5.1/unittest2/__main__.py       |   10 +
 .../pythonSrc/unittest2-0.5.1/unittest2/case.py | 1084 ++++++++++
 .../unittest2-0.5.1/unittest2/collector.py      |    9 +
 .../unittest2-0.5.1/unittest2/compatibility.py  |   64 +
 .../unittest2-0.5.1/unittest2/loader.py         |  322 +++
 .../pythonSrc/unittest2-0.5.1/unittest2/main.py |  241 +++
 .../unittest2-0.5.1/unittest2/result.py         |  183 ++
 .../unittest2-0.5.1/unittest2/runner.py         |  206 ++
 .../unittest2-0.5.1/unittest2/signals.py        |   57 +
 .../unittest2-0.5.1/unittest2/suite.py          |  287 +++
 .../unittest2-0.5.1/unittest2/test/__init__.py  |    1 +
 .../unittest2-0.5.1/unittest2/test/dummy.py     |    0
 .../unittest2-0.5.1/unittest2/test/support.py   |  177 ++
 .../unittest2/test/test_assertions.py           |  254 +++
 .../unittest2/test/test_break.py                |  260 +++
 .../unittest2-0.5.1/unittest2/test/test_case.py | 1065 ++++++++++
 .../unittest2/test/test_discovery.py            |  371 ++++
 .../unittest2/test/test_functiontestcase.py     |  149 ++
 .../unittest2/test/test_loader.py               | 1286 ++++++++++++
 .../unittest2/test/test_new_tests.py            |   46 +
 .../unittest2/test/test_program.py              |  239 +++
 .../unittest2/test/test_result.py               |  416 ++++
 .../unittest2/test/test_runner.py               |  129 ++
 .../unittest2/test/test_setups.py               |  502 +++++
 .../unittest2/test/test_skipping.py             |  143 ++
 .../unittest2/test/test_suite.py                |  341 +++
 .../unittest2/test/test_unittest2_with.py       |  143 ++
 .../pythonSrc/unittest2-0.5.1/unittest2/util.py |   99 +
 562 files changed, 55504 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/80e25b46/tools/bin/Makefile
----------------------------------------------------------------------
diff --git a/tools/bin/Makefile b/tools/bin/Makefile
index 273b599..46a2df6 100644
--- a/tools/bin/Makefile
+++ b/tools/bin/Makefile
@@ -36,7 +36,7 @@ PYLIB_SRC=$(SRC)/pythonSrc
 LIB_DIR=$(SRC)/lib
 PYLIB_DIR=$(SRC)/ext
 
-all: stream pygresql
+all: lockfile pygresql stream pychecker psi unittest2
 
 #
 # Python Libraries
@@ -65,6 +65,76 @@ pygresql:
 	cp -r $(PYLIB_SRC)/$(PYGRESQL_DIR)/build/lib.*/* $(PYLIB_DIR)/pygresql
 	touch $(PYLIB_DIR)/__init__.py
 
+#
+# LOCKFILE
+#
+# note the awk commands are used to eliminate references to code in __init__.py
+# that we don't use and also to insert a comment to tell pylint not to complain
+# about these files since we are not in a position to correct those warnings.
+#
+LOCKFILE_VERSION=0.9.1
+LOCKFILE_DIR=lockfile-$(LOCKFILE_VERSION)
+LOCKFILE_SRC=$(PYLIB_SRC)/$(LOCKFILE_DIR)/build/lib/lockfile
+LOCKFILE_DST=$(PYLIB_DIR)/lockfile
+lockfile:
+	@echo "--- lockfile"
+	cd $(PYLIB_SRC)/$(LOCKFILE_DIR)/ && python setup.py build
+	mkdir -p $(PYLIB_DIR)/lockfile
+	awk 'BEGIN{print "# pylint: disable-all"} /^if hasattr/ {exit} { print }' < $(LOCKFILE_SRC)/__init__.py    > $(LOCKFILE_DST)/__init__.py
+	awk 'BEGIN{print "# pylint: disable-all"} { print }'                      < $(LOCKFILE_SRC)/pidlockfile.py > $(LOCKFILE_DST)/pidlockfile.py
+
+#
+# PYCRYPTO
+#
+PYCRYPTO_VERSION=2.0.1
+PYCRYPTO_DIR=pycrypto-$(PYCRYPTO_VERSION)
+
+pycrypto:
+	@echo "--- pycrypto"
+	cd $(PYLIB_SRC)/$(PYCRYPTO_DIR)/ && CC="$(CC)" CFLAGS="${CFLAGS}" LDFLAGS="${LDFLAGS}" python setup.py build
+	cp -r $(PYLIB_SRC)/$(PYCRYPTO_DIR)/build/lib.*/Crypto $(PYLIB_DIR)
+
+#
+# PSI
+#
+PSI_VERSION=0.3b2_gp
+PSI_DIR=PSI-$(PSI_VERSION)
+
+psi:
+	@echo "--- psi"
+	cd $(PYLIB_SRC)/$(PSI_DIR)/ && CC="$(CC)" CFLAGS="${CFLAGS}" LDFLAGS="${LDFLAGS}" python setup.py build
+	cp -r $(PYLIB_SRC)/$(PSI_DIR)/build/lib.*/psi $(PYLIB_DIR)
+
+#
+# PYCHECKER
+#
+PYCHECKER_VERSION=0.8.18
+PYCHECKER_DIR=pychecker-$(PYCHECKER_VERSION)
+
+pychecker:
+	@echo "--- pychecker"
+	cd $(PYLIB_SRC)/$(PYCHECKER_DIR)/ && python setup.py build
+	cp -r $(PYLIB_SRC)/$(PYCHECKER_DIR)/build/lib/pychecker  $(PYLIB_DIR)/
+
+UNITTEST2_VERSION=0.5.1
+UNITTEST2_DIR=unittest2-${UNITTEST2_VERSION}
+unittest2:
+	@echo "--- unittest2"
+	cd $(PYLIB_SRC)/$(UNITTEST2_DIR)/ && python setup.py build
+	cp -r $(PYLIB_SRC)/$(UNITTEST2_DIR)/build/lib/unittest2  $(PYLIB_DIR)/
+
+#
+# PARAMIKO
+#
+PARAMIKO_VERSION=1.7.6-9
+PARAMIKO_DIR=paramiko-$(PARAMIKO_VERSION)
+paramiko:
+	@echo "--- paramiko"
+	cd $(PYLIB_SRC)/ && $(TAR) xzf $(PARAMIKO_DIR).tar.gz
+	cd $(PYLIB_SRC)/$(PARAMIKO_DIR)/ && python setup.py build
+	cp -r $(PYLIB_SRC)/$(PARAMIKO_DIR)/build/lib/paramiko  $(PYLIB_DIR)/
+
+
 PYTHON_FILES=`grep -l --exclude=Makefile --exclude=gplogfilter --exclude=gpcheckos --exclude=gpgenfsmap.py --exclude=throttlingD.py "/bin/env python" *`\
 			 `grep -l "/bin/env python" $(SRC)/../sbin/*`\
 			 `find ./gppylib -name "*.py"`\

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/80e25b46/tools/bin/ext/__init__.py
----------------------------------------------------------------------
diff --git a/tools/bin/ext/__init__.py b/tools/bin/ext/__init__.py
index e69de29..b6c84f9 100644
--- a/tools/bin/ext/__init__.py
+++ b/tools/bin/ext/__init__.py
@@ -0,0 +1,306 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from error import *
+
+from tokens import *
+from events import *
+from nodes import *
+
+from loader import *
+from dumper import *
+
+try:
+    from cyaml import *
+except ImportError:
+    pass
+
+def scan(stream, Loader=Loader):
+    """
+    Scan a YAML stream and produce scanning tokens.
+    """
+    loader = Loader(stream)
+    while loader.check_token():
+        yield loader.get_token()
+
+def parse(stream, Loader=Loader):
+    """
+    Parse a YAML stream and produce parsing events.
+    """
+    loader = Loader(stream)
+    while loader.check_event():
+        yield loader.get_event()
+
+def compose(stream, Loader=Loader):
+    """
+    Parse the first YAML document in a stream
+    and produce the corresponding representation tree.
+    """
+    loader = Loader(stream)
+    if loader.check_node():
+        return loader.get_node()
+
+def compose_all(stream, Loader=Loader):
+    """
+    Parse all YAML documents in a stream
+    and produce corresponsing representation trees.
+    """
+    loader = Loader(stream)
+    while loader.check_node():
+        yield loader.get_node()
+
+def load_all(stream, Loader=Loader):
+    """
+    Parse all YAML documents in a stream
+    and produce corresponding Python objects.
+    """
+    loader = Loader(stream)
+    while loader.check_data():
+        yield loader.get_data()
+
+def load(stream, Loader=Loader):
+    """
+    Parse the first YAML document in a stream
+    and produce the corresponding Python object.
+    """
+    loader = Loader(stream)
+    if loader.check_data():
+        return loader.get_data()
+
+def safe_load_all(stream):
+    """
+    Parse all YAML documents in a stream
+    and produce corresponding Python objects.
+    Resolve only basic YAML tags.
+    """
+    return load_all(stream, SafeLoader)
+
+def safe_load(stream):
+    """
+    Parse the first YAML document in a stream
+    and produce the corresponding Python object.
+    Resolve only basic YAML tags.
+    """
+    return load(stream, SafeLoader)
+
+def emit(events, stream=None, Dumper=Dumper,
+        canonical=None, indent=None, width=None,
+        allow_unicode=None, line_break=None):
+    """
+    Emit YAML parsing events into a stream.
+    If stream is None, return the produced string instead.
+    """
+    getvalue = None
+    if stream is None:
+        try:
+            from cStringIO import StringIO
+        except ImportError:
+            from StringIO import StringIO
+        stream = StringIO()
+        getvalue = stream.getvalue
+    dumper = Dumper(stream, canonical=canonical, indent=indent, width=width,
+            allow_unicode=allow_unicode, line_break=line_break)
+    for event in events:
+        dumper.emit(event)
+    if getvalue:
+        return getvalue()
+
+def serialize_all(nodes, stream=None, Dumper=Dumper,
+        canonical=None, indent=None, width=None,
+        allow_unicode=None, line_break=None,
+        encoding='utf-8', explicit_start=None, explicit_end=None,
+        version=None, tags=None):
+    """
+    Serialize a sequence of representation trees into a YAML stream.
+    If stream is None, return the produced string instead.
+    """
+    getvalue = None
+    if stream is None:
+        try:
+            from cStringIO import StringIO
+        except ImportError:
+            from StringIO import StringIO
+        stream = StringIO()
+        getvalue = stream.getvalue
+    dumper = Dumper(stream, canonical=canonical, indent=indent, width=width,
+            allow_unicode=allow_unicode, line_break=line_break,
+            encoding=encoding, version=version, tags=tags,
+            explicit_start=explicit_start, explicit_end=explicit_end)
+    dumper.open()
+    for node in nodes:
+        dumper.serialize(node)
+    dumper.close()
+    if getvalue:
+        return getvalue()
+
+def serialize(node, stream=None, Dumper=Dumper, **kwds):
+    """
+    Serialize a representation tree into a YAML stream.
+    If stream is None, return the produced string instead.
+    """
+    return serialize_all([node], stream, Dumper=Dumper, **kwds)
+
+def dump_all(documents, stream=None, Dumper=Dumper,
+        default_style=None, default_flow_style=None,
+        canonical=None, indent=None, width=None,
+        allow_unicode=None, line_break=None,
+        encoding='utf-8', explicit_start=None, explicit_end=None,
+        version=None, tags=None):
+    """
+    Serialize a sequence of Python objects into a YAML stream.
+    If stream is None, return the produced string instead.
+    """
+    getvalue = None
+    if stream is None:
+        try:
+            from cStringIO import StringIO
+        except ImportError:
+            from StringIO import StringIO
+        stream = StringIO()
+        getvalue = stream.getvalue
+    dumper = Dumper(stream, default_style=default_style,
+            default_flow_style=default_flow_style,
+            canonical=canonical, indent=indent, width=width,
+            allow_unicode=allow_unicode, line_break=line_break,
+            encoding=encoding, version=version, tags=tags,
+            explicit_start=explicit_start, explicit_end=explicit_end)
+    dumper.open()
+    for data in documents:
+        dumper.represent(data)
+    dumper.close()
+    if getvalue:
+        return getvalue()
+
+def dump(data, stream=None, Dumper=Dumper, **kwds):
+    """
+    Serialize a Python object into a YAML stream.
+    If stream is None, return the produced string instead.
+    """
+    return dump_all([data], stream, Dumper=Dumper, **kwds)
+
+def safe_dump_all(documents, stream=None, **kwds):
+    """
+    Serialize a sequence of Python objects into a YAML stream.
+    Produce only basic YAML tags.
+    If stream is None, return the produced string instead.
+    """
+    return dump_all(documents, stream, Dumper=SafeDumper, **kwds)
+
+def safe_dump(data, stream=None, **kwds):
+    """
+    Serialize a Python object into a YAML stream.
+    Produce only basic YAML tags.
+    If stream is None, return the produced string instead.
+    """
+    return dump_all([data], stream, Dumper=SafeDumper, **kwds)
+
+def add_implicit_resolver(tag, regexp, first=None,
+        Loader=Loader, Dumper=Dumper):
+    """
+    Add an implicit scalar detector.
+    If an implicit scalar value matches the given regexp,
+    the corresponding tag is assigned to the scalar.
+    first is a sequence of possible initial characters or None.
+    """
+    Loader.add_implicit_resolver(tag, regexp, first)
+    Dumper.add_implicit_resolver(tag, regexp, first)
+
+def add_path_resolver(tag, path, kind=None, Loader=Loader, Dumper=Dumper):
+    """
+    Add a path based resolver for the given tag.
+    A path is a list of keys that forms a path
+    to a node in the representation tree.
+    Keys can be string values, integers, or None.
+    """
+    Loader.add_path_resolver(tag, path, kind)
+    Dumper.add_path_resolver(tag, path, kind)
+
+def add_constructor(tag, constructor, Loader=Loader):
+    """
+    Add a constructor for the given tag.
+    Constructor is a function that accepts a Loader instance
+    and a node object and produces the corresponding Python object.
+    """
+    Loader.add_constructor(tag, constructor)
+
+def add_multi_constructor(tag_prefix, multi_constructor, Loader=Loader):
+    """
+    Add a multi-constructor for the given tag prefix.
+    Multi-constructor is called for a node if its tag starts with tag_prefix.
+    Multi-constructor accepts a Loader instance, a tag suffix,
+    and a node object and produces the corresponding Python object.
+    """
+    Loader.add_multi_constructor(tag_prefix, multi_constructor)
+
+def add_representer(data_type, representer, Dumper=Dumper):
+    """
+    Add a representer for the given type.
+    Representer is a function accepting a Dumper instance
+    and an instance of the given data type
+    and producing the corresponding representation node.
+    """
+    Dumper.add_representer(data_type, representer)
+
+def add_multi_representer(data_type, multi_representer, Dumper=Dumper):
+    """
+    Add a representer for the given type.
+    Multi-representer is a function accepting a Dumper instance
+    and an instance of the given data type or subtype
+    and producing the corresponding representation node.
+    """
+    Dumper.add_multi_representer(data_type, multi_representer)
+
+class YAMLObjectMetaclass(type):
+    """
+    The metaclass for YAMLObject.
+    """
+    def __init__(cls, name, bases, kwds):
+        super(YAMLObjectMetaclass, cls).__init__(name, bases, kwds)
+        if 'yaml_tag' in kwds and kwds['yaml_tag'] is not None:
+            cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml)
+            cls.yaml_dumper.add_representer(cls, cls.to_yaml)
+
+class YAMLObject(object):
+    """
+    An object that can dump itself to a YAML stream
+    and load itself from a YAML stream.
+    """
+
+    __metaclass__ = YAMLObjectMetaclass
+    __slots__ = ()  # no direct instantiation, so allow immutable subclasses
+
+    yaml_loader = Loader
+    yaml_dumper = Dumper
+
+    yaml_tag = None
+    yaml_flow_style = None
+
+    def from_yaml(cls, loader, node):
+        """
+        Convert a representation node to a Python object.
+        """
+        return loader.construct_yaml_object(node, cls)
+    from_yaml = classmethod(from_yaml)
+
+    def to_yaml(cls, dumper, data):
+        """
+        Convert a Python object to a representation node.
+        """
+        return dumper.represent_yaml_object(cls.yaml_tag, data, cls,
+                flow_style=cls.yaml_flow_style)
+    to_yaml = classmethod(to_yaml)
+

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/80e25b46/tools/bin/ext/figleaf/__init__.py
----------------------------------------------------------------------
diff --git a/tools/bin/ext/figleaf/__init__.py b/tools/bin/ext/figleaf/__init__.py
new file mode 100644
index 0000000..5761930
--- /dev/null
+++ b/tools/bin/ext/figleaf/__init__.py
@@ -0,0 +1,325 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+figleaf is another tool to trace Python code coverage.
+
+figleaf uses the sys.settrace hook to record which statements are
+executed by the CPython interpreter; this record can then be saved
+into a file, or otherwise communicated back to a reporting script.
+
+figleaf differs from the gold standard of Python coverage tools
+('coverage.py') in several ways.  First and foremost, figleaf uses the
+same criterion for "interesting" lines of code as the sys.settrace
+function, which obviates some of the complexity in coverage.py (but
+does mean that your "loc" count goes down).  Second, figleaf does not
+record code executed in the Python standard library, which results in
+a significant speedup.  And third, the format in which the coverage
+format is saved is very simple and easy to work with.
+
+You might want to use figleaf if you're recording coverage from
+multiple types of tests and need to aggregate the coverage in
+interesting ways, and/or control when coverage is recorded.
+coverage.py is a better choice for command-line execution, and its
+reporting is a fair bit nicer.
+
+Command line usage: ::
+
+  figleaf <python file to execute> <args to python file>
+
+The figleaf output is saved into the file '.figleaf', which is an
+*aggregate* of coverage reports from all figleaf runs from this
+directory.  '.figleaf' contains a pickled dictionary of sets; the keys
+are source code filenames, and the sets contain all line numbers
+executed by the Python interpreter. See the docs or command-line
+programs in bin/ for more information.
+
+High level API: ::
+
+ * ``start(ignore_lib=True)`` -- start recording code coverage.
+ * ``stop()``                 -- stop recording code coverage.
+ * ``get_trace_obj()``        -- return the (singleton) trace object.
+ * ``get_info()``             -- get the coverage dictionary
+
+Classes & functions worth knowing about (lower level API):
+
+ * ``get_lines(fp)`` -- return the set of interesting lines in the fp.
+ * ``combine_coverage(d1, d2)`` -- combine coverage info from two dicts.
+ * ``read_coverage(filename)`` -- load the coverage dictionary
+ * ``write_coverage(filename)`` -- write the coverage out.
+ * ``annotate_coverage(...)`` -- annotate a Python file with its coverage info.
+
+Known problems:
+
+ -- module docstrings are *covered* but not found.
+
+AUTHOR: C. Titus Brown, titus@idyll.org, with contributions from Iain Lowe.
+
+'figleaf' is Copyright (C) 2006, 2007 C. Titus Brown.  It is under the
+BSD license.
+"""
+__version__ = "0.6.1"
+
+# __all__ == @CTB
+
+import sys
+import os
+from cPickle import dump, load
+from optparse import OptionParser
+
+import internals
+
+# use builtin sets if in >= 2.4, otherwise use 'sets' module.
+try:
+    set()
+except NameError:
+    from sets import Set as set
+
+def get_lines(fp):
+    """
+    Return the set of interesting lines in the source code read from
+    this file handle.
+    """
+    # rstrip is a workaround for http://bugs.python.org/issue4262
+    src = fp.read().rstrip() + "\n"
+    code = compile(src, "", "exec")
+    
+    return internals.get_interesting_lines(code)
+
+def combine_coverage(d1, d2):
+    """
+    Given two coverage dictionaries, combine the recorded coverage
+    and return a new dictionary.
+    """
+    keys = set(d1.keys())
+    keys.update(set(d2.keys()))
+
+    new_d = {}
+    for k in keys:
+        v = d1.get(k, set())
+        v2 = d2.get(k, set())
+
+        s = set(v)
+        s.update(v2)
+        new_d[k] = s
+
+    return new_d
+
+def write_coverage(filename, append=True):
+    """
+    Write the current coverage info out to the given filename.  If
+    'append' is false, destroy any previously recorded coverage info.
+    """
+    if _t is None:
+        return
+
+    data = internals.CoverageData(_t)
+
+    d = data.gather_files()
+
+    # sum existing coverage?
+    if append:
+        old = {}
+        fp = None
+        try:
+            fp = open(filename)
+        except IOError:
+            pass
+
+        if fp:
+            old = load(fp)
+            fp.close()
+            d = combine_coverage(d, old)
+
+    # ok, save.
+    outfp = open(filename, 'w')
+    try:
+        dump(d, outfp)
+    finally:
+        outfp.close()
+
+def read_coverage(filename):
+    """
+    Read a coverage dictionary in from the given file.
+    """
+    fp = open(filename)
+    try:
+        d = load(fp)
+    finally:
+        fp.close()
+
+    return d
+
+def dump_pickled_coverage(out_fp):
+    """
+    Dump coverage information in pickled format into the given file handle.
+    """
+    dump(_t, out_fp)
+
+def load_pickled_coverage(in_fp):
+    """
+    Replace (overwrite) coverage information from the given file handle.
+    """
+    global _t
+    _t = load(in_fp)
+
+def annotate_coverage(in_fp, out_fp, covered, all_lines,
+                      mark_possible_lines=False):
+    """
+    A simple example coverage annotator that outputs text.
+    """
+    for i, line in enumerate(in_fp):
+        i = i + 1
+
+        if i in covered:
+            symbol = '>'
+        elif i in all_lines:
+            symbol = '!'
+        else:
+            symbol = ' '
+
+        symbol2 = ''
+        if mark_possible_lines:
+            symbol2 = ' '
+            if i in all_lines:
+                symbol2 = '-'
+
+        out_fp.write('%s%s %s' % (symbol, symbol2, line,))
+
+def get_data():
+    if _t:
+        return internals.CoverageData(_t)
+
+#######################
+
+#
+# singleton functions/top-level API
+#
+
+_t = None
+
+def init(exclude_path=None, include_only=None):
+    from internals import CodeTracer
+    
+    global _t
+    if _t is None:
+        _t = CodeTracer(exclude_path, include_only)
+
+def start(ignore_python_lib=True):
+    """
+    Start tracing code coverage.  If 'ignore_python_lib' is True on
+    initial call, ignore all files that live below the same directory as
+    the 'os' module.
+    """
+    global _t
+    if not _t:
+        exclude_path = None
+        if ignore_python_lib:
+            exclude_path = os.path.realpath(os.path.dirname(os.__file__))
+
+        init(exclude_path, None)
+    
+    _t.start()
+
+def start_section(name):
+    global _t
+    _t.start_section(name)
+    
+def stop_section():
+    global _t
+    _t.stop_section()
+
+def stop():
+    """
+    Stop tracing code coverage.
+    """
+    global _t
+    if _t is not None:
+        _t.stop()
+
+def get_trace_obj():
+    """
+    Return the (singleton) trace object, if it exists.
+    """
+    return _t
+
+def get_info(section_name=None):
+    """
+    Get the coverage dictionary from the trace object.
+    """
+    if _t:
+        return get_data().gather_files(section_name)
+
+#############
+
+def display_ast():
+    l = internals.LineGrabber(open(sys.argv[1]))
+    l.pretty_print()
+    print l.lines
+
+def main():
+    """
+    Execute the given Python file with coverage, making it look like it is
+    __main__.
+    """
+    ignore_pylibs = False
+
+    # gather args
+
+    n = 1
+    figleaf_args = []
+    for n in range(1, len(sys.argv)):
+        arg = sys.argv[n]
+        if arg.startswith('-'):
+            figleaf_args.append(arg)
+        else:
+            break
+
+    remaining_args = sys.argv[n:]
+
+    usage = "usage: %prog [options] [python_script arg1 arg2 ...]"
+    option_parser = OptionParser(usage=usage)
+
+    option_parser.add_option('-i', '--ignore-pylibs', action="store_true",
+                             dest="ignore_pylibs", default=False,
+                             help="ignore Python library modules")
+
+    (options, args) = option_parser.parse_args(args=figleaf_args)
+    assert len(args) == 0
+
+    if not remaining_args:
+        option_parser.error("you must specify a python script to run!")
+
+    ignore_pylibs = options.ignore_pylibs
+
+    ## Reset system args so that the subsequently exec'd file can read
+    ## from sys.argv
+    
+    sys.argv = remaining_args
+
+    sys.path[0] = os.path.dirname(sys.argv[0])
+
+    cwd = os.getcwd()
+
+    start(ignore_pylibs)        # START code coverage
+
+    import __main__
+    try:
+        execfile(sys.argv[0], __main__.__dict__)
+    finally:
+        stop()                          # STOP code coverage
+
+        write_coverage(os.path.join(cwd, '.figleaf'))

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/80e25b46/tools/bin/ext/figleaf/_lib.py
----------------------------------------------------------------------
diff --git a/tools/bin/ext/figleaf/_lib.py b/tools/bin/ext/figleaf/_lib.py
new file mode 100644
index 0000000..3ee49f8
--- /dev/null
+++ b/tools/bin/ext/figleaf/_lib.py
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import os.path, sys
+libdir = os.path.join(os.path.dirname(__file__), '../')
+libdir = os.path.normpath(libdir)
+
+if libdir not in sys.path:
+    sys.path.insert(0, libdir)

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/80e25b46/tools/bin/ext/figleaf/annotate.py
----------------------------------------------------------------------
diff --git a/tools/bin/ext/figleaf/annotate.py b/tools/bin/ext/figleaf/annotate.py
new file mode 100644
index 0000000..43099f6
--- /dev/null
+++ b/tools/bin/ext/figleaf/annotate.py
@@ -0,0 +1,241 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Common functions for annotating files with figleaf coverage information.
+"""
+import sys, os
+from optparse import OptionParser
+import ConfigParser
+import re
+import logging
+
+import figleaf
+
+thisdir = os.path.dirname(__file__)
+
+try:                                    # 2.3 compatibility
+    logging.basicConfig(format='%(message)s', level=logging.WARNING)
+except TypeError:
+    pass
+
+logger = logging.getLogger('figleaf.annotate')
+
+DEFAULT_CONFIGURE_FILE = ".figleafrc"
+
+### utilities
+
+def safe_conf_get(conf, section, name, default):
+    try:
+        val = conf.get(section, name)
+    except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+        val = default
+
+    return val
+
+def configure(parser):
+    """
+    Configure the optparse.OptionParser object with defaults, optionally
+    loaded from a configuration file.
+    """
+    CONFIG_FILE = os.environ.get('FIGLEAFRC', DEFAULT_CONFIGURE_FILE)
+    
+    parser.add_option("-c", "--coverage-file", action="store",
+                       type="string", dest="coverage_file",
+                       help="File containing figleaf coverage information.")
+    
+    parser.add_option("-s", "--sections-file", action="store",
+                       type="string", dest="sections_file",
+                       help="File containing figleaf sections coverage info.")
+
+    parser.add_option("-v", "--verbose", action="store_true",
+                      dest="verbose")
+
+    conf_file = ConfigParser.ConfigParser()
+    conf_file.read(CONFIG_FILE)         # ignores if not present
+
+    default_coverage_file = safe_conf_get(conf_file,
+                                          'figleaf', 'coverage_file',
+                                          '.figleaf')
+    default_sections_file = safe_conf_get(conf_file,
+                                          'figleaf', 'sections_file',
+                                          '.figleaf_sections')
+    default_verbose = int(safe_conf_get(conf_file, 'figleaf', 'verbose',
+                                        0))
+
+    parser.set_defaults(coverage_file=default_coverage_file,
+                        sections_file=default_sections_file,
+                        verbose=default_verbose)
+
+def filter_coverage(coverage, re_match):
+    """
+    ...
+    """
+    if not re_match:
+        return coverage
+
+    regexp = re.compile(re_match)
+    
+    d = {}
+    for filename, lines in coverage.items():
+        if regexp.match(filename):
+            d[filename] = lines
+            
+    return d
+
+### commands
+
+def list(options, match=""):
+    """
+    List the filenames in the coverage file, optionally limiting it to
+    those files matching to the regexp 'match'.
+    """
+    if options.verbose:
+        print>>sys.stderr, '** Reading coverage from coverage file %s' % \
+                           (options.coverage_file,)
+        if match:
+            print>>sys.stderr, '** Filtering against regexp "%s"' % (match,)
+        
+    coverage = figleaf.read_coverage(options.coverage_file)
+    coverage = filter_coverage(coverage, match)
+
+    for filename in coverage.keys():
+        print filename
+
+def list_sections(options, match=""):
+    """
+    List the filenames in the coverage file, optionally limiting it to
+    those files matching to the regexp 'match'.
+    """
+    if options.verbose:
+        print>>sys.stderr, '** Reading sections info from sections file %s' % \
+                           (options.sections_file,)
+        if match:
+            print>>sys.stderr, '** Filtering against regexp "%s"' % (match,)
+
+    fp = open(options.sections_file)
+    figleaf.load_pickled_coverage(fp) # @CTB
+
+    data = figleaf.internals.CoverageData(figleaf._t)
+    coverage = data.gather_files()
+    coverage = filter_coverage(coverage, match)
+
+    for filename in coverage.keys():
+        print filename
+
+###
+
+def read_exclude_patterns(filename):
+    """
+    Read in exclusion patterns from a file; these are just regexps.
+    """
+    if not filename:
+        return []
+
+    exclude_patterns = []
+
+    fp = open(filename)
+    for line in fp:
+        line = line.rstrip()
+        if line and not line.startswith('#'):
+            pattern = re.compile(line)
+        exclude_patterns.append(pattern)
+
+    return exclude_patterns
+
+def read_files_list(filename):
+    """
+    Read in a list of files from a file; these are relative or absolute paths.
+    """
+    s = {}
+    for line in open(filename):
+        f = line.strip()
+        s[os.path.abspath(f)] = 1
+
+    return s
+
+def filter_files(filenames, exclude_patterns = [], files_list = {}):
+    files_list = dict(files_list)       # make copy
+
+    # list of files specified?
+    if files_list:
+        for filename in files_list.keys():
+            yield filename
+
+        filenames = [ os.path.abspath(x) for x in filenames ]
+        for filename in filenames:
+            try:
+                del files_list[filename]
+            except KeyError:
+                logger.info('SKIPPING %s -- not in files list' % (filename,))
+            
+        return
+
+    ### no files list given -- handle differently
+
+    for filename in filenames:
+        abspath = os.path.abspath(filename)
+        
+        # check to see if we match anything in the exclude_patterns list
+        skip = False
+        for pattern in exclude_patterns:
+            if pattern.search(filename):
+                logger.info('SKIPPING %s -- matches exclusion pattern' % \
+                            (filename,))
+                skip = True
+                break
+
+        if skip:
+            continue
+
+        # next, check to see if we're part of the figleaf package.
+        if thisdir in filename:
+            logger.debug('SKIPPING %s -- part of the figleaf package' % \
+                         (filename,))
+            continue
+
+        # also, check for <string> (source file from things like 'exec'):
+        if filename == '<string>':
+            continue
+
+        # miscellaneous other things: doctests
+        if filename.startswith('<doctest '):
+            continue
+
+        yield filename
+
+###
+
+def main():
+    parser = OptionParser()
+    configure(parser)
+    
+    options, args = parser.parse_args()
+
+    if not len(args):
+        print "ERROR: You must specify a command like 'list' or 'report'.  Use"
+        print "\n    %s -h\n" % (sys.argv[0],)
+        print "for help on commands and options."
+        sys.exit(-1)
+        
+    cmd = args.pop(0)
+
+    if cmd == 'list':
+        list(options, *args)
+    elif cmd == 'list_sections':
+        list_sections(options, *args)
+
+    sys.exit(0)

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/80e25b46/tools/bin/ext/figleaf/annotate_cover.py
----------------------------------------------------------------------
diff --git a/tools/bin/ext/figleaf/annotate_cover.py b/tools/bin/ext/figleaf/annotate_cover.py
new file mode 100644
index 0000000..ac72bfa
--- /dev/null
+++ b/tools/bin/ext/figleaf/annotate_cover.py
@@ -0,0 +1,159 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import figleaf
+import os
+import re
+
+from annotate import read_exclude_patterns, filter_files, logger
+
+def report_as_cover(coverage, exclude_patterns=[], ):
+    ### now, output.
+
+    keys = coverage.keys()
+    info_dict = {}
+    
+    for k in filter_files(keys):
+        try:
+            pyfile = open(k, 'rU')
+            lines = figleaf.get_lines(pyfile)
+        except IOError:
+            logger.warning('CANNOT OPEN: %s' % k)
+            continue
+        except KeyboardInterrupt:
+            raise
+        except Exception, e:
+            logger.error('ERROR: file %s, exception %s' % (pyfile, str(e)))
+            continue
+
+        # ok, got all the info.  now annotate file ==> html.
+
+        covered = coverage[k]
+        pyfile = open(k)
+        (n_covered, n_lines, output) = make_cover_lines(lines, covered, pyfile)
+
+
+        try:
+            pcnt = n_covered * 100. / n_lines
+        except ZeroDivisionError:
+            pcnt = 100
+        info_dict[k] = (n_lines, n_covered, pcnt)
+
+        outfile = make_cover_filename(k)
+        try:
+            outfp = open(outfile, 'w')
+            outfp.write("\n".join(output))
+            outfp.write("\n")
+            outfp.close()
+        except IOError:
+            logger.warning('cannot open filename %s' % (outfile,))
+            continue
+
+        logger.info('reported on %s' % (outfile,))
+
+    ### print a summary, too.
+
+    info_dict_items = info_dict.items()
+
+    def sort_by_pcnt(a, b):
+        a = a[1][2]
+        b = b[1][2]
+
+        return -cmp(a,b)
+
+    info_dict_items.sort(sort_by_pcnt)
+
+    logger.info('reported on %d file(s) total\n' % len(info_dict))
+    return len(info_dict)
+
+def make_cover_lines(line_info, coverage_info, fp):
+    n_covered = n_lines = 0
+    output = []
+    
+    for i, line in enumerate(fp):
+        is_covered = False
+        is_line = False
+
+        i += 1
+
+        if i in coverage_info:
+            is_covered = True
+            prefix = '+'
+
+            n_covered += 1
+            n_lines += 1
+        elif i in line_info:
+            prefix = '-'
+            is_line = True
+
+            n_lines += 1
+        else:
+            prefix = '0'
+
+        line = line.rstrip()
+        output.append(prefix + ' ' + line)
+    
+    return (n_covered, n_lines, output)
+
+def make_cover_filename(orig):
+    return orig + '.cover'
+
+def main():
+    import sys
+    import logging
+    from optparse import OptionParser
+    
+    ###
+
+    option_parser = OptionParser()
+
+    option_parser.add_option('-x', '--exclude-patterns', action="store",
+                             dest="exclude_patterns_file",
+                             help="file containing regexp patterns to exclude")
+
+    option_parser.add_option('-q', '--quiet', action='store_true',
+                             dest='quiet',
+         help="file containig regexp patterns of files to exclude from report")
+    
+    option_parser.add_option('-D', '--debug', action='store_true',
+                             dest='debug',
+                             help='Show all debugging messages')
+
+    (options, args) = option_parser.parse_args()
+
+    if options.quiet:
+        logging.disable(logging.DEBUG)
+
+    if options.debug:
+        logger.setLevel(logging.DEBUG)
+
+    ### load
+
+    if not args:
+        args = ['.figleaf']
+
+    coverage = {}
+    for filename in args:
+        logger.debug("loading coverage info from '%s'\n" % (filename,))
+        d = figleaf.read_coverage(filename)
+        coverage = figleaf.combine_coverage(coverage, d)
+
+    if not coverage:
+        logger.warning('EXITING -- no coverage info!\n')
+        sys.exit(-1)
+
+    exclude = read_exclude_patterns(options.exclude_patterns_file)
+    report_as_cover(coverage, exclude)

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/80e25b46/tools/bin/ext/figleaf/annotate_html.py
----------------------------------------------------------------------
diff --git a/tools/bin/ext/figleaf/annotate_html.py b/tools/bin/ext/figleaf/annotate_html.py
new file mode 100644
index 0000000..594d07a
--- /dev/null
+++ b/tools/bin/ext/figleaf/annotate_html.py
@@ -0,0 +1,292 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import figleaf
+import os
+import re
+
+# use builtin sets if in >= 2.4, otherwise use 'sets' module.
+try:
+    set()
+except NameError:
+    from sets import Set as set
+
+from figleaf.annotate import read_exclude_patterns, filter_files, logger, \
+     read_files_list
+
+###
+
+def annotate_file(fp, lines, covered):
+    # initialize
+    n_covered = n_lines = 0
+
+    output = []
+    for i, line in enumerate(fp):
+        is_covered = False
+        is_line = False
+
+        i += 1
+
+        if i in covered:
+            is_covered = True
+
+            n_covered += 1
+            n_lines += 1
+        elif i in lines:
+            is_line = True
+
+            n_lines += 1
+
+        color = 'black'
+        if is_covered:
+            color = 'green'
+        elif is_line:
+            color = 'red'
+
+        line = escape_html(line.rstrip())
+        output.append('<font color="%s">%4d. %s</font>' % (color, i, line))
+
+    try:
+        percent = n_covered * 100. / n_lines
+    except ZeroDivisionError:
+        percent = 100
+
+    return output, n_covered, n_lines, percent
+
+def write_html_summary(info_dict, directory):
+    info_dict_items = info_dict.items()
+
+    def sort_by_percent(a, b):
+        a = a[1][2]
+        b = b[1][2]
+
+        return -cmp(a,b)
+    info_dict_items.sort(sort_by_percent)
+
+    summary_lines = sum([ v[0] for (k, v) in info_dict_items])
+    summary_cover = sum([ v[1] for (k, v) in info_dict_items])
+
+    summary_percent = 100
+    if summary_lines:
+        summary_percent = float(summary_cover) * 100. / float(summary_lines)
+
+
+    percents = [ float(v[1]) * 100. / float(v[0])
+                 for (k, v) in info_dict_items if v[0] ]
+    
+    percent_90 = [ x for x in percents if x >= 90 ]
+    percent_75 = [ x for x in percents if x >= 75 ]
+    percent_50 = [ x for x in percents if x >= 50 ]
+
+    ### write out summary.
+
+    index_fp = open('%s/index.html' % (directory,), 'w')
+    index_fp.write('''
+<html>
+<title>figleaf code coverage report</title>
+<h2>Summary</h2>
+%d files total: %d files &gt; 90%%, %d files &gt; 75%%, %d files &gt; 50%%
+<p>
+<table border=1>
+<tr>
+ <th>Filename</th><th># lines</th><th># covered</th><th>%% covered</th>
+</tr>
+
+<tr>
+ <td><b>totals:</b></td>
+ <td><b>%d</b></td>
+ <td><b>%d</b></td>
+ <td><b>%.1f%%</b></td>
+</tr>
+
+<tr></tr>
+
+''' % (len(percents), len(percent_90), len(percent_75), len(percent_50),
+       summary_lines, summary_cover, summary_percent,))
+
+    for filename, (n_lines, n_covered, percent_covered,) in info_dict_items:
+        html_outfile = make_html_filename(filename)
+
+        index_fp.write('''
+<tr>
+ <td><a href="./%s">%s</a></td>
+ <td>%d</td>
+ <td>%d</td>
+ <td>%.1f</td>
+</tr>
+''' % (html_outfile, filename, n_lines, n_covered, percent_covered,))
+
+    index_fp.write('</table>\n')
+    index_fp.close()
+    
+
+def report_as_html(coverage, directory, exclude_patterns, files_list):
+    """
+    Write an HTML report on all of the files, plus a summary.
+    """
+
+    ### now, output.
+
+    keys = coverage.keys()
+    info_dict = {}
+    for pyfile in filter_files(keys, exclude_patterns, files_list):
+
+        try:
+            fp = open(pyfile, 'rU')
+            lines = figleaf.get_lines(fp)
+        except KeyboardInterrupt:
+            raise
+        except IOError:
+            logger.error('CANNOT OPEN: %s' % (pyfile,))
+            continue
+        except Exception, e:
+            logger.error('ERROR: file %s, exception %s' % (pyfile, str(e)))
+            continue
+
+        #
+        # ok, we want to annotate this file.  now annotate file ==> html.
+        #
+
+        # initialize
+        covered = coverage.get(pyfile, set())
+
+        # rewind
+        fp.seek(0)
+
+        # annotate
+        output, n_covered, n_lines, percent = annotate_file(fp, lines, covered)
+
+        # summarize
+        info_dict[pyfile] = (n_lines, n_covered, percent)
+
+        # write out the file
+        html_outfile = make_html_filename(pyfile)
+        html_outfile = os.path.join(directory, html_outfile)
+        html_outfp = open(html_outfile, 'w')
+        
+        html_outfp.write('source file: <b>%s</b><br>\n' % (pyfile,))
+        html_outfp.write('''
+
+file stats: <b>%d lines, %d executed: %.1f%% covered</b>
+<pre>
+%s
+</pre>
+
+''' % (n_lines, n_covered, percent, "\n".join(output)))
+            
+        html_outfp.close()
+
+        logger.info('reported on %s' % (pyfile,))
+
+    ### print a summary, too.
+    write_html_summary(info_dict, directory)
+
+    logger.info('reported on %d file(s) total\n' % len(info_dict))
+
+def prepare_reportdir(dirname):
+    "Create output directory."
+    try:
+        os.mkdir(dirname)
+    except OSError:                         # already exists
+        pass
+
+def make_html_filename(orig):
+    "'escape' original paths into a single filename"
+
+    orig = os.path.abspath(orig)
+#    orig = os.path.basename(orig)
+    orig = os.path.splitdrive(orig)[1]
+    orig = orig.replace('_', '__')
+    orig = orig.replace(os.path.sep, '_')
+    orig += '.html'
+    return orig
+
+def escape_html(s):
+    s = s.replace("&", "&amp;")
+    s = s.replace("<", "&lt;")
+    s = s.replace(">", "&gt;")
+    s = s.replace('"', "&quot;")
+    return s
+
+def main():
+    import sys
+    import logging
+    from optparse import OptionParser
+    ###
+
+    usage = "usage: %prog [options] [coverage files ... ]"
+    option_parser = OptionParser(usage=usage)
+
+    option_parser.add_option('-x', '--exclude-patterns', action="store",
+                             dest="exclude_patterns_file",
+        help="file containing regexp patterns of files to exclude from report")
+
+    option_parser.add_option('-f', '--files-list', action="store",
+                             dest="files_list",
+                             help="file containing filenames to report on")
+
+    option_parser.add_option('-d', '--output-directory', action='store',
+                             dest="output_dir",
+                             default = "html",
+                             help="directory for HTML output")
+
+    option_parser.add_option('-q', '--quiet', action='store_true',
+                             dest='quiet',
+                             help='Suppress all but error messages')
+    
+    option_parser.add_option('-D', '--debug', action='store_true',
+                             dest='debug',
+                             help='Show all debugging messages')
+
+    (options, args) = option_parser.parse_args()
+
+    if options.quiet:
+        logging.disable(logging.DEBUG)
+        
+    if options.debug:
+        logger.setLevel(logging.DEBUG)
+
+    ### load/combine
+
+    if not args:
+        args = ['.figleaf']
+
+    coverage = {}
+    for filename in args:
+        logger.debug("loading coverage info from '%s'\n" % (filename,))
+        try:
+            d = figleaf.read_coverage(filename)
+            coverage = figleaf.combine_coverage(coverage, d)
+        except IOError:
+            logger.error("cannot open filename '%s'\n" % (filename,))
+
+    if not coverage:
+        logger.warning('EXITING -- no coverage info!\n')
+        sys.exit(-1)
+
+    exclude = []
+    if options.exclude_patterns_file:
+        exclude = read_exclude_patterns(options.exclude_patterns_file)
+
+    files_list = {}
+    if options.files_list:
+        files_list = read_files_list(options.files_list)
+
+    ### make directory
+    prepare_reportdir(options.output_dir)
+    report_as_html(coverage, options.output_dir, exclude, files_list)
+
+    print 'figleaf: HTML output written to %s' % (options.output_dir,)

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/80e25b46/tools/bin/ext/figleaf/annotate_sections.py
----------------------------------------------------------------------
diff --git a/tools/bin/ext/figleaf/annotate_sections.py b/tools/bin/ext/figleaf/annotate_sections.py
new file mode 100644
index 0000000..6d7fb89
--- /dev/null
+++ b/tools/bin/ext/figleaf/annotate_sections.py
@@ -0,0 +1,95 @@
+#! /usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import figleaf
+from figleaf import internals
+from sets import Set as set
+import sys
+from cPickle import load
+import os
+from optparse import OptionParser
+
+def main():
+    #### OPTIONS
+
+    parser = OptionParser()
+
+    parser.add_option('-c', '--coverage', nargs=1, action="store",
+                      dest="coverage_file", 
+                      help = 'load coverage info from this file',
+                      default='.figleaf_sections')
+
+    ####
+
+    (options, args) = parser.parse_args(sys.argv[1:])
+    coverage_file = options.coverage_file
+    
+    figleaf.load_pickled_coverage(open(coverage_file))
+    data = internals.CoverageData(figleaf._t)
+    full_cov = data.gather_files()
+
+    for filename in args:
+        annotate_file_with_sections(filename, data, full_cov)
+
+def annotate_file_with_sections(short, data, full_cov):
+    full = os.path.abspath(short)
+
+    tags = {}
+    sections = data.gather_sections(full)
+    sections.update(data.gather_sections(short))
+
+    print data.sections
+
+    print '*** PROCESSING:', short, '\n\t==>', short + '.sections'
+    for tag, cov in sections.items():
+        if cov:
+            tags[tag] = cov
+
+    if not tags:
+        print '*** No coverage info for file', short
+
+    tag_names = tags.keys()
+    tag_names.sort()
+    tag_names.reverse()
+
+    tags["-- all coverage --"] = full_cov.get(full, set())
+    tag_names.insert(0, "-- all coverage --")
+
+    n_tags = len(tag_names)
+    
+    fp = open(short + '.sections', 'w')
+
+    for i, tag in enumerate(tag_names):
+        fp.write('%s%s\n' % ('| ' * i, tag))
+    fp.write('| ' * n_tags)
+    fp.write('\n\n')
+
+    source = open(full)
+    for n, line in enumerate(source):
+        marks = ""
+        for tag in tag_names:
+            cov = tags[tag]
+
+            symbol = '  '
+            if (n+1) in cov:
+                symbol = '+ '
+
+            marks += symbol
+
+        fp.write('%s  | %s' % (marks, line))
+    
+    fp.close()

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/80e25b46/tools/bin/ext/figleaf/figleaf2html
----------------------------------------------------------------------
diff --git a/tools/bin/ext/figleaf/figleaf2html b/tools/bin/ext/figleaf/figleaf2html
new file mode 100755
index 0000000..58636b0
--- /dev/null
+++ b/tools/bin/ext/figleaf/figleaf2html
@@ -0,0 +1,7 @@
+#! /usr/bin/env python
+"""
+Output an HTML-ized coverage report.
+"""
+import _lib
+import figleaf.annotate_html
+figleaf.annotate_html.main()


Mime
View raw message