impala-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbap...@apache.org
Subject [1/2] incubator-impala git commit: IMPALA-4207: test infra: move Hive options from connection to cluster options
Date Thu, 29 Sep 2016 15:40:35 GMT
Repository: incubator-impala
Updated Branches:
  refs/heads/master 241c7e019 -> a5e84ac01


IMPALA-4207: test infra: move Hive options from connection to cluster options

Various test tools and frameworks, including the stress test, random
query generator, and nested types loader, share common modules. This
change

  IMPALA-3980: qgen: re-enable Hive as a target database

made changes to tests.comparison.cli_options, the shared command line
option module, and to tests.comparison.cluster, the shared module for
modeling various Impala clusters. Those changes were for the random
query generator, but didn't take into account the other shared entry
points. It was possible to call some of those entry points in such a way
as to produce an exception, because the Hive-related options are now
required for miniclusters, but the Hive-related options weren't always
being initialized in those entry points.

The simple fix is to say that, because Hive settings are now needed to
create Minicluster objects, make the Hive options initialized with
cluster options, not connection options. While I was making these
changes, I fixed all flake8 problems in this file.

Testing:

- qgen/minicluster unit tests (regression test)
- full private data load job, including load_nested.py (bug
  verification)
- data_generator.py run (regression test), long enough to verify
  connection to the minicluster, using both Hive and Impala
- discrepancy_searcher.py run (regression test), long enough verify
  connection to the minicluster, using both Hive and Impala
- concurrent_select.py (in typical mode using a CM host, this is a
  regression check; from the command line against the minicluster, this
  is a bug verification)

Change-Id: I2a2915e6db85ddb3d8e1bce8035eccd0c9324b4b
Reviewed-on: http://gerrit.cloudera.org:8080/4555
Reviewed-by: Michael Brown <mikeb@cloudera.com>
Reviewed-by: Ishaan Joshi <ishaan@cloudera.com>
Tested-by: Internal Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/a35e4380
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/a35e4380
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/a35e4380

Branch: refs/heads/master
Commit: a35e4380966346b47a81eeb2500e01bf97fa8f95
Parents: 241c7e0
Author: Michael Brown <mikeb@cloudera.com>
Authored: Wed Sep 28 08:29:19 2016 -0700
Committer: Internal Jenkins <cloudera-hudson@gerrit.cloudera.org>
Committed: Thu Sep 29 02:10:17 2016 +0000

----------------------------------------------------------------------
 tests/comparison/cli_options.py | 199 +++++++++++++++++++++--------------
 tests/comparison/cluster.py     |   3 +
 2 files changed, 124 insertions(+), 78 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a35e4380/tests/comparison/cli_options.py
----------------------------------------------------------------------
diff --git a/tests/comparison/cli_options.py b/tests/comparison/cli_options.py
index 92901f4..70740e6 100644
--- a/tests/comparison/cli_options.py
+++ b/tests/comparison/cli_options.py
@@ -24,22 +24,32 @@ from getpass import getuser
 from tempfile import gettempdir
 
 import db_connection
-from cluster import CmCluster, DEFAULT_HIVE_HOST, DEFAULT_HIVE_PORT, MiniCluster, \
-  MiniHiveCluster
+from cluster import (
+    CmCluster,
+    DEFAULT_HIVE_HOST,
+    DEFAULT_HIVE_PASSWORD,
+    DEFAULT_HIVE_PORT,
+    DEFAULT_HIVE_USER,
+    MiniCluster,
+    MiniHiveCluster,
+)
 from db_types import TYPES
 
-def add_logging_options(section, default_debug_log_file=None):
+
+def add_logging_options(parser, default_debug_log_file=None):
   if not default_debug_log_file:
     default_debug_log_file = os.path.join(
         gettempdir(), os.path.basename(sys.modules["__main__"].__file__) + ".log")
-  section.add_argument('--log-level', default='INFO',
+  parser.add_argument(
+      '--log-level', default='INFO',
       help='The log level to use.', choices=('DEBUG', 'INFO', 'WARN', 'ERROR'))
-  section.add_argument('--debug-log-file', default=default_debug_log_file,
+  parser.add_argument(
+      '--debug-log-file', default=default_debug_log_file,
       help='Path to debug log file.')
 
 
 def configure_logging(log_level, debug_log_file=None, log_thread_id=False,
-    log_process_id=False):
+                      log_process_id=False):
   root_logger = logging.getLogger()
   root_logger.setLevel(logging.DEBUG)
 
@@ -65,8 +75,8 @@ def configure_logging(log_level, debug_log_file=None, log_thread_id=False,
   def create_third_party_filter(level):
     def filter_record(record):
       name = record.name
-      if name.startswith("impala.") or name.startswith("paramiko.") \
-          or name.startswith("hdfs") or name.startswith("requests"):
+      if name.startswith("impala.") or name.startswith("paramiko.") or \
+         name.startswith("hdfs") or name.startswith("requests"):
         return record.levelno >= level
       return True
     log_filter = logging.Filter()
@@ -77,47 +87,79 @@ def configure_logging(log_level, debug_log_file=None, log_thread_id=False,
     file_logger.addFilter(create_third_party_filter(logging.INFO))
 
 
-def add_ssh_options(section):
-  section.add_argument('--ssh-user', metavar='user name', default=getuser(),
+def add_ssh_options(parser):
+  parser.add_argument(
+      '--ssh-user', metavar='user name', default=getuser(),
       help='The user name to use for SSH connections to cluster nodes.')
-  section.add_argument('--ssh-key-file', metavar='path to file',
+  parser.add_argument(
+      '--ssh-key-file', metavar='path to file',
       help='Specify an additional SSH key other than the defaults in ~/.ssh.')
-  section.add_argument('--ssh-port', metavar='number', type=int, default=22,
+  parser.add_argument(
+      '--ssh-port', metavar='number', type=int, default=22,
       help='The port number to use when connecting through SSH.')
 
 
-def add_db_name_option(section):
-  section.add_argument('--db-name', default='randomness',
+def add_db_name_option(parser):
+  parser.add_argument(
+      '--db-name', default='randomness',
       help='The name of the database to use. Ex: functional.')
 
 
-def add_cluster_options(section):
-  add_minicluster_options(section)
-  add_cm_options(section)
-  add_ssh_options(section)
-  section.add_argument("--hadoop-user-name", default=getuser(),
-      help="The user name to use when interacting with hadoop.")
+def add_cluster_options(parser):
+  add_minicluster_options(parser)
+  add_cm_options(parser)
+  add_ssh_options(parser)
+  parser.add_argument(
+      '--hadoop-user-name', default=getuser(),
+      help='The user name to use when interacting with hadoop.')
+
+
+def add_minicluster_options(parser):
+  group = parser.add_argument_group('Hive Options')
+  group.add_argument(
+      '--use-hive', action='store_true', default=False,
+      help='Use Hive (Impala will be skipped)')
+  group.add_argument(
+      '--hive-host', default=DEFAULT_HIVE_HOST,
+      help='The name of the host running the HS2')
+  group.add_argument(
+      '--hive-port', default=DEFAULT_HIVE_PORT, type=int,
+      help='The port of HiveServer2')
+  group.add_argument(
+      '--hive-user', default=DEFAULT_HIVE_USER,
+      help='The user name to use when connecting to HiveServer2')
+  group.add_argument(
+      '--hive-password', default=DEFAULT_HIVE_PASSWORD,
+      help='The password to use when connecting to HiveServer2')
+  parser.add_argument_group(group)
 
-def add_minicluster_options(section):
-  section.add_argument('--minicluster-num-impalads', default=3, type=int,
-      metavar='num impalads', help='The number of impalads in the mini cluster.')
+  parser.add_argument(
+      '--minicluster-num-impalads', default=3, type=int, metavar='num impalads',
+      help='The number of impalads in the mini cluster.')
 
-def add_cm_options(section):
-  section.add_argument('--cm-host', metavar='host name',
+
+def add_cm_options(parser):
+  parser.add_argument(
+      '--cm-host', metavar='host name',
       help='The host name of the CM server.')
-  section.add_argument('--cm-port', default=7180, type=int, metavar='port number',
+  parser.add_argument(
+      '--cm-port', default=7180, type=int, metavar='port number',
       help='The port of the CM server.')
-  section.add_argument('--cm-user', default="admin", metavar='user name',
+  parser.add_argument(
+      '--cm-user', default="admin", metavar='user name',
       help='The name of the CM user.')
-  section.add_argument('--cm-password', default="admin", metavar='password',
+  parser.add_argument(
+      '--cm-password', default="admin", metavar='password',
       help='The password for the CM user.')
-  section.add_argument('--cm-cluster-name', metavar='name',
+  parser.add_argument(
+      '--cm-cluster-name', metavar='name',
       help='If CM manages multiple clusters, use this to specify which cluster to use.')
 
 
 def create_cluster(args):
   if args.cm_host:
-    cluster = CmCluster(args.cm_host, user=args.cm_user, password=args.cm_password,
+    cluster = CmCluster(
+        args.cm_host, user=args.cm_user, password=args.cm_password,
         cluster_name=args.cm_cluster_name, ssh_user=args.ssh_user, ssh_port=args.ssh_port,
         ssh_key_file=args.ssh_key_file)
   elif args.use_hive:
@@ -128,74 +170,73 @@ def create_cluster(args):
   return cluster
 
 
-def add_storage_format_options(section):
+def add_storage_format_options(parser):
   storage_formats = ['avro', 'parquet', 'rcfile', 'sequencefile', 'textfile']
-  section.add_argument('--storage-file-formats', default=','.join(storage_formats),
+  parser.add_argument(
+      '--storage-file-formats', default=','.join(storage_formats),
       help='A comma separated list of storage formats to use.')
 
 
-def add_data_types_options(section):
-  section.add_argument('--data-types',
-      default=','.join(type_.__name__ for type_ in TYPES),
+def add_data_types_options(parser):
+  parser.add_argument(
+      '--data-types', default=','.join(type_.__name__ for type_ in TYPES),
       help='A comma separated list of data types to use.')
 
 
-def add_timeout_option(section):
-  section.add_argument('--timeout', default=(3 * 60), type=int,
-      help='Query timeout in seconds')
+def add_timeout_option(parser):
+  parser.add_argument(
+      '--timeout', default=(3 * 60), type=int, help='Query timeout in seconds')
 
 
 def add_connection_option_groups(parser):
 
-  group = parser.add_argument_group("Hive Options")
-  group.add_argument('--use-hive', action='store_true', default=False,
-      help='Use Hive (Impala will be skipped)')
-  group.add_argument('--hive-host', default=DEFAULT_HIVE_HOST,
-      help="The name of the host running the HS2")
-  group.add_argument("--hive-port", default=DEFAULT_HIVE_PORT, type=int,
-      help="The port of HiveServer2")
-  group.add_argument('--hive-user', default='hive',
-      help="The user name to use when connecting to HiveServer2")
-  group.add_argument('--hive-password', default='hive',
-      help="The password to use when connecting to HiveServer2")
-  parser.add_argument_group(group)
-
   group = parser.add_argument_group('MySQL Options')
-  group.add_argument('--use-mysql', action='store_true',
-      help='Use MySQL')
-  group.add_argument('--mysql-host', default='localhost',
+  group.add_argument(
+      '--use-mysql', action='store_true', help='Use MySQL')
+  group.add_argument(
+      '--mysql-host', default='localhost',
       help='The name of the host running the MySQL database.')
-  group.add_argument('--mysql-port', default=3306, type=int,
+  group.add_argument(
+      '--mysql-port', default=3306, type=int,
       help='The port of the host running the MySQL database.')
-  group.add_argument('--mysql-user', default='root',
+  group.add_argument(
+      '--mysql-user', default='root',
       help='The user name to use when connecting to the MySQL database.')
-  group.add_argument('--mysql-password',
+  group.add_argument(
+      '--mysql-password',
       help='The password to use when connecting to the MySQL database.')
   parser.add_argument_group(group)
 
   group = parser.add_argument_group('Oracle Options')
-  group.add_argument('--use-oracle', action='store_true',
-      help='Use Oracle')
-  group.add_argument('--oracle-host', default='localhost',
+  group.add_argument('--use-oracle', action='store_true', help='Use Oracle')
+  group.add_argument(
+      '--oracle-host', default='localhost',
       help='The name of the host running the Oracle database.')
-  group.add_argument('--oracle-port', default=1521, type=int,
+  group.add_argument(
+      '--oracle-port', default=1521, type=int,
       help='The port of the host running the Oracle database.')
-  group.add_argument('--oracle-user', default='system',
+  group.add_argument(
+      '--oracle-user', default='system',
       help='The user name to use when connecting to the Oracle database.')
-  group.add_argument('--oracle-password',
+  group.add_argument(
+      '--oracle-password',
       help='The password to use when connecting to the Oracle database.')
   parser.add_argument_group(group)
 
   group = parser.add_argument_group('Postgresql Options')
-  group.add_argument('--use-postgresql', action='store_true',
-      help='Use Postgresql')
-  group.add_argument('--postgresql-host', default='localhost',
+  group.add_argument(
+      '--use-postgresql', action='store_true', help='Use Postgresql')
+  group.add_argument(
+      '--postgresql-host', default='localhost',
       help='The name of the host running the Postgresql database.')
-  group.add_argument('--postgresql-port', default=5432, type=int,
+  group.add_argument(
+      '--postgresql-port', default=5432, type=int,
       help='The port of the host running the Postgresql database.')
-  group.add_argument('--postgresql-user', default='postgres',
+  group.add_argument(
+      '--postgresql-user', default='postgres',
       help='The user name to use when connecting to the Postgresql database.')
-  group.add_argument('--postgresql-password',
+  group.add_argument(
+      '--postgresql-password',
       help='The password to use when connecting to the Postgresql database.')
   parser.add_argument_group(group)
 
@@ -209,8 +250,8 @@ def get_db_type(args):
   if args.use_postgresql:
     db_types.append(db_connection.POSTGRESQL)
   if not db_types:
-    raise Exception("At least one of --use-mysql, --use-oracle, or --use-postgresql"
-        "must be used")
+    raise Exception(
+        "At least one of --use-mysql, --use-oracle, or --use-postgresql must be used")
   elif len(db_types) > 1:
     raise Exception("Too many databases requested: %s" % db_types)
   return db_types[0]
@@ -228,9 +269,10 @@ def create_connection(args, db_type=None, db_name=None):
   elif db_type == db_connection.HIVE:
     conn_class = db_connection.HiveConnection
   else:
-    raise Exception('Unexpected db_type: %s; expected one of %s.'
-        % (db_type, ', '.join([db_connection.POSTGRESQL, db_connection.MYSQL,
-              db_connection.ORACLE])))
+    raise Exception(
+        'Unexpected db_type: %s; expected one of %s.' % (
+            db_type, ', '.join([db_connection.POSTGRESQL, db_connection.MYSQL,
+                                db_connection.ORACLE])))
   prefix = db_type.lower()
   return conn_class(
       user_name=getattr(args, prefix + '_user'),
@@ -240,9 +282,10 @@ def create_connection(args, db_type=None, db_name=None):
       db_name=db_name)
 
 
-def add_kerberos_options(section):
-  section.add_argument("--use-kerberos", action="store_true",
+def add_kerberos_options(parser):
+  parser.add_argument(
+      "--use-kerberos", action="store_true",
       help="Use kerberos when communicating with Impala. This requires that kinit has"
       " already been done before running this script.")
-  section.add_argument("--kerberos-principal", default=getuser(),
-      help="The principal name to use.")
+  parser.add_argument(
+      "--kerberos-principal", default=getuser(), help="The principal name to use.")

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a35e4380/tests/comparison/cluster.py
----------------------------------------------------------------------
diff --git a/tests/comparison/cluster.py b/tests/comparison/cluster.py
index a009e92..bfb8fa4 100644
--- a/tests/comparison/cluster.py
+++ b/tests/comparison/cluster.py
@@ -54,6 +54,9 @@ LOG = logging.getLogger(os.path.splitext(os.path.basename(__file__))[0])
 
 DEFAULT_HIVE_HOST = '127.0.0.1'
 DEFAULT_HIVE_PORT = 11050
+DEFAULT_HIVE_USER = 'hive'
+DEFAULT_HIVE_PASSWORD = 'hive'
+
 DEFAULT_TIMEOUT = 300
 
 class Cluster(object):


Mime
View raw message