impala-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tarmstr...@apache.org
Subject [32/50] [abbrv] incubator-impala git commit: IMPALA-4454: test_kudu.TestShowCreateTable flaky
Date Thu, 17 Nov 2016 16:09:43 GMT
IMPALA-4454: test_kudu.TestShowCreateTable flaky

The cause of the flakiness is Kudu CREATE TABLE operations
that are sometimes taking a long time, leading to timeouts
in the hiveserver2 connection. This patch adds the ability
for tests using the 'conn' pytest fixture to specify a
timeout to connect(), and sets a timeout of 5 minutes for
this test.

Change-Id: I2727c27ff66140ac4043bcad332cd4e1d72b255f
Reviewed-on: http://gerrit.cloudera.org:8080/5040
Reviewed-by: Michael Brown <mikeb@cloudera.com>
Reviewed-by: Alex Behm <alex.behm@cloudera.com>
Tested-by: Internal Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/d15f86cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/d15f86cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/d15f86cb

Branch: refs/heads/hadoop-next
Commit: d15f86cb6fa5eb482b039f036dab3d979c7513c8
Parents: c68734a
Author: Thomas Tauber-Marshall <tmarshall@cloudera.com>
Authored: Thu Nov 10 15:13:56 2016 -0800
Committer: Internal Jenkins <cloudera-hudson@gerrit.cloudera.org>
Committed: Fri Nov 11 20:04:01 2016 +0000

----------------------------------------------------------------------
 tests/conftest.py             | 14 +++++++++-----
 tests/query_test/test_kudu.py |  5 +++++
 2 files changed, 14 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/d15f86cb/tests/conftest.py
----------------------------------------------------------------------
diff --git a/tests/conftest.py b/tests/conftest.py
index c75f117..6bbbf78 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -36,6 +36,7 @@ from tests.util.filesystem_utils import FILESYSTEM, ISILON_WEBHDFS_PORT
 logging.basicConfig(level=logging.INFO, format='%(threadName)s: %(message)s')
 LOG = logging.getLogger('test_configuration')
 
+DEFAULT_CONN_TIMEOUT = 45
 
 def _get_default_nn_http_addr():
   """Return the namenode ip and webhdfs port if the default shouldn't be used"""
@@ -320,6 +321,7 @@ def conn(request):
          provided by get_db_name(), it must not exist. Classes that use both
          auto_create_db() and get_db_name() should generate a random name in
          get_db_name() and cache it.
+       - get_conn_timeout(): The timeout, in seconds, to use for this connection.
      The returned connection will have a 'db_name' property.
 
      See the 'unique_database' fixture above if you want to use Impala's custom python
@@ -327,8 +329,10 @@ def conn(request):
   """
   db_name = __call_cls_method_if_exists(request.cls, "get_db_name")
   use_unique_conn = __call_cls_method_if_exists(request.cls, "auto_create_db")
+  timeout = __call_cls_method_if_exists(request.cls, "get_conn_timeout") or \
+      DEFAULT_CONN_TIMEOUT
   if use_unique_conn:
-    with __unique_conn(db_name=db_name) as conn:
+    with __unique_conn(db_name=db_name, timeout=timeout) as conn:
       yield conn
   else:
     with __auto_closed_conn(db_name=db_name) as conn:
@@ -345,7 +349,7 @@ def __call_cls_method_if_exists(cls, method_name):
 
 
 @contextlib.contextmanager
-def __unique_conn(db_name=None):
+def __unique_conn(db_name=None, timeout=DEFAULT_CONN_TIMEOUT):
   """Connects to Impala and creates a new database, then returns a connection to it.
      This is intended to be used in a "with" block. Upon exit, the database will be
      dropped. A database name can be provided by 'db_name', a database by that name
@@ -362,7 +366,7 @@ def __unique_conn(db_name=None):
   with __auto_closed_conn() as conn:
     with __auto_closed_cursor(conn) as cur:
       cur.execute("CREATE DATABASE %s" % db_name)
-  with __auto_closed_conn(db_name=db_name) as conn:
+  with __auto_closed_conn(db_name=db_name, timeout=timeout) as conn:
     try:
       yield conn
     finally:
@@ -378,13 +382,13 @@ def __unique_conn(db_name=None):
 
 
 @contextlib.contextmanager
-def __auto_closed_conn(db_name=None):
+def __auto_closed_conn(db_name=None, timeout=DEFAULT_CONN_TIMEOUT):
   """Returns a connection to Impala. This is intended to be used in a "with" block. The
      connection will be closed upon exiting the block.
 
      The returned connection will have a 'db_name' property.
   """
-  conn = impala_connect(database=db_name)
+  conn = impala_connect(database=db_name, timeout=timeout)
   try:
     conn.db_name = db_name
     yield conn

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/d15f86cb/tests/query_test/test_kudu.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_kudu.py b/tests/query_test/test_kudu.py
index 2a88137..f5579ed 100644
--- a/tests/query_test/test_kudu.py
+++ b/tests/query_test/test_kudu.py
@@ -200,6 +200,11 @@ class TestCreateExternalTable(KuduTestSuite):
 
 class TestShowCreateTable(KuduTestSuite):
 
+  @classmethod
+  def get_conn_timeout(cls):
+    # For IMPALA-4454
+    return 60 * 5 # 5 minutes
+
   def assert_show_create_equals(self, cursor, create_sql, show_create_sql):
     """Executes 'create_sql' to create a table, then runs "SHOW CREATE TABLE" and checks
        that the output is the same as 'show_create_sql'. 'create_sql' and


Mime
View raw message