impala-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tarmstr...@apache.org
Subject [1/4] incubator-impala git commit: IMPALA-5830: SET_DENY_RESERVATION_PROBABILITY test
Date Wed, 30 Aug 2017 04:08:49 GMT
Repository: incubator-impala
Updated Branches:
  refs/heads/master 6a9ca345b -> cc4816b3d


IMPALA-5830: SET_DENY_RESERVATION_PROBABILITY test

Add a targeted test that confirms that setting the query option will
force spilling.

Testing:
Ran test_spilling locally.

Change-Id: Ida6b55b2dee0779b1739af5d75943518ec40d6ce
Reviewed-on: http://gerrit.cloudera.org:8080/7809
Reviewed-by: Tim Armstrong <tarmstrong@cloudera.com>
Tested-by: Impala Public Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/caefd861
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/caefd861
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/caefd861

Branch: refs/heads/master
Commit: caefd86136e7d8d5d3cd390402abf3370dc6cc6f
Parents: 6a9ca34
Author: Tim Armstrong <tarmstrong@cloudera.com>
Authored: Thu Aug 24 13:26:46 2017 -0700
Committer: Impala Public Jenkins <impala-public-jenkins@gerrit.cloudera.org>
Committed: Tue Aug 29 23:01:10 2017 +0000

----------------------------------------------------------------------
 .../QueryTest/disable-unsafe-spills.test        | 11 ----
 .../QueryTest/spilling-query-options.test       | 54 ++++++++++++++++++++
 tests/common/impala_test_suite.py               |  2 +-
 tests/query_test/test_spilling.py               | 41 +++++++++++----
 4 files changed, 85 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/caefd861/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test
b/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test
deleted file mode 100644
index a9b8f35..0000000
--- a/testdata/workloads/functional-query/queries/QueryTest/disable-unsafe-spills.test
+++ /dev/null
@@ -1,11 +0,0 @@
-====
----- QUERY
-# tpch_avro does not have stats computed, so if we set disable_unsafe_spills we should
-# not spill to disk.
-set disable_unsafe_spills=true;
-set buffer_pool_limit=40m;
-select distinct *
-from tpch_avro.orders
----- CATCH
-Could not free memory by spilling to disk: spilling was disabled by planner. Re-enable spilling
by setting the query option DISABLE_UNSAFE_SPILLS=false
-====

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/caefd861/testdata/workloads/functional-query/queries/QueryTest/spilling-query-options.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-query-options.test
b/testdata/workloads/functional-query/queries/QueryTest/spilling-query-options.test
new file mode 100644
index 0000000..5b9a641
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-query-options.test
@@ -0,0 +1,54 @@
+====
+---- QUERY
+# tpch_avro does not have stats computed, so if we set disable_unsafe_spills we should
+# not spill to disk.
+set disable_unsafe_spills=true;
+set buffer_pool_limit=40m;
+select distinct *
+from tpch_avro.orders
+---- CATCH
+Could not free memory by spilling to disk: spilling was disabled by planner. Re-enable spilling
by setting the query option DISABLE_UNSAFE_SPILLS=false
+====
+---- QUERY
+# IMPALA-5823: make sure that SET_DENY_RESERVATION_PROBABILITY takes effect on PREPARE.
+set debug_action="-1:PREPARE:SET_DENY_RESERVATION_PROBABILITY@1.0";
+select count(*) from (
+  select distinct o_orderdate, o_custkey, o_comment
+  from tpch_parquet.orders where o_orderkey < 500000) v;
+---- TYPES
+BIGINT
+---- RESULTS
+124999
+---- RUNTIME_PROFILE
+row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
+row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\)
+====
+---- QUERY
+# IMPALA-5823: make sure that SET_DENY_RESERVATION_PROBABILITY takes effect on OPEN.
+set debug_action="-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0";
+select count(*) from (
+  select distinct o_orderdate, o_custkey, o_comment
+  from tpch_parquet.orders where o_orderkey < 500000) v;
+---- TYPES
+BIGINT
+---- RESULTS
+124999
+---- RUNTIME_PROFILE
+row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
+row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\)
+====
+---- QUERY
+# IMPALA-5823: make sure that SET_DENY_RESERVATION_PROBABILITY takes effect on GETNEXT.
+# This won't affect the merge aggregation, which accumulates its memory in Open(), but
+# will affect the streaming aggregation.
+set debug_action="-1:GETNEXT:SET_DENY_RESERVATION_PROBABILITY@1.0";
+select count(*) from (
+  select distinct o_orderdate, o_custkey, o_comment
+  from tpch_parquet.orders where o_orderkey < 500000) v;
+---- TYPES
+BIGINT
+---- RESULTS
+124999
+---- RUNTIME_PROFILE
+row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\)
+====

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/caefd861/tests/common/impala_test_suite.py
----------------------------------------------------------------------
diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index d5841b2..1b7d043 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -210,7 +210,7 @@ class ImpalaTestSuite(BaseTestSuite):
       if not query_option in self.default_query_options:
         continue
       default_val = self.default_query_options[query_option]
-      query_str = 'SET '+ query_option + '=' + default_val + ';'
+      query_str = 'SET '+ query_option + '="' + default_val + '"'
       try:
         impalad_client.execute(query_str)
       except Exception as e:

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/caefd861/tests/query_test/test_spilling.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_spilling.py b/tests/query_test/test_spilling.py
index c818b65..0b36429 100644
--- a/tests/query_test/test_spilling.py
+++ b/tests/query_test/test_spilling.py
@@ -30,14 +30,14 @@ DEBUG_ACTION_DIMS = [None,
 
 @pytest.mark.xfail(pytest.config.option.testing_remote_cluster,
                    reason='Queries may not spill on larger clusters')
-class TestSpilling(ImpalaTestSuite):
+class TestSpillingDebugActionDimensions(ImpalaTestSuite):
   @classmethod
   def get_workload(self):
     return 'functional-query'
 
   @classmethod
   def add_test_dimensions(cls):
-    super(TestSpilling, cls).add_test_dimensions()
+    super(TestSpillingDebugActionDimensions, cls).add_test_dimensions()
     cls.ImpalaTestMatrix.clear_constraints()
     cls.ImpalaTestMatrix.add_dimension(create_parquet_dimension('tpch'))
     # Tests are calibrated so that they can execute and spill with this page size.
@@ -60,19 +60,38 @@ class TestSpilling(ImpalaTestSuite):
     """Test spilling null-aware anti-joins"""
     self.run_test_case('QueryTest/spilling-naaj', vector)
 
+  def test_spilling_sorts_exhaustive(self, vector):
+    if self.exploration_strategy() != 'exhaustive':
+      pytest.skip("only run large sorts on exhaustive")
+    self.run_test_case('QueryTest/spilling-sorts-exhaustive', vector)
+
+
+@pytest.mark.xfail(pytest.config.option.testing_remote_cluster,
+                   reason='Queries may not spill on larger clusters')
+class TestSpillingNoDebugActionDimensions(ImpalaTestSuite):
+  """Spilling tests to which we don't want to apply the debug_action dimension."""
+  @classmethod
+  def get_workload(self):
+    return 'functional-query'
+
+  @classmethod
+  def add_test_dimensions(cls):
+    super(TestSpillingNoDebugActionDimensions, cls).add_test_dimensions()
+    cls.ImpalaTestMatrix.clear_constraints()
+    cls.ImpalaTestMatrix.add_dimension(create_parquet_dimension('tpch'))
+    # Tests are calibrated so that they can execute and spill with this page size.
+    cls.ImpalaTestMatrix.add_dimension(
+        create_exec_option_dimension_from_dict({'default_spillable_buffer_size' : ['256k']}))
+
   def test_spilling_naaj_no_deny_reservation(self, vector):
     """
     Null-aware anti-join tests that depend on getting more than the minimum reservation
     and therefore will not reliably pass with the deny reservation debug action enabled.
     """
-    if vector.get_value('exec_option')['debug_action'] is None:
-      self.run_test_case('QueryTest/spilling-naaj-no-deny-reservation', vector)
+    self.run_test_case('QueryTest/spilling-naaj-no-deny-reservation', vector)
 
-  def test_spilling_sorts_exhaustive(self, vector):
-    if self.exploration_strategy() != 'exhaustive':
-      pytest.skip("only run large sorts on exhaustive")
-    self.run_test_case('QueryTest/spilling-sorts-exhaustive', vector)
+  def test_spilling_query_options(self, vector):
+    """Test that spilling-related query options work end-to-end. These tests rely on
+      setting debug_action to alternative values via query options."""
+    self.run_test_case('QueryTest/spilling-query-options', vector)
 
-  def test_disable_unsafe_spills(self, vector):
-    """Test that the disable_unsafe_spills query options works end-to-end."""
-    self.run_test_case('QueryTest/disable-unsafe-spills', vector)


Mime
View raw message