impala-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tarmstr...@apache.org
Subject [3/3] incubator-impala git commit: IMPALA-3669: test_scratch_disk fails on S3
Date Fri, 10 Jun 2016 00:31:06 GMT
IMPALA-3669: test_scratch_disk fails on S3

Make the test deterministic by using max_block_mgr_memory instead of
mem_limit, so that the non-deterministic scanner memory usage does not
influence the spilling behaviour of the queries.

Testing:
Ran the test locally to confirm that it succeeded. Also manually
computed the memory requirement. The data size to be sorted is ~220MB,
so with a 64MB block manager limit per node, at least one node must
spill.

Change-Id: I9525a029ac020bb5b8bea210a741c9f9c5ec3c75
Reviewed-on: http://gerrit.cloudera.org:8080/3318
Reviewed-by: Dan Hecht <dhecht@cloudera.com>
Reviewed-by: Michael Brown <mikeb@cloudera.com>
Reviewed-by: Silvius Rus <srus@cloudera.com>
Tested-by: Tim Armstrong <tarmstrong@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/c1093ed8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/c1093ed8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/c1093ed8

Branch: refs/heads/master
Commit: c1093ed8611e8cc2702f81575e6c7193e0580386
Parents: 4e7172f
Author: Tim Armstrong <tarmstrong@cloudera.com>
Authored: Mon Jun 6 14:35:03 2016 -0700
Committer: Tim Armstrong <tarmstrong@cloudera.com>
Committed: Thu Jun 9 17:31:00 2016 -0700

----------------------------------------------------------------------
 tests/custom_cluster/test_scratch_disk.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c1093ed8/tests/custom_cluster/test_scratch_disk.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_scratch_disk.py b/tests/custom_cluster/test_scratch_disk.py
index 611bb89..7a48f1a 100644
--- a/tests/custom_cluster/test_scratch_disk.py
+++ b/tests/custom_cluster/test_scratch_disk.py
@@ -38,9 +38,10 @@ class TestScratchDir(CustomClusterTestSuite):
   in_mem_query = """
       select o_orderdate, o_custkey, o_comment from tpch.orders
       """
-  # Memory limit that is low enough to get Impala to spill to disk when executing
-  # spill_query and high enough that we can execute in_mem_query without spilling.
-  mem_limit = "200m"
+  # Block manager memory limit that is low enough to force Impala to spill to disk when
+  # executing spill_query and high enough that we can execute in_mem_query without
+  # spilling.
+  max_block_mgr_memory = "64m"
 
   def count_nonempty_dirs(self, dirs):
     count = 0
@@ -99,7 +100,7 @@ class TestScratchDir(CustomClusterTestSuite):
     self.assert_impalad_log_contains("INFO", "Using scratch directory ",
                                     expected_count=1)
     exec_option = vector.get_value('exec_option')
-    exec_option['mem_limit'] = self.mem_limit
+    exec_option['max_block_mgr_memory'] = self.max_block_mgr_memory
     impalad = self.cluster.get_any_impalad()
     client = impalad.service.create_beeswax_client()
     self.execute_query_expect_success(client, self.spill_query, exec_option)
@@ -113,7 +114,7 @@ class TestScratchDir(CustomClusterTestSuite):
     self.assert_impalad_log_contains("WARNING",
         "Running without spill to disk: no scratch directories provided\.")
     exec_option = vector.get_value('exec_option')
-    exec_option['mem_limit'] = self.mem_limit
+    exec_option['max_block_mgr_memory'] = self.max_block_mgr_memory
     impalad = self.cluster.get_any_impalad()
     client = impalad.service.create_beeswax_client()
     # Expect spill to disk to fail
@@ -132,7 +133,7 @@ class TestScratchDir(CustomClusterTestSuite):
     self.assert_impalad_log_contains("WARNING", "Could not remove and recreate directory
"
             + ".*: cannot use it for scratch\. Error was: .*", expected_count=5)
     exec_option = vector.get_value('exec_option')
-    exec_option['mem_limit'] = self.mem_limit
+    exec_option['max_block_mgr_memory'] = self.max_block_mgr_memory
     impalad = self.cluster.get_any_impalad()
     client = impalad.service.create_beeswax_client()
     # Expect spill to disk to fail
@@ -153,7 +154,7 @@ class TestScratchDir(CustomClusterTestSuite):
         + "Encountered exception while verifying existence of directory path",
         expected_count=5)
     exec_option = vector.get_value('exec_option')
-    exec_option['mem_limit'] = self.mem_limit
+    exec_option['max_block_mgr_memory'] = self.max_block_mgr_memory
     impalad = self.cluster.get_any_impalad()
     client = impalad.service.create_beeswax_client()
     # Expect spill to disk to fail


Mime
View raw message