impala-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From he...@apache.org
Subject [2/3] incubator-impala git commit: IMPALA-3200: More end-to-end tests
Date Wed, 16 Aug 2017 16:18:42 GMT
IMPALA-3200: More end-to-end tests

* Test that query startup fails if there is insufficient reservation on
 one backend.
* Verify that free buffers and clean pages shrink to zero when idle.

Change-Id: I3a3b6adf5b97b16594c6c4855d5fa831c5a90e10
Reviewed-on: http://gerrit.cloudera.org:8080/7668
Reviewed-by: Tim Armstrong <tarmstrong@cloudera.com>
Tested-by: Impala Public Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/f74f665b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/f74f665b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/f74f665b

Branch: refs/heads/master
Commit: f74f665b6cf4425ad890c9e1e79aced883ccbf7a
Parents: e4a0e2f
Author: Tim Armstrong <tarmstrong@cloudera.com>
Authored: Mon Aug 14 13:57:22 2017 -0700
Committer: Impala Public Jenkins <impala-public-jenkins@gerrit.cloudera.org>
Committed: Wed Aug 16 08:15:30 2017 +0000

----------------------------------------------------------------------
 tests/custom_cluster/test_mem_reservations.py | 55 ++++++++++++++++++----
 tests/verifiers/metric_verifier.py            |  6 ++-
 2 files changed, 52 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f74f665b/tests/custom_cluster/test_mem_reservations.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_mem_reservations.py b/tests/custom_cluster/test_mem_reservations.py
index d8a19e8..bf1641b 100644
--- a/tests/custom_cluster/test_mem_reservations.py
+++ b/tests/custom_cluster/test_mem_reservations.py
@@ -19,6 +19,8 @@ import pytest
 import threading
 
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+from tests.common.impala_test_suite import LOG
+from tests.verifiers.metric_verifier import MetricVerifier
 
 class TestMemReservations(CustomClusterTestSuite):
   """Tests for memory reservations that require custom cluster arguments."""
@@ -28,7 +30,8 @@ class TestMemReservations(CustomClusterTestSuite):
     return 'functional-query'
 
   @pytest.mark.execute_serially
-  @CustomClusterTestSuite.with_args(impalad_args="--buffer_pool_limit=2g")
+  @CustomClusterTestSuite.with_args(
+      impalad_args="--buffer_pool_limit=2g --memory_maintenance_sleep_time_ms=100")
   def test_per_backend_min_reservation(self, vector):
     """Tests that the per-backend minimum reservations are used (IMPALA-4833).
        The test sets the buffer_pool_limit very low (2gb), and then runs a query against
@@ -37,12 +40,15 @@ class TestMemReservations(CustomClusterTestSuite):
        per-backend minimum reservations are not used, then one of the queries fails to
        acquire its minimum reservation. This was verified to fail before IMPALA-4833, and
        succeeds after.
+
+       Memory maintenance sleep time is set low so we can verify that buffers are
+       released.
     """
     assert len(self.cluster.impalads) == 3
 
     # This query will have scan fragments on all nodes, but the coordinator fragment
     # has 6 analytic nodes, 5 sort nodes, and an aggregation.
-    QUERY = """
+    COORDINATOR_QUERY = """
     select max(t.c1), avg(t.c2), min(t.c3), avg(c4), avg(c5), avg(c6)
     from (select
         max(tinyint_col) over (order by int_col) c1,
@@ -54,15 +60,22 @@ class TestMemReservations(CustomClusterTestSuite):
         from functional.alltypes) t;
         """
 
-    # Set the DEFAULT_SPILLABLE_BUFFER_SIZE and MIN_SPILLABLE_BUFFER_SIZE to 64MiB
-    # so that the coordinator node requires ~1.2gb and the other backends require ~200mb.
+    # This query has two grouping aggregations on each node.
+    SYMMETRIC_QUERY = """
+    select count(*)
+    from (select distinct * from functional.alltypes) v"""
+
+    # Set the DEFAULT_SPILLABLE_BUFFER_SIZE and MIN_SPILLABLE_BUFFER_SIZE to 64MiB.
+    # so that for COORDINATOR_QUERY, the coordinator node requires ~1.2gb and the
+    # other backends require ~200mb and for SYMMETRIC_QUERY all backends require
+    # ~1.05gb.
     CONFIG_MAP = {'DEFAULT_SPILLABLE_BUFFER_SIZE': '67108864',
                   'MIN_SPILLABLE_BUFFER_SIZE': '67108864'}
 
-    # Create two threads to submit QUERY to two different coordinators concurrently.
     class QuerySubmitThread(threading.Thread):
-      def __init__(self, coordinator):
+      def __init__(self, query, coordinator):
         super(QuerySubmitThread, self).__init__()
+        self.query = query
         self.coordinator = coordinator
         self.error = None
 
@@ -71,7 +84,7 @@ class TestMemReservations(CustomClusterTestSuite):
         try:
           client.set_configuration(CONFIG_MAP)
           for i in xrange(20):
-            result = client.execute(QUERY)
+            result = client.execute(self.query)
             assert result.success
             assert len(result.data) == 1
         except Exception, e:
@@ -79,8 +92,34 @@ class TestMemReservations(CustomClusterTestSuite):
         finally:
           client.close()
 
-    threads = [QuerySubmitThread(self.cluster.impalads[i]) for i in xrange(2)]
+    # Create two threads to submit COORDINATOR_QUERY to two different coordinators concurrently.
+    # They should both succeed.
+    threads = [QuerySubmitThread(COORDINATOR_QUERY, self.cluster.impalads[i])
+              for i in xrange(2)]
     for t in threads: t.start()
     for t in threads:
       t.join()
       assert t.error is None
+
+    # Create two threads to submit COORDINATOR_QUERY to one coordinator and
+    # SYMMETRIC_QUERY to another coordinator. One of the queries should fail because
+    # memory would be overcommitted on daemon 0.
+    threads = [QuerySubmitThread(COORDINATOR_QUERY, self.cluster.impalads[0]),
+               QuerySubmitThread(SYMMETRIC_QUERY, self.cluster.impalads[1])]
+    for t in threads: t.start()
+    num_errors = 0
+    for t in threads:
+      t.join()
+      if t.error is not None:
+        assert "Failed to get minimum memory reservation" in t.error
+        LOG.info("Query failed with error: %s", t.error)
+        LOG.info(t.query)
+        num_errors += 1
+    assert num_errors == 1
+
+    # Check that free buffers are released over time. We set the memory maintenance sleep
+    # time very low above so this should happen quickly.
+    verifiers = [MetricVerifier(i.service) for i in self.cluster.impalads]
+    for v in verifiers:
+      v.wait_for_metric("buffer-pool.free-buffers", 0, timeout=60)
+      v.wait_for_metric("buffer-pool.free-buffer-bytes", 0, timeout=60)

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f74f665b/tests/verifiers/metric_verifier.py
----------------------------------------------------------------------
diff --git a/tests/verifiers/metric_verifier.py b/tests/verifiers/metric_verifier.py
index d5753fd..3331135 100644
--- a/tests/verifiers/metric_verifier.py
+++ b/tests/verifiers/metric_verifier.py
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Verfier for common impalad metrics
+# Verifier for common impalad metrics
 
 # List of metrics that should be equal to zero when there are no outstanding queries.
 METRIC_LIST = [
@@ -30,6 +30,10 @@ METRIC_LIST = [
                "impala-server.num-files-open-for-insert",
                # Disable checking of num-missing-volume-id due to IMPALA-467
                # "impala-server.scan-ranges.num-missing-volume-id",
+               # Buffer pool pages belong to specific queries. Therefore there should be
+               # no clean pages if there are no queries running.
+               "buffer-pool.clean-pages",
+               "buffer-pool.clean-page-bytes"
                ]
 
 class MetricVerifier(object):


Mime
View raw message