impala-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tarmstr...@apache.org
Subject [3/7] incubator-impala git commit: IMPALA-5838: Improve errors on AC buffer mem rejection
Date Thu, 31 Aug 2017 05:32:22 GMT
IMPALA-5838: Improve errors on AC buffer mem rejection

The error message returned when a query is rejected due to
insufficient buffer memory is misleading. It recommended a
mem_limit which would be high enough, but changing the
mem_limit may result in changing the plan, which may result
in further changes to the buffer memory requirement.

In particular, this can happen when the planner compares the
expected hash table size to the mem_limit, and decides to
choose a partitioned join over a broadcast join.

While we might consider other code changes to improve this,
for now lets just be clear in the error message.

Testing:
* Adds tests that verify the expected behavior with the new
  error message.

Change-Id: I3dc3517195508d86078a8a4b537ae7d2f52fbcb7
Reviewed-on: http://gerrit.cloudera.org:8080/7834
Reviewed-by: Matthew Jacobs <mj@cloudera.com>
Tested-by: Impala Public Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/99fe9b3f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/99fe9b3f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/99fe9b3f

Branch: refs/heads/release-2.10.0
Commit: 99fe9b3fd602180d63cbfe73ac2c9171c31ae455
Parents: 2a7c8b9
Author: Matthew Jacobs <mj@cloudera.com>
Authored: Fri Aug 25 13:41:05 2017 -0700
Committer: Tim Armstrong <tarmstrong@cloudera.com>
Committed: Wed Aug 30 14:54:49 2017 -0700

----------------------------------------------------------------------
 be/src/scheduling/admission-controller.cc       | 27 ++++++-------
 .../admission-reject-min-reservation.test       | 42 ++++++++++++++++++++
 .../custom_cluster/test_admission_controller.py | 23 +----------
 3 files changed, 57 insertions(+), 35 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/99fe9b3f/be/src/scheduling/admission-controller.cc
----------------------------------------------------------------------
diff --git a/be/src/scheduling/admission-controller.cc b/be/src/scheduling/admission-controller.cc
index 339935d..1aadf22 100644
--- a/be/src/scheduling/admission-controller.cc
+++ b/be/src/scheduling/admission-controller.cc
@@ -115,18 +115,19 @@ const string PROFILE_INFO_VAL_QUEUE_DETAIL = "waited $0 ms, reason:
$1";
 // Error status string details
 const string REASON_MEM_LIMIT_TOO_LOW_FOR_RESERVATION =
     "minimum memory reservation is greater than memory available to the query "
-    "for buffer reservations. Mem available for buffer reservations based on mem_limit: "
-    "$0, memory reservation needed: $1. Set mem_limit to at least $2. See the query "
-    "profile for more information.";
+    "for buffer reservations. Memory reservation needed given the current plan: $0. Set "
+    "mem_limit to at least $1. Note that changing the mem_limit may also change the "
+    "plan. See the query profile for more information about the per-node memory "
+    "requirements.";
 const string REASON_BUFFER_LIMIT_TOO_LOW_FOR_RESERVATION =
     "minimum memory reservation is greater than memory available to the query "
-    "for buffer reservations. Mem available for buffer reservations based on "
-    "buffer_pool_limit: $0, memory reservation needed: $1. See the query profile for "
-    "more information.";
+    "for buffer reservations. Increase the buffer_pool_limit to $0. See the query "
+    "profile for more information about the per-node memory requirements.";
 const string REASON_MIN_RESERVATION_OVER_POOL_MEM =
-    "minimum memory reservation needed is greater than pool max mem resources. pool "
-    "max mem resources: $0, cluster-wide memory reservation needed: $1. See the query "
-    "profile for more information.";
+    "minimum memory reservation needed is greater than pool max mem resources. Pool "
+    "max mem resources: $0. Cluster-wide memory reservation needed: $1. Increase the "
+    "pool max mem resources. See the query profile for more information about the "
+    "per-node memory requirements.";
 const string REASON_DISABLED_MAX_MEM_RESOURCES =
     "disabled by pool max mem resources set to 0";
 const string REASON_DISABLED_REQUESTS_LIMIT = "disabled by requests limit set to 0";
@@ -415,10 +416,9 @@ bool AdmissionController::RejectImmediately(QuerySchedule* schedule,
   // Checks related to the min buffer reservation against configured query memory limits:
   if (schedule->query_options().__isset.buffer_pool_limit &&
       schedule->query_options().buffer_pool_limit > 0) {
-    const int64_t buffer_pool_limit = schedule->query_options().buffer_pool_limit;
-    if (max_min_reservation_bytes > buffer_pool_limit) {
+    if (max_min_reservation_bytes > schedule->query_options().buffer_pool_limit) {
       *rejection_reason = Substitute(REASON_BUFFER_LIMIT_TOO_LOW_FOR_RESERVATION,
-          PrintBytes(buffer_pool_limit), PrintBytes(max_min_reservation_bytes));
+          PrintBytes(max_min_reservation_bytes));
       return true;
     }
   } else if (schedule->query_options().__isset.mem_limit &&
@@ -430,8 +430,7 @@ bool AdmissionController::RejectImmediately(QuerySchedule* schedule,
       const int64_t required_mem_limit =
           ReservationUtil::GetMinMemLimitFromReservation(max_min_reservation_bytes);
       *rejection_reason = Substitute(REASON_MEM_LIMIT_TOO_LOW_FOR_RESERVATION,
-          PrintBytes(mem_limit), PrintBytes(max_min_reservation_bytes),
-          PrintBytes(required_mem_limit));
+          PrintBytes(max_min_reservation_bytes), PrintBytes(required_mem_limit));
       return true;
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/99fe9b3f/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
b/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
new file mode 100644
index 0000000..940fd6e
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
@@ -0,0 +1,42 @@
+====
+---- QUERY
+set mem_limit=10mb;
+select distinct * from functional_parquet.alltypesagg
+---- CATCH
+minimum memory reservation is greater than memory available to the
+ query for buffer reservations. Memory reservation needed given the
+ current plan: 67.00 MB. Set mem_limit to at least 142.00 MB.
+====
+---- QUERY
+set mem_limit=150mb;
+select distinct * from functional_parquet.alltypesagg
+---- CATCH
+minimum memory reservation needed is greater than pool max mem resources.
+ Pool max mem resources: 10.00 MB.
+ Cluster-wide memory reservation needed: 201.00 MB
+====
+---- QUERY
+set buffer_pool_limit=10mb;
+select distinct * from functional_parquet.alltypesagg
+---- CATCH
+minimum memory reservation is greater than memory available to the query
+ for buffer reservations. Increase the buffer_pool_limit to 67.00 MB.
+====
+---- QUERY
+set mem_limit=1024;
+select count(*)
+from tpch_parquet.lineitem join tpch_parquet.orders on l_orderkey = o_orderkey
+---- CATCH
+minimum memory reservation is greater than memory available to the
+ query for buffer reservations. Memory reservation needed given the
+ current plan: 4.75 MB. Set mem_limit to at least 79.75 MB.
+====
+---- QUERY
+set mem_limit=80mb;
+select count(*)
+from tpch_parquet.lineitem join tpch_parquet.orders on l_orderkey = o_orderkey
+---- CATCH
+minimum memory reservation is greater than memory available to the
+ query for buffer reservations. Memory reservation needed given the
+ current plan: 17.00 MB. Set mem_limit to at least 92.00 MB.
+====

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/99fe9b3f/tests/custom_cluster/test_admission_controller.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py
index cf072b9..de97e7c 100644
--- a/tests/custom_cluster/test_admission_controller.py
+++ b/tests/custom_cluster/test_admission_controller.py
@@ -321,7 +321,7 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
       impalad_args=impalad_admission_ctrl_flags(max_requests=1, max_queued=1,
           pool_max_mem=10 * 1024 * 1024, proc_mem_limit=1024 * 1024 * 1024),
       statestored_args=_STATESTORED_ARGS)
-  def test_reject_min_reservation(self):
+  def test_reject_min_reservation(self, vector):
     """Test that the query will be rejected by admission control if:
        a) the largest per-backend min buffer reservation is larger than the query mem
           limit
@@ -330,26 +330,7 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
        c) the cluster-wide min-buffer reservation size is larger than the pool memory
           resources.
     """
-    query = "select distinct * from functional_parquet.alltypesagg"
-    opts = {'mem_limit': '10MB', 'num_nodes': '1'}
-    ex = self.execute_query_expect_failure(self.client, query, opts)
-    assert ("minimum memory reservation is greater than memory available to the query "
-        "for buffer reservations. Mem available for buffer reservations based on "
-        "mem_limit: 10.00 MB, memory reservation needed: 34.00 MB. "
-        "Set mem_limit to at least 109.00 MB.") in str(ex)
-
-    query = "select distinct * from functional_parquet.alltypesagg"
-    opts = {'buffer_pool_limit': '10MB', 'num_nodes': '1'}
-    ex = self.execute_query_expect_failure(self.client, query, opts)
-    assert ("minimum memory reservation is greater than memory available to the query "
-        "for buffer reservations. Mem available for buffer reservations based on "
-        "buffer_pool_limit: 10.00 MB, memory reservation needed: 34.00 MB.") in str(ex)
-
-    opts = {'mem_limit': '150MB', 'num_nodes': '1'}
-    ex = self.execute_query_expect_failure(self.client, query, opts)
-    assert ("minimum memory reservation needed is greater than pool max mem resources. "
-        "pool max mem resources: 10.00 MB, cluster-wide memory reservation needed: "
-        "34.00 MB") in str(ex)
+    self.run_test_case('QueryTest/admission-reject-min-reservation', vector)
 
   # Process mem_limit used in test_mem_limit_upper_bound
   PROC_MEM_TEST_LIMIT = 1024 * 1024 * 1024


Mime
View raw message