ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nc...@apache.org
Subject [18/50] [abbrv] ambari git commit: AMBARI-20033. Typecasting to 'long' from earlier 'float', before setting them the following configs for recommendation : (1). 'llap_concurrency', (2). 'llap_concurrency' max value and (3). 'hive.llap.daemon.num.executor
Date Fri, 17 Feb 2017 22:06:17 GMT
AMBARI-20033. Typecasting to 'long' from earlier 'float', before setting them the following
configs for recommendation : (1). 'llap_concurrency', (2). 'llap_concurrency' max value and
(3). 'hive.llap.daemon.num.executors' max value.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/90235286
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/90235286
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/90235286

Branch: refs/heads/branch-feature-AMBARI-12556
Commit: 90235286c8a4aebb1c5159be89509f6679a6e1da
Parents: 95d096d
Author: Swapan Shridhar <sshridhar@hortonworks.com>
Authored: Wed Feb 15 13:37:54 2017 -0800
Committer: Swapan Shridhar <sshridhar@hortonworks.com>
Committed: Wed Feb 15 13:37:54 2017 -0800

----------------------------------------------------------------------
 .../common-services/YARN/3.0.0.3.0/service_advisor.py | 14 +++++++++-----
 .../stacks/HDP/2.5/services/stack_advisor.py          | 13 ++++++++-----
 2 files changed, 17 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/90235286/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
index aecf1e3..6e4e55f 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/service_advisor.py
@@ -23,6 +23,7 @@ import os
 import traceback
 import inspect
 import socket
+import math
 from math import floor, ceil
 
 # Local imports
@@ -699,12 +700,15 @@ class YARNRecommender(service_advisor.ServiceAdvisor):
                                mem_per_thread_for_llap, normalized_tez_am_container_size))
       if llap_concurrency == 0:
         llap_concurrency = 1
+        Logger.info("DBG: Readjusted 'llap_concurrency' to : 1. Earlier calculated value
: 0")
 
       if llap_concurrency * normalized_tez_am_container_size > hive_tez_am_cap_available:
-        llap_concurrency = floor(hive_tez_am_cap_available / normalized_tez_am_container_size)
+        llap_concurrency = long(math.floor(hive_tez_am_cap_available / normalized_tez_am_container_size))
+        Logger.info("DBG: Readjusted 'llap_concurrency' to : {0}, as llap_concurrency({1})
* normalized_tez_am_container_size({2}) > hive_tez_am_cap_available({3}))"
+                    .format(llap_concurrency, llap_concurrency, normalized_tez_am_container_size,
hive_tez_am_cap_available))
 
         if llap_concurrency <= 0:
-          Logger.warning("Calculated 'LLAP Concurrent Queries' = {0}. Expected value >=
1.".format(llap_concurrency))
+          Logger.warning("DBG: Calculated 'LLAP Concurrent Queries' = {0}. Expected value
>= 1.".format(llap_concurrency))
           self.recommendDefaultLlapConfiguration(configurations, services, hosts)
           return
         Logger.info("DBG: Adjusted 'llap_concurrency' : {0}, using following: hive_tez_am_cap_available
: {1}, normalized_tez_am_container_size: "
@@ -730,8 +734,8 @@ class YARNRecommender(service_advisor.ServiceAdvisor):
                 ": {2}, MIN_EXECUTOR_TO_AM_RATIO : {3}, MAX_CONCURRENT_QUERIES : {4}".format(max_llap_concurreny_limit,
max_executors_per_node,
                                                                                         
    num_llap_nodes_requested, MIN_EXECUTOR_TO_AM_RATIO,
                                                                                         
    MAX_CONCURRENT_QUERIES))
-    max_llap_concurreny = min(max_llap_concurreny_limit, floor(llap_mem_for_tezAm_and_daemons
/ (MIN_EXECUTOR_TO_AM_RATIO *
-                                                                                        
             mem_per_thread_for_llap + normalized_tez_am_container_size)))
+    max_llap_concurreny = long(min(max_llap_concurreny_limit, floor(llap_mem_for_tezAm_and_daemons
/ (MIN_EXECUTOR_TO_AM_RATIO *
+                                                                                        
             mem_per_thread_for_llap + normalized_tez_am_container_size))))
     Logger.info("DBG: Calculated 'max_llap_concurreny' : {0}, using following : max_llap_concurreny_limit
: {1}, llap_mem_for_tezAm_and_daemons : "
                 "{2}, MIN_EXECUTOR_TO_AM_RATIO : {3}, mem_per_thread_for_llap : {4}, normalized_tez_am_container_size
: "
                 "{5}".format(max_llap_concurreny, max_llap_concurreny_limit, llap_mem_for_tezAm_and_daemons,
MIN_EXECUTOR_TO_AM_RATIO,
@@ -873,7 +877,7 @@ class YARNRecommender(service_advisor.ServiceAdvisor):
     num_executors_per_node = long(num_executors_per_node)
     putHiveInteractiveSiteProperty('hive.llap.daemon.num.executors', num_executors_per_node)
     putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors', "minimum",
1)
-    putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors', "maximum",
float(num_executors_per_node_max))
+    putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors', "maximum",
long(num_executors_per_node_max))
 
     # 'hive.llap.io.threadpool.size' config value is to be set same as value calculated for
     # 'hive.llap.daemon.num.executors' at all times.

http://git-wip-us.apache.org/repos/asf/ambari/blob/90235286/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 4de9a41..52ada52 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -1031,12 +1031,15 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
                                  mem_per_thread_for_llap, normalized_tez_am_container_size))
       if llap_concurrency == 0:
         llap_concurrency = 1
+        Logger.info("DBG: Readjusted 'llap_concurrency' to : 1. Earlier calculated value
: 0")
 
       if llap_concurrency * normalized_tez_am_container_size > hive_tez_am_cap_available:
-        llap_concurrency = math.floor(hive_tez_am_cap_available / normalized_tez_am_container_size)
+        llap_concurrency = long(math.floor(hive_tez_am_cap_available / normalized_tez_am_container_size))
+        Logger.info("DBG: Readjusted 'llap_concurrency' to : {0}, as llap_concurrency({1})
* normalized_tez_am_container_size({2}) > hive_tez_am_cap_available({3}))"
+                    .format(llap_concurrency, llap_concurrency, normalized_tez_am_container_size,
hive_tez_am_cap_available))
 
         if llap_concurrency <= 0:
-          Logger.warning("Calculated 'LLAP Concurrent Queries' = {0}. Expected value >=
1.".format(llap_concurrency))
+          Logger.warning("DBG: Calculated 'LLAP Concurrent Queries' = {0}. Expected value
>= 1.".format(llap_concurrency))
           self.recommendDefaultLlapConfiguration(configurations, services, hosts)
           return
         Logger.info("DBG: Adjusted 'llap_concurrency' : {0}, using following: hive_tez_am_cap_available
: {1}, normalized_tez_am_container_size: "
@@ -1062,8 +1065,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
                   ": {2}, MIN_EXECUTOR_TO_AM_RATIO : {3}, MAX_CONCURRENT_QUERIES : {4}".format(max_llap_concurreny_limit,
max_executors_per_node,
                                                                                         
      num_llap_nodes_requested, MIN_EXECUTOR_TO_AM_RATIO,
                                                                                         
      MAX_CONCURRENT_QUERIES))
-    max_llap_concurreny = min(max_llap_concurreny_limit, math.floor(llap_mem_for_tezAm_and_daemons
/ (MIN_EXECUTOR_TO_AM_RATIO *
-                                                                                        
             mem_per_thread_for_llap + normalized_tez_am_container_size)))
+    max_llap_concurreny = long(min(max_llap_concurreny_limit, math.floor(llap_mem_for_tezAm_and_daemons
/ (MIN_EXECUTOR_TO_AM_RATIO *
+                                                                                        
             mem_per_thread_for_llap + normalized_tez_am_container_size))))
     Logger.info("DBG: Calculated 'max_llap_concurreny' : {0}, using following : max_llap_concurreny_limit
: {1}, llap_mem_for_tezAm_and_daemons : "
                   "{2}, MIN_EXECUTOR_TO_AM_RATIO : {3}, mem_per_thread_for_llap : {4}, normalized_tez_am_container_size
: "
                   "{5}".format(max_llap_concurreny, max_llap_concurreny_limit, llap_mem_for_tezAm_and_daemons,
MIN_EXECUTOR_TO_AM_RATIO,
@@ -1209,7 +1212,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
     Logger.info("DBG: Putting num_executors_per_node as {0}".format(num_executors_per_node))
     putHiveInteractiveSiteProperty('hive.llap.daemon.num.executors', num_executors_per_node)
     putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors', "minimum",
1)
-    putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors', "maximum",
float(num_executors_per_node_max))
+    putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors', "maximum",
long(num_executors_per_node_max))
 
     # 'hive.llap.io.threadpool.size' config value is to be set same as value calculated for
     # 'hive.llap.daemon.num.executors' at all times.


Mime
View raw message