ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swa...@apache.org
Subject ambari git commit: AMBARI-17283. Display LLAP calculated configs as 'Labels' (non-editable) on 'Hive Server Interactive' Panel and Textboxes (editable) in HIVE Advanced section.
Date Fri, 17 Jun 2016 08:30:04 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.4 2f6d7fbc9 -> 752dc9d8a


AMBARI-17283. Display LLAP calculated configs as 'Labels' (non-editable) on 'Hive Server Interactive'
Panel and Textboxes (editable) in HIVE Advanced section.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/752dc9d8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/752dc9d8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/752dc9d8

Branch: refs/heads/branch-2.4
Commit: 752dc9d8ad132f2be95683160cf6fb53f4667acf
Parents: 2f6d7fb
Author: Swapan Shridhar <sshridhar@hortonworks.com>
Authored: Fri Jun 17 01:28:26 2016 -0700
Committer: Swapan Shridhar <sshridhar@hortonworks.com>
Committed: Fri Jun 17 01:28:26 2016 -0700

----------------------------------------------------------------------
 .../HIVE/configuration/hive-interactive-env.xml |  31 +++++
 .../configuration/hive-interactive-site.xml     |  81 ++++++++++++
 .../HDP/2.5/services/HIVE/themes/theme.json     |  22 ++--
 .../stacks/HDP/2.5/services/stack_advisor.py    | 100 ++++++++++-----
 .../stacks/2.5/common/test_stack_advisor.py     | 123 ++++++++++++++++++-
 5 files changed, 311 insertions(+), 46 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/752dc9d8/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
index 4d34455..b460464 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
@@ -215,6 +215,37 @@
     <display-name>LLAP app java opts</display-name>
     <on-ambari-upgrade add="true"/>
   </property>
+
+
+  <!--
+  Note: This config is added for displaying 'num_llap_nodes' value as a 'label' on Hive Server
Interactive panel and will always
+  have same value as 'num_llap_nodes'. 'num_llap_nodes' will stay editable in Advanced section.
+  -->
+
+  <property>
+    <name>num_llap_nodes_copy</name>
+    <value>1</value>
+    <description>The number of Hive LLAP daemons to run.</description>
+    <display-name>Number of LLAP Daemons</display-name>
+    <value-attributes>
+      <type>int</type>
+      <minimum>1</minimum>
+    </value-attributes>
+    <depends-on>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>enable_hive_interactive</name>
+      </property>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>num_llap_nodes</name>
+      </property>
+    </depends-on>
+    <on-ambari-upgrade add="true"/>
+  </property>
+
+
+
   <!-- hive-env.sh -->
   <property>
     <name>content</name>

http://git-wip-us.apache.org/repos/asf/ambari/blob/752dc9d8/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
index 1b2eaf5..9c7e865 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
@@ -570,4 +570,85 @@ limitations under the License.
     <value>none</value>
     <on-ambari-upgrade add="true"/>
   </property>
+
+  <!--
+  Note: Properties added here are just for displaying them as 'label' on Hive Server Interactive
panel.
+  -->
+  <property>
+    <name>hive.llap.daemon.num.executors_copy</name>
+    <!--
+      Added for displaying 'hive.llap.daemon.num.executors' value as a 'label' on Hive Server
Interactive panel and will always
+      have same value as 'hive.llap.daemon.num.executors'. 'hive.llap.daemon.num.executors_copy'
will stay editable in Advanced section.
+    -->
+    <display-name>Maximum CPUs per Daemon</display-name>
+    <description>The maximum number of CPUs a single LLAP daemon will use. Usually
this should be equal to the number of available CPUs.</description>
+    <value>1</value>
+    <value-attributes>
+      <type>int</type>
+    </value-attributes>
+    <depends-on>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.llap.daemon.num.executors</name>
+      </property>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>enable_hive_interactive</name>
+      </property>
+    </depends-on>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>hive.llap.daemon.yarn.container.mb_copy</name>
+    <!--
+      Added for displaying 'hive.llap.daemon.yarn.container.mb' value as a 'label' on Hive
Server Interactive panel and will always
+      have same value as 'hive.llap.daemon.yarn.container.mb'. 'hive.llap.daemon.yarn.container.mb'
will stay editable in Advanced section.
+    -->
+    <display-name>Memory per daemon</display-name>
+    <description>Total memory used by individual LLAP daemons. This includes memory
for the cache as well as for the query execution.</description>
+    <value>{{yarn_scheduler_allocation_minimum_mb}}</value>
+    <value-attributes>
+      <type>int</type>
+      <unit>MB</unit>
+      <overridable>false</overridable>
+    </value-attributes>
+    <depends-on>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>enable_hive_interactive</name>
+      </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.llap.daemon.yarn.container.mb</name>
+      </property>
+    </depends-on>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>hive.llap.io.memory.size_copy</name>
+    <!--
+       Added for displaying 'hive.llap.io.memory.size' value as a 'label' on Hive Server
Interactive panel and will always
+      have same value as 'hive.llap.io.memory.size'. 'hive.llap.io.memory.size' will stay
editable in Advanced section.
+    -->
+    <display-name>In-Memory Cache per Daemon</display-name>
+    <description>The amount of memory reserved for Hive's optimized in-memory cache.</description>
+    <value>0</value>
+    <value-attributes>
+      <type>int</type>
+      <unit>MB</unit>
+      <overridable>false</overridable>
+    </value-attributes>
+    <depends-on>
+      <property>
+        <type>hive-interactive-env</type>
+        <name>enable_hive_interactive</name>
+      </property>
+      <property>
+        <type>hive-interactive-site</type>
+        <name>hive.llap.io.memory.size</name>
+      </property>
+    </depends-on>
+    <on-ambari-upgrade add="true"/>
+  </property>
+
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/752dc9d8/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
index 395ae43..81277a8 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
@@ -89,7 +89,7 @@
           ]
         },
         {
-          "config": "hive-interactive-env/num_llap_nodes",
+          "config": "hive-interactive-env/num_llap_nodes_copy",
           "subsection-name": "interactive-query-row1-col1",
           "depends-on": [
             {
@@ -111,7 +111,7 @@
           ]
         },
         {
-          "config": "hive-interactive-site/hive.llap.daemon.yarn.container.mb",
+          "config": "hive-interactive-site/hive.llap.daemon.yarn.container.mb_copy",
           "subsection-name": "interactive-query-row1-col1",
           "depends-on": [
             {
@@ -133,7 +133,7 @@
           ]
         },
         {
-          "config": "hive-interactive-site/hive.llap.io.memory.size",
+          "config": "hive-interactive-site/hive.llap.io.memory.size_copy",
           "subsection-name": "interactive-query-row1-col1",
           "depends-on": [
             {
@@ -155,7 +155,7 @@
           ]
         },
         {
-          "config": "hive-interactive-site/hive.llap.daemon.num.executors",
+          "config": "hive-interactive-site/hive.llap.daemon.num.executors_copy",
           "subsection-name": "interactive-query-row1-col1",
           "depends-on": [
             {
@@ -210,9 +210,9 @@
         }
       },
       {
-        "config": "hive-interactive-env/num_llap_nodes",
+        "config": "hive-interactive-env/num_llap_nodes_copy",
         "widget": {
-          "type": "text-field",
+          "type": "label",
           "units": [
             {
               "unit-name": "int"
@@ -232,7 +232,7 @@
         }
       },
       {
-        "config": "hive-interactive-site/hive.llap.daemon.yarn.container.mb",
+        "config": "hive-interactive-site/hive.llap.daemon.yarn.container.mb_copy",
         "widget": {
           "type": "label",
           "units": [
@@ -243,9 +243,9 @@
         }
       },
       {
-        "config": "hive-interactive-site/hive.llap.io.memory.size",
+        "config": "hive-interactive-site/hive.llap.io.memory.size_copy",
         "widget": {
-          "type": "text-field",
+          "type": "label",
           "units": [
             {
               "unit-name": "MB"
@@ -254,9 +254,9 @@
         }
       },
       {
-        "config": "hive-interactive-site/hive.llap.daemon.num.executors",
+        "config": "hive-interactive-site/hive.llap.daemon.num.executors_copy",
         "widget": {
-          "type": "text-field",
+          "type": "label",
           "units": [
             {
               "unit-name": "int"

http://git-wip-us.apache.org/repos/asf/ambari/blob/752dc9d8/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 01522b2..76654c3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -402,12 +402,32 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
       putHiveInteractiveEnvProperty('enable_hive_interactive', 'false')
       putHiveInteractiveEnvPropertyAttribute("llap_queue_capacity", "visible", "false")
 
-    if self.HIVE_INTERACTIVE_SITE in services['configurations'] and \
-        'hive.llap.zk.sm.connectionString' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']:
-      # Fill the property 'hive.llap.zk.sm.connectionString' required by Hive Server Interactive
(HiveServer2)
-      zookeeper_host_port = self.getZKHostPortString(services)
-      if zookeeper_host_port:
-        putHiveInteractiveSiteProperty("hive.llap.zk.sm.connectionString", zookeeper_host_port)
+    if self.HIVE_INTERACTIVE_SITE in services['configurations']:
+      if 'hive.llap.zk.sm.connectionString' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']:
+        # Fill the property 'hive.llap.zk.sm.connectionString' required by Hive Server Interactive
(HiveServer2)
+        zookeeper_host_port = self.getZKHostPortString(services)
+        if zookeeper_host_port:
+          putHiveInteractiveSiteProperty("hive.llap.zk.sm.connectionString", zookeeper_host_port)
+
+      # Copy the value of 'hive.llap.daemon.yarn.container.mb', 'hive.llap.daemon.num.executors',
'hive.llap.io.memory.size' and 'num_llap_nodes'
+      # to their '*_copy' configs as they are used to display the values as 'label' on Hive
Server interactive panel.
+      if 'hive.llap.daemon.yarn.container.mb' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']:
+        hive_llap_yarn_cont_mb = services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']['hive.llap.daemon.yarn.container.mb']
+        putHiveInteractiveSiteProperty("hive.llap.daemon.yarn.container.mb_copy", hive_llap_yarn_cont_mb)
+        Logger.info("Adjusted config 'hive.llap.daemon.yarn.container.mb_copy' value same
as config 'hive.llap.daemon.yarn.container.mb' value({0})".format(hive_llap_yarn_cont_mb))
+      if 'hive.llap.daemon.num.executors' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']:
+        hive_llap_daemon_num_executors = services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']['hive.llap.daemon.num.executors']
+        putHiveInteractiveSiteProperty("hive.llap.daemon.num.executors_copy", hive_llap_daemon_num_executors)
+        Logger.info("Adjusted config 'hive.llap.daemon.num.executors_copy' value same as
config 'hive.llap.daemon.num.executors' value({0})".format(hive_llap_daemon_num_executors))
+      if 'hive.llap.io.memory.size' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']:
+        hive_llap_io_mem_size = services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']['hive.llap.io.memory.size']
+        putHiveInteractiveSiteProperty("hive.llap.io.memory.size_copy", hive_llap_io_mem_size)
+        Logger.info("Adjusted config 'hive.llap.io.memory.size_copy' value same as config
'hive.llap.io.memory.size' value({0})".format(hive_llap_io_mem_size))
+    if 'hive-interactive-env' in services['configurations'] and \
+      'num_llap_nodes' in services['configurations']['hive-interactive-env']['properties']:
+      num_llap_node = services['configurations']['hive-interactive-env']['properties']['num_llap_nodes']
+      putHiveInteractiveEnvProperty("num_llap_nodes_copy", num_llap_node)
+      Logger.info("Adjusted config 'num_llap_nodes_copy' value same as config 'num_llap_nodes'
value({0})".format(num_llap_node))
     pass
 
   def recommendYARNConfigurations(self, configurations, clusterData, services, hosts):
@@ -427,9 +447,10 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
         self.checkAndStopLlapQueue(services, configurations, LLAP_QUEUE_NAME)
 
   """
-  Entry point for updating Hive's 'LLAP app' configs namely : (1). num_llap_nodes (2). hive.llap.daemon.yarn.container.mb
-    (3). hive.llap.daemon.num.executors (4). hive.llap.io.memory.size (5). llap_heap_size
(6). slider_am_container_size,
-    and (7). hive.server2.tez.sessions.per.default.queue
+  Entry point for updating Hive's 'LLAP app' configs namely : (1). num_llap_nodes, num_llap_nodes_copy
+    (2). hive.llap.daemon.yarn.container.mb, hive.llap.daemon.yarn.container.mb_copy (3).
hive.llap.daemon.num.executors,
+    hive.llap.daemon.num.executors_copy (4). hive.llap.io.memory.size, hive.llap.io.memory.size_copy
(5). llap_heap_size
+    (6). slider_am_container_size, and (7). hive.server2.tez.sessions.per.default.queue
 
     The trigger point for updating LLAP configs (mentioned above) is change in values of
any of the following:
     (1). 'enable_hive_interactive' set to 'true' (2). 'llap_queue_capacity' (3). 'hive.server2.tez.sessions.per.default.queue'
@@ -437,6 +458,10 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
 
     If change in value for 'llap_queue_capacity' or 'hive.server2.tez.sessions.per.default.queue'
is detected, that config
     value is not calulated, but read and use in calculation for dependent configs.
+
+    Note : Configs with name as '*_copy' will have same value as their conterpart configs
(eg: 'num_llap_nodes'
+    for 'num_llap_nodes_copy'). '*_copy' configs are added so that we can show them as label
on UI in Hive Server Interactive
+    Panel and their counterparts shown in 'Advanced' section as editable ones.
   """
   def updateLlapConfigs(self, configurations, services, hosts, llap_queue_name):
     putHiveInteractiveSiteProperty = self.putProperty(configurations, self.HIVE_INTERACTIVE_SITE,
services)
@@ -452,10 +477,20 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
     # Update 'hive.llap.daemon.queue.name' prop combo entries and llap capacity slider visibility.
     self.setLlapDaemonQueuePropAttributesAndCapSliderVisibility(services, configurations)
 
+    read_llap_daemon_yarn_cont_mb = long(self.get_yarn_min_container_size(services, configurations))
+    putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', read_llap_daemon_yarn_cont_mb)
     # initial memory setting to make sure hive.llap.daemon.yarn.container.mb >= yarn.scheduler.minimum-allocation-mb
-    Logger.debug("Setting hive.llap.daemon.yarn.container.mb to yarn min container size as
initial size "
+    Logger.debug("Adjusted 'hive.llap.daemon.yarn.container.mb' to yarn min container size
as initial size "
                  "(" + str(self.get_yarn_min_container_size(services, configurations)) +
" MB).")
-    putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', long(self.get_yarn_min_container_size(services,
configurations)))
+    # This is a stopgap thing, to be sure that on 1st invocation, value for 'hive.llap.daemon.yarn.container.mb_copy'
+    # is same as value set by 'hive.llap.daemon.yarn.container.mb'.
+    if 'hive-interactive-site' not in configurations or \
+      ('hive-interactive-site' in configurations and \
+           'hive.llap.daemon.yarn.container.mb_copy' not in configurations['hive-interactive-site']['properties']):
+      putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb_copy', read_llap_daemon_yarn_cont_mb)
+      Logger.debug("Adjusted 'hive.llap.daemon.yarn.container.mb_copy' to yarn min container
size as initial size "
+                   "(" + str(self.get_yarn_min_container_size(services, configurations))
+ " MB).")
+
 
     try:
       if self.HIVE_INTERACTIVE_SITE in services['configurations'] and \
@@ -560,15 +595,16 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
         # Calculate 'total memory available for llap daemons' across cluster
         total_am_capacity_required = tez_am_container_size * llap_concurrency + slider_am_container_size
         cap_available_for_daemons = total_llap_queue_size - total_am_capacity_required
-        if cap_available_for_daemons < yarn_min_container_size :
-          raise Fail("'Capacity available for LLAP daemons'({0}) < 'YARN minimum container
size'({1}). Invalid configuration detected. "
-                     "Increase LLAP queue size.".format(cap_available_for_daemons, yarn_min_container_size))
         Logger.info("Calculated cap_available_for_daemons : {0}, using following : current_selected_queue_for_llap_cap
: {1}, "
                     "yarn_nm_mem_in_mb : {2}, total_cluster_capacity : {3}, total_llap_queue_size
: {4}, tez_am_container_size"
                     " : {5}, yarn_min_container_size : {6}, llap_concurrency : {7}, total_am_capacity_required
: {8}"
                     .format(cap_available_for_daemons, current_selected_queue_for_llap_cap,
yarn_nm_mem_in_mb, total_cluster_capacity,
                             total_llap_queue_size, tez_am_container_size, yarn_min_container_size,
llap_concurrency,
                             total_am_capacity_required))
+        if cap_available_for_daemons < yarn_min_container_size :
+          raise Fail("'Capacity available for LLAP daemons'({0}) < 'YARN minimum container
size'({1}). Invalid configuration detected. "
+                     "Increase LLAP queue size.".format(cap_available_for_daemons, yarn_min_container_size))
+
 
 
         # Calculate value for 'num_llap_nodes', an across cluster config.
@@ -639,13 +675,22 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
 
         putHiveInteractiveEnvProperty('num_llap_nodes', num_llap_nodes)
         Logger.info("LLAP config 'num_llap_nodes' updated. Current: {0}".format(num_llap_nodes))
+        # Copy the value of 'num_llap_nodes' to 'num_llap_nodes_copy' as 'num_llap_nodes_copy'
config is used to display
+        #  it as Label on Hive Server interactive panel.
+        putHiveInteractiveEnvProperty('num_llap_nodes_copy', num_llap_nodes)
 
         llap_container_size = long(llap_container_size)
         putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', llap_container_size)
         Logger.info("LLAP config 'hive.llap.daemon.yarn.container.mb' updated. Current: {0}".format(llap_container_size))
+        # Copy the value of 'hive.llap.daemon.yarn.container.mb' to 'hive.llap.daemon.yarn.container.mb_copy'
as
+        # 'hive.llap.daemon.yarn.container.mb_copy' config is used to display it as Label
on Hive Server interactive panel.
+        putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb_copy', llap_container_size)
 
         num_executors_per_node = long(num_executors_per_node)
         putHiveInteractiveSiteProperty('hive.llap.daemon.num.executors', num_executors_per_node)
+        # Copy the value of 'hive.llap.daemon.num.executors' to 'hive.llap.daemon.num.executors_copy'
as
+        # 'hive.llap.daemon.num.executors_copy' config is used to display it as Label on
Hive Server interactive panel.
+        putHiveInteractiveSiteProperty('hive.llap.daemon.num.executors_copy', num_executors_per_node)
         Logger.info("LLAP config 'hive.llap.daemon.num.executors' updated. Current: {0}".format(num_executors_per_node))
         # 'hive.llap.io.threadpool.size' config value is to be set same as value calculated
for
         # 'hive.llap.daemon.num.executors' at all times.
@@ -655,6 +700,9 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
         cache_size_per_node = long(cache_size_per_node)
         putHiveInteractiveSiteProperty('hive.llap.io.memory.size', cache_size_per_node)
         Logger.info("LLAP config 'hive.llap.io.memory.size' updated. Current: {0}".format(cache_size_per_node))
+        # Copy the value of 'hive.llap.io.memory.size' to 'hive.llap.io.memory.size_copy'
as
+        # 'hive.llap.io.memory.size_copy' config is used to display it as Label on Hive Server
interactive panel.
+        putHiveInteractiveSiteProperty('hive.llap.io.memory.size_copy', cache_size_per_node)
         llap_io_enabled = 'false'
         if cache_size_per_node >= 64:
           llap_io_enabled = 'true'
@@ -691,17 +739,26 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
         putHiveInteractiveEnvProperty('num_llap_nodes', 0)
         putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "minimum", 1)
         putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "maximum", node_manager_cnt)
+        putHiveInteractiveEnvProperty('num_llap_nodes_copy', 0)
+        putHiveInteractiveEnvPropertyAttribute('num_llap_nodes_copy', "minimum", 1)
+        putHiveInteractiveEnvPropertyAttribute('num_llap_nodes_copy', "maximum", node_manager_cnt)
 
         putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', yarn_min_container_size)
         putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.yarn.container.mb', "minimum",
yarn_min_container_size)
+        putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb_copy', yarn_min_container_size)
+        putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.yarn.container.mb_copy',
"minimum", yarn_min_container_size)
 
         putHiveInteractiveSiteProperty('hive.llap.daemon.num.executors', 0)
         putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors', "minimum",
1)
+        putHiveInteractiveSiteProperty('hive.llap.daemon.num.executors_copy', 0)
+        putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors_copy', "minimum",
1)
+
         putHiveInteractiveSiteProperty('hive.llap.io.threadpool.size', 0)
 
         putHiveInteractiveSiteProperty('hive.llap.io.threadpool.size', 0)
 
         putHiveInteractiveSiteProperty('hive.llap.io.memory.size', 0)
+        putHiveInteractiveSiteProperty('hive.llap.io.memory.size_copy', 0)
 
         putHiveInteractiveEnvProperty('llap_heap_size', 0)
 
@@ -769,21 +826,6 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
 
 
   """
-  Returns current value of cache per node for LLAP (hive.llap.io.memory.size). If value can't
be retrieved, return 0.
-  """
-  def get_cache_size_per_node_for_llap_nodes(self, services):
-    if 'hive.llap.io.memory.size' in services['configurations']['hive-interactive-site']['properties']:
-      cache_size_per_node = float(
-        services['configurations']['hive-interactive-site']['properties']['hive.llap.io.memory.size'])
-      return cache_size_per_node
-    else:
-      Logger.error("Couldn't retrieve Hive Server interactive's 'hive.llap.io.memory.size'
config.")
-      # Not doing raise as the Exception that catches it will set all other LLAP configs
related
-      # to LLAP package as 0, a way to tell that calulations couldn't be done. This is not
the intention here.
-      # Just keep cache 0, if it couldn't be retrieved.
-      return 0
-
-  """
   Returns current value of number of LLAP nodes in cluster (num_llap_nodes)
   """
   def get_num_llap_nodes(self, services):

http://git-wip-us.apache.org/repos/asf/ambari/blob/752dc9d8/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index f66140d..1bc53ea 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -597,6 +597,8 @@ class TestHDP25StackAdvisor(TestCase):
 
 
 
+
+
   # Tests related to 'recommendYARNConfigurations()'
 
 
@@ -3538,6 +3540,13 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_size'],
'512')
     self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'],
'llap')
 
+    # Check '*_copy' configs, used for displaying it as label on UI.
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'],
'1024')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'],
'1')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'],
'512')
+    self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'],
'1')
+
+
 
   # Test 20: (1). 'default' and 'llap' (State : RUNNING) queue exists at root level in capacity-scheduler,
and
   #         'capacity-scheduler' configs are passed-in as single "/n" separated string  and
@@ -3729,6 +3738,12 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'],
                       {'visible': 'false'})
 
+    # Check '*_copy' configs, used for displaying it as label on UI.
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'],
'5120')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'],
'1')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'],
'4779')
+    self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'],
'1')
+
 
 
 
@@ -3925,6 +3940,12 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'],
                       {'maximum': '100', 'minimum': '20', 'visible': 'true'})
 
+    # Check '*_copy' configs, used for displaying it as label on UI.
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'],
'9216')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'],
'1')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'],
'8192')
+    self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'],
'1')
+
 
 
   ####################### 'Three Node Managers' cluster - tests for calculating llap configs
################
@@ -4122,6 +4143,12 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'],
                       {'maximum': '100', 'minimum': '20', 'visible': 'true'})
 
+    # Check '*_copy' configs, used for displaying it as label on UI.
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'],
'40960')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'],
'4')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'],
'36864')
+    self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'],
'2')
+
 
 
   # Test 23: (1). 'default' and 'llap' (State : RUNNING) queue exists at root level in capacity-scheduler,
and
@@ -4314,6 +4341,12 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'],
                       {'maximum': '100', 'minimum': '20', 'visible': 'true'})
 
+    # Check '*_copy' configs, used for displaying it as label on UI.
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'],
'10230')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'],
'3')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'],
'7158')
+    self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'],
'2')
+
 
 
   # Test 24: (1). 'default' and 'llap' (State : RUNNING) queue exists at root level in capacity-scheduler,
and
@@ -4504,6 +4537,12 @@ class TestHDP25StackAdvisor(TestCase):
                       {'maximum': '100', 'minimum': '20', 'visible': 'true'})
 
 
+    # Check '*_copy' configs, used for displaying it as label on UI.
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'],
'61440')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'],
'3')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'],
'58368')
+    self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'],
'1')
+
 
 
 
@@ -4701,6 +4740,12 @@ class TestHDP25StackAdvisor(TestCase):
                       {'maximum': '100', 'minimum': '20', 'visible': 'true'})
 
 
+    # Check '*_copy' configs, used for displaying it as label on UI.
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'],
'39936')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'],
'4')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'],
'35840')
+    self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'],
'3')
+
 
 
   # Test 26: (1). 'default' and 'llap' (State : RUNNING) queue exists at root level in capacity-scheduler,
and
@@ -4894,6 +4939,12 @@ class TestHDP25StackAdvisor(TestCase):
                       {'maximum': '100', 'minimum': '20', 'visible': 'true'})
 
 
+    # Check '*_copy' configs, used for displaying it as label on UI.
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'],
'10230')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'],
'10')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'],
'6820')
+    self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'],
'3')
+
 
 
   # Test 27: (1). 'default' and 'llap' (State : RUNNING) queue exists at root level in capacity-scheduler,
and
@@ -5081,6 +5132,12 @@ class TestHDP25StackAdvisor(TestCase):
                       {'maximum': '100', 'minimum': '25', 'visible': 'true'})
 
 
+    # Check '*_copy' configs, used for displaying it as label on UI.
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'],
'164480')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'],
'3')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'],
'161408')
+    self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'],
'1')
+
 
 
 
@@ -5261,6 +5318,12 @@ class TestHDP25StackAdvisor(TestCase):
                       {'maximum': '100', 'minimum': '49', 'visible': 'true'})
 
 
+    # Check '*_copy' configs, used for displaying it as label on UI.
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'],
'164480')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'],
'3')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'],
'161408')
+    self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'],
'1')
+
 
 
 
@@ -5443,6 +5506,12 @@ class TestHDP25StackAdvisor(TestCase):
                       {'maximum': '100', 'minimum': '20', 'visible': 'true'})
 
 
+    # Check '*_copy' configs, used for displaying it as label on UI.
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'],
'246720')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'],
'3')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'],
'243648')
+    self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'],
'1')
+
 
 
   # Test 30: (1). Multiple queue exist at various depths in capacity-scheduler, and 'capacity-scheduler'
configs are
@@ -5654,6 +5723,12 @@ class TestHDP25StackAdvisor(TestCase):
 
 
 
+    # Check '*_copy' configs, used for displaying it as label on UI.
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'],
'10240')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'],
'3')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'],
'7168')
+    self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'],
'4')
+
 
 
 
@@ -5865,6 +5940,12 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'],
                       {'visible': 'false'})
 
+    # Check '*_copy' configs, used for displaying it as label on UI.
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'],
'512')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'],
'0')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'],
'0')
+    self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'],
'0')
+
 
 
 
@@ -5971,14 +6052,18 @@ class TestHDP25StackAdvisor(TestCase):
           {
             'properties': {
               'enable_hive_interactive': 'true',
-              'llap_queue_capacity':'50'
+              'llap_queue_capacity':'50',
+              'num_llap_nodes' : '1'
             }
           },
         "hive-interactive-site":
           {
             'properties': {
               'hive.llap.daemon.queue.name': 'llap',
-              'hive.server2.tez.sessions.per.default.queue': '1'
+              'hive.server2.tez.sessions.per.default.queue': '1',
+              'hive.llap.daemon.num.executors' : '1',
+              'hive.llap.daemon.yarn.container.mb' : '10240',
+              'hive.llap.io.memory.size' : '512'
             }
           },
         "hive-env":
@@ -6036,6 +6121,12 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'],
                       {'maximum': '100', 'minimum': '100', 'visible': 'true'})
 
+    # Check '*_copy' configs, used for displaying it as label on UI.
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'],
'10240')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'],
'1')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'],
'512')
+    self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'],
'1')
+
 
 
 
@@ -6162,7 +6253,8 @@ class TestHDP25StackAdvisor(TestCase):
           {
             'properties': {
               'enable_hive_interactive': 'true',
-              'llap_queue_capacity':'0'
+              'llap_queue_capacity':'0',
+              'num_llap_nodes': '2'
             }
           },
         "hive-env":
@@ -6193,7 +6285,10 @@ class TestHDP25StackAdvisor(TestCase):
           {
             'properties': {
               'hive.llap.daemon.queue.name': 'llap',
-              'hive.server2.tez.sessions.per.default.queue': '1'
+              'hive.server2.tez.sessions.per.default.queue': '1',
+              'hive.llap.daemon.num.executors' : '1',
+              'hive.llap.daemon.yarn.container.mb' : '8192',
+              'hive.llap.io.memory.size' : '0'
             }
           },
       }
@@ -6220,6 +6315,12 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'],
                       {'visible': 'false'})
 
+    # Check '*_copy' configs, used for displaying it as label on UI.
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'],
'8192')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'],
'1')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'],
'0')
+    self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'],
'2')
+
 
 
 
@@ -6327,14 +6428,18 @@ class TestHDP25StackAdvisor(TestCase):
           {
             'properties': {
               'enable_hive_interactive': 'true',
-              'llap_queue_capacity':'50'
+              'llap_queue_capacity':'50',
+              'num_llap_nodes': '1'
             }
           },
         "hive-interactive-site":
           {
             'properties': {
               'hive.llap.daemon.queue.name': 'default',
-              'hive.server2.tez.sessions.per.default.queue': '1'
+              'hive.server2.tez.sessions.per.default.queue': '1',
+              'hive.llap.daemon.num.executors' : '1',
+              'hive.llap.daemon.yarn.container.mb' : '4096',
+              'hive.llap.io.memory.size' : '512'
             }
           },
         "hive-env":
@@ -6384,6 +6489,12 @@ class TestHDP25StackAdvisor(TestCase):
                       self.expected_hive_interactive_site_default['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'])
     self.assertEquals(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'],
'default')
 
+    # Check '*_copy' configs, used for displaying it as label on UI.
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'],
'4096')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'],
'1')
+    self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'],
'512')
+    self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'],
'1')
+
 
 
 


Mime
View raw message