ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From odiache...@apache.org
Subject ambari git commit: AMBARI-14165. Populate hawq's yarn-client xml under /usr/local/hawq/etc(Bhuvnesh Chaudhary via odiachenko).
Date Thu, 03 Dec 2015 20:23:53 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 3b4d8f187 -> 19c1e4021


AMBARI-14165. Populate hawq's yarn-client xml under /usr/local/hawq/etc(Bhuvnesh Chaudhary
via odiachenko).


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/19c1e402
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/19c1e402
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/19c1e402

Branch: refs/heads/trunk
Commit: 19c1e4021b2d2d285ec65d3f9fa92265a261b7cd
Parents: 3b4d8f1
Author: Oleksandr Diachenko <odiachenko@pivotal.io>
Authored: Thu Dec 3 12:23:42 2015 -0800
Committer: Oleksandr Diachenko <odiachenko@pivotal.io>
Committed: Thu Dec 3 12:23:42 2015 -0800

----------------------------------------------------------------------
 .../HAWQ/2.0.0/configuration/yarn-client.xml    | 99 ++++++++++++++++++++
 .../HAWQ/2.0.0/package/scripts/common.py        | 55 +++++++++++
 .../HAWQ/2.0.0/package/scripts/constants.py     |  1 +
 .../HAWQ/2.0.0/package/scripts/params.py        |  4 +-
 4 files changed, 157 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/19c1e402/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/yarn-client.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/yarn-client.xml
b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/yarn-client.xml
new file mode 100755
index 0000000..a2296a1
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/yarn-client.xml
@@ -0,0 +1,99 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<configuration>
+	<!-- RPC client configuration -->
+      <property>
+          <name>rpc.client.timeout</name>
+          <value>3600000</value>
+          <description>
+          Timeout interval of a RPC invocation in millisecond. Default is 3600000.
+          </description>
+        </property>
+        <property>
+          <name>rpc.client.connect.tcpnodelay</name>
+          <value>true</value>
+          <description>
+          Whether to set socket TCP_NODELAY to true when connecting to RPC server. Default
is true.
+          </description>
+        </property>
+
+        <property>
+          <name>rpc.client.max.idle</name>
+          <value>10000</value>
+          <description>
+          Max idle time of a RPC connection in milliseconds. Default is 10000.
+          </description>
+        </property>
+        
+        <property>
+          <name>rpc.client.ping.interval</name>
+          <value>10000</value>
+          <description>
+          Interval at which the RPC client sends a heart beat to server. 0 means disable,
default is 10000.
+          </description>
+        </property>
+        
+        <property>
+          <name>rpc.client.connect.timeout</name>
+          <value>600000</value>
+          <description>
+          Timeout interval in milliseconds for which RPC client tries to setup the connection.
Default is 600000.
+          </description>
+        </property>
+        
+        <property>
+          <name>rpc.client.connect.retry</name>
+          <value>10</value>
+          <description>
+          Max retry attempts if the RPC client fails to setup the connection to server. Default
is 10.
+          </description>
+        </property>
+        
+        <property>
+          <name>rpc.client.read.timeout</name>
+          <value>3600000</value>
+          <description>
+          Timeout interval in milliseconds when the RPC client is trying to read from server.
Default is 3600000.
+          </description>
+        </property>
+
+        <property>
+          <name>rpc.client.write.timeout</name>
+          <value>3600000</value>
+          <description>
+          Timeout interval in milliseconds when the RPC client is trying to write to server.
Default is 3600000.
+          </description>
+        </property>
+        
+        <property>
+          <name>rpc.client.socket.linger.timeout</name>
+          <value>-1</value>
+          <description>
+          Set value to socket SO_LINGER when connecting to RPC server. -1 means default OS
value. Default is -1.
+          </description>
+        </property>
+        
+        <property>
+          <name>yarn.client.failover.max.attempts</name>
+          <value>15</value>
+          <description>
+          If multiple resource managers are configured, it is the max retry attempts for
which yarn client tries to issue a RPC call. Default is 15.
+          </description>
+        </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/19c1e402/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py
b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py
index 31ef2fa..c62ac12 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py
@@ -22,6 +22,7 @@ import crypt
 import filecmp
 from resource_management.libraries.resources.xml_config import XmlConfig
 from resource_management.core.resources.system import Execute, Directory, File
+from resource_management.libraries.script.config_dictionary import ConfigDictionary
 from resource_management.core.logger import Logger
 from resource_management.core.system import System
 from resource_management.core.exceptions import Fail
@@ -88,9 +89,63 @@ def setup_common_configurations():
             owner=constants.hawq_user,
             group=constants.hawq_group,
             mode=0644)
+  if "yarn-site" in params.config["configurations"]:
+    __update_yarn_client()
   __set_osparams()
 
 
+def __update_yarn_client():
+  """
+  Writes yarn-client.xml on the local filesystem on hawq nodes.
+  If yarn ha is enabled, appends related parameters to yarn-client.xml
+  """
+  import params
+
+  yarn_client_dict = params.yarn_client.copy()
+  if params.yarn_ha_enabled:
+    # Temporary logic, this logic will be moved in ambari-web to expose these parameters
on UI once Yarn HA is enabled
+    rm_ids = [rm_id.strip() for rm_id in params.config['configurations']['yarn-site']['yarn.resourcemanager.ha.rm-ids'].split(',')]
+    rm_id1 = rm_ids[0]
+    rm_id2 = rm_ids[1]
+    # Identify the hostname for yarn resource managers
+    rm_host1= params.config['configurations']['yarn-site']['yarn.resourcemanager.hostname.{0}'.format(rm_id1)]
+    rm_host2= params.config['configurations']['yarn-site']['yarn.resourcemanager.hostname.{0}'.format(rm_id2)]
+    # Ambari does not update yarn.resourcemanager.address.${rm_id} and yarn.resourcemanager.scheduler.address.${rm_id}
+    # property as its derived automatically at yarn.
+    # Hawq uses these properties to use yarn ha. If these properties are defined at Ambari
use them, else derive them.
+    # Use port 8032 to derive hawq.resourcemanager.address.${rm_id}:port value if needed
+    rm_default_port = 8032
+    # Use port 8030 to derive hawq.resourcemanager.scheduler.address.${rm_id}:port value
if needed
+    rm_scheduler_default_port = 8030
+
+    rm_address_host1 = params.config['configurations']['yarn-site'].get('yarn.resourcemanager.address.{0}'.format(rm_id1))
+    if rm_address_host1 is None:
+      rm_address_host1 = "{0}:{1}".format(rm_host1, rm_default_port)
+
+    rm_address_host2 = params.config['configurations']['yarn-site'].get('yarn.resourcemanager.address.{0}'.format(rm_id2))
+    if rm_address_host2 is None:
+      rm_address_host2 = "{0}:{1}".format(rm_host2, rm_default_port)
+
+    rm_scheduler_address_host1 = params.config['configurations']['yarn-site'].get('yarn.resourcemanager.scheduler.address.{0}'.format(rm_id1))
+    if rm_scheduler_address_host1 is None:
+      rm_scheduler_address_host1 = "{0}:{1}".format(rm_host1, rm_scheduler_default_port)
+
+    rm_scheduler_address_host2 = params.config['configurations']['yarn-site'].get('yarn.resourcemanager.scheduler.address.{0}'.format(rm_id2))
+    if rm_scheduler_address_host2 is None:
+      rm_scheduler_address_host2 = "{0}:{1}".format(rm_host2, rm_scheduler_default_port)
+
+    yarn_client_dict['yarn.resourcemanager.ha'] = "{0},{1}".format(rm_address_host1, rm_address_host2)
+    yarn_client_dict['yarn.resourcemanager.scheduler.ha'] = "{0},{1}".format(rm_scheduler_address_host1,
rm_scheduler_address_host2)
+
+  XmlConfig("yarn-client.xml",
+            conf_dir=constants.hawq_config_dir,
+            configurations=ConfigDictionary(yarn_client_dict),
+            configuration_attributes=params.config['configuration_attributes']['yarn-client'],
+            owner=constants.hawq_user,
+            group=constants.hawq_group,
+            mode=0644)
+
+
 def __substitute_hostnames_in_hawq_site():
   """
   Temporary function to replace localhost with actual HAWQ component hostnames.

http://git-wip-us.apache.org/repos/asf/ambari/blob/19c1e402/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py
b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py
index b56fd59..ff8f4be 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py
@@ -23,6 +23,7 @@ SEGMENT = "segment"
 START = "start"
 INIT = "init"
 STOP = "stop"
+YARN = "yarn"
 
 # Users
 root_user = "root"

http://git-wip-us.apache.org/repos/asf/ambari/blob/19c1e402/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py
index fb449b9..c8e97b8 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py
@@ -70,6 +70,7 @@ namenode_host= __get_namenode_host()
 # YARN
 # Note: YARN is not mandatory for HAWQ. It is required only when the users set HAWQ to use
YARN as resource manager
 rm_host = __get_component_host('rm_host')
+yarn_ha_enabled = default('/configurations/yarn-site/yarn.resourcemanager.ha.enabled', False)
 
 # Config files
 gpcheck_content = config['configurations']['gpcheck-env']['content']
@@ -79,6 +80,7 @@ hawq_limits = config['configurations']['hawq-limits-env']
 hawq_sysctl = config['configurations']['hawq-sysctl-env']
 
 hawq_site = config['configurations']['hawq-site']
+yarn_client = config['configurations']['yarn-client']
 hawq_master_dir = hawq_site.get('hawq_master_directory')
 hawq_segment_dir = hawq_site.get('hawq_segment_directory')
 hawq_master_temp_dir = hawq_site.get('hawq_master_temp_directory')
@@ -88,5 +90,3 @@ hawq_segment_temp_dir = hawq_site.get('hawq_segment_temp_directory')
 hawq_hdfs_data_dir = "/{0}".format(hawq_site.get('hawq_dfs_url').split('/', 1)[1])
 hawq_master_address_port = hawq_site.get('hawq_master_address_port')
 hawq_segment_address_port = hawq_site.get('hawq_segment_address_port')
-
-


Mime
View raw message