ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject git commit: AMBARI-8064 YARN/MR/Tez configs update (dsen)
Date Sat, 01 Nov 2014 19:34:45 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 74877bf7c -> 3d7b434cf


AMBARI-8064 YARN/MR/Tez configs update (dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3d7b434c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3d7b434c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3d7b434c

Branch: refs/heads/trunk
Commit: 3d7b434cf3323a7b5b607f776b8b107c37d7e5b8
Parents: 74877bf
Author: Dmytro Sen <dsen@apache.org>
Authored: Sat Nov 1 21:34:27 2014 +0200
Committer: Dmytro Sen <dsen@apache.org>
Committed: Sat Nov 1 21:34:27 2014 +0200

----------------------------------------------------------------------
 .../services/HDFS/configuration/core-site.xml   |  31 +++
 .../YARN/configuration-mapred/mapred-site.xml   |  36 +++
 .../YARN/configuration/capacity-scheduler.xml   |  45 +++
 .../services/YARN/configuration/yarn-site.xml   | 271 +++++++++++++++++++
 4 files changed, 383 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/3d7b434c/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/core-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/core-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/core-site.xml
new file mode 100644
index 0000000..ece7835
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/core-site.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+ <!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+ 
+        http://www.apache.org/licenses/LICENSE-2.0
+ 
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+ -->
+ 
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration supports_final="true" xmlns:xi="http://www.w3.org/2001/XInclude">
+
+  <property>
+    <name>hadoop.http.authentication.simple.anonymous.allowed</name>
+    <value>true</value>
+    <description></description>
+  </property>
+
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/3d7b434c/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml
index 6a5d7db..b61a6a2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml
@@ -77,4 +77,40 @@
     <description></description>
   </property>
 
+  <property>
+    <name>mapreduce.reduce.shuffle.fetch.retry.enabled</name>
+    <value>1</value>
+    <description></description>
+  </property>
+
+  <property>
+    <name>mapreduce.reduce.shuffle.fetch.retry.interval-ms</name>
+    <value>1000</value>
+    <description></description>
+  </property>
+
+  <property>
+    <name>mapreduce.reduce.shuffle.fetch.retry.timeout-ms</name>
+    <value>30000</value>
+    <description></description>
+  </property>
+
+  <property>
+    <name>mapreduce.job.emit-timeline-data</name>
+    <value>false</value>
+    <description></description>
+  </property>
+
+  <property>
+    <name>mapreduce.job.log4j-properties-file</name>
+    <value>/etc/hadoop/conf/log4j.properties</value>
+    <description></description>
+  </property>
+
+  <property>
+    <name>mapreduce.jobhistory.bind-host</name>
+    <value>0.0.0.0</value>
+    <description></description>
+  </property>
+
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/3d7b434c/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/capacity-scheduler.xml
b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/capacity-scheduler.xml
new file mode 100644
index 0000000..63d56b0
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/capacity-scheduler.xml
@@ -0,0 +1,45 @@
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<configuration supports_final="false">
+  <property>
+    <name>yarn.scheduler.capacity.resource-calculator</name>
+    <value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.scheduler.capacity.root.accessible-node-labels</name>
+    <value>*</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.scheduler.capacity.root.accessible-node-labels.default.capacity</name>
+    <value>-1</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity</name>
+    <value>-1</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.scheduler.capacity.root.default-node-label-expression</name>
+    <value> </value>
+    <description></description>
+  </property>
+
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/3d7b434c/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml
index dc1ba0c..b1648cd 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml
@@ -40,4 +40,275 @@
       List of hostname:port pairs defining the zookeeper quorum binding for the registry
     </description>
   </property>
+  <property>
+    <name>yarn.nodemanager.recovery.enabled</name>
+    <value>1</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.nodemanager.recovery.dir</name>
+    <value>/hadoop/yarn/yarn-nm-recovery</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.client.nodemanager-connect.retry-interval-ms</name>
+    <value>10000</value>
+    <description>Time interval between each attempt to connect to NM</description>
+  </property>
+  <property>
+    <name>yarn.client.nodemanager-connect.max-wait-ms</name>
+    <value>900000</value>
+    <description>Max time to wait to establish a connection to NM</description>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.recovery.enabled</name>
+    <value>1</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.work-preserving-recovery.enabled</name>
+    <value>1</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.store.class</name>
+    <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.zk-address</name>
+    <value>127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002</value>
+    <description>
+      List Host:Port of the ZooKeeper servers to be used by the RM. comma separated host:port
pairs, each corresponding to a zk server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002"
If the optional chroot suffix is used the example would look like: "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002/app/a"
where the client would be rooted at "/app/a" and all paths would be relative to this root
- ie getting/setting/etc...  "/foo/bar" would result in operations being run on "/app/a/foo/bar"
(from the server perspective).
+    </description>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.zk-state-store.parent-path</name>
+    <value>/rmstore</value>
+    <description>Full path of the ZooKeeper znode where RM state will be stored. This
must be supplied when using org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore
as the value for yarn.resourcemanager.store.class</description>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.zk-acl</name>
+    <value>world:anyone:rwcda </value>
+    <description>ACL's to be used for ZooKeeper znodes.</description>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.zk-state-store.root-node.acl</name>
+    <value> </value>
+    <description>
+      ACLs to be used for the root znode when using ZKRMStateStore in a HA
+      scenario for fencing.
+
+      ZKRMStateStore supports implicit fencing to allow a single
+      ResourceManager write-access to the store. For fencing, the
+      ResourceManagers in the cluster share read-write-admin privileges on the
+      root node, but the Active ResourceManager claims exclusive create-delete
+      permissions.
+
+      By default, when this property is not set, we use the ACLs from
+      yarn.resourcemanager.zk-acl for shared admin access and
+      rm-address:random-number for username-based exclusive create-delete
+      access.
+
+      This property allows users to set ACLs of their choice instead of using
+      the default mechanism. For fencing to work, the ACLs should be
+      carefully set differently on each ResourceManger such that all the
+      ResourceManagers have shared admin access and the Active ResourceManger
+      takes over (exclusively) the create-delete access.
+    </description>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms</name>
+    <value>10000</value>
+    <description>Set the amount of time RM waits before allocating new containers on
work-preserving-recovery. Such wait period gives RM a chance to settle down resyncing with
NMs in the cluster on recovery, before assigning new containers to applications.</description>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.connect.retry-interval.ms</name>
+    <value>30000</value>
+    <description>How often to try connecting to the ResourceManager.</description>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.connect.max-wait.ms</name>
+    <value>900000</value>
+    <description>Maximum time to wait to establish connection to ResourceManager</description>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.zk-retry-interval-ms</name>
+    <value>1000</value>
+    <description>"Retry interval in milliseconds when connecting to ZooKeeper.
+      When HA is enabled, the value here is NOT used. It is generated
+      automatically from yarn.resourcemanager.zk-timeout-ms and
+      yarn.resourcemanager.zk-num-retries."
+    </description>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.zk-num-retries</name>
+    <value>1000</value>
+    <description>Number of times RM tries to connect to ZooKeeper.</description>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.zk-timeout-ms</name>
+    <value>10000</value>
+    <description>ZooKeeper session timeout in milliseconds. Session expiration is managed
by the ZooKeeper cluster itself, not by the client. This value is used by the cluster to determine
when the client's session expires. Expirations happens when the cluster does not hear from
the client within the specified session timeout period (i.e. no heartbeat).</description>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.state-store.max-completed-applications</name>
+    <value>${yarn.resourcemanager.max-completed-applications}</value>
+    <description>The maximum number of completed applications RM state store keeps,
less than or equals to ${yarn.resourcemanager.max-completed-applications}. By default, it
equals to ${yarn.resourcemanager.max-completed-applications}. This ensures that the applications
kept in the state store are consistent with the applications remembered in RM memory. Any
values larger than ${yarn.resourcemanager.max-completed-applications} will be reset to ${yarn.resourcemanager.max-completed-applications}.
Note that this value impacts the RM recovery performance.Typically,  a smaller value indicates
better performance on RM recovery.</description>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.fs.state-store.retry-policy-spec</name>
+    <value>2000, 500</value>
+    <description>hdfs client retry policy specification. hdfs client retry is always
enabled. Specified in pairs of sleep-time and number-of-retries and (t0, n0), (t1, n1), ...,
the first n0 retries sleep t0 milliseconds on average, the following n1 retries sleep t1 milliseconds
on average, and so on.</description>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.fs.state-store.uri</name>
+    <value> </value>
+    <description>RI pointing to the location of the FileSystem path where RM state
will be stored. This must be supplied when using org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore
as the value for yarn.resourcemanager.store.class </description>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.ha.enabled</name>
+    <value>0</value>
+    <description>enable RM HA or not</description>
+  </property>
+  <property>
+    <name>yarn.nodemanager.linux-container-executor.resources-handler.class</name>
+    <value>org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler</value>
+    <description>Pre-requisite to use CGroups</description>
+  </property>
+  <property>
+    <name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name>
+    <value>hadoop-yarn</value>
+    <description>Name of the Cgroups hierarchy under which all YARN jobs will be launched</description>
+  </property>
+  <property>
+    <name>yarn.nodemanager.linux-container-executor.cgroups.mount</name>
+    <value>1</value>
+    <description>If true, YARN will automount the CGroup, however the directory needs
to already exist; else, the cgroup should be mounted by the admin</description>
+  </property>
+  <property>
+    <name>yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage</name>
+    <value>0</value>
+    <description>Strictly limit CPU resource usage to allocated usage even if spare
CPU is available</description>
+  </property>
+  <property>
+    <name>yarn.nodemanager.resource.cpu-vcores</name>
+    <value>8</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
+    <value>100</value>
+    <description>The amount of CPU allocated for YARN containers - only effective when
used with CGroups</description>
+  </property>
+  <property>
+    <name>yarn.node-labels.manager-class</name>
+    <value>org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager</value>
+    <description>If user want to enable this feature, specify it to "org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager</description>
+  </property>
+  <property>
+    <name>yarn.node-labels.fs-store.retry-policy-spec</name>
+    <value>2000, 500</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb</name>
+    <value>1000</value>
+    <description>This is related to disk size on the machines, admins should set one
of yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb or yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage
but not both. If both are set, the more conservative value will be used</description>
+  </property>
+  <property>
+    <name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
+    <value>90</value>
+    <description>This is related to disk size on the machines, admins should set one
of yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb or yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage
but not both. If both are set, the more conservative value will be used</description>
+  </property>
+  <property>
+    <name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name>
+    <value>-1</value>
+    <description>Defines how often NMs wake up to upload log files. The default value
is -1. By default, the logs will be uploaded whenthe application is finished. By setting this
configure, logs can be uploaded periodically when the application is running. The minimum
rolling-interval-seconds can be set is 3600.</description>
+  </property>
+  <property>
+    <name>yarn.nodemanager.log-aggregation.debug-enabled</name>
+    <value>0</value>
+    <description>This configuration is for debug and test purpose. By setting this
configuration as true. We can break the lower bound of yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</description>
+  </property>
+  <property>
+    <name>yarn.nodemanager.log-aggregation.num-log-files-per-app</name>
+    <value>30</value>
+    <description>This is temporary solution. The configuration will be deleted once,
we find a more scalable method to only write a single log file per LRS.</description>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.system-metrics-publisher.enabled</name>
+    <value>1</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size</name>
+    <value>10</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.timeline-service.client.max-retries</name>
+    <value>30</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.timeline-service.client.retry-interval-ms</name>
+    <value>1000</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.timeline-service.ttl-enable</name>
+    <value>1</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.timeline-service.leveldb-timeline-store.path</name>
+    <value>/hadoop/yarn/timeline</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.timeline-service.leveldb-timeline-store.read-cache-size</name>
+    <value>104857600</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size</name>
+    <value>10000</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size</name>
+    <value>10000</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.timeline-service.http-authentication.type</name>
+    <value>simple</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.timeline-service.http-authentication.simple.anonymous.allowed</name>
+    <value>1</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled</name>
+    <value>0</value>
+    <description></description>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.bind-host</name>
+    <value>0.0.0.0</value>
+    <description>Default value is 0.0.0.0, when this is set the service will bind on
all interfaces.  I think these two options (blank, "0.0.0.0" sans quotes) should be the two
available values, with blank as the default.</description>
+  </property>
+  <property>
+    <name>yarn.nodemanager.bind-host</name>
+    <value>0.0.0.0</value>
+    <description>Default value is 0.0.0.0, when this is set the service will bind on
all interfaces.  I think these two options (blank, "0.0.0.0" sans quotes) should be the two
available values, with blank as the default.</description>
+  </property>
+  <property>
+    <name>yarn.timeline-service.bind-host</name>
+    <value>0.0.0.0</value>
+    <description>Default value is 0.0.0.0, when this is set the service will bind on
all interfaces.  I think these two options (blank, "0.0.0.0" sans quotes) should be the two
available values, with blank as the default.</description>
+  </property>
 </configuration>


Mime
View raw message