Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 86F9F10BF7 for ; Fri, 2 Jan 2015 12:26:29 +0000 (UTC) Received: (qmail 38628 invoked by uid 500); 2 Jan 2015 12:26:30 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 38600 invoked by uid 500); 2 Jan 2015 12:26:30 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 38590 invoked by uid 99); 2 Jan 2015 12:26:30 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 02 Jan 2015 12:26:30 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id E5ED7A3C1A7; Fri, 2 Jan 2015 12:26:29 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: dsen@apache.org To: commits@ambari.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: ambari git commit: AMBARI-8967 Metrics: Issues when installing AMS (dsen) Date: Fri, 2 Jan 2015 12:26:29 +0000 (UTC) Repository: ambari Updated Branches: refs/heads/trunk 536901a5d -> 6dff2c162 AMBARI-8967 Metrics: Issues when installing AMS (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6dff2c16 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6dff2c16 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6dff2c16 Branch: refs/heads/trunk Commit: 6dff2c162d435c949390689a9d6b16f1a9e7b5d6 Parents: 536901a Author: Dmytro Sen Authored: Fri Jan 2 12:57:00 2015 +0200 Committer: Dmytro Sen Committed: Fri Jan 2 12:57:00 2015 +0200 ---------------------------------------------------------------------- .../loadsimulator/net/RestMetricsSender.java | 2 +- .../AMS/0.1.0/configuration/ams-hbase-env.xml | 136 +++++++++---------- .../AMS/0.1.0/configuration/ams-site.xml | 2 +- .../AMS/0.1.0/package/scripts/ams.py | 6 + .../AMS/0.1.0/package/scripts/params.py | 3 + .../stacks/HDP/2.2/services/stack_advisor.py | 8 +- 6 files changed, 83 insertions(+), 74 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6dff2c16/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java index 5130ae3..8657436 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java @@ -32,7 +32,7 @@ import java.net.ProtocolException; public class RestMetricsSender implements MetricsSender { private final static Logger LOG = LoggerFactory.getLogger(RestMetricsSender.class); - private final static String COLLECTOR_URL = "http://%s:8188/ws/v1/timeline/metrics"; + private final static String COLLECTOR_URL = "http://%s:6188/ws/v1/timeline/metrics"; private final String collectorServiceAddress; /** http://git-wip-us.apache.org/repos/asf/ambari/blob/6dff2c16/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-env.xml b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-env.xml index 7a61c60..1cf80c8 100644 --- a/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-env.xml +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-hbase-env.xml @@ -63,74 +63,74 @@ content This is the jinja template for hbase-env.sh file - # Set environment variables here. - - # The java implementation to use. Java 1.6 required. - export JAVA_HOME={{java64_home}} - - # HBase Configuration directory - export HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{hbase_conf_dir}}} - - # Extra Java CLASSPATH elements. Optional. - export HBASE_CLASSPATH=${HBASE_CLASSPATH} - - if [ -f "/usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar" ]; then - export HBASE_CLASSPATH=${HBASE_CLASSPATH}:/usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar - fi - - # The maximum amount of heap to use, in MB. Default is 1000. - # export HBASE_HEAPSIZE=1000 - - # Extra Java runtime options. - # Below are what we set by default. May only work with SUN JVM. - # For more on why as well as other possible settings, - # see http://wiki.apache.org/hadoop/PerformanceTuning - export HBASE_OPTS="-XX:+UseConcMarkSweepGC -XX:ErrorFile={{hbase_log_dir}}/hs_err_pid%p.log" - export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`" - # Uncomment below to enable java garbage collection logging. - # export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log" - - # Uncomment and adjust to enable JMX exporting - # See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. - # More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html - # - # export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" - export HBASE_MASTER_OPTS="-Xmx{{master_heapsize}}" - export HBASE_REGIONSERVER_OPTS="-Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}" - # export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" - # export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" - - # File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. - export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers - - # Extra ssh options. Empty by default. - # export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" - - # Where log files are stored. $HBASE_HOME/logs by default. - export HBASE_LOG_DIR={{hbase_log_dir}} - - # A string representing this instance of hbase. $USER by default. - # export HBASE_IDENT_STRING=$USER - - # The scheduling priority for daemon processes. See 'man nice'. - # export HBASE_NICENESS=10 - - # The directory where pid files are stored. /tmp by default. - export HBASE_PID_DIR={{hbase_pid_dir}} - - # Seconds to sleep between slave commands. Unset by default. This - # can be useful in large clusters, where, e.g., slave rsyncs can - # otherwise arrive faster than the master can service them. - # export HBASE_SLAVE_SLEEP=0.1 - - # Tell HBase whether it should manage it's own instance of Zookeeper or not. - export HBASE_MANAGES_ZK=false - - {% if security_enabled %} - export HBASE_OPTS="$HBASE_OPTS -Djava.security.auth.login.config={{client_jaas_config_file}}" - export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Djava.security.auth.login.config={{master_jaas_config_file}}" - export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Djava.security.auth.login.config={{regionserver_jaas_config_file}}" - {% endif %} +# Set environment variables here. + +# The java implementation to use. Java 1.6 required. +export JAVA_HOME={{java64_home}} + +# HBase Configuration directory +export HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{hbase_conf_dir}}} + +# Extra Java CLASSPATH elements. Optional. +export HBASE_CLASSPATH=${HBASE_CLASSPATH} + +if [ -f "/usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar" ]; then + export HBASE_CLASSPATH=${HBASE_CLASSPATH}:/usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar +fi + +# The maximum amount of heap to use, in MB. Default is 1000. +export HBASE_HEAPSIZE={{hbase_heapsize}} + +# Extra Java runtime options. +# Below are what we set by default. May only work with SUN JVM. +# For more on why as well as other possible settings, +# see http://wiki.apache.org/hadoop/PerformanceTuning +export HBASE_OPTS="-XX:+UseConcMarkSweepGC -XX:ErrorFile={{hbase_log_dir}}/hs_err_pid%p.log" +export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{hbase_log_dir}}/gc.log-`date +'%Y%m%d%H%M'`" +# Uncomment below to enable java garbage collection logging. +# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log" + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# +# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +export HBASE_MASTER_OPTS="-Xmx{{master_heapsize}}" +export HBASE_REGIONSERVER_OPTS="-Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}" +# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" + +# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers + +# Extra ssh options. Empty by default. +# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + +# Where log files are stored. $HBASE_HOME/logs by default. +export HBASE_LOG_DIR={{hbase_log_dir}} + +# A string representing this instance of hbase. $USER by default. +# export HBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export HBASE_NICENESS=10 + +# The directory where pid files are stored. /tmp by default. +export HBASE_PID_DIR={{hbase_pid_dir}} + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of Zookeeper or not. +export HBASE_MANAGES_ZK=false + +{% if security_enabled %} +export HBASE_OPTS="$HBASE_OPTS -Djava.security.auth.login.config={{client_jaas_config_file}}" +export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Djava.security.auth.login.config={{master_jaas_config_file}}" +export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Djava.security.auth.login.config={{regionserver_jaas_config_file}}" +{% endif %} http://git-wip-us.apache.org/repos/asf/ambari/blob/6dff2c16/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-site.xml b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-site.xml index 2f8281a..3cac412 100644 --- a/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-site.xml +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/configuration/ams-site.xml @@ -47,7 +47,7 @@ timeline.metrics.aggregator.checkpoint.dir - /tmp + /var/lib/ambari-metrics-collector/checkpoint Directory to store aggregator checkpoints. Change to a permanent location so that checkpoint ar not lost. http://git-wip-us.apache.org/repos/asf/ambari/blob/6dff2c16/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/ams.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/ams.py b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/ams.py index 1f09d7d..4a3a3f1 100644 --- a/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/ams.py +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/ams.py @@ -31,6 +31,12 @@ def ams(name=None): recursive=True ) + Directory(params.ams_checkpoint_dir, + owner=params.ams_user, + group=params.user_group, + recursive=True + ) + XmlConfig("ams-site.xml", conf_dir=params.ams_collector_conf_dir, configurations=params.config['configurations']['ams-site'], http://git-wip-us.apache.org/repos/asf/ambari/blob/6dff2c16/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/params.py index afbbf4d..0932f3a 100644 --- a/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/AMS/0.1.0/package/scripts/params.py @@ -95,7 +95,10 @@ regionserver_heapsize = config['configurations']['ams-hbase-env']['hbase_regions regionserver_xmn_max = config['configurations']['ams-hbase-env']['hbase_regionserver_xmn_max'] regionserver_xmn_percent = config['configurations']['ams-hbase-env']['hbase_regionserver_xmn_ratio'] regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max) +# For embedded mode +hbase_heapsize = master_heapsize +ams_checkpoint_dir = config['configurations']['ams-site']['timeline.metrics.aggregator.checkpoint.dir'] hbase_pid_dir = status_params.hbase_pid_dir hbase_tmp_dir = config['configurations']['ams-hbase-site']['hbase.tmp.dir'] # TODO UPGRADE default, update site during upgrade http://git-wip-us.apache.org/repos/asf/ambari/blob/6dff2c16/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 7f09dfa..fbfc850 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -137,13 +137,13 @@ class HDP22StackAdvisor(HDP21StackAdvisor): regionServerItem = self.validatorLessThenDefaultValue(properties, recommendedDefaults, "hbase_regionserver_heapsize") masterItem = self.validatorLessThenDefaultValue(properties, recommendedDefaults, "hbase_master_heapsize") - if regionServerItem is None and masterItem is None: - hbase_regionserver_heapsize = formatXmxSizeToBytes(properties["hbase_regionserver_heapsize"]) + if masterItem is None: hbase_master_heapsize = formatXmxSizeToBytes(properties["hbase_master_heapsize"]) # TODO Add AMS Collector Xmx property to ams-env - # Collector + HBASE Master + HBASE RegionServer HeapSize - requiredMemory = 1073741824 + hbase_regionserver_heapsize + hbase_master_heapsize + # Collector 512m + HBASE Master heapsize + # For standalone HBase, master's heap memory is used by regionserver as well + requiredMemory = 536870912 + hbase_master_heapsize amsCollectorHosts = self.getComponentHostNames(services, "AMS", "METRIC_COLLECTOR") for collectorHostName in amsCollectorHosts: