ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mpapirkovs...@apache.org
Subject [1/5] ambari git commit: AMBARI-7681. Add Metrics Service to common services stack. (mpapirkovskyy)
Date Wed, 19 Nov 2014 00:38:17 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-metrics-dev 3c23bee67 -> a3ed7a3fd


http://git-wip-us.apache.org/repos/asf/ambari/blob/a3ed7a3f/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/params.py
b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/params.py
index c31caf2..6488a79 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/params.py
@@ -26,6 +26,23 @@ import status_params
 config = Script.get_config()
 exec_tmp_dir = Script.get_tmp_dir()
 
+#AMS data
+ams_user=status_params.ams_user
+ams_pid_dir="/var/run/ambari-metrics"
+
+ams_collector_script = "/usr/sbin/ambari-metrics-collector"
+ams_collector_conf_dir = "/etc/ambari-metrics-collector/conf"
+ams_collector_pid_dir = status_params.ams_collector_pid_dir
+ams_collector_hosts = default("/clusterHostInfo/metric_collector_hosts", [])
+ams_collector_host_single = ams_collector_hosts[0] #TODO cardinality is 1+ so we can have
more than one host
+ams_collector_port = 8188
+
+
+ams_monitor_conf_dir = "/etc/ambari-metrics-monitor/conf/"
+ams_monitor_dir = "/usr/lib/python2.6/site-packages/resource_monitoring"
+ams_monitor_pid_dir = status_params.ams_monitor_pid_dir
+ams_monitor_script = "/usr/sbin/ambari-metrics-monitor"
+
 #RPM versioning support
 rpm_version = default("/configurations/hadoop-env/rpm_version", None)
 
@@ -36,12 +53,14 @@ if rpm_version is not None:
 
 #hadoop params
 if rpm_version is not None:
+  hadoop_native_lib = format("/usr/hdp/current/hadoop-client/lib/native/")
   hadoop_bin_dir = format("/usr/hdp/current/hadoop/bin")
   daemon_script = format('/usr/hdp/current/hbase/bin/hbase-daemon.sh')
   region_mover = format('/usr/hdp/current/hbase/bin/region_mover.rb')
-  region_drainer = format('/usr/hdp/currenthbase/bin/draining_servers.rb')
+  region_drainer = format('/usr/hdp/current/hbase/bin/draining_servers.rb')
   hbase_cmd = format('/usr/hdp/current/hbase/bin/hbase')
 else:
+  hadoop_native_lib = format("/usr/lib/hadoop/lib/native")
   hadoop_bin_dir = "/usr/bin"
   daemon_script = "/usr/lib/hbase/bin/hbase-daemon.sh"
   region_mover = "/usr/lib/hbase/bin/region_mover.rb"
@@ -49,7 +68,8 @@ else:
   hbase_cmd = "/usr/lib/hbase/bin/hbase"
 
 hadoop_conf_dir = "/etc/hadoop/conf"
-hbase_conf_dir = "/etc/hbase/conf"
+#hbase_conf_dir = "/etc/ams-hbase/conf"
+hbase_conf_dir = ams_collector_conf_dir
 hbase_excluded_hosts = config['commandParams']['excluded_hosts']
 hbase_drain_only = config['commandParams']['mark_draining_only']
 hbase_included_hosts = config['commandParams']['included_hosts']
@@ -65,32 +85,34 @@ metric_prop_file_name = "hadoop-metrics2-hbase.properties"
 # not supporting 32 bit jdk.
 java64_home = config['hostLevelParams']['java_home']
 
-log_dir = config['configurations']['hbase-env']['hbase_log_dir']
-master_heapsize = config['configurations']['hbase-env']['hbase_master_heapsize']
+hbase_log_dir = config['configurations']['ams-hbase-env']['hbase_log_dir']
+master_heapsize = config['configurations']['ams-hbase-env']['hbase_master_heapsize']
 
-regionserver_heapsize = config['configurations']['hbase-env']['hbase_regionserver_heapsize']
-regionserver_xmn_max = config['configurations']['hbase-env']['hbase_regionserver_xmn_max']
-regionserver_xmn_percent = config['configurations']['hbase-env']['hbase_regionserver_xmn_ratio']
+regionserver_heapsize = config['configurations']['ams-hbase-env']['hbase_regionserver_heapsize']
+regionserver_xmn_max = config['configurations']['ams-hbase-env']['hbase_regionserver_xmn_max']
+regionserver_xmn_percent = config['configurations']['ams-hbase-env']['hbase_regionserver_xmn_ratio']
 regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent,
regionserver_xmn_max)
 
-pid_dir = status_params.pid_dir
-tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir']
+hbase_pid_dir = status_params.hbase_pid_dir
+hbase_tmp_dir = config['configurations']['ams-hbase-site']['hbase.tmp.dir']
 # TODO UPGRADE default, update site during upgrade
-_local_dir_conf = default('/configurations/hbase-site/hbase.local.dir', "${hbase.tmp.dir}/local")
-local_dir = substitute_vars(_local_dir_conf, config['configurations']['hbase-site'])
+_local_dir_conf = default('/configurations/ams-hbase-site/hbase.local.dir', "${hbase.tmp.dir}/local")
+local_dir = substitute_vars(_local_dir_conf, config['configurations']['ams-hbase-site'])
 
 client_jaas_config_file = format("{hbase_conf_dir}/hbase_client_jaas.conf")
 master_jaas_config_file = format("{hbase_conf_dir}/hbase_master_jaas.conf")
 regionserver_jaas_config_file = format("{hbase_conf_dir}/hbase_regionserver_jaas.conf")
 
-ganglia_server_hosts = default('/clusterHostInfo/ganglia_server_host', []) # is not passed
when ganglia is not present
-ganglia_server_host = '' if len(ganglia_server_hosts) == 0 else ganglia_server_hosts[0]
+# ganglia_server_hosts = default('/clusterHostInfo/ganglia_server_host', []) # is not passed
when ganglia is not present
+# ganglia_server_host = '' if len(ganglia_server_hosts) == 0 else ganglia_server_hosts[0]
 
 # if hbase is selected the hbase_rs_hosts, should not be empty, but still default just in
case
-if 'slave_hosts' in config['clusterHostInfo']:
-  rs_hosts = default('/clusterHostInfo/hbase_rs_hosts', '/clusterHostInfo/slave_hosts') #if
hbase_rs_hosts not given it is assumed that region servers on same nodes as slaves
-else:
-  rs_hosts = default('/clusterHostInfo/hbase_rs_hosts', '/clusterHostInfo/all_hosts') 
+# if 'slave_hosts' in config['clusterHostInfo']:
+#   rs_hosts = default('/clusterHostInfo/hbase_rs_hosts', '/clusterHostInfo/slave_hosts')
#if hbase_rs_hosts not given it is assumed that region servers on same nodes as slaves
+# else:
+#   rs_hosts = default('/clusterHostInfo/hbase_rs_hosts', '/clusterHostInfo/all_hosts')
+
+rs_hosts = ["localhost"]
 
 smoke_test_user = config['configurations']['cluster-env']['smokeuser']
 smokeuser_permissions = "RWXCA"
@@ -99,28 +121,30 @@ user_group = config['configurations']['cluster-env']["user_group"]
 
 if security_enabled:
   _hostname_lowercase = config['hostname'].lower()
-  master_jaas_princ = config['configurations']['hbase-site']['hbase.master.kerberos.principal'].replace('_HOST',_hostname_lowercase)
-  regionserver_jaas_princ = config['configurations']['hbase-site']['hbase.regionserver.kerberos.principal'].replace('_HOST',_hostname_lowercase)
+  master_jaas_princ = config['configurations']['ams-hbase-site']['hbase.master.kerberos.principal'].replace('_HOST',_hostname_lowercase)
+  regionserver_jaas_princ = config['configurations']['ams-hbase-site']['hbase.regionserver.kerberos.principal'].replace('_HOST',_hostname_lowercase)
 
-master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file']
-regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file']
+master_keytab_path = config['configurations']['ams-hbase-site']['hbase.master.keytab.file']
+regionserver_keytab_path = config['configurations']['ams-hbase-site']['hbase.regionserver.keytab.file']
 smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
-hbase_user_keytab = config['configurations']['hbase-env']['hbase_user_keytab']
+hbase_user_keytab = config['configurations']['ams-hbase-env']['hbase_user_keytab']
 kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
-if security_enabled:
-  kinit_cmd = format("{kinit_path_local} -kt {hbase_user_keytab} {hbase_user};")
-else:
-  kinit_cmd = ""
+
+# if security_enabled:
+#   kinit_cmd = format("{kinit_path_local} -kt {hbase_user_keytab} {hbase_user};")
+# else:
+#   kinit_cmd = ""
 
 #log4j.properties
-if (('hbase-log4j' in config['configurations']) and ('content' in config['configurations']['hbase-log4j'])):
-  log4j_props = config['configurations']['hbase-log4j']['content']
+if (('ams-hbase-log4j' in config['configurations']) and ('content' in config['configurations']['ams-hbase-log4j'])):
+  log4j_props = config['configurations']['ams-hbase-log4j']['content']
 else:
   log4j_props = None
-  
-hbase_env_sh_template = config['configurations']['hbase-env']['content']
 
-hbase_hdfs_root_dir = config['configurations']['hbase-site']['hbase.rootdir']
+hbase_env_sh_template = config['configurations']['ams-hbase-env']['content']
+ams_env_sh_template = config['configurations']['ams-env']['content']
+
+hbase_hdfs_root_dir = config['configurations']['ams-hbase-site']['hbase.rootdir']
 hbase_staging_dir = "/apps/hbase/staging"
 #for create_hdfs_directory
 hostname = config["hostname"]
@@ -131,12 +155,15 @@ kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin",
"/
 import functools
 #create partial functions with common arguments for every HdfsDirectory call
 #to create hdfs directory we need to call params.HdfsDirectory in code
-HdfsDirectory = functools.partial(
-  HdfsDirectory,
-  conf_dir=hadoop_conf_dir,
-  hdfs_user=hdfs_user,
-  security_enabled = security_enabled,
-  keytab = hdfs_user_keytab,
-  kinit_path_local = kinit_path_local,
-  bin_dir = hadoop_bin_dir
-)
+# HdfsDirectory = functools.partial(
+#   HdfsDirectory,
+#   conf_dir=hadoop_conf_dir,
+#   hdfs_user=hdfs_user,
+#   security_enabled = security_enabled,
+#   keytab = hdfs_user_keytab,
+#   kinit_path_local = kinit_path_local,
+#   bin_dir = hadoop_bin_dir
+# )
+
+
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/a3ed7a3f/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/service_check.py
b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/service_check.py
new file mode 100644
index 0000000..5561e10
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/service_check.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""

http://git-wip-us.apache.org/repos/asf/ambari/blob/a3ed7a3f/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/status_params.py
b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/status_params.py
new file mode 100644
index 0000000..0e949eb
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/scripts/status_params.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+config = Script.get_config()
+
+hbase_pid_dir = config['configurations']['ams-hbase-env']['hbase_pid_dir']
+hbase_user = config['configurations']['ams-hbase-env']['hbase_user']
+ams_user = config['configurations']['ams-env']['ams_user']
+
+ams_monitor_pid_dir = "/var/run/ambari-metrics-monitor"
+ams_collector_pid_dir = "/var/run/ambari-metrics-collector"

http://git-wip-us.apache.org/repos/asf/ambari/blob/a3ed7a3f/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_groups.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_groups.conf.j2
b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_groups.conf.j2
new file mode 100644
index 0000000..f86343f
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_groups.conf.j2
@@ -0,0 +1,55 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+{
+   "host_metric_groups": {
+      "cpu_info": {
+         "collect_every": "15",
+         "metrics": [
+            {
+               "name": "cpu_user",
+               "value_threshold": "1.0"
+            }
+         ]
+      },
+      "disk_info": {
+         "collect_every": "30",
+         "metrics": [
+            {
+               "name": "disk_free",
+               "value_threshold": "5.0"
+            }
+         ]
+      },
+      "network_info": {
+         "collect_every": "20",
+         "metrics": [
+            {
+               "name": "bytes_out",
+               "value_threshold": "128"
+            }
+         ]
+      }
+   },
+   "process_metric_groups": {
+      "": {
+         "collect_every": "15",
+         "metrics": []
+      }
+   }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/a3ed7a3f/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_monitor.ini.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_monitor.ini.j2
b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_monitor.ini.j2
new file mode 100644
index 0000000..5e672fc
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/AMS/package/templates/metric_monitor.ini.j2
@@ -0,0 +1,30 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+[default]
+debug_level = INFO
+metrics_server = {{ams_collector_host_single}}:{{ams_collector_port}}
+enable_time_threshold = false
+enable_value_threshold = false
+
+[emitter]
+send_interval = 60
+
+[collector]
+collector_sleep_interval = 5
+max_queue_size = 5000


Mime
View raw message