ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nc...@apache.org
Subject [28/50] ambari git commit: AMBARI-20424. AMS should skip network stats from virtual network interfaces. (mpapirkovskyy)
Date Tue, 14 Mar 2017 18:42:12 GMT
AMBARI-20424. AMS should skip network stats from virtual network interfaces. (mpapirkovskyy)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/db3c09bb
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/db3c09bb
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/db3c09bb

Branch: refs/heads/branch-feature-AMBARI-12556
Commit: db3c09bb6f64c8ad46301b8828f62a130d56dd52
Parents: 1cffdcd
Author: Myroslav Papirkovskyi <mpapyrkovskyy@hortonworks.com>
Authored: Mon Mar 13 18:38:49 2017 +0200
Committer: Myroslav Papirkovskyi <mpapyrkovskyy@hortonworks.com>
Committed: Mon Mar 13 18:38:49 2017 +0200

----------------------------------------------------------------------
 .../conf/unix/metric_monitor.ini                |   2 +
 .../src/main/python/core/config_reader.py       |   7 +-
 .../src/main/python/core/host_info.py           |  41 ++++--
 .../src/test/python/core/TestHostInfo.py        | 147 ++++++++++++++++++-
 .../0.1.0/configuration/ams-env.xml             |  16 ++
 .../0.1.0/package/scripts/params.py             |  23 +--
 .../package/templates/metric_monitor.ini.j2     |   2 +
 7 files changed, 212 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/db3c09bb/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/metric_monitor.ini
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/metric_monitor.ini b/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/metric_monitor.ini
index e98c65c..7fe7397 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/metric_monitor.ini
+++ b/ambari-metrics/ambari-metrics-host-monitoring/conf/unix/metric_monitor.ini
@@ -22,6 +22,8 @@ metrics_servers = localhost
 enable_time_threshold = false
 enable_value_threshold = false
 skip_disk_patterns =
+skip_virtual_interfaces = false
+skip_network_interfaces_patterns =
 
 [emitter]
 send_interval = 60

http://git-wip-us.apache.org/repos/asf/ambari/blob/db3c09bb/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
index 7010187..5686c50 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py
@@ -22,7 +22,6 @@ import ConfigParser
 import StringIO
 import json
 import os
-import ast
 from ambari_commons import OSConst
 from ambari_commons.os_family_impl import OsFamilyImpl
 
@@ -254,3 +253,9 @@ class Configuration:
 
   def get_disk_metrics_skip_pattern(self):
     return self.get("default", "skip_disk_patterns")
+
+  def get_virtual_interfaces_skip(self):
+    return self.get("default", "skip_virtual_interfaces")
+
+  def get_network_interfaces_skip_pattern(self):
+    return self.get("default", "skip_network_interfaces_patterns")

http://git-wip-us.apache.org/repos/asf/ambari/blob/db3c09bb/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py
b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py
index 90f73fa..035c833 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py
@@ -19,14 +19,14 @@ limitations under the License.
 '''
 
 import logging
-import psutil
+import operator
 import os
 import platform
-import time
-import threading
-import socket
-import operator
+import psutil
 import re
+import socket
+import threading
+import time
 from collections import namedtuple
 
 logger = logging.getLogger()
@@ -153,13 +153,22 @@ class HostInfo():
 
     net_stats = psutil.net_io_counters(True)
     new_net_stats = {}
+
+    skip_virtual_interfaces = self.get_virtual_network_interfaces() if self.__config.get_virtual_interfaces_skip()
== 'True' else []
+    skip_network_patterns = self.__config.get_network_interfaces_skip_pattern()
+    skip_network_patterns_list = skip_network_patterns.split(',') if skip_network_patterns
and skip_network_patterns != 'None' else []
     for interface, values in net_stats.iteritems():
-      if interface != 'lo':
-        new_net_stats = {'bytes_out': new_net_stats.get('bytes_out', 0) + values.bytes_sent,
-                         'bytes_in': new_net_stats.get('bytes_in', 0) + values.bytes_recv,
-                         'pkts_out': new_net_stats.get('pkts_out', 0) + values.packets_sent,
-                         'pkts_in': new_net_stats.get('pkts_in', 0) + values.packets_recv
-        }
+      if interface != 'lo' and not interface in skip_virtual_interfaces:
+        ignore_network = False
+        for p in skip_network_patterns_list:
+          if re.match(p, interface):
+            ignore_network = True
+        if not ignore_network:
+          new_net_stats = {'bytes_out': new_net_stats.get('bytes_out', 0) + values.bytes_sent,
+                           'bytes_in': new_net_stats.get('bytes_in', 0) + values.bytes_recv,
+                           'pkts_out': new_net_stats.get('pkts_out', 0) + values.packets_sent,
+                           'pkts_in': new_net_stats.get('pkts_in', 0) + values.packets_recv
+          }
 
     with self.__last_network_lock:
       result = dict((k, (v - self.__last_network_data.get(k, 0)) / delta) for k, v in new_net_stats.iteritems())
@@ -354,3 +363,13 @@ class HostInfo():
 
   def get_ip_address(self):
     return socket.gethostbyname(socket.getfqdn())
+
+  def get_virtual_network_interfaces(self):
+    sys_net_path = "/sys/class/net"
+    net_devices = []
+    if os.path.isdir(sys_net_path):
+      links = [f for f in os.listdir(sys_net_path) if os.path.islink(os.path.join(sys_net_path,
f))]
+      for link in links:
+        if "devices/virtual" in os.readlink(os.path.join(sys_net_path, link)):
+          net_devices.append(link)
+    return net_devices

http://git-wip-us.apache.org/repos/asf/ambari/blob/db3c09bb/ambari-metrics/ambari-metrics-host-monitoring/src/test/python/core/TestHostInfo.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/test/python/core/TestHostInfo.py
b/ambari-metrics/ambari-metrics-host-monitoring/src/test/python/core/TestHostInfo.py
index 63a1ae1..1b249c7 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/src/test/python/core/TestHostInfo.py
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/test/python/core/TestHostInfo.py
@@ -18,13 +18,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 '''
 
+import collections
 import logging
-from host_info import HostInfo
 import platform
-from unittest import TestCase
+from host_info import HostInfo
 from mock.mock import patch, MagicMock
-from core.config_reader import Configuration
-import collections
+from unittest import TestCase
 
 logger = logging.getLogger()
 
@@ -205,5 +204,145 @@ class TestHostInfo(TestCase):
     self.assertEqual(disk_counter_per_disk['sdisk_sdb1_read_merged_count'], 16)
     self.assertEqual(disk_counter_per_disk['sdisk_sdb1_write_merged_count'], 17)
 
+  @patch.object(HostInfo, "get_virtual_network_interfaces", new = MagicMock(return_value
= ['etc11', 'etc2']))
+  @patch("psutil.net_io_counters")
+  def test_get_network_info_virtual_devices(self, net_io_counters):
+    Stats = collections.namedtuple('interface', ['bytes_sent', 'bytes_recv',
+                                                  'packets_sent', 'packets_recv'
+                                                  ])
+
+    net_stats = Stats(bytes_sent = 0, bytes_recv = 1,
+                          packets_sent = 2, packets_recv = 3
+    )
+
+    all_net_stats = { 'etc11' : net_stats, 'etc2' : net_stats }
+    net_io_counters.return_value = all_net_stats
+
+    c = MagicMock()
+
+    #skip virtual devices
+    c.get_virtual_interfaces_skip.return_value = 'True'
+    hostinfo = HostInfo(c)
+    network_info = hostinfo.get_network_info()
+
+    self.assertEqual(network_info, {})
+
+    #do not skip virtual devices
+    c.get_virtual_interfaces_skip.return_value = 'False'
+    hostinfo = HostInfo(c)
+    network_info = hostinfo.get_network_info()
+
+    #len({'bytes_in': ..., 'pkts_in': ..., 'pkts_out': ..., 'bytes_out': ...}) == 4
+    self.assertEqual(len(network_info), 4)
+
+  @patch("psutil.net_io_counters")
+  def test_get_network_info_skip_by_pattern(self, net_io_counters):
+    Stats = collections.namedtuple('interface', ['bytes_sent', 'bytes_recv',
+                                                  'packets_sent', 'packets_recv'
+                                                  ])
+
+    net_stats = Stats(bytes_sent = 0, bytes_recv = 1,
+                          packets_sent = 2, packets_recv = 3
+    )
+
+    all_net_stats = { 'etc11' : net_stats, 'etc2' : net_stats }
+    net_io_counters.return_value = all_net_stats
+
+    c = MagicMock()
+
+    #skip all by pattern
+    c.get_virtual_interfaces_skip.return_value = 'False'
+    c.get_network_interfaces_skip_pattern.return_value = "^etc\d*$"
+    hostinfo = HostInfo(c)
+    network_info = hostinfo.get_network_info()
+
+    self.assertEqual(network_info, {})
+
+    #skip one by pattern
+    c.get_network_interfaces_skip_pattern.return_value = "^etc\d{1}$"
+    hostinfo = HostInfo(c)
+    network_info = hostinfo.get_network_info()
+
+    self.assertEqual(len(network_info), 4)
+
+    all_net_stats = { 'etc2' : net_stats }
+    net_io_counters.return_value = all_net_stats
+    c.get_network_interfaces_skip_pattern.return_value = "^etc\d{1}$"
+    hostinfo = HostInfo(c)
+    network_info = hostinfo.get_network_info()
+
+    self.assertEqual(network_info, {})
+
+    #skip by 'None' pattern
+    c.get_network_interfaces_skip_pattern.return_value = "None"
+    hostinfo = HostInfo(c)
+    network_info = hostinfo.get_network_info()
+
+    self.assertEqual(len(network_info), 4)
+
+  @patch.object(HostInfo, "get_virtual_network_interfaces", new = MagicMock(return_value
= ['etc11', 'etc2']))
+  @patch("psutil.net_io_counters")
+  def test_get_network_info_skip_by_pattern_and_virtual(self, net_io_counters):
+    Stats = collections.namedtuple('interface', ['bytes_sent', 'bytes_recv',
+                                                  'packets_sent', 'packets_recv'
+                                                  ])
+
+    net_stats = Stats(bytes_sent = 0, bytes_recv = 1,
+                          packets_sent = 2, packets_recv = 3
+    )
+
+    all_net_stats = { 'etc11' : net_stats, 'etc2' : net_stats, 'etc333' : net_stats }
+    net_io_counters.return_value = all_net_stats
+
+    c = MagicMock()
+
+    #skip only one by pattern and other as virtual
+    c.get_virtual_interfaces_skip.return_value = 'True'
+    c.get_network_interfaces_skip_pattern.return_value = "^etc\d{3}$"
+    hostinfo = HostInfo(c)
+    network_info = hostinfo.get_network_info()
+
+    self.assertEqual(network_info, {})
+
+  @patch("os.path.isdir")
+  @patch("os.listdir")
+  @patch("os.readlink")
+  @patch("os.path.islink")
+  def test_get_virtual_network_interfaces(self, islink, readlink, listdir, isdir):
+    hostinfo = HostInfo(MagicMock())
+
+    #virtual net device is present
+    isdir.return_value = True
+    listdir.return_value = ['virtual_net_dev']
+    readlink.return_value = "../..devices/virtual/int6"
+    islink.return_value = True
+
+    virtual_net_devices = hostinfo.get_virtual_network_interfaces()
+    self.assertEqual(virtual_net_devices, ['virtual_net_dev'])
+
+    #virtual net device is not present
+    isdir.return_value = True
+    listdir.return_value = ['virtual_net_dev']
+    readlink.return_value = "../..devices/pp01.000/virtio1/int6"
+    islink.return_value = True
+
+    virtual_net_devices = hostinfo.get_virtual_network_interfaces()
+    self.assertEqual(virtual_net_devices, [])
+
+    #symlinks not present
+    isdir.return_value = True
+    listdir.return_value = ['virtual_net_dev']
+    readlink.return_value = "../..devices/virtual/int6"
+    islink.return_value = False
+
+    virtual_net_devices = hostinfo.get_virtual_network_interfaces()
+    self.assertEqual(virtual_net_devices, [])
+
+    #sysfs not available
+    isdir.return_value = False
+    virtual_net_devices = hostinfo.get_virtual_network_interfaces()
+
+    self.assertEqual(virtual_net_devices, [])
+
 
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/db3c09bb/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml
index a33656e..d79e357 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml
@@ -101,6 +101,22 @@
     <on-ambari-upgrade add="true"/>
   </property>
   <property>
+    <name>timeline.metrics.skip.network.interfaces.patterns</name>
+    <value>None</value>
+    <description>
+      Comma separated list of network interfaces to be ignored while collecting network metrics
+    </description>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>timeline.metrics.skip.virtual.interfaces</name>
+    <value>false</value>
+    <description>
+      Skips virtual network interfaces while collecting network metrics
+    </description>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
     <name>content</name>
     <display-name>ams-env template</display-name>
     <value>

http://git-wip-us.apache.org/repos/asf/ambari/blob/db3c09bb/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
index fcfe088..5672ee5 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
@@ -18,23 +18,24 @@ limitations under the License.
 
 """
 
-from functions import calc_xmn_from_xms
-from functions import check_append_heap_property
-from functions import trim_heap_property
+import ConfigParser
+import os
+
+from ambari_commons import OSCheck
+from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames
 from resource_management.core.logger import Logger
-from resource_management.libraries.script.script import Script
-from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
 from resource_management.libraries import functions
-from resource_management.libraries.functions.expect import expect
 from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.expect import expect
 from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
 from resource_management.libraries.functions.substitute_vars import substitute_vars
 from resource_management.libraries.resources.hdfs_resource import HdfsResource
-from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames
+
 import status_params
-from ambari_commons import OSCheck
-import ConfigParser
-import os
+from functions import calc_xmn_from_xms
+from functions import check_append_heap_property
+from functions import trim_heap_property
 
 if OSCheck.is_windows_family():
   from params_windows import *
@@ -204,6 +205,8 @@ metrics_collector_heapsize = default('/configurations/ams-env/metrics_collector_
 metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sink.report.interval",
60)
 metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period",
10)
 skip_disk_metrics_patterns = default("/configurations/ams-env/timeline.metrics.skip.disk.metrics.patterns",
None)
+skip_network_interfaces_patterns = default("/configurations/ams-env/timeline.metrics.skip.network.interfaces.patterns",
None)
+skip_virtual_interfaces = default("/configurations/ams-env/timeline.metrics.skip.virtual.interfaces",
False)
 
 hbase_log_dir = config['configurations']['ams-hbase-env']['hbase_log_dir']
 hbase_classpath_additional = default("/configurations/ams-hbase-env/hbase_classpath_additional",
None)

http://git-wip-us.apache.org/repos/asf/ambari/blob/db3c09bb/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2
index 3823912..1505f9b 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2
@@ -23,6 +23,8 @@ metrics_servers = {{ams_collector_hosts}}
 enable_time_threshold = false
 enable_value_threshold = false
 skip_disk_patterns = {{skip_disk_metrics_patterns}}
+skip_virtual_interfaces = {{skip_virtual_interfaces}}
+skip_network_interfaces_patterns = {{skip_network_interfaces_patterns}}
 
 [emitter]
 send_interval = {{metrics_report_interval}}


Mime
View raw message