ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swa...@apache.org
Subject ambari git commit: AMBARI-13108. Kafka metrics result in skewed split point distribiution. (swagle)
Date Wed, 16 Sep 2015 19:14:23 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 e1efd557e -> 0dfe5769f


AMBARI-13108. Kafka metrics result in skewed split point distribiution. (swagle)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0dfe5769
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0dfe5769
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0dfe5769

Branch: refs/heads/branch-2.1
Commit: 0dfe5769ff449c3f520a6dfb782b7256304d5cde
Parents: e1efd55
Author: Siddharth Wagle <swagle@hortonworks.com>
Authored: Wed Sep 16 12:14:05 2015 -0700
Committer: Siddharth Wagle <swagle@hortonworks.com>
Committed: Wed Sep 16 12:14:15 2015 -0700

----------------------------------------------------------------------
 .../server/configuration/Configuration.java     |  2 +-
 .../0.1.0/package/scripts/split_points.py       | 33 ++++++++++++++++++--
 2 files changed, 31 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/0dfe5769/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index 813b3d9..e3686ac 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -422,7 +422,7 @@ public class Configuration {
   private static final String TIMELINE_METRICS_REQUEST_READ_TIMEOUT = "server.timeline.metrics.cache.read.timeout.millis";
   private static final String DEFAULT_TIMELINE_METRICS_REQUEST_READ_TIMEOUT = "10000";
   private static final String TIMELINE_METRICS_REQUEST_INTERVAL_READ_TIMEOUT = "server.timeline.metrics.cache.interval.read.timeout.millis";
-  private static final String DEFAULT_TIMELINE_METRICS_REQUEST_INTERVAL_READ_TIMEOUT = "5000";
+  private static final String DEFAULT_TIMELINE_METRICS_REQUEST_INTERVAL_READ_TIMEOUT = "10000";
   private static final String TIMELINE_METRICS_REQUEST_CONNECT_TIMEOUT = "server.timeline.metrics.cache.connect.timeout.millis";
   private static final String DEFAULT_TIMELINE_METRICS_REQUEST_CONNECT_TIMEOUT = "5000";
   private static final String TIMELINE_METRICS_REQUEST_CATCHUP_INTERVAL = "server.timeline.metrics.cache.catchup.interval";

http://git-wip-us.apache.org/repos/asf/ambari/blob/0dfe5769/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/split_points.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/split_points.py
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/split_points.py
index b8b38f9..a585c6f 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/split_points.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/split_points.py
@@ -28,6 +28,8 @@ import ast
 metric_filename_ext = '.txt'
 # 5 regions for higher order aggregate tables
 other_region_static_count = 5
+# Max equidistant points to return per service
+max_equidistant_points = 50
 
 b_bytes = 1
 k_bytes = 1 << 10  # 1024
@@ -72,6 +74,10 @@ class FindSplitPointsForAMSRegions():
     self.serviceMetricsDir = serviceMetricsDir
     self.services = services
     self.mode = operation_mode
+    # Add host metrics if not present as input
+    if self.services and 'HOST' not in self.services:
+      self.services.append('HOST')
+
     # Initialize before user
     self.initialize()
 
@@ -90,7 +96,7 @@ class FindSplitPointsForAMSRegions():
       if self.mode == 'distributed':
         xmx_bytes = xmx_region_bytes
 
-      memstore_max_mem = float(self.ams_hbase_site['hbase.regionserver.global.memstore.upperLimit'])
* xmx_bytes
+      memstore_max_mem = float(self.ams_hbase_site['hbase.regionserver.global.memstore.lowerLimit'])
* xmx_bytes
       memstore_flush_size = format_Xmx_size_to_bytes(self.ams_hbase_site['hbase.hregion.memstore.flush.size'])
 
       max_inmemory_regions = (memstore_max_mem / memstore_flush_size) - other_region_static_count
@@ -120,15 +126,36 @@ class FindSplitPointsForAMSRegions():
       # services arg is not passed
       if self.services is None or file.rstrip(metric_filename_ext) in self.services:
         print 'Processing file: %s' % os.path.join(self.serviceMetricsDir, file)
+        service_metrics = set()
         with open(os.path.join(self.serviceMetricsDir, file), 'r') as f:
           for metric in f:
-            metrics.add(metric.strip())
+            service_metrics.add(metric.strip())
+          pass
+        pass
+        metrics.update(self.find_equidistant_metrics(list(sorted(service_metrics))))
       pass
     pass
 
     self.metrics = sorted(metrics)
     print 'metrics length: %s' % len(self.metrics)
 
+  # Pick 50 metric points for each service that are equidistant from
+  # each other for a service
+  def find_equidistant_metrics(self, service_metrics):
+    equi_metrics = []
+    idx = len(service_metrics) / max_equidistant_points
+    if idx == 0:
+      return service_metrics
+    pass
+
+    index = idx
+    for i in range(0, max_equidistant_points - 1):
+      equi_metrics.append(service_metrics[index - 1])
+      index += idx
+    pass
+
+    return equi_metrics
+
   def get_split_points(self):
     split_points = collections.namedtuple('SplitPoints', [ 'precision', 'aggregate' ])
     split_points.precision = []
@@ -159,7 +186,7 @@ class FindSplitPointsForAMSRegions():
 def main(argv = None):
   scriptDir = os.path.realpath(os.path.dirname(argv[0]))
   serviceMetricsDir = os.path.join(scriptDir, os.pardir, 'files', 'service-metrics')
-  print 'serviceMetricsDir: %s' % serviceMetricsDir
+
   if os.path.exists(serviceMetricsDir):
     onlyargs = argv[1:]
     if len(onlyargs) < 3:


Mime
View raw message