incubator-ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nc...@apache.org
Subject [2/2] git commit: AMBARI-2910. Add various YARN time-series metrics. (ncole)
Date Thu, 15 Aug 2013 15:33:22 GMT
AMBARI-2910. Add various YARN time-series metrics. (ncole)


Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/83c4991e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/83c4991e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/83c4991e

Branch: refs/heads/trunk
Commit: 83c4991ec867ce816a3f68b9d1be4fe18a6e2f8b
Parents: 0800104
Author: Nate Cole <ncole@hortonworks.com>
Authored: Thu Aug 15 09:35:11 2013 -0400
Committer: Nate Cole <ncole@hortonworks.com>
Committed: Thu Aug 15 11:33:06 2013 -0400

----------------------------------------------------------------------
 .../hadoop-metrics2.properties-GANGLIA.erb      |     3 +
 .../templates/hadoop-metrics2.properties.erb    |     3 +
 .../main/resources/ganglia_properties_2.json    |    61 +
 .../ganglia/GangliaPropertyProviderTest.java    |    42 +-
 .../temporal_ganglia_data_yarn_queues.txt       | 15720 +++++++++++++++++
 5 files changed, 15828 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/83c4991e/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties-GANGLIA.erb
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties-GANGLIA.erb
b/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties-GANGLIA.erb
index 3ab9875..206c64c 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties-GANGLIA.erb
+++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties-GANGLIA.erb
@@ -37,3 +37,6 @@ reducetask.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host"
 resourcemanager.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host")%>:8664
 nodemanager.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host")%>:8660
 historyserver.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host")%>:8666
+
+resourcemanager.sink.ganglia.tagsForPrefix.yarn=*
+resourcemanager.sink.ganglia.tagsForPrefix.rpc=*

http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/83c4991e/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties.erb
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties.erb
b/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties.erb
index 3ab9875..206c64c 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties.erb
+++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties.erb
@@ -37,3 +37,6 @@ reducetask.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host"
 resourcemanager.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host")%>:8664
 nodemanager.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host")%>:8660
 historyserver.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host")%>:8666
+
+resourcemanager.sink.ganglia.tagsForPrefix.yarn=*
+resourcemanager.sink.ganglia.tagsForPrefix.rpc=*

http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/83c4991e/ambari-server/src/main/resources/ganglia_properties_2.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/ganglia_properties_2.json b/ambari-server/src/main/resources/ganglia_properties_2.json
index 7dc2df1..5d4284c 100644
--- a/ambari-server/src/main/resources/ganglia_properties_2.json
+++ b/ambari-server/src/main/resources/ganglia_properties_2.json
@@ -6722,7 +6722,38 @@
         "metric":"ugi.ugi.LoginFailureAvgTime",
         "pointInTime":false,
         "temporal":true
+      },
+      "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AvailableMB":{
+        "metric":"yarn.QueueMetrics.Queue=(.+).AvailableMB",
+        "pointInTime" : false,
+        "temporal" : true
+      },
+      "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/PendingMB":{
+        "metric":"yarn.QueueMetrics.Queue=(.+).PendingMB",
+        "pointInTime" : false,
+        "temporal" : true
+      },
+      "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AllocatedMB":{
+        "metric":"yarn.QueueMetrics.Queue=(.+).AllocatedMB",
+        "pointInTime" : false,
+        "temporal" : true
+      },
+      "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AllocatedContainers":{
+        "metric":"yarn.QueueMetrics.Queue=(.+).AllocatedContainers",
+        "pointInTime" : false,
+        "temporal" : true
+      },
+      "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/ReservedContainers":{
+        "metric":"yarn.QueueMetrics.Queue=(.+).ReservedContainers",
+        "pointInTime" : false,
+        "temporal" : true
+      },
+      "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/PendingContainers":{
+        "metric":"yarn.QueueMetrics.Queue=(.+).PendingContainers",
+        "pointInTime" : false,
+        "temporal" : true
       }
+
     },
     "NODEMANAGER":{
       "metrics/boottime":{
@@ -16877,6 +16908,36 @@
         "metric":"ugi.ugi.LoginFailureAvgTime",
         "pointInTime":false,
         "temporal":true
+      },
+      "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AvailableMB":{
+        "metric":"yarn.QueueMetrics.Queue=(.+).AvailableMB",
+        "pointInTime" : false,
+        "temporal" : true
+      },
+      "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/PendingMB":{
+        "metric":"yarn.QueueMetrics.Queue=(.+).PendingMB",
+        "pointInTime" : false,
+        "temporal" : true
+      },
+      "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AllocatedMB":{
+        "metric":"yarn.QueueMetrics.Queue=(.+).AllocatedMB",
+        "pointInTime" : false,
+        "temporal" : true
+      },
+      "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AllocatedContainers":{
+        "metric":"yarn.QueueMetrics.Queue=(.+).AllocatedContainers",
+        "pointInTime" : false,
+        "temporal" : true
+      },
+      "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/ReservedContainers":{
+        "metric":"yarn.QueueMetrics.Queue=(.+).ReservedContainers",
+        "pointInTime" : false,
+        "temporal" : true
+      },
+      "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/PendingContainers":{
+        "metric":"yarn.QueueMetrics.Queue=(.+).PendingContainers",
+        "pointInTime" : false,
+        "temporal" : true
       }
     },
 

http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/83c4991e/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java
index d476f04..1fc1e02 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java
@@ -56,7 +56,10 @@ public class GangliaPropertyProviderTest {
   private static final String CLUSTER_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles",
"cluster_name");
   private static final String HOST_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles",
"host_name");
   private static final String COMPONENT_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles",
"component_name");
-
+  
+  private static final String RM_CATEGORY_1 = "metrics/yarn/Queue/root/default";
+  private static final String RM_AVAILABLE_MEMORY_PROPERTY = PropertyHelper.getPropertyId(RM_CATEGORY_1,
"AvailableMB");
+  
   private ComponentSSLConfiguration configuration;
 
   @Parameterized.Parameters
@@ -534,6 +537,43 @@ public class GangliaPropertyProviderTest {
     Assert.assertEquals(11, PropertyHelper.getProperties(resource).size());
     Assert.assertNotNull(resource.getPropertyValue(FLUME_CHANNEL_CAPACITY_PROPERTY));
   }
+  
+  @Test
+  public void testPopulateResources_params_category5() throws Exception {
+    TestStreamProvider streamProvider  = new TestStreamProvider("temporal_ganglia_data_yarn_queues.txt");
+    TestGangliaHostProvider hostProvider = new TestGangliaHostProvider();
+
+    GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(
+        PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, MetricsVersion.HDP2),
+        streamProvider,
+        configuration,
+        hostProvider,
+        CLUSTER_NAME_PROPERTY_ID,
+        HOST_NAME_PROPERTY_ID,
+        COMPONENT_NAME_PROPERTY_ID);
+    
+    Resource resource = new ResourceImpl(Resource.Type.HostComponent);
+
+    resource.setProperty(HOST_NAME_PROPERTY_ID, "dev01.ambari.apache.org");
+    resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "RESOURCEMANAGER");
+
+    // only ask for one property
+    Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>();
+    temporalInfoMap.put(RM_CATEGORY_1, new TemporalInfoImpl(10L, 20L, 1L));
+    
+    Request  request = PropertyHelper.getReadRequest(Collections.singleton(RM_CATEGORY_1),
temporalInfoMap);
+
+    Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource),
request, null).size());
+
+    String expected = (configuration.isGangliaSSL() ? "https" : "http") +
+        "://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPResourceManager&h=dev01.ambari.apache.org&s=10&e=20&r=1";
+    Assert.assertEquals(expected, streamProvider.getLastSpec());    
+    
+    System.out.println(PropertyHelper.getProperties(resource));
+    
+    Assert.assertEquals(14, PropertyHelper.getProperties(resource).size());
+    Assert.assertNotNull(resource.getPropertyValue(RM_AVAILABLE_MEMORY_PROPERTY));
+  }  
 
   private static class TestGangliaHostProvider implements GangliaHostProvider {
 


Mime
View raw message