ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From avija...@apache.org
Subject ambari git commit: AMBARI-15892 : Incorrect (Negative) values are shown for memory metrics - Commit 2 (avijayan)
Date Fri, 22 Apr 2016 02:39:19 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.2.2 13d69a89a -> 9d26e3774


AMBARI-15892 : Incorrect (Negative) values are shown for memory metrics - Commit 2 (avijayan)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9d26e377
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9d26e377
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9d26e377

Branch: refs/heads/branch-2.2.2
Commit: 9d26e3774bdf122e5d1bd5342b687254dfbc7014
Parents: 13d69a8
Author: Aravindan Vijayan <avijayan@hortonworks.com>
Authored: Thu Apr 21 19:37:58 2016 -0700
Committer: Aravindan Vijayan <avijayan@hortonworks.com>
Committed: Thu Apr 21 19:38:04 2016 -0700

----------------------------------------------------------------------
 .../metrics/timeline/PhoenixHBaseAccessor.java  | 20 ++++++++++++++++----
 .../timeline/TimelineMetricConfiguration.java   |  3 +++
 .../server/upgrade/UpgradeCatalog222.java       | 14 +++++++++++++-
 .../HBASE/0.96.0.2.0/widgets.json               |  2 +-
 .../YARN/2.1.0.2.0/YARN_widgets.json            |  8 +-------
 .../stacks/HDP/2.3/services/HBASE/widgets.json  |  2 +-
 .../HDP/2.3/services/YARN/YARN_widgets.json     |  8 +-------
 .../server/upgrade/UpgradeCatalog222Test.java   | 12 ++----------
 8 files changed, 38 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/9d26e377/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
index b4c832b..9579207 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
@@ -69,6 +69,7 @@ import java.util.concurrent.TimeUnit;
 
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.AGGREGATE_TABLE_SPLIT_POINTS;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HBASE_BLOCKING_STORE_FILES;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_TABLES_DURABILITY;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_DAILY_TABLE_TTL;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.CLUSTER_HOUR_TABLE_TTL;
@@ -390,9 +391,13 @@ public class PhoenixHBaseAccessor {
             LOG.info("Setting config property " + HSTORE_COMPACTION_CLASS_KEY +
               " = " + FIFO_COMPACTION_POLICY_CLASS + " for " + tableName);
             // Need to set blockingStoreFiles to 1000 for FIFO
-            tableDescriptor.setConfiguration(BLOCKING_STORE_FILES_KEY, "1000");
+            int blockingStoreFiles = hbaseConf.getInt(HBASE_BLOCKING_STORE_FILES, 1000);
+            if (blockingStoreFiles < 1000) {
+              blockingStoreFiles = 1000;
+            }
+            tableDescriptor.setConfiguration(BLOCKING_STORE_FILES_KEY, String.valueOf(blockingStoreFiles));
             LOG.info("Setting config property " + BLOCKING_STORE_FILES_KEY +
-              " = " + 1000 + " for " + tableName);
+              " = " + blockingStoreFiles + " for " + tableName);
             modifyTable = true;
           }
           // Set back original policy if fifo disabled
@@ -402,9 +407,16 @@ public class PhoenixHBaseAccessor {
               DEFAULT_COMPACTION_POLICY_CLASS);
             LOG.info("Setting config property " + HSTORE_COMPACTION_CLASS_KEY +
               " = " + DEFAULT_COMPACTION_POLICY_CLASS + " for " + tableName);
-            tableDescriptor.setConfiguration(BLOCKING_STORE_FILES_KEY, "300");
+
+            int blockingStoreFiles = hbaseConf.getInt(HBASE_BLOCKING_STORE_FILES, 300);
+            if (blockingStoreFiles > 300) {
+              LOG.warn("HBase blocking store file set too high without FIFO " +
+                "Compaction policy enabled, restoring low value = 300.");
+              blockingStoreFiles = 300;
+            }
+            tableDescriptor.setConfiguration(BLOCKING_STORE_FILES_KEY, String.valueOf(blockingStoreFiles));
             LOG.info("Setting config property " + BLOCKING_STORE_FILES_KEY +
-              " = " + 300 + " for " + tableName);
+              " = " + blockingStoreFiles + " for " + tableName);
             modifyTable = true;
           }
           // Change TTL setting to match user configuration

http://git-wip-us.apache.org/repos/asf/ambari/blob/9d26e377/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
index 1018207..a74b05e 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
@@ -219,6 +219,9 @@ public class TimelineMetricConfiguration {
   public static final String TIMELINE_METRICS_TABLES_DURABILITY =
     "timeline.metrics.tables.durability";
 
+  public static final String HBASE_BLOCKING_STORE_FILES =
+    "hbase.hstore.blockingStoreFiles";
+
   public static final String HOST_APP_ID = "HOST";
 
   private Configuration hbaseConf;

http://git-wip-us.apache.org/repos/asf/ambari/blob/9d26e377/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java
index 8858bd5..31a3ce5 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java
@@ -107,6 +107,7 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
   public static final String PHOENIX_QUERY_KEEPALIVE_PROPERTY = "phoenix.query.keepAliveMs";
   public static final String TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED
     = "timeline.metrics.cluster.aggregator.interpolation.enabled";
+  public static final String TIMELINE_METRICS_SINK_COLLECTION_PERIOD = "timeline.metrics.sink.collection.period";
 
   public static final String AMS_SERVICE_NAME = "AMBARI_METRICS";
   public static final String AMS_COLLECTOR_COMPONENT_NAME = "METRICS_COLLECTOR";
@@ -399,6 +400,13 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
               newProperties.put(TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED,
String.valueOf(true));
             }
 
+            if (!amsSiteProperties.containsKey(TIMELINE_METRICS_SINK_COLLECTION_PERIOD) ||
+              "60".equals(amsSiteProperties.get(TIMELINE_METRICS_SINK_COLLECTION_PERIOD)))
{
+
+              newProperties.put(TIMELINE_METRICS_SINK_COLLECTION_PERIOD, "10");
+              LOG.info("Setting value of " + TIMELINE_METRICS_SINK_COLLECTION_PERIOD + "
: 10");
+            }
+
             updateConfigurationPropertiesForCluster(cluster, AMS_SITE, newProperties, true,
true);
           }
 
@@ -456,7 +464,7 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
     Map<String, List<String>> widgetMap = new HashMap<>();
     Map<String, String> sectionLayoutMap = new HashMap<>();
 
-    List<String> yarnSummaryWidgets = new ArrayList<>(Arrays.asList("Container
Failures", "App Failures"));
+    List<String> yarnSummaryWidgets = new ArrayList<>(Arrays.asList("Container
Failures", "App Failures", "Cluster Memory"));
     widgetMap.put("YARN_SUMMARY", yarnSummaryWidgets);
     sectionLayoutMap.put("YARN_SUMMARY", "default_yarn_dashboard");
 
@@ -581,6 +589,10 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
               if (targetWidgetLayoutInfo != null) {
                 entityToUpdate.setMetrics(gson.toJson(targetWidgetLayoutInfo.getMetricsInfo()));
                 entityToUpdate.setWidgetValues(gson.toJson(targetWidgetLayoutInfo.getValues()));
+                if ("HBASE".equals(serviceName) && "Reads and Writes".equals(widgetName))
{
+                  entityToUpdate.setDescription(targetWidgetLayoutInfo.getDescription());
+                  LOG.info("Update description for HBase Reads and Writes widget");
+                }
                 widgetDAO.merge(entityToUpdate);
               } else {
                 LOG.warn("Unable to find widget layout info for " + widgetName +

http://git-wip-us.apache.org/repos/asf/ambari/blob/9d26e377/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/widgets.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/widgets.json
b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/widgets.json
index ede9f49..4120ff2 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/widgets.json
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/widgets.json
@@ -7,7 +7,7 @@
       "widgetLayoutInfo": [
         {
           "widget_name": "Reads and Writes",
-          "description": "Count of read and write requests on all regions in the cluster.",
+          "description": "Rate (per second) of read and write requests on all regions in
the cluster.",
           "default_section_name": "HBASE_SUMMARY",
           "widget_type": "GRAPH",
           "is_visible": true,

http://git-wip-us.apache.org/repos/asf/ambari/blob/9d26e377/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/YARN_widgets.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/YARN_widgets.json
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/YARN_widgets.json
index 442d9a2..4b76a17 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/YARN_widgets.json
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/YARN_widgets.json
@@ -230,18 +230,12 @@
               "metric_path": "metrics/memory/mem_free._avg",
               "service_name": "YARN",
               "component_name": "NODEMANAGER"
-            },
-            {
-              "name": "mem_cached._sum",
-              "metric_path": "metrics/memory/mem_cached._avg",
-              "service_name": "YARN",
-              "component_name": "NODEMANAGER"
             }
           ],
           "values": [
             {
               "name": "Memory utilization",
-              "value": "${((mem_total._sum - mem_free._sum - mem_cached._sum)/mem_total._sum)
* 100}"
+              "value": "${((mem_total._sum - mem_free._sum)/mem_total._sum) * 100}"
             }
           ],
           "properties": {

http://git-wip-us.apache.org/repos/asf/ambari/blob/9d26e377/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/widgets.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/widgets.json b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/widgets.json
index d8f4e6a..ae47833 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/widgets.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/widgets.json
@@ -7,7 +7,7 @@
       "widgetLayoutInfo": [
         {
           "widget_name": "Reads and Writes",
-          "description": "Count of read and write requests on all regions in the cluster.",
+          "description": "Rate (per second) of read and write requests on all regions in
the cluster.",
           "widget_type": "GRAPH",
           "is_visible": true,
           "metrics": [

http://git-wip-us.apache.org/repos/asf/ambari/blob/9d26e377/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/YARN_widgets.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/YARN_widgets.json
b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/YARN_widgets.json
index c2acf1f..782f21d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/YARN_widgets.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/YARN_widgets.json
@@ -259,18 +259,12 @@
               "metric_path": "metrics/memory/mem_free._avg",
               "service_name": "YARN",
               "component_name": "NODEMANAGER"
-            },
-            {
-              "name": "mem_cached._sum",
-              "metric_path": "metrics/memory/mem_cached._avg",
-              "service_name": "YARN",
-              "component_name": "NODEMANAGER"
             }
           ],
           "values": [
             {
               "name": "Memory utilization",
-              "value": "${((mem_total._sum - mem_free._sum - mem_cached._sum)/mem_total._sum)
* 100}"
+              "value": "${((mem_total._sum - mem_free._sum)/mem_total._sum) * 100}"
             }
           ],
           "properties": {

http://git-wip-us.apache.org/repos/asf/ambari/blob/9d26e377/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java
b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java
index 83b995b..4302349 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java
@@ -349,14 +349,10 @@ public class UpgradeCatalog222Test {
         put("timeline.metrics.cluster.aggregator.daily.checkpointCutOffMultiplier", String.valueOf(1));
         put("timeline.metrics.service.operation.mode", "distributed");
         put("timeline.metrics.host.aggregator.ttl", String.valueOf(86400));
-        put("timeline.metrics.host.aggregator.minute.ttl", String.valueOf(604800));
-        put("timeline.metrics.host.aggregator.hourly.ttl", String.valueOf(2592000));
-        put("timeline.metrics.host.aggregator.daily.ttl", String.valueOf(31536000));
         put("timeline.metrics.cluster.aggregator.second.ttl", String.valueOf(21600)); //Less
than 1 day
         put("timeline.metrics.cluster.aggregator.minute.ttl", String.valueOf(7776000));
-        put("timeline.metrics.cluster.aggregator.hourly.ttl", String.valueOf(31536000));
-        put("timeline.metrics.cluster.aggregator.daily.ttl", String.valueOf(63072000));
         put("timeline.metrics.service.webapp.address", "0.0.0.0:6188");
+        put("timeline.metrics.sink.collection.period", "60");
       }
     };
     Map<String, String> newPropertiesAmsSite = new HashMap<String, String>()
{
@@ -365,16 +361,12 @@ public class UpgradeCatalog222Test {
         put("timeline.metrics.cluster.aggregator.daily.checkpointCutOffMultiplier", String.valueOf(2));
         put("timeline.metrics.service.watcher.disabled", String.valueOf(false));
         put("timeline.metrics.host.aggregator.ttl", String.valueOf(3 * 86400));
-        put("timeline.metrics.host.aggregator.minute.ttl", String.valueOf(7 * 86400));
-        put("timeline.metrics.host.aggregator.hourly.ttl", String.valueOf(30 * 86400));
-        put("timeline.metrics.host.aggregator.daily.ttl", String.valueOf(365 * 86400));
         put("timeline.metrics.cluster.aggregator.second.ttl", String.valueOf(21600));
         put("timeline.metrics.cluster.aggregator.minute.ttl", String.valueOf(30 * 86400));
-        put("timeline.metrics.cluster.aggregator.hourly.ttl", String.valueOf(365 * 86400));
-        put("timeline.metrics.cluster.aggregator.daily.ttl", String.valueOf(730 * 86400));
         put("timeline.metrics.service.operation.mode", "distributed");
         put("timeline.metrics.service.webapp.address", "host1:6188");
         put("timeline.metrics.cluster.aggregator.interpolation.enabled", String.valueOf(true));
+        put("timeline.metrics.sink.collection.period", "10");
       }
     };
     EasyMockSupport easyMockSupport = new EasyMockSupport();


Mime
View raw message