ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From avija...@apache.org
Subject [04/28] ambari git commit: AMBARI-15902. Refactor Metadata manager for supporting distributed collector. (swagle)
Date Tue, 15 Nov 2016 19:13:57 GMT
AMBARI-15902. Refactor Metadata manager for supporting distributed collector. (swagle)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/798a70e7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/798a70e7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/798a70e7

Branch: refs/heads/branch-2.5
Commit: 798a70e76eebb71d5b62accbecbb8e5cdeda1d52
Parents: 14a4f97
Author: Aravindan Vijayan <avijayan@hortonworks.com>
Authored: Mon Nov 14 20:25:59 2016 -0800
Committer: Aravindan Vijayan <avijayan@hortonworks.com>
Committed: Tue Nov 15 11:02:12 2016 -0800

----------------------------------------------------------------------
 .../timeline/TimelineMetricConfiguration.java   |   3 +
 .../TimelineMetricMetadataManager.java          |  61 ++++++++--
 .../discovery/TimelineMetricMetadataSync.java   |  72 +++++++++++-
 .../timeline/discovery/TestMetadataManager.java |   4 +-
 .../timeline/discovery/TestMetadataSync.java    | 116 +++++++++++++++++++
 .../server/upgrade/UpgradeCatalog240.java       |  26 +++--
 .../0.1.0/configuration/ams-site.xml            |   8 ++
 7 files changed, 266 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/798a70e7/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
index c4a3fee..9a4ed58 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
@@ -252,6 +252,9 @@ public class TimelineMetricConfiguration {
   public static final String TIMELINE_METRIC_AGGREGATION_SQL_FILTERS =
     "timeline.metrics.cluster.aggregation.sql.filters";
 
+  public static final String TIMELINE_METRIC_METADATA_FILTERS =
+    "timeline.metrics.service.metadata.filters";
+
   public static final String HOST_APP_ID = "HOST";
 
   public static final String DEFAULT_INSTANCE_PORT = "12001";

http://git-wip-us.apache.org/repos/asf/ambari/blob/798a70e7/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java
index 3053252..3e4fc5f 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery;
 
 import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -27,8 +28,11 @@ import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
 
 import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
@@ -36,9 +40,11 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.DISABLE_METRIC_METADATA_MGMT;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.METRICS_METADATA_SYNC_INIT_DELAY;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.METRICS_METADATA_SYNC_SCHEDULE_DELAY;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_METADATA_FILTERS;
 
 public class TimelineMetricMetadataManager {
   private static final Log LOG = LogFactory.getLog(TimelineMetricMetadataManager.class);
@@ -56,10 +62,19 @@ public class TimelineMetricMetadataManager {
   private PhoenixHBaseAccessor hBaseAccessor;
   private Configuration metricsConf;
 
+  TimelineMetricMetadataSync metricMetadataSync;
+  // Filter metrics names matching given patterns, from metadata
+  final List<String> metricNameFilters = new ArrayList<>();
+
   public TimelineMetricMetadataManager(PhoenixHBaseAccessor hBaseAccessor,
                                        Configuration metricsConf) {
     this.hBaseAccessor = hBaseAccessor;
     this.metricsConf = metricsConf;
+
+    String patternStrings = metricsConf.get(TIMELINE_METRIC_METADATA_FILTERS);
+    if (!StringUtils.isEmpty(patternStrings)) {
+      metricNameFilters.addAll(Arrays.asList(patternStrings.split(",")));
+    }
   }
 
   /**
@@ -69,21 +84,21 @@ public class TimelineMetricMetadataManager {
     if (metricsConf.getBoolean(DISABLE_METRIC_METADATA_MGMT, false)) {
       isDisabled = true;
     } else {
+      metricMetadataSync = new TimelineMetricMetadataSync(this);
       // Schedule the executor to sync to store
-      executorService.scheduleWithFixedDelay(new TimelineMetricMetadataSync(this),
+      executorService.scheduleWithFixedDelay(metricMetadataSync,
         metricsConf.getInt(METRICS_METADATA_SYNC_INIT_DELAY, 120), // 2 minutes
         metricsConf.getInt(METRICS_METADATA_SYNC_SCHEDULE_DELAY, 300), // 5 minutes
         TimeUnit.SECONDS);
       // Read from store and initialize map
       try {
-        Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadata =
-          hBaseAccessor.getTimelineMetricMetadata();
+        Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadata = getMetadataFromStore();
 
         LOG.info("Retrieved " + metadata.size() + ", metadata objects from store.");
         // Store in the cache
         METADATA_CACHE.putAll(metadata);
 
-        Map<String, Set<String>> hostedAppData = hBaseAccessor.getHostedAppsMetadata();
+        Map<String, Set<String>> hostedAppData = getHostedAppsFromStore();
 
         LOG.info("Retrieved " + hostedAppData.size() + " host objects from store.");
         HOSTED_APPS_MAP.putAll(hostedAppData);
@@ -115,10 +130,26 @@ public class TimelineMetricMetadataManager {
   }
 
   /**
+   * Test metric name for valid patterns and return true/false
+   */
+  boolean skipMetadataCache(String metricName) {
+    for (String pattern : metricNameFilters) {
+      if (metricName.contains(pattern)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
    * Update value in metadata cache
    * @param metadata @TimelineMetricMetadata
    */
   public void putIfModifiedTimelineMetricMetadata(TimelineMetricMetadata metadata) {
+    if (skipMetadataCache(metadata.getMetricName())) {
+      return;
+    }
+
     TimelineMetricMetadataKey key = new TimelineMetricMetadataKey(
       metadata.getMetricName(), metadata.getAppId());
 
@@ -176,16 +207,28 @@ public class TimelineMetricMetadataManager {
     );
   }
 
+  public boolean isDisabled() {
+    return isDisabled;
+  }
+
+  boolean isDistributedModeEnabled() {
+    return metricsConf.get("timeline.metrics.service.operation.mode", "").equals("distributed");
+  }
+
   /**
-   * Fetch hosted apps from store
+   * Fetch metrics metadata from store
    * @throws SQLException
    */
-  Map<String, Set<String>> getPersistedHostedAppsData() throws SQLException {
-    return hBaseAccessor.getHostedAppsMetadata();
+  Map<TimelineMetricMetadataKey, TimelineMetricMetadata> getMetadataFromStore() throws
SQLException {
+    return hBaseAccessor.getTimelineMetricMetadata();
   }
 
-  public boolean isDisabled() {
-    return isDisabled;
+  /**
+   * Fetch hosted apps from store
+   * @throws SQLException
+   */
+  Map<String, Set<String>> getHostedAppsFromStore() throws SQLException {
+    return hBaseAccessor.getHostedAppsMetadata();
   }
 
   private boolean supportAggregates(TimelineMetric metric) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/798a70e7/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataSync.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataSync.java
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataSync.java
index 54ea200..25b525a 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataSync.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataSync.java
@@ -41,6 +41,22 @@ public class TimelineMetricMetadataSync implements Runnable {
 
   @Override
   public void run() {
+    LOG.debug("Persisting metric metadata...");
+    persistMetricMetadata();
+    LOG.debug("Persisting hosted apps metadata...");
+    persistHostAppsMetadata();
+    if (cacheManager.isDistributedModeEnabled()) {
+      LOG.debug("Refreshing metric metadata...");
+      refreshMetricMetadata();
+      LOG.debug("Refreshing hosted apps metadata...");
+      refreshHostAppsMetadata();
+    }
+  }
+
+  /**
+   * Find metrics not persisted to store and persist them
+   */
+  private void persistMetricMetadata() {
     List<TimelineMetricMetadata> metadataToPersist = new ArrayList<>();
     // Find all entries to persist
     for (TimelineMetricMetadata metadata : cacheManager.getMetadataCache().values()) {
@@ -70,11 +86,38 @@ public class TimelineMetricMetadataSync implements Runnable {
         cacheManager.getMetadataCache().put(key, metadata);
       }
     }
-    // Sync hosted apps data is needed
+  }
+
+  /**
+   * Read all metric metadata and update cached values - HA mode
+   */
+  private void refreshMetricMetadata() {
+    Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadataFromStore = null;
+    try {
+      metadataFromStore = cacheManager.getMetadataFromStore();
+    } catch (SQLException e) {
+      LOG.warn("Error refreshing metadata from store.", e);
+    }
+    if (metadataFromStore != null) {
+      Map<TimelineMetricMetadataKey, TimelineMetricMetadata> cachedMetadata =
+        cacheManager.getMetadataCache();
+
+      for (Map.Entry<TimelineMetricMetadataKey, TimelineMetricMetadata> metadataEntry
: metadataFromStore.entrySet()) {
+        if (!cachedMetadata.containsKey(metadataEntry.getKey())) {
+          cachedMetadata.put(metadataEntry.getKey(), metadataEntry.getValue());
+        }
+      }
+    }
+  }
+
+  /**
+   * Sync hosted apps data if needed
+   */
+  private void persistHostAppsMetadata() {
     if (cacheManager.syncHostedAppsMetadata()) {
       Map<String, Set<String>> persistedData = null;
       try {
-        persistedData = cacheManager.getPersistedHostedAppsData();
+        persistedData = cacheManager.getHostedAppsFromStore();
       } catch (SQLException e) {
         LOG.warn("Failed on fetching hosted apps data from store.", e);
         return; // Something wrong with store
@@ -86,8 +129,8 @@ public class TimelineMetricMetadataSync implements Runnable {
         for (Map.Entry<String, Set<String>> cacheEntry : cachedData.entrySet())
{
           // No persistence / stale data in store
           if (persistedData == null || persistedData.isEmpty() ||
-              !persistedData.containsKey(cacheEntry.getKey()) ||
-              !persistedData.get(cacheEntry.getKey()).containsAll(cacheEntry.getValue()))
{
+            !persistedData.containsKey(cacheEntry.getKey()) ||
+            !persistedData.get(cacheEntry.getKey()).containsAll(cacheEntry.getValue())) {
             dataToSync.put(cacheEntry.getKey(), cacheEntry.getValue());
           }
         }
@@ -102,4 +145,25 @@ public class TimelineMetricMetadataSync implements Runnable {
 
     }
   }
+
+  /**
+   * Read all hosted apps metadata and update cached values - HA
+   */
+  private void refreshHostAppsMetadata() {
+    Map<String, Set<String>> hostedAppsDataFromStore = null;
+    try {
+      hostedAppsDataFromStore = cacheManager.getHostedAppsFromStore();
+    } catch (SQLException e) {
+      LOG.warn("Error refreshing metadata from store.", e);
+    }
+    if (hostedAppsDataFromStore != null) {
+      Map<String, Set<String>> cachedData = cacheManager.getHostedAppsCache();
+
+      for (Map.Entry<String, Set<String>> storeEntry : hostedAppsDataFromStore.entrySet())
{
+        if (!cachedData.containsKey(storeEntry.getKey())) {
+          cachedData.put(storeEntry.getKey(), storeEntry.getValue());
+        }
+      }
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/798a70e7/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java
b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java
index e99c001..2809640 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java
@@ -122,13 +122,11 @@ public class TestMetadataManager extends AbstractMiniHBaseClusterTest
{
     Assert.assertEquals(value2, savedData.get(key2));
 
     Map<String, Set<String>> cachedHostData = metadataManager.getHostedAppsCache();
-    Map<String, Set<String>> savedHostData = metadataManager.getPersistedHostedAppsData();
+    Map<String, Set<String>> savedHostData = metadataManager.getHostedAppsFromStore();
     Assert.assertEquals(cachedData.size(), savedData.size());
     Assert.assertEquals("dummy_app1", cachedHostData.get("dummy_host1").iterator().next());
     Assert.assertEquals("dummy_app2", cachedHostData.get("dummy_host2").iterator().next());
     Assert.assertEquals("dummy_app1", savedHostData.get("dummy_host1").iterator().next());
     Assert.assertEquals("dummy_app2", savedHostData.get("dummy_host2").iterator().next());
   }
-
-
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/798a70e7/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataSync.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataSync.java
b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataSync.java
new file mode 100644
index 0000000..78f2bfe
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataSync.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery;
+
+import junit.framework.Assert;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor;
+import org.junit.Test;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import static org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata.MetricType.GAUGE;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_METADATA_FILTERS;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+
+public class TestMetadataSync {
+  @Test
+  public void testRefreshMetadataOnWrite() throws Exception {
+    Configuration configuration = createNiceMock(Configuration.class);
+    PhoenixHBaseAccessor hBaseAccessor = createNiceMock(PhoenixHBaseAccessor.class);
+
+    final TimelineMetricMetadata testMetadata1 = new TimelineMetricMetadata(
+      "m1", "a1", "", GAUGE.name(), System.currentTimeMillis(), true);
+    final TimelineMetricMetadata testMetadata2 = new TimelineMetricMetadata(
+      "m2", "a2", "", GAUGE.name(), System.currentTimeMillis(), true);
+
+    Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadata =
+      new HashMap<TimelineMetricMetadataKey, TimelineMetricMetadata>() {{
+        put(new TimelineMetricMetadataKey("m1", "a1"), testMetadata1);
+        put(new TimelineMetricMetadataKey("m2", "a2"), testMetadata2);
+      }};
+
+    Map<String, Set<String>> hostedApps = new HashMap<String, Set<String>>()
{{
+      put("h1", new HashSet<>(Arrays.asList("a1")));
+      put("h2", new HashSet<>(Arrays.asList("a1", "a2")));
+    }};
+
+    expect(configuration.get("timeline.metrics.service.operation.mode", "")).andReturn("distributed");
+    expect(hBaseAccessor.getTimelineMetricMetadata()).andReturn(metadata);
+    expect(hBaseAccessor.getHostedAppsMetadata()).andReturn(hostedApps);
+
+    replay(configuration, hBaseAccessor);
+
+    TimelineMetricMetadataManager metadataManager = new
+      TimelineMetricMetadataManager(hBaseAccessor, configuration);
+
+    metadataManager.metricMetadataSync = new TimelineMetricMetadataSync(metadataManager);
+
+    metadataManager.metricMetadataSync.run();
+
+    verify(configuration, hBaseAccessor);
+
+    metadata = metadataManager.getMetadataCache();
+    Assert.assertEquals(2, metadata.size());
+    Assert.assertTrue(metadata.containsKey(new TimelineMetricMetadataKey("m1", "a1")));
+    Assert.assertTrue(metadata.containsKey(new TimelineMetricMetadataKey("m2", "a2")));
+
+    hostedApps = metadataManager.getHostedAppsCache();
+    Assert.assertEquals(2, hostedApps.size());
+    Assert.assertEquals(1, hostedApps.get("h1").size());
+    Assert.assertEquals(2, hostedApps.get("h2").size());
+  }
+
+  @Test
+  public void testFilterByRegexOnMetricName() throws Exception {
+    Configuration configuration = createNiceMock(Configuration.class);
+    PhoenixHBaseAccessor hBaseAccessor = createNiceMock(PhoenixHBaseAccessor.class);
+
+    TimelineMetricMetadata metadata1 = new TimelineMetricMetadata(
+      "xxx.abc.yyy", "a1", "", GAUGE.name(), System.currentTimeMillis(), true);
+    TimelineMetricMetadata metadata2 = new TimelineMetricMetadata(
+      "xxx.cdef.yyy", "a2", "", GAUGE.name(), System.currentTimeMillis(), true);
+    TimelineMetricMetadata metadata3 = new TimelineMetricMetadata(
+      "xxx.pqr.zzz", "a3", "", GAUGE.name(), System.currentTimeMillis(), true);
+
+    expect(configuration.get(TIMELINE_METRIC_METADATA_FILTERS)).andReturn("abc,cde");
+
+    replay(configuration, hBaseAccessor);
+
+    TimelineMetricMetadataManager metadataManager = new
+      TimelineMetricMetadataManager(hBaseAccessor, configuration);
+
+    metadataManager.putIfModifiedTimelineMetricMetadata(metadata1);
+    metadataManager.putIfModifiedTimelineMetricMetadata(metadata2);
+    metadataManager.putIfModifiedTimelineMetricMetadata(metadata3);
+
+    verify(configuration, hBaseAccessor);
+
+    Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadata =
+      metadataManager.getMetadataCache();
+
+    Assert.assertEquals(1, metadata.size());
+    Assert.assertEquals("xxx.pqr.zzz", metadata.keySet().iterator().next().getMetricName());
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/798a70e7/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
index cfd25e7..25b6360 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
@@ -90,7 +90,7 @@ import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.jdbc.support.JdbcUtils;
-
+import org.apache.commons.lang.StringUtils;
 import javax.persistence.EntityManager;
 import javax.persistence.Query;
 import javax.persistence.TypedQuery;
@@ -1441,10 +1441,10 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
    */
   protected void updateAlertCurrentTable() throws SQLException {
     dbAccessor.addColumn(ALERT_CURRENT_TABLE,
-            new DBColumnInfo(ALERT_CURRENT_OCCURRENCES_COLUMN, Long.class, null, 1, false));
+      new DBColumnInfo(ALERT_CURRENT_OCCURRENCES_COLUMN, Long.class, null, 1, false));
 
     dbAccessor.addColumn(ALERT_CURRENT_TABLE, new DBColumnInfo(ALERT_CURRENT_FIRMNESS_COLUMN,
-            String.class, 255, AlertFirmness.HARD.name(), false));
+      String.class, 255, AlertFirmness.HARD.name(), false));
   }
 
   /**
@@ -1469,15 +1469,15 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
     dbAccessor.executeUpdate(String.format(updateStatement,
         2, PermissionEntity.CLUSTER_ADMINISTRATOR_PERMISSION_NAME));
     dbAccessor.executeUpdate(String.format(updateStatement,
-            3, PermissionEntity.CLUSTER_OPERATOR_PERMISSION_NAME));
+      3, PermissionEntity.CLUSTER_OPERATOR_PERMISSION_NAME));
     dbAccessor.executeUpdate(String.format(updateStatement,
-            4, PermissionEntity.SERVICE_ADMINISTRATOR_PERMISSION_NAME));
+      4, PermissionEntity.SERVICE_ADMINISTRATOR_PERMISSION_NAME));
     dbAccessor.executeUpdate(String.format(updateStatement,
-            5, PermissionEntity.SERVICE_OPERATOR_PERMISSION_NAME));
+      5, PermissionEntity.SERVICE_OPERATOR_PERMISSION_NAME));
     dbAccessor.executeUpdate(String.format(updateStatement,
-            6, PermissionEntity.CLUSTER_USER_PERMISSION_NAME));
+      6, PermissionEntity.CLUSTER_USER_PERMISSION_NAME));
     dbAccessor.executeUpdate(String.format(updateStatement,
-            7, PermissionEntity.VIEW_USER_PERMISSION_NAME));
+      7, PermissionEntity.VIEW_USER_PERMISSION_NAME));
   }
 
   /**
@@ -1911,6 +1911,16 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
               }
             }
           }
+
+          Config amsSite = cluster.getDesiredConfigByType("ams-site");
+          if (amsSite != null) {
+            String metadataFilters = amsSite.getProperties().get("timeline.metrics.service.metadata.filters");
+            if (StringUtils.isEmpty(metadataFilters) ||
+                !metadataFilters.contains("ContainerResource")) {
+              updateConfigurationProperties("ams-site",
+                Collections.singletonMap("timeline.metrics.service.metadata.filters", "ContainerResource"),
true, false);
+            }
+          }
         }
       }
     }

http://git-wip-us.apache.org/repos/asf/ambari/blob/798a70e7/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
index a5860e0..e217011 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
@@ -708,4 +708,12 @@
     </description>
     <on-ambari-upgrade add="true"/>
   </property>
+  <property>
+    <name>timeline.metrics.service.metadata.filters</name>
+    <value>ContainerResource</value>
+    <description>
+      Commas separated list of regular expressions that match metric names
+      which prevents certain metrics from ending up in metadata cache.
+    </description>
+  </property>
 </configuration>


Mime
View raw message