Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 5E6A7196BF for ; Thu, 21 Apr 2016 13:28:08 +0000 (UTC) Received: (qmail 14569 invoked by uid 500); 21 Apr 2016 13:28:08 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 14457 invoked by uid 500); 21 Apr 2016 13:28:08 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 14397 invoked by uid 99); 21 Apr 2016 13:28:08 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 21 Apr 2016 13:28:08 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 11F1BDFF93; Thu, 21 Apr 2016 13:28:08 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: dmitriusan@apache.org To: commits@ambari.apache.org Date: Thu, 21 Apr 2016 13:28:08 -0000 Message-Id: <2faef43910784bd5805df3fea2859347@git.apache.org> In-Reply-To: <37dcafed34aa47e1b2a19e9131fec3cc@git.apache.org> References: <37dcafed34aa47e1b2a19e9131fec3cc@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/3] ambari git commit: AMBARI-15994. Restart button took about 5 minutes to appear after a config change for zookeeper (dlysnichenko) AMBARI-15994. Restart button took about 5 minutes to appear after a config change for zookeeper (dlysnichenko) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2bc94a04 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2bc94a04 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2bc94a04 Branch: refs/heads/branch-2.2 Commit: 2bc94a04b1603aa6b2be7a9ffe7e661c01d996cf Parents: 4e8670e Author: Lisnichenko Dmitro Authored: Thu Apr 21 16:26:48 2016 +0300 Committer: Lisnichenko Dmitro Committed: Thu Apr 21 16:26:48 2016 +0300 ---------------------------------------------------------------------- .../server/configuration/Configuration.java | 10 ++ .../ambari/server/state/ConfigHelper.java | 30 +++-- .../org/apache/ambari/server/state/Host.java | 9 ++ .../ambari/server/state/host/HostImpl.java | 21 +++- .../ambari/server/state/ConfigHelperTest.java | 125 +++++++++++++++++++ 5 files changed, 184 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2bc94a04/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java index 0446469..6bcf539 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java @@ -185,6 +185,7 @@ public class Configuration { public static final String SERVER_HRC_STATUS_SUMMARY_CACHE_SIZE = "server.hrcStatusSummary.cache.size"; public static final String SERVER_HRC_STATUS_SUMMARY_CACHE_EXPIRY_DURATION = "server.hrcStatusSummary.cache.expiryDuration"; public static final String SERVER_STALE_CONFIG_CACHE_ENABLED_KEY = "server.cache.isStale.enabled"; + public static final String SERVER_STALE_CONFIG_CACHE_EXPIRATION_KEY = "server.cache.isStale.expiration"; public static final String SERVER_PERSISTENCE_TYPE_KEY = "server.persistence.type"; public static final String SERVER_JDBC_USER_NAME_KEY = "server.jdbc.user.name"; public static final String SERVER_JDBC_USER_PASSWD_KEY = "server.jdbc.user.passwd"; @@ -382,6 +383,7 @@ public class Configuration { private static final long SERVER_EC_CACHE_SIZE_DEFAULT = 10000L; private static final String SERVER_STALE_CONFIG_CACHE_ENABLED_DEFAULT = "true"; + private static final String SERVER_STALE_CONFIG_CACHE_EXPIRATION_DEFAULT = "60"; private static final String SERVER_JDBC_USER_NAME_DEFAULT = "ambari"; private static final String SERVER_JDBC_USER_PASSWD_DEFAULT = "bigdata"; private static final String SERVER_JDBC_RCA_USER_NAME_DEFAULT = "mapred"; @@ -1875,6 +1877,14 @@ public class Configuration { } /** + * @return expiration time of stale config cache + */ + public Integer staleConfigCacheExpiration() { + return Integer.parseInt(properties.getProperty(SERVER_STALE_CONFIG_CACHE_EXPIRATION_KEY, + SERVER_STALE_CONFIG_CACHE_EXPIRATION_DEFAULT)); + } + + /** * @return a string array of suffixes used to validate repo URLs. */ public String[] getRepoValidationSuffixes(String osType) { http://git-wip-us.apache.org/repos/asf/ambari/blob/2bc94a04/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java index b21cbbd..7ddaee4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java @@ -62,7 +62,7 @@ public class ConfigHelper { private static final String DELETED = "DELETED_"; public static final String CLUSTER_DEFAULT_TAG = "tag"; private final boolean STALE_CONFIGS_CACHE_ENABLED; - private final int STALE_CONFIGS_CACHE_EXPIRATION_TIME = 300; + private final int STALE_CONFIGS_CACHE_EXPIRATION_TIME; private final Cache staleConfigsCache; private static final Logger LOG = @@ -89,6 +89,7 @@ public class ConfigHelper { ambariMetaInfo = metaInfo; this.clusterDAO = clusterDAO; STALE_CONFIGS_CACHE_ENABLED = configuration.isStaleConfigCacheEnabled(); + STALE_CONFIGS_CACHE_EXPIRATION_TIME = configuration.staleConfigCacheExpiration(); staleConfigsCache = CacheBuilder.newBuilder(). expireAfterWrite(STALE_CONFIGS_CACHE_EXPIRATION_TIME, TimeUnit.SECONDS).build(); } @@ -104,9 +105,24 @@ public class ConfigHelper { public Map> getEffectiveDesiredTags( Cluster cluster, String hostName) throws AmbariException { + return getEffectiveDesiredTags(cluster, hostName, false); + } + + /** + * Gets the desired tags for a cluster and host + * + * @param cluster the cluster + * @param hostName the host name + * @param bypassCache don't use cached values + * @return a map of tag type to tag names with overrides + * @throws AmbariException + */ + public Map> getEffectiveDesiredTags( + Cluster cluster, String hostName, boolean bypassCache) throws AmbariException { + Host host = (hostName == null) ? null : clusters.getHost(hostName); - Map desiredHostConfigs = (host == null) ? null : host.getDesiredHostConfigs(cluster); - return getEffectiveDesiredTags(cluster, desiredHostConfigs); + Map desiredHostConfigs = (host == null) ? null : host.getDesiredHostConfigs(cluster, bypassCache); + return getEffectiveDesiredTags(cluster, desiredHostConfigs, bypassCache); } /** @@ -114,12 +130,13 @@ public class ConfigHelper { * * @param cluster the cluster * @param hostConfigOverrides the host overrides applied using config groups + * @param bypassCache don't use cached values * @return a map of tag type to tag names with overrides */ private Map> getEffectiveDesiredTags( - Cluster cluster, Map hostConfigOverrides) { + Cluster cluster, Map hostConfigOverrides, boolean bypassCache) { - Map clusterDesired = (cluster == null) ? new HashMap() : cluster.getDesiredConfigs(true); + Map clusterDesired = (cluster == null) ? new HashMap() : cluster.getDesiredConfigs(bypassCache); Map> resolved = new TreeMap>(); @@ -981,8 +998,7 @@ public class ConfigHelper { Cluster cluster = clusters.getClusterById(sch.getClusterId()); StackId stackId = cluster.getDesiredStackVersion(); - Map> desired = getEffectiveDesiredTags(cluster, - sch.getHostName()); + Map> desired = getEffectiveDesiredTags(cluster, sch.getHostName(), true); ServiceInfo serviceInfo = ambariMetaInfo.getService(stackId.getStackName(), stackId.getStackVersion(), sch.getServiceName()); http://git-wip-us.apache.org/repos/asf/ambari/blob/2bc94a04/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java b/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java index 34fb995..ca854db 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java @@ -373,6 +373,15 @@ public interface Host { Map getDesiredHostConfigs(Cluster cluster) throws AmbariException; /** + * Get the desired configurations for the host including overrides + * @param cluster + * @param bypassCache + * @return + * @throws AmbariException + */ + Map getDesiredHostConfigs(Cluster cluster, boolean bypassCache) throws AmbariException; + + /** * Sets the maintenance state for the host. * @param clusterId the cluster id * @param state the state http://git-wip-us.apache.org/repos/asf/ambari/blob/2bc94a04/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java index 4b747fa..d221112 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java @@ -1327,12 +1327,25 @@ public class HostImpl implements Host { */ @Override public Map getDesiredHostConfigs(Cluster cluster) throws AmbariException { + return getDesiredHostConfigs(cluster, false); + } + + /** + * Get a map of configType with all applicable config tags. + * + * @param cluster the cluster + * @param bypassCache don't use cached values + * + * @return Map of configType -> HostConfig + */ + @Override + public Map getDesiredHostConfigs(Cluster cluster, boolean bypassCache) throws AmbariException { Map hostConfigMap = new HashMap(); - Map clusterDesiredConfigs = (cluster == null) ? new HashMap() : cluster.getDesiredConfigs(); + Map clusterDesiredConfigs = (cluster == null) ? new HashMap() : cluster.getDesiredConfigs(bypassCache); if (clusterDesiredConfigs != null) { for (Map.Entry desiredConfigEntry - : clusterDesiredConfigs.entrySet()) { + : clusterDesiredConfigs.entrySet()) { HostConfig hostConfig = new HostConfig(); hostConfig.setDefaultVersionTag(desiredConfigEntry.getValue().getTag()); hostConfigMap.put(desiredConfigEntry.getKey(), hostConfig); @@ -1344,7 +1357,7 @@ public class HostImpl implements Host { if (configGroups != null && !configGroups.isEmpty()) { for (ConfigGroup configGroup : configGroups.values()) { for (Map.Entry configEntry : configGroup - .getConfigurations().entrySet()) { + .getConfigurations().entrySet()) { String configType = configEntry.getKey(); // HostConfig config holds configType -> versionTag, per config group @@ -1364,7 +1377,7 @@ public class HostImpl implements Host { } Config config = configEntry.getValue(); hostConfig.getConfigGroupOverrides().put(configGroup.getId(), - config.getTag()); + config.getTag()); } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/2bc94a04/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java index 86b119e..f79c823 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java @@ -31,7 +31,11 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import javax.persistence.EntityManager; @@ -224,6 +228,22 @@ public class ConfigHelperTest { return configGroup.getId(); } + void applyConfig(Map properties, String configType, String configTag) throws Exception { + ConfigurationRequest cr = new ConfigurationRequest(); + cr.setClusterName(clusterName); + cr.setType(configType); + cr.setVersionTag(configTag); + cr.setProperties(properties); + + final ClusterRequest clusterRequest = + new ClusterRequest(cluster.getClusterId(), clusterName, + cluster.getDesiredStackVersion().getStackVersion(), null); + + clusterRequest.setDesiredConfig(Collections.singletonList(cr)); + managementController.updateClusters(new HashSet() {{ + add(clusterRequest); + }}, null); + } @Test public void testEffectiveTagsForHost() throws Exception { final Config config = new ConfigImpl("core-site"); @@ -745,6 +765,111 @@ public class ConfigHelperTest { verify(sch); } + @Test + public void testCalculateIsStaleConfigsParallel() throws Exception{ + Map schReturn = new HashMap(); + HostConfig hc = new HostConfig(); + // Put a different version to check for change + hc.setDefaultVersionTag("version2"); + schReturn.put("flume-conf", hc); + + // set up mocks + final ServiceComponentHost sch = createNiceMock(ServiceComponentHost.class); + // set up expectations + expect(sch.getActualConfigs()).andReturn(schReturn).anyTimes(); + expect(sch.getHostName()).andReturn("h1").anyTimes(); + expect(sch.getClusterId()).andReturn(1l).anyTimes(); + expect(sch.getServiceName()).andReturn("FLUME").anyTimes(); + expect(sch.getServiceComponentName()).andReturn("FLUME_HANDLER").anyTimes(); + replay(sch); + // Cluster level config changes + + final Config config1 = cluster.getDesiredConfigByType("flume-conf"); + + applyConfig(new HashMap(){{ + put("property", "1"); + }}, "flume-conf", "version2"); + + final Config config2 = cluster.getDesiredConfigByType("flume-conf"); + + applyConfig(new HashMap(){{ + put("property", "2"); + }}, "flume-conf", "version3"); + + final Config config3 = cluster.getDesiredConfigByType("flume-conf"); + + cluster.addDesiredConfig("admin", new HashSet(){{add(config1);}}); + + final AtomicBoolean mustBeStale = new AtomicBoolean(); + mustBeStale.set(false); + + final AtomicBoolean failed = new AtomicBoolean(); + failed.set(false); + + final AtomicBoolean finished = new AtomicBoolean(); + finished.set(false); + // avoid situations when not checked previous mustBeStale value yes and applied new config version + final Lock checkLock = new ReentrantLock(); + + // parallel thread that will compare actual stale config with expected accordingly to desired configs, checks if + // isStaleConfigs bypassing every cache and returns correct information. + Thread parallel = new Thread(new Runnable() { + @Override + public void run() { + while(!finished.get()){ + checkLock.lock(); + try { + boolean isStale = configHelper.isStaleConfigs(sch); + if(mustBeStale.get() != isStale){ + failed.set(true); + break; + } + } catch (AmbariException e) { + e.printStackTrace(); + } finally { + checkLock.unlock(); + } + } + } + }); + + parallel.start(); + + Random r = new Random(); + for(int i=0; i< 1000; i++){ + try { + checkLock.lock(); + switch(r.nextInt(3)) { + case 0: { + cluster.addDesiredConfig("admin", new HashSet(){{add(config1);}}); + mustBeStale.set(true); + checkLock.unlock(); + break; + } + case 1: { + cluster.addDesiredConfig("admin", new HashSet(){{add(config2);}}); + mustBeStale.set(false); + checkLock.unlock(); + break; + } + case 2: { + cluster.addDesiredConfig("admin", new HashSet(){{add(config3);}}); + mustBeStale.set(true); + checkLock.unlock(); + break; + } + } + } catch (Exception e){ + checkLock.unlock(); + } + if(!parallel.isAlive()) { + break; + } + } + finished.set(true); + parallel.join(); + Assert.assertFalse(failed.get()); + } } public static class RunWithCustomModule {