ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dmitriu...@apache.org
Subject [3/3] ambari git commit: AMBARI-15994. Restart button took about 5 minutes to appear after a config change for zookeeper (dlysnichenko)
Date Thu, 21 Apr 2016 13:28:09 GMT
AMBARI-15994. Restart button took about 5 minutes to appear after a config change for zookeeper
(dlysnichenko)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/894894ad
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/894894ad
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/894894ad

Branch: refs/heads/branch-2.2.2
Commit: 894894ade9abb1e5337808ca168c97da6bc4d9ac
Parents: f33c18b
Author: Lisnichenko Dmitro <dlysnichenko@hortonworks.com>
Authored: Thu Apr 21 16:26:48 2016 +0300
Committer: Lisnichenko Dmitro <dlysnichenko@hortonworks.com>
Committed: Thu Apr 21 16:27:05 2016 +0300

----------------------------------------------------------------------
 .../server/configuration/Configuration.java     |  10 ++
 .../ambari/server/state/ConfigHelper.java       |  30 +++--
 .../org/apache/ambari/server/state/Host.java    |   9 ++
 .../ambari/server/state/host/HostImpl.java      |  21 +++-
 .../ambari/server/state/ConfigHelperTest.java   | 125 +++++++++++++++++++
 5 files changed, 184 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/894894ad/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index 0446469..6bcf539 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -185,6 +185,7 @@ public class Configuration {
   public static final String SERVER_HRC_STATUS_SUMMARY_CACHE_SIZE = "server.hrcStatusSummary.cache.size";
   public static final String SERVER_HRC_STATUS_SUMMARY_CACHE_EXPIRY_DURATION = "server.hrcStatusSummary.cache.expiryDuration";
   public static final String SERVER_STALE_CONFIG_CACHE_ENABLED_KEY = "server.cache.isStale.enabled";
+  public static final String SERVER_STALE_CONFIG_CACHE_EXPIRATION_KEY = "server.cache.isStale.expiration";
   public static final String SERVER_PERSISTENCE_TYPE_KEY = "server.persistence.type";
   public static final String SERVER_JDBC_USER_NAME_KEY = "server.jdbc.user.name";
   public static final String SERVER_JDBC_USER_PASSWD_KEY = "server.jdbc.user.passwd";
@@ -382,6 +383,7 @@ public class Configuration {
 
   private static final long SERVER_EC_CACHE_SIZE_DEFAULT = 10000L;
   private static final String SERVER_STALE_CONFIG_CACHE_ENABLED_DEFAULT = "true";
+  private static final String SERVER_STALE_CONFIG_CACHE_EXPIRATION_DEFAULT = "60";
   private static final String SERVER_JDBC_USER_NAME_DEFAULT = "ambari";
   private static final String SERVER_JDBC_USER_PASSWD_DEFAULT = "bigdata";
   private static final String SERVER_JDBC_RCA_USER_NAME_DEFAULT = "mapred";
@@ -1875,6 +1877,14 @@ public class Configuration {
   }
 
   /**
+   * @return expiration time of stale config cache
+   */
+  public Integer staleConfigCacheExpiration() {
+    return Integer.parseInt(properties.getProperty(SERVER_STALE_CONFIG_CACHE_EXPIRATION_KEY,
+        SERVER_STALE_CONFIG_CACHE_EXPIRATION_DEFAULT));
+  }
+
+  /**
    * @return a string array of suffixes used to validate repo URLs.
    */
   public String[] getRepoValidationSuffixes(String osType) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/894894ad/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index b21cbbd..7ddaee4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -62,7 +62,7 @@ public class ConfigHelper {
   private static final String DELETED = "DELETED_";
   public static final String CLUSTER_DEFAULT_TAG = "tag";
   private final boolean STALE_CONFIGS_CACHE_ENABLED;
-  private final int STALE_CONFIGS_CACHE_EXPIRATION_TIME = 300;
+  private final int STALE_CONFIGS_CACHE_EXPIRATION_TIME;
   private final Cache<ServiceComponentHost, Boolean> staleConfigsCache;
 
   private static final Logger LOG =
@@ -89,6 +89,7 @@ public class ConfigHelper {
     ambariMetaInfo = metaInfo;
     this.clusterDAO = clusterDAO;
     STALE_CONFIGS_CACHE_ENABLED = configuration.isStaleConfigCacheEnabled();
+    STALE_CONFIGS_CACHE_EXPIRATION_TIME = configuration.staleConfigCacheExpiration();
     staleConfigsCache = CacheBuilder.newBuilder().
         expireAfterWrite(STALE_CONFIGS_CACHE_EXPIRATION_TIME, TimeUnit.SECONDS).build();
   }
@@ -104,9 +105,24 @@ public class ConfigHelper {
   public Map<String, Map<String, String>> getEffectiveDesiredTags(
       Cluster cluster, String hostName) throws AmbariException {
 
+    return getEffectiveDesiredTags(cluster, hostName, false);
+  }
+
+  /**
+   * Gets the desired tags for a cluster and host
+   *
+   * @param cluster  the cluster
+   * @param hostName the host name
+   * @param bypassCache don't use cached values
+   * @return a map of tag type to tag names with overrides
+   * @throws AmbariException
+   */
+  public Map<String, Map<String, String>> getEffectiveDesiredTags(
+      Cluster cluster, String hostName, boolean bypassCache) throws AmbariException {
+
     Host host = (hostName == null) ? null : clusters.getHost(hostName);
-    Map<String, HostConfig> desiredHostConfigs = (host == null) ? null : host.getDesiredHostConfigs(cluster);
-    return getEffectiveDesiredTags(cluster, desiredHostConfigs);
+    Map<String, HostConfig> desiredHostConfigs = (host == null) ? null : host.getDesiredHostConfigs(cluster,
bypassCache);
+    return getEffectiveDesiredTags(cluster, desiredHostConfigs, bypassCache);
   }
 
   /**
@@ -114,12 +130,13 @@ public class ConfigHelper {
    *
    * @param cluster             the cluster
    * @param hostConfigOverrides the host overrides applied using config groups
+   * @param bypassCache         don't use cached values
    * @return a map of tag type to tag names with overrides
    */
   private Map<String, Map<String, String>> getEffectiveDesiredTags(
-      Cluster cluster, Map<String, HostConfig> hostConfigOverrides) {
+      Cluster cluster, Map<String, HostConfig> hostConfigOverrides, boolean bypassCache)
{
 
-    Map<String, DesiredConfig> clusterDesired = (cluster == null) ? new HashMap<String,
DesiredConfig>() : cluster.getDesiredConfigs(true);
+    Map<String, DesiredConfig> clusterDesired = (cluster == null) ? new HashMap<String,
DesiredConfig>() : cluster.getDesiredConfigs(bypassCache);
 
     Map<String, Map<String, String>> resolved = new TreeMap<String, Map<String,
String>>();
 
@@ -981,8 +998,7 @@ public class ConfigHelper {
     Cluster cluster = clusters.getClusterById(sch.getClusterId());
     StackId stackId = cluster.getDesiredStackVersion();
 
-    Map<String, Map<String, String>> desired = getEffectiveDesiredTags(cluster,
-        sch.getHostName());
+    Map<String, Map<String, String>> desired = getEffectiveDesiredTags(cluster,
sch.getHostName(), true);
 
     ServiceInfo serviceInfo = ambariMetaInfo.getService(stackId.getStackName(),
         stackId.getStackVersion(), sch.getServiceName());

http://git-wip-us.apache.org/repos/asf/ambari/blob/894894ad/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java b/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java
index 34fb995..ca854db 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java
@@ -373,6 +373,15 @@ public interface Host {
   Map<String, HostConfig> getDesiredHostConfigs(Cluster cluster) throws AmbariException;
 
   /**
+   * Get the desired configurations for the host including overrides
+   * @param cluster
+   * @param bypassCache
+   * @return
+   * @throws AmbariException
+   */
+  Map<String, HostConfig> getDesiredHostConfigs(Cluster cluster, boolean bypassCache)
throws AmbariException;
+
+  /**
    * Sets the maintenance state for the host.
    * @param clusterId the cluster id
    * @param state the state

http://git-wip-us.apache.org/repos/asf/ambari/blob/894894ad/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
index 4b747fa..d221112 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
@@ -1327,12 +1327,25 @@ public class HostImpl implements Host {
    */
   @Override
   public Map<String, HostConfig> getDesiredHostConfigs(Cluster cluster) throws AmbariException
{
+    return getDesiredHostConfigs(cluster, false);
+  }
+
+  /**
+   * Get a map of configType with all applicable config tags.
+   *
+   * @param cluster  the cluster
+   * @param bypassCache don't use cached values
+   *
+   * @return Map of configType -> HostConfig
+   */
+  @Override
+  public Map<String, HostConfig> getDesiredHostConfigs(Cluster cluster, boolean bypassCache)
throws AmbariException {
     Map<String, HostConfig> hostConfigMap = new HashMap<String, HostConfig>();
-    Map<String, DesiredConfig> clusterDesiredConfigs = (cluster == null) ? new HashMap<String,
DesiredConfig>() : cluster.getDesiredConfigs();
+    Map<String, DesiredConfig> clusterDesiredConfigs = (cluster == null) ? new HashMap<String,
DesiredConfig>() : cluster.getDesiredConfigs(bypassCache);
 
     if (clusterDesiredConfigs != null) {
       for (Map.Entry<String, DesiredConfig> desiredConfigEntry
-              : clusterDesiredConfigs.entrySet()) {
+          : clusterDesiredConfigs.entrySet()) {
         HostConfig hostConfig = new HostConfig();
         hostConfig.setDefaultVersionTag(desiredConfigEntry.getValue().getTag());
         hostConfigMap.put(desiredConfigEntry.getKey(), hostConfig);
@@ -1344,7 +1357,7 @@ public class HostImpl implements Host {
     if (configGroups != null && !configGroups.isEmpty()) {
       for (ConfigGroup configGroup : configGroups.values()) {
         for (Map.Entry<String, Config> configEntry : configGroup
-                .getConfigurations().entrySet()) {
+            .getConfigurations().entrySet()) {
 
           String configType = configEntry.getKey();
           // HostConfig config holds configType -> versionTag, per config group
@@ -1364,7 +1377,7 @@ public class HostImpl implements Host {
           }
           Config config = configEntry.getValue();
           hostConfig.getConfigGroupOverrides().put(configGroup.getId(),
-                  config.getTag());
+              config.getTag());
         }
       }
     }

http://git-wip-us.apache.org/repos/asf/ambari/blob/894894ad/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
index 86b119e..f79c823 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
@@ -31,7 +31,11 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 
 import javax.persistence.EntityManager;
 
@@ -224,6 +228,22 @@ public class ConfigHelperTest {
       return configGroup.getId();
     }
 
+    void applyConfig(Map<String, String> properties, String configType, String configTag)
throws Exception {
+      ConfigurationRequest cr = new ConfigurationRequest();
+      cr.setClusterName(clusterName);
+      cr.setType(configType);
+      cr.setVersionTag(configTag);
+      cr.setProperties(properties);
+
+      final ClusterRequest clusterRequest =
+          new ClusterRequest(cluster.getClusterId(), clusterName,
+              cluster.getDesiredStackVersion().getStackVersion(), null);
+
+      clusterRequest.setDesiredConfig(Collections.singletonList(cr));
+      managementController.updateClusters(new HashSet<ClusterRequest>() {{
+        add(clusterRequest);
+      }}, null);
+    }
     @Test
     public void testEffectiveTagsForHost() throws Exception {
       final Config config = new ConfigImpl("core-site");
@@ -745,6 +765,111 @@ public class ConfigHelperTest {
 
       verify(sch);
     }
+    @Test
+    public void testCalculateIsStaleConfigsParallel() throws Exception{
+      Map<String, HostConfig> schReturn = new HashMap<String, HostConfig>();
+      HostConfig hc = new HostConfig();
+      // Put a different version to check for change
+      hc.setDefaultVersionTag("version2");
+      schReturn.put("flume-conf", hc);
+
+      // set up mocks
+      final ServiceComponentHost sch = createNiceMock(ServiceComponentHost.class);
+      // set up expectations
+      expect(sch.getActualConfigs()).andReturn(schReturn).anyTimes();
+      expect(sch.getHostName()).andReturn("h1").anyTimes();
+      expect(sch.getClusterId()).andReturn(1l).anyTimes();
+      expect(sch.getServiceName()).andReturn("FLUME").anyTimes();
+      expect(sch.getServiceComponentName()).andReturn("FLUME_HANDLER").anyTimes();
+      replay(sch);
+      // Cluster level config changes
+
+      final Config config1 = cluster.getDesiredConfigByType("flume-conf");
+
+      applyConfig(new HashMap<String, String>(){{
+        put("property", "1");
+      }}, "flume-conf", "version2");
+
+      final Config config2 = cluster.getDesiredConfigByType("flume-conf");
+
+      applyConfig(new HashMap<String, String>(){{
+        put("property", "2");
+      }}, "flume-conf", "version3");
+
+      final Config config3 = cluster.getDesiredConfigByType("flume-conf");
+
+      cluster.addDesiredConfig("admin", new HashSet<Config>(){{add(config1);}});
+
+      final AtomicBoolean mustBeStale = new AtomicBoolean();
+      mustBeStale.set(false);
+
+      final AtomicBoolean failed = new AtomicBoolean();
+      failed.set(false);
+
+      final AtomicBoolean finished = new AtomicBoolean();
+      finished.set(false);
+      // avoid situations when not checked previous mustBeStale value yes and applied new
config version
+      final Lock checkLock = new ReentrantLock();
+
+      // parallel thread that will compare actual stale config with expected accordingly
to desired configs, checks if
+      // isStaleConfigs bypassing every cache and returns correct information.
+      Thread parallel = new Thread(new Runnable() {
+        @Override
+        public void run() {
+          while(!finished.get()){
+            checkLock.lock();
+            try {
+              boolean isStale = configHelper.isStaleConfigs(sch);
+              if(mustBeStale.get() != isStale){
+                failed.set(true);
+                break;
+              }
+            } catch (AmbariException e) {
+              e.printStackTrace();
+            } finally {
+              checkLock.unlock();
+            }
+          }
+        }
+      });
+
+      parallel.start();
+
+      Random r = new Random();
+      for(int i=0; i< 1000; i++){
+        try {
+          checkLock.lock();
+          switch(r.nextInt(3)) {
+            case 0: {
+              cluster.addDesiredConfig("admin", new HashSet<Config>(){{add(config1);}});
+              mustBeStale.set(true);
+              checkLock.unlock();
+              break;
+            }
+            case 1: {
+              cluster.addDesiredConfig("admin", new HashSet<Config>(){{add(config2);}});
+              mustBeStale.set(false);
+              checkLock.unlock();
+              break;
+            }
+            case 2: {
+              cluster.addDesiredConfig("admin", new HashSet<Config>(){{add(config3);}});
+              mustBeStale.set(true);
+              checkLock.unlock();
+              break;
+            }
+          }
+        } catch (Exception e){
+          checkLock.unlock();
+        }
+        if(!parallel.isAlive()) {
+          break;
+        }
+      }
+      finished.set(true);
+      parallel.join();
+      Assert.assertFalse(failed.get());
+    }
   }
 
   public static class RunWithCustomModule {


Mime
View raw message