ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dmitriu...@apache.org
Subject [1/2] git commit: AMBARI-7761. stale_configs parameter works incorrectly for Flume (dlysnichenko)
Date Tue, 14 Oct 2014 13:34:10 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-1.7.0 b941191c5 -> d9d9a5450
  refs/heads/trunk a50ac9d13 -> abd9fcacb


AMBARI-7761. stale_configs parameter works incorrectly for Flume (dlysnichenko)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/abd9fcac
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/abd9fcac
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/abd9fcac

Branch: refs/heads/trunk
Commit: abd9fcacbf05c6fed8509ee689c9c342d28ce960
Parents: a50ac9d
Author: Lisnichenko Dmitro <dlysnichenko@hortonworks.com>
Authored: Tue Oct 14 16:32:09 2014 +0300
Committer: Lisnichenko Dmitro <dlysnichenko@hortonworks.com>
Committed: Tue Oct 14 16:32:09 2014 +0300

----------------------------------------------------------------------
 .../ambari/server/state/ConfigHelper.java       | 40 +++++++++--
 .../api/util/StackExtensionHelperTest.java      |  2 +-
 .../AmbariManagementControllerTest.java         |  2 +-
 .../ambari/server/state/ConfigHelperTest.java   | 73 ++++++++++++++++++--
 .../services/FLUME/configuration/flume-conf.xml | 31 +++++++++
 .../services/FLUME/configuration/flume-env.xml  | 40 +++++++++++
 .../FLUME/configuration/flume-log4j.xml         | 31 +++++++++
 .../HDP/2.0.6/services/FLUME/metainfo.xml       | 69 ++++++++++++++++++
 8 files changed, 275 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/abd9fcac/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index 8b724b1..e15a62a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -47,6 +47,7 @@ import org.apache.ambari.server.controller.ConfigurationRequest;
 import org.apache.ambari.server.orm.dao.ClusterDAO;
 import org.apache.ambari.server.orm.entities.ClusterConfigEntity;
 import org.apache.ambari.server.state.PropertyInfo.PropertyType;
+import org.apache.ambari.server.state.configgroup.ConfigGroup;
 import org.apache.ambari.server.upgrade.UpgradeCatalog170;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -670,8 +671,8 @@ public class ConfigHelper {
         // desired and actual both define the type
         HostConfig hc = actual.get(type);
         Map<String, String> actualTags = buildTags(hc);
-        
-        if (!isTagChanged(tags, actualTags)) {
+
+        if (!isTagChanged(tags, actualTags, hasGroupSpecificConfigsForType(cluster, sch.getHostName(),
type))) {
           stale = false;
         } else if (type.equals(Configuration.GLOBAL_CONFIG_TAG)) {
           // tags are changed, need to find out what has changed,
@@ -691,6 +692,31 @@ public class ConfigHelper {
   }
 
   /**
+   * Determines if the hostname has group specific configs for the type specified
+   *
+   * @param cluster
+   * @param hostname of the host to look for
+   * @param type     the type to look for (e.g. flume-conf)
+   * @return <code>true</code> if the hostname has group specific configuration
for the type
+   */
+  private boolean hasGroupSpecificConfigsForType(Cluster cluster, String hostname, String
type) {
+    try {
+      Map<Long, ConfigGroup> configGroups = cluster.getConfigGroupsByHostname(hostname);
+      if (configGroups != null && !configGroups.isEmpty()) {
+        for (ConfigGroup configGroup : configGroups.values()) {
+          Config config = configGroup.getConfigurations().get(type);
+          if (config != null) {
+            return true;
+          }
+        }
+      }
+    } catch (AmbariException ambariException) {
+      LOG.warn("Could not determine group configuration for host. Details: " + ambariException.getMessage());
+    }
+    return false;
+  }
+
+  /**
    * @return <code>true</code> if any service on the stack defines a property
    * for the type.
    */
@@ -761,10 +787,16 @@ public class ConfigHelper {
   /**
    * @return true if the tags are different in any way, even if not-specified
    */
-  private boolean isTagChanged(Map<String, String> desiredTags, Map<String, String>
actualTags) {
-    if (!actualTags.get(CLUSTER_DEFAULT_TAG).equals(desiredTags.get(CLUSTER_DEFAULT_TAG)))
+  private boolean isTagChanged(Map<String, String> desiredTags, Map<String, String>
actualTags, boolean groupSpecificConfigs) {
+    if (!actualTags.get(CLUSTER_DEFAULT_TAG).equals(desiredTags.get(CLUSTER_DEFAULT_TAG))
&& !groupSpecificConfigs)
       return true;
 
+    // if the host has group specific configs for type we should ignore the cluster level
configs and compare specifics
+    if (groupSpecificConfigs) {
+      actualTags.remove(CLUSTER_DEFAULT_TAG);
+      desiredTags.remove(CLUSTER_DEFAULT_TAG);
+    }
+
     Set<String> desiredSet = new HashSet<String>(desiredTags.values());
     Set<String> actualSet = new HashSet<String>(actualTags.values());
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/abd9fcac/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java
index 84ba6aa..acd595d 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java
@@ -298,7 +298,7 @@ public class StackExtensionHelperTest {
     helper.populateServicesForStack(stackInfo);
     helper.fillInfo();
     List<ServiceInfo> allServices = helper.getAllApplicableServices(stackInfo);
-    assertEquals(12, allServices.size());
+    assertEquals(13, allServices.size());
     for (ServiceInfo serviceInfo : allServices) {
       if (serviceInfo.getName().equals("NAGIOS")) {
         assertTrue(serviceInfo.isMonitoringService());

http://git-wip-us.apache.org/repos/asf/ambari/blob/abd9fcac/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index c9c8258..db96059 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -7113,7 +7113,7 @@ public class AmbariManagementControllerTest {
   public void testGetStackServices() throws Exception {
     StackServiceRequest request = new StackServiceRequest(STACK_NAME, NEW_STACK_VERSION,
null);
     Set<StackServiceResponse> responses = controller.getStackServices(Collections.singleton(request));
-    Assert.assertEquals(11, responses.size());
+    Assert.assertEquals(12, responses.size());
 
 
     StackServiceRequest requestWithParams = new StackServiceRequest(STACK_NAME, NEW_STACK_VERSION,
SERVICE_NAME);

http://git-wip-us.apache.org/repos/asf/ambari/blob/abd9fcac/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
index 73004ba..1c01ec5 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
@@ -17,6 +17,11 @@
  */
 package org.apache.ambari.server.state;
 
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+
 import com.google.inject.Guice;
 import com.google.inject.Injector;
 import com.google.inject.persist.PersistService;
@@ -35,12 +40,14 @@ import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
+import java.util.ArrayList;
+import java.util.List;
+
+
 
 public class ConfigHelperTest {
   private Clusters clusters;
@@ -101,6 +108,22 @@ public class ConfigHelperTest {
     managementController.updateClusters(new HashSet<ClusterRequest>()
     {{ add(clusterRequest1); }}, null);
 
+    // flume-conf
+
+    ConfigurationRequest cr2 = new ConfigurationRequest();
+    cr2.setClusterName(clusterName);
+    cr2.setType("flume-conf");
+    cr2.setVersionTag("version1");
+
+
+    final ClusterRequest clusterRequest2 =
+      new ClusterRequest(cluster.getClusterId(), clusterName,
+        cluster.getDesiredStackVersion().getStackVersion(), null);
+
+    clusterRequest2.setDesiredConfig(Collections.singletonList(cr2));
+    managementController.updateClusters(new HashSet<ClusterRequest>()
+    {{ add(clusterRequest2); }}, null);
+
     // global
     cr.setType("global");
     cr.setVersionTag("version1");
@@ -115,13 +138,13 @@ public class ConfigHelperTest {
       put("attribute2", attrs);
     }});
 
-    final ClusterRequest clusterRequest2 =
+    final ClusterRequest clusterRequest3 =
       new ClusterRequest(cluster.getClusterId(), clusterName,
         cluster.getDesiredStackVersion().getStackVersion(), null);
 
-    clusterRequest2.setDesiredConfig(Collections.singletonList(cr));
+    clusterRequest3.setDesiredConfig(Collections.singletonList(cr));
     managementController.updateClusters(new HashSet<ClusterRequest>()
-    {{ add(clusterRequest2); }}, null);
+    {{ add(clusterRequest3); }}, null);
   }
 
   @After
@@ -248,10 +271,10 @@ public class ConfigHelperTest {
 
     Map<String, Map<String, Map<String, String>>> effectiveAttributes =
configHelper
         .getEffectiveConfigAttributes(cluster,
-            configHelper.getEffectiveDesiredTags(cluster, "h1"));
+          configHelper.getEffectiveDesiredTags(cluster, "h1"));
 
     Assert.assertNotNull(effectiveAttributes);
-    Assert.assertEquals(2, effectiveAttributes.size());
+    Assert.assertEquals(3, effectiveAttributes.size());
 
     Assert.assertTrue(effectiveAttributes.containsKey("global"));
     Map<String, Map<String, String>> globalAttrs = effectiveAttributes.get("global");
@@ -494,4 +517,40 @@ public class ConfigHelperTest {
     Assert.assertEquals("true", finalResultAttributes.get("c"));
     Assert.assertEquals("true", finalResultAttributes.get("d"));
   }
+
+  @Test
+  public void testCalculateIsStaleConfigs() throws Exception {
+
+    Map<String, HostConfig> schReturn = new HashMap<String, HostConfig>();
+    HostConfig hc = new HostConfig();
+    // Put a different version to check for change
+    hc.setDefaultVersionTag("version2");
+    schReturn.put("flume-conf", hc);
+    // set up mocks
+    ServiceComponentHost sch = createNiceMock(ServiceComponentHost.class);
+    // set up expectations
+    expect(sch.getActualConfigs()).andReturn(schReturn).times(3);
+    expect(sch.getHostName()).andReturn("h1").times(6);
+    expect(sch.getClusterId()).andReturn(1l).times(3);
+    expect(sch.getServiceName()).andReturn("FLUME").times(3);
+    expect(sch.getServiceComponentName()).andReturn("FLUME_HANDLER").times(3);
+    replay(sch);
+    // Cluster level config changes
+    Assert.assertTrue(configHelper.isStaleConfigs(sch));
+    HostConfig hc2 = new HostConfig();
+    hc2.setDefaultVersionTag("version1");
+    schReturn.put("flume-conf", hc2);
+    // invalidate cache to test new sch
+    configHelper.invalidateStaleConfigsCache();
+    // Cluster level same configs
+    Assert.assertFalse(configHelper.isStaleConfigs(sch));
+    // Cluster level same configs but group specific configs for host have been updated
+    List<String> hosts = new ArrayList<String>();
+    hosts.add("h1");
+    List<Config> configs = new ArrayList<Config>();
+    configs.add(new ConfigImpl("flume-conf"));
+    addConfigGroup("configGroup1", "FLUME", hosts, configs);
+    Assert.assertTrue(configHelper.isStaleConfigs(sch));
+    verify(sch);
+  }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/abd9fcac/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-conf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-conf.xml
b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-conf.xml
new file mode 100644
index 0000000..74a4c15
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-conf.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration supports_final="false">
+  <property>
+    <name>content</name>
+    <description>Describe all the Flume agent configurations</description>
+    <value>
+# Flume agent config
+    </value>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/abd9fcac/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-env.xml
b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-env.xml
new file mode 100644
index 0000000..7b11bde
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-env.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+  <property>
+    <name>flume_conf_dir</name>
+    <value>/etc/flume/conf</value>
+    <description>Location to save configuration files</description>
+  </property>
+  <property>
+    <name>flume_log_dir</name>
+    <value>/var/log/flume</value>
+    <description>Location to save log files</description>
+  </property>
+  <property>
+    <name>flume_user</name>
+    <value>flume</value>
+    <property-type>USER</property-type>
+    <description>Flume User</description>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/abd9fcac/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-log4j.xml
b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-log4j.xml
new file mode 100644
index 0000000..8c6ac27
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/configuration/flume-log4j.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration supports_final="false">
+  <property>
+    <name>content</name>
+    <description>Custom log4j.properties</description>
+    <value>
+# Flume log4j config
+    </value>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/abd9fcac/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/metainfo.xml
b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/metainfo.xml
new file mode 100644
index 0000000..4f6bbd9
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/FLUME/metainfo.xml
@@ -0,0 +1,69 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<metainfo>
+  <schemaVersion>2.0</schemaVersion>
+  <services>
+    <service>
+      <name>FLUME</name>
+      <displayName>Flume</displayName>
+      <comment>Data management and processing platform</comment>
+      <version>1.4.0.2.0</version>
+      <components>
+        <component>
+          <name>FLUME_HANDLER</name>
+          <displayName>Flume</displayName>
+          <category>SLAVE</category>
+          <cardinality>1+</cardinality>
+          <commandScript>
+            <script>scripts/flume_handler.py</script>
+            <scriptType>PYTHON</scriptType>
+            <timeout>600</timeout>
+          </commandScript>
+        </component>
+      </components>
+
+      <osSpecifics>
+        <osSpecific>
+          <osFamily>any</osFamily>
+          <packages>
+            <package>
+              <name>flume</name>
+            </package>
+          </packages>
+        </osSpecific>
+      </osSpecifics>
+
+      <commandScript>
+        <script>scripts/flume_check.py</script>
+        <scriptType>PYTHON</scriptType>
+        <timeout>300</timeout>
+      </commandScript>
+      
+      <requiredServices>
+        <service>HDFS</service>
+      </requiredServices>
+
+      <configuration-dependencies>
+        <config-type>flume-env</config-type>
+        <config-type>flume-conf</config-type>
+        <config-type>flume-log4j</config-type>
+      </configuration-dependencies>
+
+    </service>
+  </services>
+</metainfo>


Mime
View raw message