ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aonis...@apache.org
Subject [2/2] ambari git commit: AMBARI-8187. Download client configs gives incorrect content for certain components (aonishuk)
Date Thu, 06 Nov 2014 18:39:36 GMT
AMBARI-8187. Download client configs gives incorrect content for certain components (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e58eda61
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e58eda61
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e58eda61

Branch: refs/heads/branch-1.7.0
Commit: e58eda61e4c94a9ffc20c6c6667fa3d183a647bf
Parents: d8d6bf4
Author: Andrew Onishuk <aonishuk@hortonworks.com>
Authored: Thu Nov 6 20:39:28 2014 +0200
Committer: Andrew Onishuk <aonishuk@hortonworks.com>
Committed: Thu Nov 6 20:39:28 2014 +0200

----------------------------------------------------------------------
 .../resource_management/libraries/script/script.py    | 14 ++++++++++++++
 .../internal/ClientConfigResourceProvider.java        |  6 +++++-
 .../stacks/HDP/2.0.6/services/HBASE/metainfo.xml      |  5 +++++
 .../stacks/HDP/2.1/services/FALCON/metainfo.xml       |  4 ++--
 .../test/python/stacks/1.3.2/HDFS/test_hdfs_client.py |  6 ++++++
 .../stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py   |  6 ++++++
 .../src/test/python/stacks/1.3.2/configs/default.json |  1 +
 .../test/python/stacks/2.0.6/HDFS/test_hdfs_client.py |  6 ++++++
 .../src/test/python/stacks/2.0.6/configs/default.json |  1 +
 9 files changed, 46 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/e58eda61/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py
b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index 4caaa78..001922d 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -30,6 +30,7 @@ from contextlib import closing
 
 
 from resource_management.libraries.resources import XmlConfig
+from resource_management.libraries.resources import PropertiesFile
 from resource_management.core.resources import File, Directory
 from resource_management.core.source import InlineTemplate
 
@@ -250,6 +251,10 @@ class Script(object):
     import params
     return {'configurations':params.config['configurations'][dict],
             'configuration_attributes':params.config['configuration_attributes'][dict]}
+    
+  def generate_configs_get_xml_file_dict(self, filename, dict):
+    import params
+    return params.config['configurations'][dict]
 
   def generate_configs(self, env):
     """
@@ -260,6 +265,8 @@ class Script(object):
     env.set_params(params)
     xml_configs_list = params.config['commandParams']['xml_configs_list']
     env_configs_list = params.config['commandParams']['env_configs_list']
+    properties_configs_list = params.config['commandParams']['properties_configs_list']
+    
     conf_tmp_dir = tempfile.mkdtemp()
     output_filename = os.path.join(self.get_tmp_dir(),params.config['commandParams']['output_file'])
 
@@ -274,6 +281,13 @@ class Script(object):
       for filename,dicts in file_dict.iteritems():
         File(os.path.join(conf_tmp_dir, filename),
              content=InlineTemplate(self.generate_configs_get_template_file_content(filename,
dicts)))
+        
+    for file_dict in properties_configs_list:
+      for filename, dict in file_dict.iteritems():
+        PropertiesFile(os.path.join(conf_tmp_dir, filename),
+          properties=self.generate_configs_get_xml_file_dict(filename, dict)
+        )
+      
     with closing(tarfile.open(output_filename, "w:gz")) as tar:
       tar.add(conf_tmp_dir, arcname=os.path.basename("."))
       tar.close()

http://git-wip-us.apache.org/repos/asf/ambari/blob/e58eda61/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
index 339ebc4..8f59581 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
@@ -265,6 +265,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
       Map<String, Object> commandParams = new HashMap<String, Object>();
       List<Map<String, String>> xmlConfigs = new LinkedList<Map<String,
String>>();
       List<Map<String, String>> envConfigs = new LinkedList<Map<String,
String>>();
+      List<Map<String, String>> propertiesConfigs = new LinkedList<Map<String,
String>>();
 
       //Fill file-dictionary configs from metainfo
       for (ClientConfigFileDefinition clientConfigFile : clientConfigFiles) {
@@ -272,13 +273,16 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
         fileDict.put(clientConfigFile.getFileName(), clientConfigFile.getDictionaryName());
         if (clientConfigFile.getType().equals("xml")) {
           xmlConfigs.add(fileDict);
-        } else {
+        } else if (clientConfigFile.getType().equals("env")) {
           envConfigs.add(fileDict);
+        } else if (clientConfigFile.getType().equals("properties")) {
+          propertiesConfigs.add(fileDict);
         }
       }
 
       commandParams.put("xml_configs_list", xmlConfigs);
       commandParams.put("env_configs_list", envConfigs);
+      commandParams.put("properties_configs_list", propertiesConfigs);
       commandParams.put("output_file", componentName + "-configs.tar.gz");
 
       Map<String, Object> jsonContent = new TreeMap<String, Object>();

http://git-wip-us.apache.org/repos/asf/ambari/blob/e58eda61/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/metainfo.xml
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/metainfo.xml
index 0816591..fd290df 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/metainfo.xml
@@ -97,6 +97,11 @@
               <dictionaryName>hbase-env</dictionaryName>
             </configFile>
             <configFile>
+              <type>xml</type>
+              <fileName>hbase-policy.xml</fileName>
+              <dictionaryName>hbase-policy</dictionaryName>
+            </configFile>
+            <configFile>
               <type>env</type>
               <fileName>log4j.properties</fileName>
               <dictionaryName>hbase-log4j</dictionaryName>

http://git-wip-us.apache.org/repos/asf/ambari/blob/e58eda61/ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/metainfo.xml
b/ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/metainfo.xml
index b4e6883..78336e6 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/metainfo.xml
@@ -41,12 +41,12 @@
               <dictionaryName>falcon-env</dictionaryName>
             </configFile>
             <configFile>
-              <type>xml</type>
+              <type>properties</type>
               <fileName>runtime.properties</fileName>
               <dictionaryName>falcon-runtime.properties</dictionaryName>
             </configFile>
             <configFile>
-              <type>xml</type>
+              <type>properties</type>
               <fileName>startup.properties</fileName>
               <dictionaryName>falcon-startup.properties</dictionaryName>
             </configFile>

http://git-wip-us.apache.org/repos/asf/ambari/blob/e58eda61/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_hdfs_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_hdfs_client.py b/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_hdfs_client.py
index 80da9b6..f670ac4 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_hdfs_client.py
+++ b/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_hdfs_client.py
@@ -52,6 +52,12 @@ class Test(RMFTestCase):
     self.assertResourceCalled('File', '/tmp/123/log4j.properties',
                               content = InlineTemplate("log4jproperties\nline2log4jproperties\nline2\nambari.jobhistory.database=jdbc:postgresql://c6401.ambari.apache.org/ambarirca\nambari.jobhistory.driver=org.postgresql.Driver\nambari.jobhistory.user=mapred\nambari.jobhistory.password=mapred\nambari.jobhistory.logger=${hadoop.root.logger}\n\nlog4j.appender.JHA=org.apache.ambari.log4j.hadoop.mapreduce.jobhistory.JobHistoryAppender\nlog4j.appender.JHA.database=jdbc:postgresql://c6401.ambari.apache.org/ambarirca\nlog4j.appender.JHA.driver=org.postgresql.Driver\nlog4j.appender.JHA.user=mapred\nlog4j.appender.JHA.password=mapred\n\nlog4j.logger.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=DEBUG,JHA\nlog4j.additivity.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=true\n\n"),
                               )
+    self.assertResourceCalled('PropertiesFile', '/tmp/123/runtime.properties',
+                              properties = UnknownConfigurationMock(),
+    )
+    self.assertResourceCalled('PropertiesFile', '/tmp/123/startup.properties',
+                              properties = UnknownConfigurationMock(),
+    )
     self.assertResourceCalled('Directory', '/tmp/123',
                               action = ['delete'],
                               )

http://git-wip-us.apache.org/repos/asf/ambari/blob/e58eda61/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py
b/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py
index 2081c36..1621b42 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py
+++ b/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py
@@ -196,6 +196,12 @@ class TestMapreduceClient(RMFTestCase):
     self.assertResourceCalled('File', '/tmp/123/log4j.properties',
                               content = InlineTemplate("log4jproperties\nline2log4jproperties\nline2\nambari.jobhistory.database=jdbc:postgresql://c6401.ambari.apache.org/ambarirca\nambari.jobhistory.driver=org.postgresql.Driver\nambari.jobhistory.user=mapred\nambari.jobhistory.password=mapred\nambari.jobhistory.logger=${hadoop.root.logger}\n\nlog4j.appender.JHA=org.apache.ambari.log4j.hadoop.mapreduce.jobhistory.JobHistoryAppender\nlog4j.appender.JHA.database=jdbc:postgresql://c6401.ambari.apache.org/ambarirca\nlog4j.appender.JHA.driver=org.postgresql.Driver\nlog4j.appender.JHA.user=mapred\nlog4j.appender.JHA.password=mapred\n\nlog4j.logger.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=DEBUG,JHA\nlog4j.additivity.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=true\n\n"),
                               )
+    self.assertResourceCalled('PropertiesFile', '/tmp/123/runtime.properties',
+                              properties = UnknownConfigurationMock(),
+    )
+    self.assertResourceCalled('PropertiesFile', '/tmp/123/startup.properties',
+                              properties = UnknownConfigurationMock(),
+    )
     self.assertResourceCalled('Directory', '/tmp/123',
                               action = ['delete'],
                               )

http://git-wip-us.apache.org/repos/asf/ambari/blob/e58eda61/ambari-server/src/test/python/stacks/1.3.2/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/configs/default.json b/ambari-server/src/test/python/stacks/1.3.2/configs/default.json
index 0256444..a39d51d 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/configs/default.json
+++ b/ambari-server/src/test/python/stacks/1.3.2/configs/default.json
@@ -35,6 +35,7 @@
         "update_exclude_file_only" : "false",
         "xml_configs_list":[{"core-site.xml":"core-site"},{"mapred-site.xml":"mapred-site"}],
         "env_configs_list":[{"log4j.properties":"hdfs-log4j,mapreduce-log4j"}],
+        "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}],
         "output_file":"MAPREDUCE_CLIENT-configs.tar.gz"
     },
     "taskId": 18, 

http://git-wip-us.apache.org/repos/asf/ambari/blob/e58eda61/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
index 0a4bfb5..7c9e42b 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
@@ -52,6 +52,12 @@ class Test(RMFTestCase):
                               content = InlineTemplate(self.getConfig()['configurations']['hdfs-log4j']['content']+
                                                        self.getConfig()['configurations']['yarn-log4j']['content']),
                               )
+    self.assertResourceCalled('PropertiesFile', '/tmp/123/runtime.properties',
+                              properties = UnknownConfigurationMock(),
+    )
+    self.assertResourceCalled('PropertiesFile', '/tmp/123/startup.properties',
+                              properties = UnknownConfigurationMock(),
+    )
     self.assertResourceCalled('Directory', '/tmp/123',
                               action = ['delete'],
                               )

http://git-wip-us.apache.org/repos/asf/ambari/blob/e58eda61/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
index 2dc4a8d..6b68e0b 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
@@ -32,6 +32,7 @@
         "update_exclude_file_only" : "false",
         "xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}],
         "env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}],
+        "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}],
         "output_file":"HDFS_CLIENT-configs.tar.gz"
     },
     "taskId": 152, 


Mime
View raw message