ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From alejan...@apache.org
Subject ambari git commit: AMBARI-8130. Provide ability to download client configurations in Windows (Florian Barca via alejandro)
Date Fri, 07 Nov 2014 00:40:21 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-windows-dev 2868c2051 -> 9cff20bf1


AMBARI-8130. Provide ability to download client configurations in Windows (Florian Barca via
alejandro)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9cff20bf
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9cff20bf
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9cff20bf

Branch: refs/heads/branch-windows-dev
Commit: 9cff20bf1634a8b242a8b150c41ae5a93888adcb
Parents: 2868c20
Author: Alejandro Fernandez <afernandez@hortonworks.com>
Authored: Thu Nov 6 16:39:31 2014 -0800
Committer: Alejandro Fernandez <afernandez@hortonworks.com>
Committed: Thu Nov 6 16:39:31 2014 -0800

----------------------------------------------------------------------
 .../libraries/functions/tar_archive.py          | 30 ++++++++++
 .../libraries/functions/zip_archive.py          | 40 +++++++++++++
 .../libraries/providers/xml_config.py           |  6 +-
 .../libraries/script/script.py                  | 59 ++++++++++----------
 ambari-server/conf/unix/ambari.properties       |  1 +
 ambari-server/conf/windows/ambari.properties    |  1 +
 .../src/main/assemblies/server-windows.xml      |  8 +++
 .../server/api/services/ComponentService.java   |  6 +-
 .../server/configuration/Configuration.java     | 19 +++++++
 .../internal/ClientConfigResourceProvider.java  |  7 ++-
 10 files changed, 141 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
new file mode 100644
index 0000000..efbf933
--- /dev/null
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import os
+import tarfile
+from contextlib import closing
+
+def archive_dir(output_filename, input_dir):
+  with closing(tarfile.open(output_filename, "w:gz")) as tar:
+    try:
+      tar.add(input_dir, arcname=os.path.basename("."))
+    finally:
+      tar.close()

http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-common/src/main/python/resource_management/libraries/functions/zip_archive.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/zip_archive.py
b/ambari-common/src/main/python/resource_management/libraries/functions/zip_archive.py
new file mode 100644
index 0000000..cab3627
--- /dev/null
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/zip_archive.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import os
+import zipfile
+
+def _zip_dir(zip, root):
+  for dirname, dirnames, filenames in os.walk(root):
+    for filename in filenames:
+      if len(dirname) > len(root):
+        rel_path = os.path.relpath(dirname, root)
+        arch_name = rel_path + os.sep + filename
+      else:
+        arch_name = filename
+      zip.write(os.path.join(dirname, filename), arch_name)
+
+
+def archive_dir(output_filename, input_dir):
+  zipf = zipfile.ZipFile(output_filename, 'w')
+  try:
+    _zip_dir(zipf, input_dir)
+  finally:
+    zipf.close()

http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py
b/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py
index 34b43e3..a7b2c21 100644
--- a/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py
+++ b/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py
@@ -49,11 +49,11 @@ class XmlConfigProvider(Provider):
   </configuration>''', extra_imports=[time], configurations_dict=self.resource.configurations,
                                     configuration_attrs=self.resource.configuration_attributes)
 
-
-    Logger.info(format("Generating config: {xml_config_provider_config_dir}/{filename}"))
+    xml_config_dest_file_path = os.path.join(xml_config_provider_config_dir, filename)
+    Logger.info("Generating config: {}".format(xml_config_dest_file_path))
 
     with Environment.get_instance_copy() as env:
-      File (format("{xml_config_provider_config_dir}/{filename}"),
+      File (xml_config_dest_file_path,
         content = config_content,
         owner = self.resource.owner,
         group = self.resource.group,

http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py
b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index 059dd57..d58502a 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -17,7 +17,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 '''
-import tarfile
 import tempfile
 
 __all__ = ["Script"]
@@ -27,7 +26,6 @@ import sys
 import json
 import logging
 import platform
-from contextlib import closing
 
 from resource_management.libraries.resources import XmlConfig
 from resource_management.core.resources import File, Directory
@@ -41,6 +39,9 @@ IS_WINDOWS = platform.system() == "Windows"
 if IS_WINDOWS:
   from resource_management.libraries.functions.install_hdp_msi import install_windows_msi
   from resource_management.libraries.functions.reload_windows_env import reload_windows_env
+  from resource_management.libraries.functions.zip_archive import archive_dir
+else:
+  from resource_management.libraries.functions.tar_archive import archive_dir
 
 USAGE = """Usage: {0} <COMMAND> <JSON_CONFIG> <BASEDIR> <STROUTPUT>
<LOGGING_LEVEL> <TMP_DIR>
 
@@ -280,43 +281,45 @@ class Script(object):
     self.fail_with_error('configure method isn\'t implemented')
 
   def generate_configs_get_template_file_content(self, filename, dicts):
-    import params
+    config = self.get_config()
     content = ''
     for dict in dicts.split(','):
-      if dict.strip() in params.config['configurations']:
-        content += params.config['configurations'][dict.strip()]['content']
+      if dict.strip() in config['configurations']:
+        content += config['configurations'][dict.strip()]['content']
 
     return content
 
   def generate_configs_get_xml_file_content(self, filename, dict):
-    import params
-    return {'configurations':params.config['configurations'][dict],
-            'configuration_attributes':params.config['configuration_attributes'][dict]}
+    config = self.get_config()
+    return {'configurations':config['configurations'][dict],
+            'configuration_attributes':config['configuration_attributes'][dict]}
 
   def generate_configs(self, env):
     """
     Generates config files and stores them as an archive in tmp_dir
     based on xml_configs_list and env_configs_list from commandParams
     """
-    import params
-    env.set_params(params)
-    xml_configs_list = params.config['commandParams']['xml_configs_list']
-    env_configs_list = params.config['commandParams']['env_configs_list']
-    conf_tmp_dir = tempfile.mkdtemp()
-    output_filename = os.path.join(self.get_tmp_dir(),params.config['commandParams']['output_file'])
+    config = self.get_config()
+
+    xml_configs_list = config['commandParams']['xml_configs_list']
+    env_configs_list = config['commandParams']['env_configs_list']
 
     Directory(self.get_tmp_dir(), recursive=True)
-    for file_dict in xml_configs_list:
-      for filename, dict in file_dict.iteritems():
-        XmlConfig(filename,
-                  conf_dir=conf_tmp_dir,
-                  **self.generate_configs_get_xml_file_content(filename, dict)
-        )
-    for file_dict in env_configs_list:
-      for filename,dicts in file_dict.iteritems():
-        File(os.path.join(conf_tmp_dir, filename),
-             content=InlineTemplate(self.generate_configs_get_template_file_content(filename,
dicts)))
-    with closing(tarfile.open(output_filename, "w:gz")) as tar:
-      tar.add(conf_tmp_dir, arcname=os.path.basename("."))
-      tar.close()
-    Directory(conf_tmp_dir, action="delete")
+
+    conf_tmp_dir = tempfile.mkdtemp(dir=self.get_tmp_dir())
+    output_filename = os.path.join(self.get_tmp_dir(), config['commandParams']['output_file'])
+
+    try:
+      for file_dict in xml_configs_list:
+        for filename, dict in file_dict.iteritems():
+          XmlConfig(filename,
+                    conf_dir=conf_tmp_dir,
+                    **self.generate_configs_get_xml_file_content(filename, dict)
+          )
+      for file_dict in env_configs_list:
+        for filename,dicts in file_dict.iteritems():
+          File(os.path.join(conf_tmp_dir, filename),
+               content=InlineTemplate(self.generate_configs_get_template_file_content(filename,
dicts)))
+      archive_dir(output_filename, conf_tmp_dir)
+    finally:
+      Directory(conf_tmp_dir, action="delete")

http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-server/conf/unix/ambari.properties
----------------------------------------------------------------------
diff --git a/ambari-server/conf/unix/ambari.properties b/ambari-server/conf/unix/ambari.properties
index 531d717..771f711 100644
--- a/ambari-server/conf/unix/ambari.properties
+++ b/ambari-server/conf/unix/ambari.properties
@@ -33,6 +33,7 @@ bootstrap.setup_agent.script=/usr/lib/python2.6/site-packages/ambari_server/setu
 recommendations.dir=/var/run/ambari-server/stack-recommendations
 stackadvisor.script=/var/lib/ambari-server/resources/scripts/stack_advisor.py
 server.tmp.dir=/var/lib/ambari-server/tmp
+ambari.python.wrap=ambari-python-wrap
 
 api.authenticate=true
 server.connection.max.idle.millis=900000

http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-server/conf/windows/ambari.properties
----------------------------------------------------------------------
diff --git a/ambari-server/conf/windows/ambari.properties b/ambari-server/conf/windows/ambari.properties
index f67e7e2..fd3a7ba 100644
--- a/ambari-server/conf/windows/ambari.properties
+++ b/ambari-server/conf/windows/ambari.properties
@@ -52,6 +52,7 @@ recommendations.dir=\\var\\run\\ambari-server\\stack-recommendations
 stackadvisor.script=resources\\scripts\\stack_advisor.py
 server.tmp.dir=\\var\\run\\ambari-server\\tmp
 views.dir=resources\\views
+ambari.python.wrap=python.exe
 
 # Default timeout in seconds before task is killed
 agent.task.timeout=600

http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-server/src/main/assemblies/server-windows.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/assemblies/server-windows.xml b/ambari-server/src/main/assemblies/server-windows.xml
index 84029dd..e48d3d8 100644
--- a/ambari-server/src/main/assemblies/server-windows.xml
+++ b/ambari-server/src/main/assemblies/server-windows.xml
@@ -117,6 +117,14 @@
       <directory>${basedir}/../ambari-common/src/main/python/ambari_commons</directory>
       <outputDirectory>ambari-server-${project.version}/sbin/ambari_commons</outputDirectory>
     </fileSet>
+    <fileSet>
+      <directory>${basedir}/../ambari-common/src/main/python/ambari_jinja2/ambari_jinja2</directory>
+      <outputDirectory>ambari-server-${project.version}/sbin/ambari_jinja2</outputDirectory>
+    </fileSet>
+    <fileSet>
+      <directory>${basedir}/../ambari-common/src/main/python/resource_management</directory>
+      <outputDirectory>ambari-server-${project.version}/sbin/resource_management</outputDirectory>
+    </fileSet>
      <!--
     <fileSet>
       <directory>${project.build.directory}/web/</directory>

http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java
index 5510697..e7e0029 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java
@@ -234,7 +234,7 @@ public class ComponentService extends BaseService {
     Response.ResponseBuilder rb = Response.status(Response.Status.OK);
     Configuration configs = new Configuration();
     String tmpDir = configs.getProperty(Configuration.SERVER_TMP_DIR_KEY);
-    File file = new File(tmpDir+File.separator+componentName+"-configs.tar.gz");
+    File file = new File(tmpDir + File.separator + componentName + "-configs" + Configuration.DEF_ARCHIVE_EXTENSION);
     InputStream resultInputStream = null;
     try {
       resultInputStream = new FileInputStream(file);
@@ -242,8 +242,8 @@ public class ComponentService extends BaseService {
       e.printStackTrace();
     }
 
-    String contentType = "application/x-ustar";
-    String outputFileName = componentName + "-configs.tar.gz";
+    String contentType = Configuration.DEF_ARCHIVE_CONTENT_TYPE;
+    String outputFileName = componentName + "-configs" + Configuration.DEF_ARCHIVE_EXTENSION;
     rb.header("Content-Disposition",  "attachment; filename=\"" + outputFileName + "\"");
     rb.entity(resultInputStream);
     return rb.type(contentType).build();

http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index c1b5dcb..56bb5ab 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -71,6 +71,8 @@ public class Configuration {
   public static final String RECOMMENDATIONS_DIR_DEFAULT = "/var/run/ambari-server/stack-recommendations";
   public static final String STACK_ADVISOR_SCRIPT = "stackadvisor.script";
   public static final String STACK_ADVISOR_SCRIPT_DEFAULT = "/var/lib/ambari-server/resources/scripts/stack_advisor.py";
+  public static final String AMBARI_PYTHON_WRAP_KEY = "ambari.python.wrap";
+  public static final String AMBARI_PYTHON_WRAP_DEFAULT = "python";
   public static final String API_AUTHENTICATE = "api.authenticate";
   public static final String API_USE_SSL = "api.ssl";
   public static final String API_CSRF_PREVENTION_KEY = "api.csrfPrevention.enabled";
@@ -237,6 +239,10 @@ public class Configuration {
   public static final String DEFAULT_SCHEDULER_START_DELAY_SECONDS = "120";
   public static final String SERVER_TMP_DIR_KEY = "server.tmp.dir";
   public static final String SERVER_TMP_DIR_DEFAULT = "/var/lib/ambari-server/tmp";
+
+  public static final String DEF_ARCHIVE_EXTENSION;
+  public static final String DEF_ARCHIVE_CONTENT_TYPE;
+
   /**
    * This key defines whether stages of parallel requests are executed in
    * parallel or sequentally. Only stages from different requests
@@ -328,6 +334,17 @@ public class Configuration {
   private volatile boolean credentialProviderInitialized = false;
   private Map<String, String> customDbProperties = null;
 
+  static {
+    if (System.getProperty("os.name").contains("Windows")) {
+      DEF_ARCHIVE_EXTENSION = ".zip";
+      DEF_ARCHIVE_CONTENT_TYPE = "application/zip";
+    }
+    else {
+      DEF_ARCHIVE_EXTENSION = ".tar.gz";
+      DEF_ARCHIVE_CONTENT_TYPE = "application/x-ustar";
+    }
+  }
+
   public Configuration() {
     this(readConfigFile());
   }
@@ -342,6 +359,8 @@ public class Configuration {
     this.properties = properties;
 
     configsMap = new HashMap<String, String>();
+    configsMap.put(AMBARI_PYTHON_WRAP_KEY, properties.getProperty(
+        AMBARI_PYTHON_WRAP_KEY, AMBARI_PYTHON_WRAP_DEFAULT));
     configsMap.put(SRVR_TWO_WAY_SSL_KEY, properties.getProperty(
         SRVR_TWO_WAY_SSL_KEY, SRVR_TWO_WAY_SSL_DEFAULT));
     configsMap.put(SRVR_TWO_WAY_SSL_PORT_KEY, properties.getProperty(

http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
index 312f23f..1cfc1d0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
@@ -123,6 +123,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
 
     Configuration configs = new Configuration();
     String TMP_PATH = configs.getProperty(Configuration.SERVER_TMP_DIR_KEY);
+    String pythonCmd = configs.getProperty(Configuration.AMBARI_PYTHON_WRAP_KEY);
     AmbariManagementController managementController = getManagementController();
     ConfigHelper configHelper = managementController.getConfigHelper();
     Cluster cluster = null;
@@ -277,7 +278,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
 
       commandParams.put("xml_configs_list", xmlConfigs);
       commandParams.put("env_configs_list", envConfigs);
-      commandParams.put("output_file", componentName + "-configs.tar.gz");
+      commandParams.put("output_file", componentName + "-configs" + Configuration.DEF_ARCHIVE_EXTENSION);
 
       Map<String, Object> jsonContent = new TreeMap<String, Object>();
       jsonContent.put("configurations", configurations);
@@ -308,7 +309,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
         throw new SystemException("Failed to write configurations to json file ", e);
       }
 
-      String cmd = "ambari-python-wrap " + commandScriptAbsolute + " generate_configs " +
jsonFileName.getAbsolutePath() + " " +
+      String cmd = pythonCmd + " " + commandScriptAbsolute + " generate_configs " + jsonFileName.getAbsolutePath()
+ " " +
               packageFolderAbsolute + " " + TMP_PATH + File.separator + "structured-out.json"
+ " INFO " + TMP_PATH;
 
       try {
@@ -331,6 +332,8 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
       throw new SystemException("Controller error ", e);
     }
 
+    Resource resource = new ResourceImpl(Resource.Type.ClientConfig);
+    resources.add(resource);
     return resources;
   }
 


Mime
View raw message