ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject [04/50] ambari git commit: AMBARI-20596. Cleanup temporary files needed for downloading client configurations response (Attila Magyar via adoroszlai)
Date Thu, 06 Apr 2017 18:57:53 GMT
AMBARI-20596. Cleanup temporary files needed for downloading client configurations response
(Attila Magyar via adoroszlai)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fc36391c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fc36391c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fc36391c

Branch: refs/heads/branch-feature-AMBARI-12556
Commit: fc36391cf6e2b6aec04476af46eceaf91ceed01b
Parents: 3bab212
Author: Attila Magyar <amagyar@hortonworks.com>
Authored: Mon Apr 3 12:05:16 2017 +0200
Committer: Attila Doroszlai <adoroszlai@hortonworks.com>
Committed: Mon Apr 3 12:05:16 2017 +0200

----------------------------------------------------------------------
 .../libraries/script/script.py                  |  8 +++--
 .../internal/ClientConfigResourceProvider.java  | 35 +++++++++++++++-----
 .../ClientConfigResourceProviderTest.java       | 11 +++---
 .../stacks/2.0.6/HDFS/test_hdfs_client.py       | 11 +++---
 4 files changed, 44 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/fc36391c/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py
b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index fad14fd..5fa9ec4 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -874,6 +874,7 @@ class Script(object):
     Directory(self.get_tmp_dir(), create_parents = True)
 
     conf_tmp_dir = tempfile.mkdtemp(dir=self.get_tmp_dir())
+    os.chmod(conf_tmp_dir, 0700)
     output_filename = os.path.join(self.get_tmp_dir(), config['commandParams']['output_file'])
 
     try:
@@ -881,22 +882,23 @@ class Script(object):
         for filename, dict in file_dict.iteritems():
           XmlConfig(filename,
                     conf_dir=conf_tmp_dir,
-                    mode=0644,
+                    mode=0600,
                     **self.generate_configs_get_xml_file_content(filename, dict)
           )
       for file_dict in env_configs_list:
         for filename,dicts in file_dict.iteritems():
           File(os.path.join(conf_tmp_dir, filename),
-               mode=0644,
+               mode=0600,
                content=InlineTemplate(self.generate_configs_get_template_file_content(filename,
dicts)))
 
       for file_dict in properties_configs_list:
         for filename, dict in file_dict.iteritems():
           PropertiesFile(os.path.join(conf_tmp_dir, filename),
-                         mode=0644,
+                         mode=0600,
                          properties=self.generate_configs_get_xml_file_dict(filename, dict)
           )
       with closing(tarfile.open(output_filename, "w:gz")) as tar:
+        os.chmod(output_filename, 0600)
         try:
           tar.add(conf_tmp_dir, arcname=os.path.basename("."))
         finally:

http://git-wip-us.apache.org/repos/asf/ambari/blob/fc36391c/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
index e98c062..e42bd45 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
@@ -213,6 +213,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
     String TMP_PATH = configMap.get(Configuration.SERVER_TMP_DIR.getKey());
     String pythonCmd = configMap.get(Configuration.AMBARI_PYTHON_WRAP.getKey());
     List<String> pythonCompressFilesCmds = new ArrayList<>();
+    List<File> commandFiles = new ArrayList<>();
 
     for (ServiceComponentHostResponse response : componentMap.values()){
 
@@ -444,8 +445,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
         jsonContent.put("clusterName", cluster.getClusterName());
         jsonConfigurations = gson.toJson(jsonContent);
 
-        File jsonFileName = new File(TMP_PATH + File.separator + componentName + "-configuration.json");
-        File tmpDirectory = new File(jsonFileName.getParent());
+        File tmpDirectory = new File(TMP_PATH);
         if (!tmpDirectory.exists()) {
           try {
             tmpDirectory.mkdirs();
@@ -455,22 +455,33 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
             throw new SystemException("Failed to get temporary directory to store configurations",
se);
           }
         }
+        File jsonFile = File.createTempFile(componentName, "-configuration.json", tmpDirectory);
+        try {
+          jsonFile.setWritable(true, true);
+          jsonFile.setReadable(true, true);
+        } catch (SecurityException e) {
+          throw new SystemException("Failed to set permission", e);
+        }
+
         PrintWriter printWriter = null;
         try {
-          printWriter = new PrintWriter(jsonFileName.getAbsolutePath());
+          printWriter = new PrintWriter(jsonFile.getAbsolutePath());
           printWriter.print(jsonConfigurations);
           printWriter.close();
         } catch (FileNotFoundException e) {
           throw new SystemException("Failed to write configurations to json file ", e);
         }
 
-        String cmd = pythonCmd + " " + commandScriptAbsolute + " generate_configs " + jsonFileName.getAbsolutePath()
+ " " +
+        String cmd = pythonCmd + " " + commandScriptAbsolute + " generate_configs " + jsonFile.getAbsolutePath()
+ " " +
           packageFolderAbsolute + " " + TMP_PATH + File.separator + "structured-out.json"
+ " INFO " + TMP_PATH;
 
+        commandFiles.add(jsonFile);
         pythonCompressFilesCmds.add(cmd);
 
       } catch (AmbariException e) {
         throw new SystemException("Controller error ", e);
+      } catch (IOException e) {
+        throw new SystemException("Controller error ", e);
       }
     }
 
@@ -483,11 +494,17 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
     ExecutorService processExecutor = Executors.newFixedThreadPool(threadPoolSize);
 
     // put all threads that starts process to compress each component config files in the
executor
-    List<CommandLineThreadWrapper> pythonCmdThreads = executeCommands(processExecutor,
pythonCompressFilesCmds);
-
-    // wait for all threads to finish
-    Integer timeout =  configs.getExternalScriptTimeout();
-    waitForAllThreadsToJoin(processExecutor, pythonCmdThreads, timeout);
+    try {
+      List<CommandLineThreadWrapper> pythonCmdThreads = executeCommands(processExecutor,
pythonCompressFilesCmds);
+
+      // wait for all threads to finish
+      Integer timeout = configs.getExternalScriptTimeout();
+      waitForAllThreadsToJoin(processExecutor, pythonCmdThreads, timeout);
+    } finally {
+      for (File each : commandFiles) {
+        each.delete();
+      }
+    }
 
     if (StringUtils.isEmpty(requestComponentName)) {
       TarUtils tarUtils;

http://git-wip-us.apache.org/repos/asf/ambari/blob/fc36391c/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java
index 8f0c66f..c2ee4d6 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java
@@ -19,6 +19,7 @@
 package org.apache.ambari.server.controller.internal;
 
 import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.anyString;
 import static org.easymock.EasyMock.createMock;
 import static org.easymock.EasyMock.createNiceMock;
 import static org.easymock.EasyMock.expect;
@@ -26,7 +27,6 @@ import static org.easymock.EasyMock.expectLastCall;
 import static org.easymock.EasyMock.replay;
 import static org.easymock.EasyMock.verify;
 import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
 import static org.powermock.api.mockito.PowerMockito.whenNew;
 
 import java.io.ByteArrayInputStream;
@@ -359,6 +359,9 @@ public class ClientConfigResourceProviderTest {
     expect(configHelper.createUserGroupsMap(stackId, cluster, desiredConfigMap)).andReturn(userGroupsMap).anyTimes();
 
     PowerMock.expectNew(File.class, new Class<?>[]{String.class}, anyObject(String.class)).andReturn(newFile).anyTimes();
+    PowerMock.mockStatic(File.class);
+    expect(File.createTempFile(anyString(), anyString(), anyObject(File.class))).andReturn(newFile);
+
     String commandLine = "ambari-python-wrap /tmp/stacks/S1/V1/PIG/package/null generate_configs
"+newFile  +
       " /tmp/stacks/S1/V1/PIG/package /var/lib/ambari-server/tmp/structured-out.json " +
             "INFO /var/lib/ambari-server/tmp";
@@ -397,8 +400,7 @@ public class ClientConfigResourceProviderTest {
 
     Set<Resource> resources = provider.getResources(request, predicate);
     assertFalse(resources.isEmpty());
-    String str = FileUtils.readFileToString(newFile);
-    assertTrue(str.contains("\"user_groups\":\"{\\\"hdfsUser\\\":[\\\"hdfsGroup\\\"]}"));
+    assertFalse(newFile.exists());
 
     // verify
     verify(managementController, clusters, cluster, ambariMetaInfo, stackId, componentInfo,commandScriptDefinition,
@@ -574,8 +576,9 @@ public class ClientConfigResourceProviderTest {
     userSet.add("hdfs");
     expect(configHelper.getPropertyValuesWithPropertyType(stackId, PropertyInfo.PropertyType.USER,
cluster, desiredConfigMap)).andReturn(userSet);
     PowerMock.expectNew(File.class, new Class<?>[]{String.class}, anyObject(String.class)).andReturn(mockFile).anyTimes();
+    PowerMock.mockStatic(File.class);
+    expect(File.createTempFile(anyString(), anyString(), anyObject(File.class))).andReturn(PowerMock.createNiceMock(File.class));
     PowerMock.createNiceMockAndExpectNew(PrintWriter.class, anyObject());
-    expect(mockFile.getParent()).andReturn("");
     PowerMock.mockStatic(Runtime.class);
     expect(mockFile.exists()).andReturn(true);
     String commandLine = "ambari-python-wrap " + commonServicesPath + "/PIG/package/null
generate_configs null " +

http://git-wip-us.apache.org/repos/asf/ambari/blob/fc36391c/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
index b2636ab..85098fa 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
@@ -32,6 +32,7 @@ from resource_management.libraries.script.script import Script
 @patch.object(tempfile,"mkdtemp", new = MagicMock(return_value='/tmp/123'))
 @patch.object(contextlib,"closing", new = MagicMock())
 @patch("os.path.exists", new = MagicMock(return_value=True))
+@patch("os.chmod", new = MagicMock(return_value=True))
 class Test(RMFTestCase):
   COMMON_SERVICES_PACKAGE_DIR = "HDFS/2.1.0.2.0/package"
   STACK_VERSION = "2.0.6"
@@ -49,25 +50,25 @@ class Test(RMFTestCase):
                               )
     self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
                               conf_dir = '/tmp/123',
-                              mode=0644,
+                              mode=0600,
                               configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site'],
                               configurations = self.getConfig()['configurations']['hdfs-site'],
                               )
     self.assertResourceCalled('File', '/tmp/123/hadoop-env.sh',
-                              mode=0644,
+                              mode=0600,
                               content = InlineTemplate(self.getConfig()['configurations']['hadoop-env']['content']),
                               )
     self.assertResourceCalled('File', '/tmp/123/log4j.properties',
-                              mode=0644,
+                              mode=0600,
                               content = InlineTemplate(self.getConfig()['configurations']['hdfs-log4j']['content']+
                                                        self.getConfig()['configurations']['yarn-log4j']['content']),
                               )
     self.assertResourceCalled('PropertiesFile', '/tmp/123/runtime.properties',
-                              mode=0644,
+                              mode=0600,
                               properties = UnknownConfigurationMock(),
     )
     self.assertResourceCalled('PropertiesFile', '/tmp/123/startup.properties',
-                              mode=0644,
+                              mode=0600,
                               properties = UnknownConfigurationMock(),
     )
     self.assertResourceCalled('Directory', '/tmp/123',


Mime
View raw message