ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aonis...@apache.org
Subject [5/5] ambari git commit: AMBARI-11034. Implement HdfsResouces and it's usage (aonishuk)
Date Tue, 12 May 2015 14:53:46 GMT
AMBARI-11034. Implement HdfsResouces and it's usage (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c2f35d48
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c2f35d48
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c2f35d48

Branch: refs/heads/trunk
Commit: c2f35d48976aa02d73a34a56be256dbd8946fa23
Parents: 6f6c383
Author: Andrew Onishuk <aonishuk@hortonworks.com>
Authored: Tue May 12 17:51:33 2015 +0300
Committer: Andrew Onishuk <aonishuk@hortonworks.com>
Committed: Tue May 12 17:52:50 2015 +0300

----------------------------------------------------------------------
 ambari-agent/pom.xml                            |   7 +
 .../resource_management/TestContentSources.py   |   6 +-
 .../resource_management/TestCopyFromLocal.py    |  68 ---
 .../TestPropertiesFileResource.py               |  10 +-
 .../TestRepositoryResource.py                   |   6 +-
 .../TestXmlConfigResource.py                    |   8 +-
 .../python/resource_management/core/source.py   |  11 +-
 .../libraries/functions/__init__.py             |   1 -
 .../libraries/functions/get_hdp_version.py      |   8 +-
 .../libraries/providers/__init__.py             |   3 +-
 .../libraries/providers/copy_from_local.py      |  94 ----
 .../libraries/providers/hdfs_directory.py       | 112 -----
 .../libraries/providers/hdfs_resource.py        | 118 +++++
 .../libraries/resources/__init__.py             |   3 +-
 .../libraries/resources/copy_from_local.py      |  41 --
 .../libraries/resources/hdfs_directory.py       |  45 --
 .../libraries/resources/hdfs_resource.py        |  76 ++++
 .../libraries/script/script.py                  |   2 +
 .../1.6.1.2.2.0/package/scripts/accumulo.py     |   6 +-
 .../package/scripts/accumulo_configuration.py   |  14 +-
 .../1.6.1.2.2.0/package/scripts/params.py       |  17 +-
 .../0.1.0/package/scripts/hbase.py              |  14 +-
 .../0.1.0/package/scripts/params.py             |  16 +-
 .../FALCON/0.5.0.2.1/package/scripts/falcon.py  |  23 +-
 .../0.5.0.2.1/package/scripts/params_linux.py   |  20 +-
 .../HBASE/0.96.0.2.0/package/scripts/hbase.py   |  14 +-
 .../0.96.0.2.0/package/scripts/params_linux.py  |  22 +-
 .../0.96.0.2.0/package/scripts/service_check.py |   1 -
 .../package/files/fast-hdfs-resource.jar        | Bin 0 -> 19285282 bytes
 .../HDFS/2.1.0.2.0/package/scripts/hdfs.py      |   5 +
 .../2.1.0.2.0/package/scripts/hdfs_namenode.py  |  15 +-
 .../package/scripts/hdfs_nfsgateway.py          |   1 -
 .../2.1.0.2.0/package/scripts/hdfs_snamenode.py |   1 -
 .../2.1.0.2.0/package/scripts/params_linux.py   |  20 +-
 .../2.1.0.2.0/package/scripts/service_check.py  |  53 +--
 .../0.12.0.2.0/package/files/templetonSmoke.sh  |  22 +-
 .../HIVE/0.12.0.2.0/package/scripts/hive.py     | 116 +++--
 .../0.12.0.2.0/package/scripts/hive_server.py   |  13 +-
 .../0.12.0.2.0/package/scripts/params_linux.py  |  64 ++-
 .../HIVE/0.12.0.2.0/package/scripts/webhcat.py  |  71 ---
 .../package/scripts/webhcat_service_check.py    |  28 +-
 .../package/templates/templeton_smoke.pig.j2    |  24 +
 .../MAHOUT/1.0.0.2.3/package/scripts/params.py  |  17 +-
 .../1.0.0.2.3/package/scripts/service_check.py  |  49 +-
 .../4.0.0.2.0/package/files/oozieSmoke2.sh      |  52 +--
 .../files/prepareOozieHdfsDirectories.sh        |  45 ++
 .../OOZIE/4.0.0.2.0/package/scripts/oozie.py    |   6 +-
 .../package/scripts/oozie_server_upgrade.py     |  16 +-
 .../4.0.0.2.0/package/scripts/params_linux.py   |  22 +-
 .../4.0.0.2.0/package/scripts/service_check.py  |  40 +-
 .../0.12.0.2.0/package/scripts/params_linux.py  |  25 +-
 .../0.12.0.2.0/package/scripts/service_check.py |  67 +--
 .../package/scripts/job_history_server.py       |  10 +-
 .../SPARK/1.2.0.2.2/package/scripts/params.py   |  21 +-
 .../1.2.0.2.2/package/scripts/setup_spark.py    |   6 +-
 .../1.2.0.2.2/package/scripts/spark_service.py  |   9 +-
 .../STORM/0.9.1.2.1/configuration/storm-env.xml |  12 +-
 .../0.4.0.2.1/package/scripts/params_linux.py   |  25 +-
 .../0.4.0.2.1/package/scripts/service_check.py  |  53 +--
 .../2.1.0.2.0/package/scripts/historyserver.py  |  25 +-
 .../2.1.0.2.0/package/scripts/install_jars.py   |  68 ++-
 .../package/scripts/mapred_service_check.py     |  27 +-
 .../2.1.0.2.0/package/scripts/params_linux.py   |  32 +-
 .../package/scripts/resourcemanager.py          |  14 +-
 .../2.1.0.2.0/package/scripts/service_check.py  |   2 +-
 .../YARN/2.1.0.2.0/package/scripts/yarn.py      |  38 +-
 .../stacks/2.0.6/HBASE/test_hbase_master.py     | 192 ++++----
 .../2.0.6/HBASE/test_hbase_regionserver.py      |  91 ----
 .../python/stacks/2.0.6/HDFS/test_datanode.py   |   6 +
 .../stacks/2.0.6/HDFS/test_journalnode.py       |   6 +
 .../python/stacks/2.0.6/HDFS/test_namenode.py   | 448 ++++++++++---------
 .../python/stacks/2.0.6/HDFS/test_nfsgateway.py |   6 +
 .../stacks/2.0.6/HDFS/test_service_check.py     |  59 +--
 .../python/stacks/2.0.6/HDFS/test_snamenode.py  |   6 +
 .../test/python/stacks/2.0.6/HDFS/test_zkfc.py  |  12 +
 .../stacks/2.0.6/HIVE/test_hive_server.py       | 329 +++++++++-----
 .../2.0.6/HIVE/test_hive_service_check.py       | 106 ++++-
 .../stacks/2.0.6/HIVE/test_webhcat_server.py    | 134 ------
 .../stacks/2.0.6/OOZIE/test_oozie_server.py     | 178 ++++++--
 .../stacks/2.0.6/OOZIE/test_service_check.py    |  45 +-
 .../stacks/2.0.6/PIG/test_pig_service_check.py  |  84 +++-
 .../stacks/2.0.6/YARN/test_historyserver.py     | 282 ++++++------
 .../stacks/2.0.6/YARN/test_mapreduce2_client.py |   4 +
 .../2.0.6/YARN/test_mapreduce2_service_check.py |  64 +--
 .../stacks/2.0.6/YARN/test_nodemanager.py       | 138 +-----
 .../stacks/2.0.6/YARN/test_resourcemanager.py   |  40 +-
 .../stacks/2.0.6/YARN/test_yarn_client.py       |   4 +
 .../python/stacks/2.0.6/configs/default.json    |  13 +-
 .../python/stacks/2.0.6/configs/secured.json    |  13 +-
 .../stacks/2.1/FALCON/test_falcon_server.py     |  47 +-
 .../stacks/2.1/FALCON/test_service_check.py     |   2 +-
 .../python/stacks/2.1/TEZ/test_service_check.py | 100 ++---
 .../stacks/2.1/YARN/test_apptimelineserver.py   |   1 +
 .../stacks/2.2/PIG/test_pig_service_check.py    | 121 +++--
 .../stacks/2.2/SPARK/test_job_history_server.py |  88 +++-
 .../test/python/stacks/2.2/configs/default.json |  13 +-
 .../test/python/stacks/2.2/configs/secured.json |  13 +-
 .../2.3/MAHOUT/test_mahout_service_check.py     |  63 +--
 .../dependency-reduced-pom.xml                  |  42 ++
 contrib/fast-hdfs-resource/pom.xml              |  86 ++++
 .../fast-hdfs-resource/resources/example.json   |  57 +++
 .../resources/test_perfomance.sh                |  46 ++
 .../ambari/fast_hdfs_resource/Resource.java     | 295 ++++++++++++
 .../ambari/fast_hdfs_resource/Runner.java       |  93 ++++
 104 files changed, 3063 insertions(+), 2144 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-agent/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-agent/pom.xml b/ambari-agent/pom.xml
index 4dde1d9..b807ba9 100644
--- a/ambari-agent/pom.xml
+++ b/ambari-agent/pom.xml
@@ -426,6 +426,12 @@
                 </source>
               </sources>
             </mapping>
+            <mapping>
+              <directory>/var/lib/ambari-agent/lib</directory>
+              <filemode>755</filemode>
+              <username>root</username>
+              <groupname>root</groupname>
+            </mapping>
           </mappings>
         </configuration>
       </plugin>
@@ -551,6 +557,7 @@
                 <path>/var/lib/${project.artifactId}/data/tmp</path>
                 <path>/var/lib/${project.artifactId}/keys</path>
                 <path>${package.log.dir}</path>
+                <path>/var/lib/${project.artifactId}/lib</path>
               </paths>
               <mapper>
                 <type>perm</type>

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-agent/src/test/python/resource_management/TestContentSources.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestContentSources.py b/ambari-agent/src/test/python/resource_management/TestContentSources.py
index d35ec9a..c144cf9 100644
--- a/ambari-agent/src/test/python/resource_management/TestContentSources.py
+++ b/ambari-agent/src/test/python/resource_management/TestContentSources.py
@@ -221,7 +221,7 @@ class TestContentSources(TestCase):
       content = template.get_content()
     self.assertEqual(open_mock.call_count, 1)
 
-    self.assertEqual(u'test template content\n', content)
+    self.assertEqual(u'test template content', content)
     open_mock.assert_called_with('/absolute/path/test.j2', 'rb')
     self.assertEqual(getmtime_mock.call_count, 1)
     getmtime_mock.assert_called_with('/absolute/path/test.j2')
@@ -234,7 +234,7 @@ class TestContentSources(TestCase):
       template = InlineTemplate("{{test_arg1}} template content", [], test_arg1 = "test")
       content = template.get_content()
 
-    self.assertEqual(u'test template content\n', content)
+    self.assertEqual(u'test template content', content)
 
   def test_template_imports(self):
     """
@@ -250,4 +250,4 @@ class TestContentSources(TestCase):
     with Environment("/base") as env:
       template = InlineTemplate("{{test_arg1}} template content {{os.path.join(path[0],path[1])}}", [os], test_arg1 = "test", path = ["/one","two"])
       content = template.get_content()
-    self.assertEqual(u'test template content /one/two\n', content)
+    self.assertEqual(u'test template content /one/two', content)

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py b/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py
deleted file mode 100644
index 1ec1858..0000000
--- a/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py
+++ /dev/null
@@ -1,68 +0,0 @@
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-from unittest import TestCase
-from mock.mock import patch, MagicMock
-from resource_management import *
-from resource_management.core import shell
-
-@patch.object(shell, "call", new = MagicMock(return_value=(1, "")))
-@patch.object(System, "os_family", new = 'redhat')
-class TestCopyFromLocal(TestCase):
-
-  @patch("resource_management.libraries.providers.execute_hadoop.ExecuteHadoopProvider")
-  def test_run_default_args(self, execute_hadoop_mock):
-    with Environment() as env:
-      CopyFromLocal('/user/testdir/*.files',
-        owner='user1',
-        dest_dir='/apps/test/',
-        kinnit_if_needed='',
-        hdfs_user='hdfs'
-      )
-      self.assertEqual(execute_hadoop_mock.call_count, 2)
-      call_arg_list = execute_hadoop_mock.call_args_list
-      self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/',
-                       call_arg_list[0][0][0].command)
-      print call_arg_list[0][0][0].arguments
-      self.assertEquals({'not_if': "ambari-sudo.sh su user1 -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]PATH=$PATH:/usr/bin hadoop fs -ls /apps/test//*.files'", 'bin_dir': '/usr/bin', 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
-                        call_arg_list[0][0][0].arguments)
-      self.assertEquals('fs -chown user1 /apps/test//*.files', call_arg_list[1][0][0].command)
-      self.assertEquals({'user': 'hdfs', 'bin_dir': '/usr/bin', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments)
-
-
-  @patch("resource_management.libraries.providers.execute_hadoop.ExecuteHadoopProvider")
-  def test_run_with_chmod(self, execute_hadoop_mock):
-    with Environment() as env:
-      CopyFromLocal('/user/testdir/*.files',
-        mode=0655,
-        owner='user1',
-        group='hdfs',
-        dest_dir='/apps/test/',
-        kinnit_if_needed='',
-        hdfs_user='hdfs'
-      )
-      self.assertEqual(execute_hadoop_mock.call_count, 3)
-      call_arg_list = execute_hadoop_mock.call_args_list
-      self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/',
-                       call_arg_list[0][0][0].command)
-      self.assertEquals({'not_if': "ambari-sudo.sh su user1 -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]PATH=$PATH:/usr/bin hadoop fs -ls /apps/test//*.files'", 'bin_dir': '/usr/bin', 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
-                        call_arg_list[0][0][0].arguments)
-      self.assertEquals('fs -chown user1:hdfs /apps/test//*.files', call_arg_list[1][0][0].command)
-      self.assertEquals({'user': 'hdfs', 'bin_dir': '/usr/bin', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments)
-
-

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-agent/src/test/python/resource_management/TestPropertiesFileResource.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestPropertiesFileResource.py b/ambari-agent/src/test/python/resource_management/TestPropertiesFileResource.py
index bb91159..5c97380 100644
--- a/ambari-agent/src/test/python/resource_management/TestPropertiesFileResource.py
+++ b/ambari-agent/src/test/python/resource_management/TestPropertiesFileResource.py
@@ -65,7 +65,7 @@ class TestPropertiesFIleResource(TestCase):
                      properties={}
       )
 
-    create_file_mock.assert_called_with('/somewhere_in_system/one_file.properties', u'# Generated by Apache Ambari. Today is Wednesday\n    \n    \n', encoding=None)
+    create_file_mock.assert_called_with('/somewhere_in_system/one_file.properties', u'# Generated by Apache Ambari. Today is Wednesday\n    \n    ', encoding=None)
     ensure_mock.assert_called()
 
 
@@ -98,7 +98,7 @@ class TestPropertiesFIleResource(TestCase):
                      properties={},
       )
 
-    create_file_mock.assert_called_with('/dir/and/dir/file.txt', u'# Generated by Apache Ambari. Some other day\n    \n    \n', encoding=None)
+    create_file_mock.assert_called_with('/dir/and/dir/file.txt', u'# Generated by Apache Ambari. Some other day\n    \n    ', encoding=None)
     ensure_mock.assert_called()
 
 
@@ -131,7 +131,7 @@ class TestPropertiesFIleResource(TestCase):
                      properties={'property1': 'value1'},
       )
 
-    create_file_mock.assert_called_with('/dir/new_file', u'# Generated by Apache Ambari. 777\n    \nproperty1=value1\n    \n', encoding=None)
+    create_file_mock.assert_called_with('/dir/new_file', u'# Generated by Apache Ambari. 777\n    \nproperty1=value1\n    ', encoding=None)
     ensure_mock.assert_called()
 
 
@@ -169,7 +169,7 @@ class TestPropertiesFIleResource(TestCase):
                      },
       )
 
-    create_file_mock.assert_called_with('/dir/new_file', u"# Generated by Apache Ambari. 777\n    \n=\nprop.1='.'yyyy-MM-dd-HH\nprop.2=INFO, openjpa\nprop.3=%d{ISO8601} %5p %c{1}:%L - %m%n\nprop.4=${oozie.log.dir}/oozie.log\nprop.empty=\n    \n", encoding=None)
+    create_file_mock.assert_called_with('/dir/new_file', u"# Generated by Apache Ambari. 777\n    \n=\nprop.1='.'yyyy-MM-dd-HH\nprop.2=INFO, openjpa\nprop.3=%d{ISO8601} %5p %c{1}:%L - %m%n\nprop.4=${oozie.log.dir}/oozie.log\nprop.empty=\n    ", encoding=None)
     ensure_mock.assert_called()
 
 
@@ -206,5 +206,5 @@ class TestPropertiesFIleResource(TestCase):
       )
 
     read_file_mock.assert_called()
-    create_file_mock.assert_called_with('/dir1/new_file', u'# Generated by Apache Ambari. 777\n    \nproperty_1=value1\n    \n', encoding=None)
+    create_file_mock.assert_called_with('/dir1/new_file', u'# Generated by Apache Ambari. 777\n    \nproperty_1=value1\n    ', encoding=None)
     ensure_mock.assert_called()

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-agent/src/test/python/resource_management/TestRepositoryResource.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestRepositoryResource.py b/ambari-agent/src/test/python/resource_management/TestRepositoryResource.py
index 6b94481..c31cc20 100644
--- a/ambari-agent/src/test/python/resource_management/TestRepositoryResource.py
+++ b/ambari-agent/src/test/python/resource_management/TestRepositoryResource.py
@@ -164,7 +164,7 @@ class TestRepositoryResource(TestCase):
       template_content = call_content[1]['content']
       
       self.assertEquals(template_name, '/tmp/1.txt')
-      self.assertEquals(template_content, 'deb http://download.base_url.org/rpm/ a b c\n')
+      self.assertEquals(template_content, 'deb http://download.base_url.org/rpm/ a b c')
       
       copy_item = str(file_mock.call_args_list[1])
       self.assertEqual(copy_item, "call('/etc/apt/sources.list.d/HDP.list', content=StaticFile('/tmp/1.txt'))")
@@ -205,7 +205,7 @@ class TestRepositoryResource(TestCase):
       template_content = call_content[1]['content']
 
       self.assertEquals(template_name, '/tmp/1.txt')
-      self.assertEquals(template_content, 'deb http://download.base_url.org/rpm/ a b c\n')
+      self.assertEquals(template_content, 'deb http://download.base_url.org/rpm/ a b c')
 
       copy_item = str(file_mock.call_args_list[1])
       self.assertEqual(copy_item, "call('/etc/apt/sources.list.d/HDP.list', content=StaticFile('/tmp/1.txt'))")
@@ -239,7 +239,7 @@ class TestRepositoryResource(TestCase):
       template_content = call_content[1]['content']
       
       self.assertEquals(template_name, '/tmp/1.txt')
-      self.assertEquals(template_content, 'deb http://download.base_url.org/rpm/ a b c\n')
+      self.assertEquals(template_content, 'deb http://download.base_url.org/rpm/ a b c')
       
       self.assertEqual(file_mock.call_count, 1)
       self.assertEqual(execute_mock.call_count, 0)

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py b/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py
index a7eaae9..6092717 100644
--- a/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py
+++ b/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py
@@ -62,7 +62,7 @@ class TestXmlConfigResource(TestCase):
                 configuration_attributes={}
                 )
 
-    create_file_mock.assert_called_with('/dir/conf/file.xml', u'<!--Wed 2014-02-->\n    <configuration>\n    \n  </configuration>\n', encoding='UTF-8')
+    create_file_mock.assert_called_with('/dir/conf/file.xml', u'<!--Wed 2014-02-->\n    <configuration>\n    \n  </configuration>', encoding='UTF-8')
 
 
   @patch("resource_management.core.providers.system._ensure_metadata")
@@ -91,7 +91,7 @@ class TestXmlConfigResource(TestCase):
                 configuration_attributes={'attr': {'property1': 'attr_value'}}
                 )
 
-    create_file_mock.assert_called_with('/dir/conf/file.xml', u'<!--Wed 2014-02-->\n    <configuration>\n    \n    <property>\n      <name>property1</name>\n      <value>value1</value>\n      <attr>attr_value</attr>\n    </property>\n    \n  </configuration>\n', encoding='UTF-8')
+    create_file_mock.assert_called_with('/dir/conf/file.xml', u'<!--Wed 2014-02-->\n    <configuration>\n    \n    <property>\n      <name>property1</name>\n      <value>value1</value>\n      <attr>attr_value</attr>\n    </property>\n    \n  </configuration>', encoding='UTF-8')
 
 
   @patch("resource_management.core.providers.system._ensure_metadata")
@@ -144,7 +144,7 @@ class TestXmlConfigResource(TestCase):
                     }
                 })
 
-    create_file_mock.assert_called_with('/dir/conf/file.xml', u'<!--Wed 2014-02-->\n    <configuration>\n    \n    <property>\n      <name></name>\n      <value></value>\n    </property>\n    \n    <property>\n      <name>prop.1</name>\n      <value>&#39;.&#39;yyyy-MM-dd-HH</value>\n      <attr1>x</attr1>\n    </property>\n    \n    <property>\n      <name>prop.2</name>\n      <value>INFO, openjpa</value>\n    </property>\n    \n    <property>\n      <name>prop.3</name>\n      <value>%d{ISO8601} %5p %c{1}:%L - %m%n</value>\n      <attr2>value3</attr2>\n    </property>\n    \n    <property>\n      <name>prop.4</name>\n      <value>${oozie.log.dir}/oozie.log</value>\n      <attr_value_empty></attr_value_empty>\n      <attr2>value4</attr2>\n    </property>\n    \n    <property>\n      <name>prop.empty</name>\n      <value></value>\n      <attr_value_empty></attr_value_empty>\n    </property>\n    \n  </configuration>\n', encoding='UTF-8')
+    create_file_mock.assert_called_with('/dir/conf/file.xml', u'<!--Wed 2014-02-->\n    <configuration>\n    \n    <property>\n      <name></name>\n      <value></value>\n    </property>\n    \n    <property>\n      <name>prop.1</name>\n      <value>&#39;.&#39;yyyy-MM-dd-HH</value>\n      <attr1>x</attr1>\n    </property>\n    \n    <property>\n      <name>prop.2</name>\n      <value>INFO, openjpa</value>\n    </property>\n    \n    <property>\n      <name>prop.3</name>\n      <value>%d{ISO8601} %5p %c{1}:%L - %m%n</value>\n      <attr2>value3</attr2>\n    </property>\n    \n    <property>\n      <name>prop.4</name>\n      <value>${oozie.log.dir}/oozie.log</value>\n      <attr_value_empty></attr_value_empty>\n      <attr2>value4</attr2>\n    </property>\n    \n    <property>\n      <name>prop.empty</name>\n      <value></value>\n      <attr_value_empty></attr_value_empty>\n    </property>\n    \n  </configuration>', encoding='UTF-8')
 
   @patch("resource_management.core.providers.system._ensure_metadata")
   @patch.object(sudo, "create_file")
@@ -177,7 +177,7 @@ class TestXmlConfigResource(TestCase):
                 configuration_attributes={}
                 )
 
-    create_file_mock.assert_called_with('/dir/conf/file.xml', u'<!--Wed 2014-02-->\n    <configuration>\n    \n    <property>\n      <name></name>\n      <value></value>\n    </property>\n    \n    <property>\n      <name>first</name>\n      <value>should be first</value>\n    </property>\n    \n    <property>\n      <name>second</name>\n      <value>should be second</value>\n    </property>\n    \n    <property>\n      <name>third</name>\n      <value>should be third</value>\n    </property>\n    \n    <property>\n      <name>z_last</name>\n      <value>should be last</value>\n    </property>\n    \n  </configuration>\n', encoding='UTF-8')
+    create_file_mock.assert_called_with('/dir/conf/file.xml', u'<!--Wed 2014-02-->\n    <configuration>\n    \n    <property>\n      <name></name>\n      <value></value>\n    </property>\n    \n    <property>\n      <name>first</name>\n      <value>should be first</value>\n    </property>\n    \n    <property>\n      <name>second</name>\n      <value>should be second</value>\n    </property>\n    \n    <property>\n      <name>third</name>\n      <value>should be third</value>\n    </property>\n    \n    <property>\n      <name>z_last</name>\n      <value>should be last</value>\n    </property>\n    \n  </configuration>', encoding='UTF-8')
 
   @patch("resource_management.libraries.providers.xml_config.File")
   @patch.object(sudo, "path_exists")

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-common/src/main/python/resource_management/core/source.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/source.py b/ambari-common/src/main/python/resource_management/core/source.py
index 9d1fc76..4eecc7d 100644
--- a/ambari-common/src/main/python/resource_management/core/source.py
+++ b/ambari-common/src/main/python/resource_management/core/source.py
@@ -141,7 +141,7 @@ else:
       self.context.update(variables)
       
       rendered = self.template.render(self.context)
-      return rendered + "\n" if not rendered.endswith('\n') else rendered
+      return rendered
     
   class InlineTemplate(Template):
     def __init__(self, name, extra_imports=[], **kwargs):
@@ -189,9 +189,14 @@ class DownloadSource(Source):
         opener = urllib2.build_opener()
       
       req = urllib2.Request(self.url)
-      web_file = opener.open(req)
+      
+      try:
+        web_file = opener.open(req)
+      except urllib2.HTTPError as ex:
+        raise Fail("Failed to download file from {0} due to HTTP error: {1}".format(self.url, str(ex)))
+      
       content = web_file.read()
-
+      
       if self.cache:
         with open(filepath, 'w') as fp:
           fp.write(content)

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py b/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
index e06d246..ae6e47b 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
@@ -40,7 +40,6 @@ from resource_management.libraries.functions.format_jvm_option import *
 from resource_management.libraries.functions.constants import *
 from resource_management.libraries.functions.get_hdp_version import *
 from resource_management.libraries.functions.get_lzo_packages import *
-from resource_management.libraries.functions.dynamic_variable_interpretation import *
 from resource_management.libraries.functions.setup_ranger_plugin import *
 
 IS_WINDOWS = platform.system() == "Windows"

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-common/src/main/python/resource_management/libraries/functions/get_hdp_version.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_hdp_version.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_hdp_version.py
index ed34c02..ec30dc4 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/get_hdp_version.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/get_hdp_version.py
@@ -32,6 +32,11 @@ def get_hdp_version(package_name):
   """
   @param package_name, name of the package, from which, function will try to get hdp version
   """
+  
+  if not os.path.exists("/usr/bin/hdp-select"):
+    Logger.info('Skipping get_hdp_version since hdp-select is not yet available')
+    return None # lazy fail
+  
   try:
     command = 'hdp-select status ' + package_name
     return_code, hdp_output = shell.call(command, timeout=20)
@@ -48,6 +53,7 @@ def get_hdp_version(package_name):
   match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)
 
   if match is None:
-    raise Fail('Failed to get extracted version')
+    Logger.info('Failed to get extracted version with hdp-select')
+    return None # lazy fail
 
   return hdp_version

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-common/src/main/python/resource_management/libraries/providers/__init__.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/__init__.py b/ambari-common/src/main/python/resource_management/libraries/providers/__init__.py
index 34b10a9..44e9ca1 100644
--- a/ambari-common/src/main/python/resource_management/libraries/providers/__init__.py
+++ b/ambari-common/src/main/python/resource_management/libraries/providers/__init__.py
@@ -42,8 +42,7 @@ PROVIDERS = dict(
     XmlConfig="resource_management.libraries.providers.xml_config.XmlConfigProvider",
     PropertiesFile="resource_management.libraries.providers.properties_file.PropertiesFileProvider",
     MonitorWebserver="resource_management.libraries.providers.monitor_webserver.MonitorWebserverProvider",
-    HdfsDirectory="resource_management.libraries.providers.hdfs_directory.HdfsDirectoryProvider",
-    CopyFromLocal="resource_management.libraries.providers.copy_from_local.CopyFromLocalProvider",
+    HdfsResource="resource_management.libraries.providers.hdfs_resource.HdfsResourceProvider",
     ModifyPropertiesFile="resource_management.libraries.providers.modify_properties_file.ModifyPropertiesFileProvider"
   ),
 )

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py b/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py
deleted file mode 100644
index bd9805c..0000000
--- a/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-import os
-from resource_management.libraries.resources.execute_hadoop import ExecuteHadoop
-from resource_management.core.providers import Provider
-from resource_management.libraries.functions.format import format
-from resource_management.core.shell import as_user
-from resource_management.core.resources.system import Execute
-
-class CopyFromLocalProvider(Provider):
-  def action_run(self):
-
-    path = self.resource.path
-    dest_dir = self.resource.dest_dir
-    dest_file = self.resource.dest_file
-    kinnit_if_needed = self.resource.kinnit_if_needed
-    user = self.resource.user   # user to perform commands as. If not provided, default to the owner
-    owner = self.resource.owner
-    group = self.resource.group
-    mode = self.resource.mode
-    hdfs_usr=self.resource.hdfs_user
-    hadoop_conf_path = self.resource.hadoop_conf_dir
-    bin_dir = self.resource.hadoop_bin_dir
-
-
-    if dest_file:
-      copy_cmd = format("fs -copyFromLocal {path} {dest_dir}/{dest_file}")
-      dest_path = dest_dir + dest_file if dest_dir.endswith(os.sep) else dest_dir + os.sep + dest_file
-    else:
-      dest_file_name = os.path.split(path)[1]
-      copy_cmd = format("fs -copyFromLocal {path} {dest_dir}")
-      dest_path = dest_dir + os.sep + dest_file_name
-    # Need to run unless as resource user
-    
-    if kinnit_if_needed:
-      Execute(kinnit_if_needed, 
-              user=user if user else owner,
-      )
-    
-    unless_cmd = as_user(format("PATH=$PATH:{bin_dir} hadoop fs -ls {dest_path}"), user if user else owner)
-
-    ExecuteHadoop(copy_cmd,
-                  not_if=unless_cmd,
-                  user=user if user else owner,
-                  bin_dir=bin_dir,
-                  conf_dir=hadoop_conf_path
-                  )
-
-    if not owner:
-      chown = None
-    else:
-      if not group:
-        chown = owner
-      else:
-        chown = format('{owner}:{group}')
-
-    if chown:
-      chown_cmd = format("fs -chown {chown} {dest_path}")
-
-      ExecuteHadoop(chown_cmd,
-                    user=hdfs_usr,
-                    bin_dir=bin_dir,
-                    conf_dir=hadoop_conf_path)
-    pass
-
-    if mode:
-      dir_mode = oct(mode)[1:]
-      chmod_cmd = format('fs -chmod {dir_mode} {dest_path}')
-
-      ExecuteHadoop(chmod_cmd,
-                    user=hdfs_usr,
-                    bin_dir=bin_dir,
-                    conf_dir=hadoop_conf_path)
-    pass

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_directory.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_directory.py b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_directory.py
deleted file mode 100644
index 9fa2de7..0000000
--- a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_directory.py
+++ /dev/null
@@ -1,112 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-import os
-
-from resource_management import *
-directories_list = [] #direcotries list for mkdir
-chmod_map = {} #(mode,recursive):dir_list map
-chown_map = {} #(owner,group,recursive):dir_list map
-class HdfsDirectoryProvider(Provider):
-  def action_create_delayed(self):
-    global delayed_directories
-    global chmod_map
-    global chown_map
-
-    if not self.resource.dir_name:
-      return
-
-    dir_name = self.resource.dir_name
-    dir_owner = self.resource.owner
-    dir_group = self.resource.group
-    dir_mode = oct(self.resource.mode)[1:] if self.resource.mode else None
-    directories_list.append(self.resource.dir_name)
-
-    recursive_chown_str = "-R" if self.resource.recursive_chown else ""
-    recursive_chmod_str = "-R" if self.resource.recursive_chmod else ""
-    # grouping directories by mode/owner/group to modify them in one 'chXXX' call
-    if dir_mode:
-      chmod_key = (dir_mode,recursive_chmod_str)
-      if chmod_map.has_key(chmod_key):
-        chmod_map[chmod_key].append(dir_name)
-      else:
-        chmod_map[chmod_key] = [dir_name]
-
-    if dir_owner:
-      owner_key = (dir_owner,dir_group,recursive_chown_str)
-      if chown_map.has_key(owner_key):
-        chown_map[owner_key].append(dir_name)
-      else:
-        chown_map[owner_key] = [dir_name]
-
-  def action_create(self):
-    global delayed_directories
-    global chmod_map
-    global chown_map
-
-    self.action_create_delayed()
-
-    hdp_conf_dir = self.resource.conf_dir
-    hdp_hdfs_user = self.resource.hdfs_user
-    secured = self.resource.security_enabled
-    keytab_file = self.resource.keytab
-    kinit_path = self.resource.kinit_path_local
-    bin_dir = self.resource.bin_dir
-
-    chmod_commands = []
-    chown_commands = []
-
-    for chmod_key, chmod_dirs in chmod_map.items():
-      mode = chmod_key[0]
-      recursive = chmod_key[1]
-      chmod_dirs_str = ' '.join(chmod_dirs)
-      chmod_commands.append(format("hadoop --config {hdp_conf_dir} fs -chmod {recursive} {mode} {chmod_dirs_str}"))
-
-    for chown_key, chown_dirs in chown_map.items():
-      owner = chown_key[0]
-      group = chown_key[1]
-      recursive = chown_key[2]
-      chown_dirs_str = ' '.join(chown_dirs)
-      if owner:
-        chown = owner
-        if group:
-          chown = format("{owner}:{group}")
-        chown_commands.append(format("hadoop --config {hdp_conf_dir} fs -chown {recursive} {chown} {chown_dirs_str}"))
-
-    if secured:
-        Execute(format("{kinit_path} -kt {keytab_file} {hdfs_principal_name}"),
-                user=hdp_hdfs_user)
-    #create all directories in one 'mkdir' call
-    dir_list_str = ' '.join(directories_list)
-    #for hadoop 2 we need to specify -p to create directories recursively
-    parent_flag = '-p'
-
-    Execute(format('hadoop --config {hdp_conf_dir} fs -mkdir {parent_flag} {dir_list_str} && {chmod_cmd} && {chown_cmd}',
-                   chmod_cmd=' && '.join(chmod_commands),
-                   chown_cmd=' && '.join(chown_commands)),
-            user=hdp_hdfs_user,
-            path=bin_dir,
-            not_if=as_user(format("hadoop --config {hdp_conf_dir} fs -ls {dir_list_str}"), hdp_hdfs_user)
-    )
-
-    directories_list[:] = []
-    chmod_map.clear()
-    chown_map.clear()

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
new file mode 100644
index 0000000..21ad8ed
--- /dev/null
+++ b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
@@ -0,0 +1,118 @@
+# !/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+import json
+from resource_management.core.environment import Environment
+from resource_management.core.base import Fail
+from resource_management.core.resources.system import Execute
+from resource_management.core.resources.system import File
+from resource_management.core.providers import Provider
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions import format
+
+
+JSON_PATH = '/var/lib/ambari-agent/data/hdfs_resources.json'
+JAR_PATH = '/var/lib/ambari-agent/lib/fast-hdfs-resource.jar'
+
+RESOURCE_TO_JSON_FIELDS = {
+  'target': 'target',
+  'type': 'type',
+  'action': 'action',
+  'source': 'source',
+  'owner': 'owner',
+  'group': 'group',
+  'mode': 'mode',
+  'recursive_chown': 'recursiveChown',
+  'recursive_chmod': 'recursiveChmod',
+  'change_permissions_for_parents': 'changePermissionforParents'
+}
+
+
+class HdfsResourceProvider(Provider):
+  def action_delayed(self, action_name):
+    resource = {}
+    env = Environment.get_instance()
+    if not 'hdfs_files' in env.config:
+      env.config['hdfs_files'] = []
+
+    # Check required parameters
+    if not self.resource.type or not self.resource.action:
+      raise Fail("Resource parameter type or action is not set.")
+
+    # Put values in dictionary-resource
+    for field_name, json_field_name in RESOURCE_TO_JSON_FIELDS.iteritems():
+      if field_name == 'action':
+        resource[json_field_name] = action_name
+      elif field_name == 'mode' and self.resource.mode:
+        resource[json_field_name] = oct(self.resource.mode)[1:]
+      elif getattr(self.resource, field_name):
+        resource[json_field_name] = getattr(self.resource, field_name)
+
+    # Add resource to create
+    env.config['hdfs_files'].append(resource)
+
+  def action_create_on_execute(self):
+    self.action_delayed("create")
+
+  def action_delete_on_execute(self):
+    self.action_delayed("delete")
+
+  def action_execute(self):
+    env = Environment.get_instance()
+
+    # Check required parameters
+    if not self.resource.user:
+      raise Fail("Resource parameter 'user' is not set.")
+
+    if not 'hdfs_files' in env.config or not env.config['hdfs_files']:
+      Logger.info("No resources to create. 'create_on_execute' or 'delete_on_execute' wasn't triggered before this 'execute' action.")
+      return
+
+    hadoop_bin_dir = self.resource.hadoop_bin_dir
+    hadoop_conf_dir = self.resource.hadoop_conf_dir
+    user = self.resource.user
+    security_enabled = self.resource.security_enabled
+    keytab_file = self.resource.keytab
+    kinit_path = self.resource.kinit_path_local
+    logoutput = self.resource.logoutput
+    jar_path=JAR_PATH
+    json_path=JSON_PATH
+
+    if security_enabled:
+      Execute(format("{kinit_path} -kt {keytab_file} {hdfs_principal_name}"),
+              user=user
+      )
+
+    # Write json file to disk
+    File(JSON_PATH,
+         owner = user,
+         content = json.dumps(env.config['hdfs_files'])
+    )
+
+    # Execute jar to create/delete resources in hadoop
+    Execute(format("hadoop --config {hadoop_conf_dir} jar {jar_path} {json_path}"),
+            user=user,
+            path=[hadoop_bin_dir],
+            logoutput=logoutput,
+    )
+
+    # Clean
+    env.config['hdfs_files'] = []

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-common/src/main/python/resource_management/libraries/resources/__init__.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/__init__.py b/ambari-common/src/main/python/resource_management/libraries/resources/__init__.py
index 596c2e2..524292f 100644
--- a/ambari-common/src/main/python/resource_management/libraries/resources/__init__.py
+++ b/ambari-common/src/main/python/resource_management/libraries/resources/__init__.py
@@ -26,7 +26,6 @@ from resource_management.libraries.resources.xml_config import *
 from resource_management.libraries.resources.properties_file import *
 from resource_management.libraries.resources.repository import *
 from resource_management.libraries.resources.monitor_webserver import *
-from resource_management.libraries.resources.hdfs_directory import *
-from resource_management.libraries.resources.copy_from_local import *
+from resource_management.libraries.resources.hdfs_resource import *
 from resource_management.libraries.resources.msi import *
 from resource_management.libraries.resources.modify_properties_file import *
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py b/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py
deleted file mode 100644
index 42050a3..0000000
--- a/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-_all__ = ["CopyFromLocal"]
-from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument
-
-class CopyFromLocal(Resource):
-  action = ForcedListArgument(default="run")
-
-  path = ResourceArgument(default=lambda obj: obj.name)
-  dest_dir = ResourceArgument(required=True)
-  dest_file = ResourceArgument()
-  owner = ResourceArgument(required=True)  # file user owner
-  group = ResourceArgument()               # file group user
-  mode = ResourceArgument()                # file ACL mode
-  kinnit_if_needed = ResourceArgument(default='')
-  user = ResourceArgument()                # user to perform commands as. If not provided, default to the owner
-  hadoop_conf_dir = ResourceArgument(default='/etc/hadoop/conf')
-  hdfs_user = ResourceArgument(default='hdfs')
-  hadoop_bin_dir = ResourceArgument(default='/usr/bin')
-
-  actions = Resource.actions + ["run"]

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_directory.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_directory.py b/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_directory.py
deleted file mode 100644
index 7888cd8..0000000
--- a/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_directory.py
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-_all__ = ["HdfsDirectory"]
-from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument
-
-class HdfsDirectory(Resource):
-  action = ForcedListArgument()
-
-  dir_name = ResourceArgument(default=lambda obj: obj.name)
-  owner = ResourceArgument()
-  group = ResourceArgument()
-  mode = ResourceArgument()
-  recursive_chown = BooleanArgument(default=False)
-  recursive_chmod = BooleanArgument(default=False)
-
-  conf_dir = ResourceArgument()
-  security_enabled = BooleanArgument(default=False)
-  keytab = ResourceArgument()
-  kinit_path_local = ResourceArgument()
-  hdfs_user = ResourceArgument()
-  bin_dir = ResourceArgument(default="")
-
-  #action 'create' immediately creates all pending directory in efficient manner
-  #action 'create_delayed' add directory to list of pending directories
-  actions = Resource.actions + ["create","create_delayed"]

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py
new file mode 100644
index 0000000..92a043f
--- /dev/null
+++ b/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_resource.py
@@ -0,0 +1,76 @@
+# !/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+_all__ = ["HdfsResource"]
+from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument
+
+"""
+Calling a lot of hadoop commands takes too much time.
+The cause is that for every call new connection initialized, with datanodes, namenode.
+
+While this resource can gather the dicteroies/files to create/delete/copyFromLocal.
+And after just with one call create all that.
+
+action = create_delayed / delete_delayed. Are for gathering information  about what you want
+to create.
+
+After everything is gathered you should execute action = execute. To perform delayed actions
+
+The resource is a replacement for the following operations:
+  1) hadoop fs -rmr
+  2) hadoop fs -copyFromLocal
+  3) hadoop fs -put
+  4) hadoop fs -mkdir
+  5) hadoop fs -touchz
+  6) hadoop fs -chmod
+  7) hadoop fs -chown
+"""
+
+
+class HdfsResource(Resource):
+  # Required: {target, type, action}
+  # path to hadoop file/directory
+  target = ResourceArgument(default=lambda obj: obj.name)
+  # "directory" or "file"
+  type = ResourceArgument()
+  # "create_delayed" or "delete_delayed" or "execute"
+  action = ForcedListArgument()
+  # if present - copies file/directory from local path {source} to hadoop path - {target}
+  source = ResourceArgument()
+  owner = ResourceArgument()
+  group = ResourceArgument()
+  mode = ResourceArgument()
+  logoutput = ResourceArgument()
+  recursive_chown = BooleanArgument(default=False)
+  recursive_chmod = BooleanArgument(default=False)
+  change_permissions_for_parents = BooleanArgument(default=False)
+
+  security_enabled = BooleanArgument(default=False)
+  keytab = ResourceArgument()
+  kinit_path_local = ResourceArgument()
+  user = ResourceArgument()
+  hadoop_bin_dir = ResourceArgument()
+  hadoop_conf_dir = ResourceArgument()
+
+  #action 'execute' immediately creates all pending files/directories in efficient manner
+  #action 'create_delayed/delete_delayed' adds file/directory to list of pending directories
+  actions = Resource.actions + ["create_on_execute", "delete_on_execute", "execute"]

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index c9ba846..8de9247 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -297,6 +297,7 @@ class Script(object):
       return None
 
     stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
+
     if stack_version_unformatted is None or stack_version_unformatted == '':
       return None
 
@@ -325,6 +326,7 @@ class Script(object):
     :return: True if the command's stack is less than the specified version
     """
     hdp_stack_version = Script.get_hdp_stack_version()
+
     if hdp_stack_version is None:
       return False
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo.py b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo.py
index e974573..5e476a5 100644
--- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo.py
+++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo.py
@@ -75,13 +75,13 @@ def accumulo(name=None # 'master' or 'tserver' or 'client'
       owner=params.accumulo_user
     )
 
-  if name in ["master","tserver"]:
+  if name == "master":
     params.HdfsDirectory(format("{params.accumulo_hdfs_root_dir}"),
-                         action="create_delayed",
+                         action="create_on_execute",
                          owner=params.accumulo_user,
     )
     params.HdfsDirectory(format("{params.accumulo_hdfs_stage_dir}"),
-                         action="create_delayed",
+                         action="create_on_execute",
                          owner=params.accumulo_user,
                          mode=0751
     )

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo_configuration.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo_configuration.py b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo_configuration.py
index 88d83d3..f26fdd0 100644
--- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo_configuration.py
+++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo_configuration.py
@@ -160,17 +160,19 @@ def setup_conf_dir(name=None): # 'master' or 'tserver' or 'monitor' or 'gc' or '
 
   # other server setup
   if name == 'master':
-    params.HdfsDirectory(format("/user/{params.accumulo_user}"),
-                         action="create_delayed",
+    params.HdfsResource(format("/user/{params.accumulo_user}"),
+                         type="directory",
+                         action="create_on_execute",
                          owner=params.accumulo_user,
                          mode=0700
     )
-    params.HdfsDirectory(format("{params.parent_dir}"),
-                         action="create_delayed",
+    params.HdfsResource(format("{params.parent_dir}"),
+                         type="directory",
+                         action="create_on_execute",
                          owner=params.accumulo_user,
                          mode=0700
     )
-    params.HdfsDirectory(None, action="create")
+    params.HdfsResource(None, action="execute")
     if params.security_enabled and params.has_secure_user_auth:
       Execute( format("{params.kinit_cmd} "
                       "{params.daemon_script} init "
@@ -184,6 +186,7 @@ def setup_conf_dir(name=None): # 'master' or 'tserver' or 'monitor' or 'gc' or '
                                      "{params.hadoop_conf_dir} fs -stat "
                                      "{params.instance_volumes}"),
                               params.accumulo_user),
+               logoutput=True,
                user=params.accumulo_user)
     else:
       passfile = format("{params.exec_tmp_dir}/pass")
@@ -205,6 +208,7 @@ def setup_conf_dir(name=None): # 'master' or 'tserver' or 'monitor' or 'gc' or '
                                        "{params.hadoop_conf_dir} fs -stat "
                                        "{params.instance_volumes}"),
                                 params.accumulo_user),
+                 logoutput=True,
                  user=params.accumulo_user)
       finally:
         os.remove(passfile)

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py
index ff5a874..99522c1 100644
--- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py
@@ -18,11 +18,12 @@ limitations under the License.
 
 """
 from resource_management.libraries.functions import conf_select
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
 from resource_management.libraries.functions import format
 from resource_management.libraries.functions.version import format_hdp_stack_version
 from resource_management.libraries.functions.default import default
 from resource_management.libraries.script.script import Script
-from resource_management.libraries.resources.hdfs_directory import HdfsDirectory
+
 
 import status_params
 
@@ -147,14 +148,14 @@ hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
 hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
 hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
 import functools
-#create partial functions with common arguments for every HdfsDirectory call
-#to create hdfs directory we need to call params.HdfsDirectory in code
-HdfsDirectory = functools.partial(
-  HdfsDirectory,
-  conf_dir=hadoop_conf_dir,
-  hdfs_user=hdfs_user,
+#create partial functions with common arguments for every HdfsResource call
+#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+  HdfsResource,
+  user=hdfs_user,
   security_enabled = security_enabled,
   keytab = hdfs_user_keytab,
   kinit_path_local = kinit_path_local,
-  bin_dir = hadoop_bin_dir
+  hadoop_bin_dir = hadoop_bin_dir,
+  hadoop_conf_dir = hadoop_conf_dir
 )

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py
index cd9f8f9..009076c 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py
@@ -167,23 +167,25 @@ def hbase(name=None # 'master' or 'regionserver' or 'client'
     hbase_TemplateConfig( format("hbase_client_jaas.conf"), user=params.hbase_user)
     hbase_TemplateConfig( format("ams_zookeeper_jaas.conf"), user=params.hbase_user)
 
-  if name in ["master","regionserver"]:
+  if name == "master":
 
     if params.is_hbase_distributed:
 
-      params.HdfsDirectory(params.hbase_root_dir,
-                           action="create_delayed",
+      params.HdfsResource(params.hbase_root_dir,
+                           type="directory",
+                           action="create_on_execute",
                            owner=params.hbase_user,
                            mode=0775
       )
 
-      params.HdfsDirectory(params.hbase_staging_dir,
-                           action="create_delayed",
+      params.HdfsResource(params.hbase_staging_dir,
+                           type="directory",
+                           action="create_on_execute",
                            owner=params.hbase_user,
                            mode=0711
       )
 
-      params.HdfsDirectory(None, action="create")
+      params.HdfsResource(None, action="execute")
 
     else:
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
index 7e516b8..4a63e3c 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
@@ -185,17 +185,17 @@ hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_nam
 kinit_path_local = functions.get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
 
 import functools
-# create partial functions with common arguments for every HdfsDirectory call
-# to create hdfs directory we need to call params.HdfsDirectory in code
-HdfsDirectory = functools.partial(
-  HdfsDirectory,
-  conf_dir=hadoop_conf_dir,
-  hdfs_user=hdfs_user,
+#create partial functions with common arguments for every HdfsResource call
+#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+  HdfsResource,
+  user=hdfs_user,
   security_enabled = security_enabled,
   keytab = hdfs_user_keytab,
   kinit_path_local = kinit_path_local,
-  bin_dir = hadoop_bin_dir
-)
+  hadoop_bin_dir = hadoop_bin_dir,
+  hadoop_conf_dir = hadoop_conf_dir
+ )
 
 
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
index ed9098c..86318f3 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
@@ -91,33 +91,36 @@ def falcon(type, action = None):
   if type == 'server':
     if action == 'config':
       if params.store_uri[0:4] == "hdfs":
-        params.HdfsDirectory(params.store_uri,
-                             action="create_delayed",
+        params.HdfsResource(params.store_uri,
+                             type="directory",
+                             action="create_on_execute",
                              owner=params.falcon_user,
                              mode=0755
         )
-      if params.store_uri[0:4] == "file":
+      elif params.store_uri[0:4] == "file":
         Directory(params.store_uri[7:],
                   owner=params.falcon_user,
                   recursive=True
         )
-      params.HdfsDirectory(params.flacon_apps_dir,
-                           action="create_delayed",
+      params.HdfsResource(params.flacon_apps_dir,
+                           type="directory",
+                           action="create_on_execute",
                            owner=params.falcon_user,
-                           mode=0777#TODO change to proper mode
+                           mode=0777 #TODO change to proper mode
       )
       if params.falcon_store_uri[0:4] == "hdfs":
-        params.HdfsDirectory(params.falcon_store_uri,
-                             action="create_delayed",
+        params.HdfsResource(params.falcon_store_uri,
+                             type="directory",
+                             action="create_on_execute",
                              owner=params.falcon_user,
                              mode=0755
         )
-      if params.falcon_store_uri[0:4] == "file":
+      elif params.falcon_store_uri[0:4] == "file":
         Directory(params.falcon_store_uri[7:],
                   owner=params.falcon_user,
                   recursive=True
         )
-      params.HdfsDirectory(None, action="create")
+      params.HdfsResource(None, action="execute")
       Directory(params.falcon_local_dir,
                 owner=params.falcon_user,
                 recursive=True,

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
index 9a5fdb5..9663d11 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
@@ -18,13 +18,14 @@ limitations under the License.
 """
 import status_params
 
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
 from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import format
 from resource_management.libraries.functions.version import format_hdp_stack_version
 from resource_management.libraries.functions.default import default
 from resource_management.libraries.functions import get_kinit_path
 from resource_management.libraries.script.script import Script
-from resource_management.libraries.resources.hdfs_directory import HdfsDirectory
+
 
 config = Script.get_config()
 
@@ -101,14 +102,15 @@ hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_nam
 smokeuser_principal =  config['configurations']['cluster-env']['smokeuser_principal_name']
 kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
 import functools
-#create partial functions with common arguments for every HdfsDirectory call
-#to create hdfs directory we need to call params.HdfsDirectory in code
-HdfsDirectory = functools.partial(
-  HdfsDirectory,
-  conf_dir=hadoop_conf_dir,
-  hdfs_user=hdfs_user,
+#create partial functions with common arguments for every HdfsResource call
+#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+  HdfsResource,
+  user=hdfs_user,
   security_enabled = security_enabled,
   keytab = hdfs_user_keytab,
   kinit_path_local = kinit_path_local,
-  bin_dir = hadoop_bin_dir
-)
+  hadoop_bin_dir = hadoop_bin_dir,
+  hadoop_conf_dir = hadoop_conf_dir
+ )
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
index 9a35a24..1bfa7e4 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
@@ -163,17 +163,19 @@ def hbase(name=None):
       group=params.user_group,
       owner=params.hbase_user
     )
-  if name in ["master","regionserver"]:
-    params.HdfsDirectory(params.hbase_hdfs_root_dir,
-                         action="create_delayed",
+  if name == "master":
+    params.HdfsResource(params.hbase_hdfs_root_dir,
+                         type="directory",
+                         action="create_on_execute",
                          owner=params.hbase_user
     )
-    params.HdfsDirectory(params.hbase_staging_dir,
-                         action="create_delayed",
+    params.HdfsResource(params.hbase_staging_dir,
+                         type="directory",
+                         action="create_on_execute",
                          owner=params.hbase_user,
                          mode=0711
     )
-    params.HdfsDirectory(None, action="create")
+    params.HdfsResource(None, action="execute")
 
 def hbase_TemplateConfig(name, tag=None):
   import params

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
index 8f23bb2..c077f54 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
@@ -24,14 +24,16 @@ from functions import calc_xmn_from_xms
 
 from ambari_commons.constants import AMBARI_SUDO_BINARY
 
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
 from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import format
 from resource_management.libraries.functions.version import format_hdp_stack_version
 from resource_management.libraries.functions.default import default
 from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions import is_empty
 from resource_management.libraries.functions import get_unique_id_and_date
 from resource_management.libraries.script.script import Script
-from resource_management.libraries.resources.hdfs_directory import HdfsDirectory
+
 from resource_management.libraries.functions.substitute_vars import substitute_vars
 
 # server configurations
@@ -139,7 +141,9 @@ if security_enabled:
   _hostname_lowercase = config['hostname'].lower()
   master_jaas_princ = config['configurations']['hbase-site']['hbase.master.kerberos.principal'].replace('_HOST',_hostname_lowercase)
   regionserver_jaas_princ = config['configurations']['hbase-site']['hbase.regionserver.kerberos.principal'].replace('_HOST',_hostname_lowercase)
-  queryserver_jaas_princ = config['configurations']['hbase-site']['phoenix.queryserver.kerberos.principal'].replace('_HOST',_hostname_lowercase)
+  _queryserver_jaas_princ = config['configurations']['hbase-site']['phoenix.queryserver.kerberos.principal']
+  if not is_empty(_queryserver_jaas_princ):
+    queryserver_jaas_princ =_queryserver_jaas_princ.replace('_HOST',_hostname_lowercase)
 
 master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file']
 regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file']
@@ -168,16 +172,16 @@ hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
 hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
 hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
 import functools
-#create partial functions with common arguments for every HdfsDirectory call
-#to create hdfs directory we need to call params.HdfsDirectory in code
-HdfsDirectory = functools.partial(
-  HdfsDirectory,
-  conf_dir=hadoop_conf_dir,
-  hdfs_user=hdfs_user,
+#create partial functions with common arguments for every HdfsResource call
+#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+  HdfsResource,
+  user=hdfs_user,
   security_enabled = security_enabled,
   keytab = hdfs_user_keytab,
   kinit_path_local = kinit_path_local,
-  bin_dir = hadoop_bin_dir
+  hadoop_bin_dir = hadoop_bin_dir,
+  hadoop_conf_dir = hadoop_conf_dir
 )
 
 # ranger host

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/service_check.py
index a60ebad..b774f19 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/service_check.py
@@ -46,7 +46,6 @@ class HbaseServiceCheckDefault(HbaseServiceCheck):
     env.set_params(params)
     
     output_file = "/apps/hbase/data/ambarismoketest"
-    test_cmd = format("fs -test -e {output_file}")
     smokeuser_kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};") if params.security_enabled else ""
     hbase_servicecheck_file = format("{exec_tmp_dir}/hbase-smoke.sh")
   

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/files/fast-hdfs-resource.jar
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/files/fast-hdfs-resource.jar b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/files/fast-hdfs-resource.jar
new file mode 100644
index 0000000..defde28
Binary files /dev/null and b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/files/fast-hdfs-resource.jar differ

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
index a615c27..95fe90c 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
@@ -98,6 +98,11 @@ def hdfs(name=None):
        owner=tc_owner,
        content=Template("slaves.j2")
   )
+
+  # for source-code of jar goto contrib/fast-hdfs-resource
+  File(format("{ambari_libs_dir}/fast-hdfs-resource.jar"),
+       content=StaticFile("fast-hdfs-resource.jar")
+  )
   
   if params.lzo_enabled and len(params.lzo_packages) > 0:
       Package(params.lzo_packages)

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py
index e36019e..453d824 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py
@@ -168,18 +168,21 @@ def create_name_dirs(directories):
 def create_hdfs_directories(check):
   import params
 
-  params.HdfsDirectory("/tmp",
-                       action="create_delayed",
+  params.HdfsResource("/tmp",
+                       type="directory",
+                       action="create_on_execute",
                        owner=params.hdfs_user,
                        mode=0777
   )
-  params.HdfsDirectory(params.smoke_hdfs_user_dir,
-                       action="create_delayed",
+  params.HdfsResource(params.smoke_hdfs_user_dir,
+                       type="directory",
+                       action="create_on_execute",
                        owner=params.smoke_user,
                        mode=params.smoke_hdfs_user_mode
   )
-  params.HdfsDirectory(None, action="create",
-                       only_if=check #skip creation when HA not active
+  params.HdfsResource(None, 
+                      action="execute",
+                      only_if=check #skip creation when HA not active
   )
 
 def format_namenode(force=None):

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_nfsgateway.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_nfsgateway.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_nfsgateway.py
index ad5ef89..ac0e24d 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_nfsgateway.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_nfsgateway.py
@@ -22,7 +22,6 @@ from resource_management.core.logger import Logger
 from resource_management.core.resources import Directory
 from resource_management.core import shell
 from utils import service
-from utils import hdfs_directory
 import subprocess,os
 
 # NFS GATEWAY is always started by root using jsvc due to rpcbind bugs

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_snamenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_snamenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_snamenode.py
index 3feff67..78ef977 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_snamenode.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_snamenode.py
@@ -19,7 +19,6 @@ limitations under the License.
 
 from resource_management import *
 from utils import service
-from utils import hdfs_directory
 from ambari_commons.os_family_impl import OsFamilyImpl, OsFamilyFuncImpl
 from ambari_commons import OSConst
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
index 18e0fa0..18aede8 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
@@ -32,7 +32,8 @@ from resource_management.libraries.functions.default import default
 from resource_management.libraries.functions import get_klist_path
 from resource_management.libraries.functions import get_kinit_path
 from resource_management.libraries.script.script import Script
-from resource_management.libraries.resources.hdfs_directory import HdfsDirectory
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+
 from resource_management.libraries.functions.format_jvm_option import format_jvm_option
 from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
 
@@ -93,7 +94,7 @@ if Script.is_hdp_stack_greater_or_equal("2.2"):
       hadoop_secure_dn_user = '""'
 
 
-
+ambari_libs_dir = "/var/lib/ambari-agent/lib"
 limits_conf_dir = "/etc/security/limits.d"
 
 if Script.is_hdp_stack_greater_or_equal("2.0") and Script.is_hdp_stack_less_than("2.1") and not OSCheck.is_suse_family():
@@ -281,18 +282,19 @@ else:
   jn_kinit_cmd = ""
 
 import functools
-#create partial functions with common arguments for every HdfsDirectory call
-#to create hdfs directory we need to call params.HdfsDirectory in code
-HdfsDirectory = functools.partial(
-  HdfsDirectory,
-  conf_dir=hadoop_conf_dir,
-  hdfs_user=hdfs_user,
+#create partial functions with common arguments for every HdfsResource call
+#to create/delete/copyfromlocal hdfs directories/files we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+  HdfsResource,
+  user=hdfs_user,
   security_enabled = security_enabled,
   keytab = hdfs_user_keytab,
   kinit_path_local = kinit_path_local,
-  bin_dir = hadoop_bin_dir
+  hadoop_bin_dir = hadoop_bin_dir,
+  hadoop_conf_dir = hadoop_conf_dir
 )
 
+
 # The logic for LZO also exists in OOZIE's params.py
 io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
 lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/service_check.py
index dd319b0..7a1e6b7 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/service_check.py
@@ -37,15 +37,6 @@ class HdfsServiceCheckDefault(HdfsServiceCheck):
 
     safemode_command = format("dfsadmin -fs {namenode_address} -safemode get | grep OFF")
 
-    create_dir_cmd = format("fs -mkdir {dir}")
-    chmod_command = format("fs -chmod 777 {dir}")
-    test_dir_exists = as_user(format("{hadoop_bin_dir}/hadoop --config {hadoop_conf_dir} fs -test -e {dir}"), params.hdfs_user)
-    cleanup_cmd = format("fs -rm {tmp_file}")
-    #cleanup put below to handle retries; if retrying there wil be a stale file
-    #that needs cleanup; exit code is fn of second command
-    create_file_cmd = format(
-      "{cleanup_cmd}; hadoop --config {hadoop_conf_dir} fs -put /etc/passwd {tmp_file}")
-    test_cmd = format("fs -test -e {tmp_file}")
     if params.security_enabled:
       Execute(format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"),
         user=params.hdfs_user
@@ -58,39 +49,23 @@ class HdfsServiceCheckDefault(HdfsServiceCheck):
                   tries=20,
                   bin_dir=params.hadoop_bin_dir
     )
-    ExecuteHadoop(create_dir_cmd,
-                  user=params.hdfs_user,
-                  logoutput=True,
-                  not_if=test_dir_exists,
-                  conf_dir=params.hadoop_conf_dir,
-                  try_sleep=3,
-                  tries=5,
-                  bin_dir=params.hadoop_bin_dir
+    params.HdfsResource(dir,
+                        type="directory",
+                        action="create_on_execute",
+                        mode=0777
     )
-    ExecuteHadoop(chmod_command,
-                  user=params.hdfs_user,
-                  logoutput=True,
-                  conf_dir=params.hadoop_conf_dir,
-                  try_sleep=3,
-                  tries=5,
-                  bin_dir=params.hadoop_bin_dir
+    params.HdfsResource(tmp_file,
+                        type="file",
+                        action="delete_on_execute",
     )
-    ExecuteHadoop(create_file_cmd,
-                  user=params.hdfs_user,
-                  logoutput=True,
-                  conf_dir=params.hadoop_conf_dir,
-                  try_sleep=3,
-                  tries=5,
-                  bin_dir=params.hadoop_bin_dir
-    )
-    ExecuteHadoop(test_cmd,
-                  user=params.hdfs_user,
-                  logoutput=True,
-                  conf_dir=params.hadoop_conf_dir,
-                  try_sleep=3,
-                  tries=5,
-                  bin_dir=params.hadoop_bin_dir
+
+    params.HdfsResource(tmp_file,
+                        type="file",
+                        source="/etc/passwd",
+                        action="create_on_execute"
     )
+    params.HdfsResource(None, action="execute")
+
     if params.has_journalnode_hosts:
       journalnode_port = params.journalnode_port
       checkWebUIFileName = "checkWebUI.py"

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2f35d48/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/files/templetonSmoke.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/files/templetonSmoke.sh b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/files/templetonSmoke.sh
index d16848d..2083312 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/files/templetonSmoke.sh
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/files/templetonSmoke.sh
@@ -23,10 +23,11 @@
 export ttonhost=$1
 export smoke_test_user=$2
 export templeton_port=$3
-export smoke_user_keytab=$4
-export security_enabled=$5
-export kinit_path_local=$6
-export smokeuser_principal=$7
+export ttonTestScript=$4
+export smoke_user_keytab=$5
+export security_enabled=$6
+export kinit_path_local=$7
+export smokeuser_principal=$8
 export ttonurl="http://${ttonhost}:${templeton_port}/templeton/v1"
 
 if [[ $security_enabled == "true" ]]; then
@@ -67,21 +68,8 @@ if [[ $security_enabled == "true" ]]; then
 fi
 
 #try pig query
-outname=${smoke_test_user}.`date +"%M%d%y"`.$$;
-ttonTestOutput="/tmp/idtest.${outname}.out";
-ttonTestInput="/tmp/idtest.${outname}.in";
 ttonTestScript="idtest.${outname}.pig"
 
-echo "A = load '$ttonTestInput' using PigStorage(':');"  > /tmp/$ttonTestScript
-echo "B = foreach A generate \$0 as id; " >> /tmp/$ttonTestScript
-echo "store B into '$ttonTestOutput';" >> /tmp/$ttonTestScript
-
-#copy pig script to hdfs
-/var/lib/ambari-agent/ambari-sudo.sh su ${smoke_test_user} -s /bin/bash - -c "hadoop dfs -copyFromLocal /tmp/$ttonTestScript /tmp/$ttonTestScript"
-
-#copy input file to hdfs
-/var/lib/ambari-agent/ambari-sudo.sh su ${smoke_test_user} -s /bin/bash - -c "hadoop dfs -copyFromLocal /etc/passwd $ttonTestInput"
-
 #create, copy post args file
 echo -n "user.name=${smoke_test_user}&file=/tmp/$ttonTestScript" > /tmp/pig_post.txt
 


Mime
View raw message