ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mpapirkovs...@apache.org
Subject ambari git commit: AMBARI-11298. Ambari set up for Falcon Hive DR. (mpapirkovskyy)
Date Thu, 21 May 2015 14:57:31 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk f592dbe39 -> 18e9d2d31


AMBARI-11298. Ambari set up for Falcon Hive DR. (mpapirkovskyy)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/18e9d2d3
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/18e9d2d3
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/18e9d2d3

Branch: refs/heads/trunk
Commit: 18e9d2d319d418296058a2f757be9503e781a849
Parents: f592dbe
Author: Myroslav Papirkovskyy <mpapyrkovskyy@hortonworks.com>
Authored: Thu May 21 17:55:10 2015 +0300
Committer: Myroslav Papirkovskyy <mpapyrkovskyy@hortonworks.com>
Committed: Thu May 21 17:57:27 2015 +0300

----------------------------------------------------------------------
 .../0.5.0.2.1/configuration/falcon-env.xml      |  7 ++++-
 .../FALCON/0.5.0.2.1/package/scripts/falcon.py  | 11 ++++++++
 .../0.5.0.2.1/package/scripts/params_linux.py   |  5 ++++
 .../FALCON/configuration/falcon-env.xml         | 27 ++++++++++++++++++++
 .../stacks/2.1/FALCON/test_falcon_server.py     | 17 ++++++++++++
 .../test/python/stacks/2.1/configs/default.json |  3 ++-
 .../test/python/stacks/2.1/configs/secured.json |  3 ++-
 .../stacks/2.2/configs/falcon-upgrade.json      |  3 ++-
 8 files changed, 72 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/18e9d2d3/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml
b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml
index 363e4f3..75178d2 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml
@@ -111,5 +111,10 @@ export FALCON_DATA_DIR={{falcon_embeddedmq_data}}
 # Where do you want to expand the war file. By Default it is in /server/webapp dir under
the base install dir.
 #export FALCON_EXPANDED_WEBAPP_DIR=
     </value>
-  </property>  
+  </property>
+  <property>
+    <name>supports_hive_dr</name>
+    <value>false</value>
+    <description>Enable support of Falcon Hive DR</description>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/18e9d2d3/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
index 86318f3..639071e 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
@@ -120,6 +120,17 @@ def falcon(type, action = None):
                   owner=params.falcon_user,
                   recursive=True
         )
+      if params.supports_hive_dr:
+        params.HdfsResource(params.dfs_data_mirroring_dir,
+                            type="directory",
+                            action="create_on_execute",
+                            owner=params.hdfs_user,
+                            group=params.user_group,
+                            recursive_chown = True,
+                            recursive_chmod = True,
+                            mode=0770,
+                            source=params.local_data_mirroring_dir
+        )
       params.HdfsResource(None, action="execute")
       Directory(params.falcon_local_dir,
                 owner=params.falcon_user,

http://git-wip-us.apache.org/repos/asf/ambari/blob/18e9d2d3/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
index 9663d11..031f5ab 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
@@ -101,6 +101,11 @@ hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
 hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
 smokeuser_principal =  config['configurations']['cluster-env']['smokeuser_principal_name']
 kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths',
None))
+
+supports_hive_dr = config['configurations']['falcon-env']['supports_hive_dr']
+local_data_mirroring_dir = "/usr/hdp/current/falcon-server/data-mirroring"
+dfs_data_mirroring_dir = "/apps/data-mirroring"
+
 import functools
 #create partial functions with common arguments for every HdfsResource call
 #to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in
code

http://git-wip-us.apache.org/repos/asf/ambari/blob/18e9d2d3/ambari-server/src/main/resources/stacks/HDP/2.3/services/FALCON/configuration/falcon-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/FALCON/configuration/falcon-env.xml
b/ambari-server/src/main/resources/stacks/HDP/2.3/services/FALCON/configuration/falcon-env.xml
new file mode 100644
index 0000000..449f8fd
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/FALCON/configuration/falcon-env.xml
@@ -0,0 +1,27 @@
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+  <property>
+    <name>supports_hive_dr</name>
+    <value>true</value>
+    <description>Enable support of Falcon Hive DR</description>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/18e9d2d3/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
index 8fa7cd2..7fc5bce 100644
--- a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
+++ b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
@@ -138,6 +138,23 @@ class TestFalconServer(RMFTestCase):
         owner = 'falcon',
         recursive = True,
     )
+    self.assertResourceCalled('HdfsResource', '/apps/data-mirroring',
+        security_enabled = False,
+        hadoop_bin_dir = '/usr/bin',
+        keytab = UnknownConfigurationMock(),
+        kinit_path_local = '/usr/bin/kinit',
+        user = 'hdfs',
+        owner = 'hdfs',
+        group='hadoop',
+        hadoop_conf_dir = '/etc/hadoop/conf',
+        type = 'directory',
+        recursive_chown = True,
+        recursive_chmod = True,
+        action = ['create_on_execute'],
+        mode = 0770,
+        source='/usr/hdp/current/falcon-server/data-mirroring'
+    )
+
     self.assertResourceCalled('HdfsResource', None,
         security_enabled = False,
         hadoop_bin_dir = '/usr/bin',

http://git-wip-us.apache.org/repos/asf/ambari/blob/18e9d2d3/ambari-server/src/test/python/stacks/2.1/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/configs/default.json b/ambari-server/src/test/python/stacks/2.1/configs/default.json
index 41a7c23..f878aad 100644
--- a/ambari-server/src/test/python/stacks/2.1/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.1/configs/default.json
@@ -666,7 +666,8 @@
             "content": "\n# The java implementation to use. If JAVA_HOME is not found we
expect java and jar to be in path\nexport JAVA_HOME={{java_home}}\n\n# any additional java
opts you want to set. This will apply to both client and server operations\n#export FALCON_OPTS=\n\n#
any additional java opts that you want to set for client only\n#export FALCON_CLIENT_OPTS=\n\n#
java heap size we want to set for the client. Default is 1024MB\n#export FALCON_CLIENT_HEAP=\n\n#
any additional opts you want to set for prisim service.\n#export FALCON_PRISM_OPTS=\n\n# java
heap size we want to set for the prisim service. Default is 1024MB\n#export FALCON_PRISM_HEAP=\n\n#
any additional opts you want to set for falcon service.\nexport FALCON_SERVER_OPTS=\"-Dfalcon.embeddedmq={{falcon_embeddedmq_enabled}}
-Dfalcon.emeddedmq.port={{falcon_emeddedmq_port}}\"\n\n# java heap size we want to set for
the falcon server. Default is 1024MB\n#export FALCON_SERVER_HEAP=\n\n# What is is considered
as falco
 n home dir. Default is the base locaion of the installed software\n#export FALCON_HOME_DIR=\n\n#
Where log files are stored. Defatult is logs directory under the base install location\nexport
FALCON_LOG_DIR={{falcon_log_dir}}\n\n# Where pid files are stored. Defatult is logs directory
under the base install location\nexport FALCON_PID_DIR={{falcon_pid_dir}}\n\n# where the falcon
active mq data is stored. Defatult is logs/data directory under the base install location\nexport
FALCON_DATA_DIR={{falcon_embeddedmq_data}}\n\n# Where do you want to expand the war file.
By Default it is in /server/webapp dir under the base install dir.\n#export FALCON_EXPANDED_WEBAPP_DIR=",

             "falcon.embeddedmq.data": "/hadoop/falcon/embeddedmq/data", 
             "falcon.embeddedmq": "true", 
-            "falcon_store_uri": "file:///hadoop/falcon/store"
+            "falcon_store_uri": "file:///hadoop/falcon/store",
+            "supports_hive_dr": "true"
         }, 
         "oozie-env": {
             "oozie_derby_database": "Derby", 

http://git-wip-us.apache.org/repos/asf/ambari/blob/18e9d2d3/ambari-server/src/test/python/stacks/2.1/configs/secured.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/configs/secured.json b/ambari-server/src/test/python/stacks/2.1/configs/secured.json
index be22980..193012e 100644
--- a/ambari-server/src/test/python/stacks/2.1/configs/secured.json
+++ b/ambari-server/src/test/python/stacks/2.1/configs/secured.json
@@ -658,7 +658,8 @@
             "content": "\n# The java implementation to use. If JAVA_HOME is not found we
expect java and jar to be in path\nexport JAVA_HOME={{java_home}}\n\n# any additional java
opts you want to set. This will apply to both client and server operations\n#export FALCON_OPTS=\n\n#
any additional java opts that you want to set for client only\n#export FALCON_CLIENT_OPTS=\n\n#
java heap size we want to set for the client. Default is 1024MB\n#export FALCON_CLIENT_HEAP=\n\n#
any additional opts you want to set for prisim service.\n#export FALCON_PRISM_OPTS=\n\n# java
heap size we want to set for the prisim service. Default is 1024MB\n#export FALCON_PRISM_HEAP=\n\n#
any additional opts you want to set for falcon service.\nexport FALCON_SERVER_OPTS=\"-Dfalcon.embeddedmq={{falcon_embeddedmq_enabled}}
-Dfalcon.emeddedmq.port={{falcon_emeddedmq_port}}\"\n\n# java heap size we want to set for
the falcon server. Default is 1024MB\n#export FALCON_SERVER_HEAP=\n\n# What is is considered
as falco
 n home dir. Default is the base locaion of the installed software\n#export FALCON_HOME_DIR=\n\n#
Where log files are stored. Defatult is logs directory under the base install location\nexport
FALCON_LOG_DIR={{falcon_log_dir}}\n\n# Where pid files are stored. Defatult is logs directory
under the base install location\nexport FALCON_PID_DIR={{falcon_pid_dir}}\n\n# where the falcon
active mq data is stored. Defatult is logs/data directory under the base install location\nexport
FALCON_DATA_DIR={{falcon_embeddedmq_data}}\n\n# Where do you want to expand the war file.
By Default it is in /server/webapp dir under the base install dir.\n#export FALCON_EXPANDED_WEBAPP_DIR=",

             "falcon.embeddedmq.data": "/hadoop/falcon/embeddedmq/data", 
             "falcon.embeddedmq": "true", 
-            "falcon_store_uri": "file:///hadoop/falcon/store"
+            "falcon_store_uri": "file:///hadoop/falcon/store",
+            "supports_hive_dr": "true"
         }, 
         "oozie-env": {
             "oozie_derby_database": "Derby", 

http://git-wip-us.apache.org/repos/asf/ambari/blob/18e9d2d3/ambari-server/src/test/python/stacks/2.2/configs/falcon-upgrade.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/configs/falcon-upgrade.json b/ambari-server/src/test/python/stacks/2.2/configs/falcon-upgrade.json
index c54a4e6..dce5cae 100644
--- a/ambari-server/src/test/python/stacks/2.2/configs/falcon-upgrade.json
+++ b/ambari-server/src/test/python/stacks/2.2/configs/falcon-upgrade.json
@@ -176,7 +176,8 @@
             "content": "\n# The java implementation to use. If JAVA_HOME is not found we
expect java and jar to be in path\nexport JAVA_HOME={{java_home}}\n\n# any additional java
opts you want to set. This will apply to both client and server operations\n#export FALCON_OPTS=\n\n#
any additional java opts that you want to set for client only\n#export FALCON_CLIENT_OPTS=\n\n#
java heap size we want to set for the client. Default is 1024MB\n#export FALCON_CLIENT_HEAP=\n\n#
any additional opts you want to set for prisim service.\n#export FALCON_PRISM_OPTS=\n\n# java
heap size we want to set for the prisim service. Default is 1024MB\n#export FALCON_PRISM_HEAP=\n\n#
any additional opts you want to set for falcon service.\nexport FALCON_SERVER_OPTS=\"-Dfalcon.embeddedmq={{falcon_embeddedmq_enabled}}
-Dfalcon.emeddedmq.port={{falcon_emeddedmq_port}}\"\n\n# java heap size we want to set for
the falcon server. Default is 1024MB\n#export FALCON_SERVER_HEAP=\n\n# What is is considered
as falco
 n home dir. Default is the base location of the installed software\n#export FALCON_HOME_DIR=\n\n#
Where log files are stored. Defatult is logs directory under the base install location\nexport
FALCON_LOG_DIR={{falcon_log_dir}}\n\n# Where pid files are stored. Defatult is logs directory
under the base install location\nexport FALCON_PID_DIR={{falcon_pid_dir}}\n\n# where the falcon
active mq data is stored. Defatult is logs/data directory under the base install location\nexport
FALCON_DATA_DIR={{falcon_embeddedmq_data}}\n\n# Where do you want to expand the war file.
By Default it is in /server/webapp dir under the base install dir.\n#export FALCON_EXPANDED_WEBAPP_DIR=",

             "falcon.embeddedmq.data": "/hadoop/falcon/embeddedmq/data", 
             "falcon.embeddedmq": "true", 
-            "falcon_store_uri": "file:///hadoop/falcon/store"
+            "falcon_store_uri": "file:///hadoop/falcon/store",
+            "supports_hive_dr": "true"
         }, 
         "core-site": {
             "proxyuser_group": "users", 


Mime
View raw message