ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smoha...@apache.org
Subject ambari git commit: AMBARI-19577. Add Livy session recovery configurations in Ambari (Saisai Shao via smohanty)
Date Fri, 10 Feb 2017 16:45:48 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk eecb5a311 -> a0ba7fbe7


AMBARI-19577. Add Livy session recovery configurations in Ambari (Saisai Shao via smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a0ba7fbe
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a0ba7fbe
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a0ba7fbe

Branch: refs/heads/trunk
Commit: a0ba7fbe7f80332bcff1cab9ea7dbe27ca59a003
Parents: eecb5a3
Author: Sumit Mohanty <smohanty@hortonworks.com>
Authored: Fri Feb 10 08:36:58 2017 -0800
Committer: Sumit Mohanty <smohanty@hortonworks.com>
Committed: Fri Feb 10 08:42:42 2017 -0800

----------------------------------------------------------------------
 .../SPARK/1.2.1/package/scripts/params.py       |  1 +
 .../SPARK/1.2.1/package/scripts/setup_livy.py   |  8 +++++
 .../SPARK2/2.0.0/package/scripts/params.py      |  1 +
 .../SPARK2/2.0.0/package/scripts/setup_livy2.py |  8 +++++
 .../services/SPARK/configuration/livy-conf.xml  | 24 +++++++++++++++
 .../SPARK2/configuration/livy2-conf.xml         | 24 +++++++++++++++
 .../python/stacks/2.5/SPARK/test_spark_livy.py  | 31 ++++++++++++++++++++
 .../stacks/2.6/SPARK2/test_spark_livy2.py       | 31 ++++++++++++++++++++
 8 files changed, 128 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
index 82cd0b2..6a59caf 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
@@ -222,6 +222,7 @@ if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY,
stac
   user_group = status_params.user_group
   livy_hdfs_user_dir = format("/user/{livy_user}")
   livy_server_pid_file = status_params.livy_server_pid_file
+  livy_recovery_dir = default("/configurations/livy-conf/livy.server.recovery.state-store.url",
"/livy-recovery")
 
   livy_server_start = format("{livy_home}/bin/livy-server start")
   livy_server_stop = format("{livy_home}/bin/livy-server stop")

http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_livy.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_livy.py
b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_livy.py
index 07b704f..32615c3 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_livy.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_livy.py
@@ -40,6 +40,14 @@ def setup_livy(env, type, upgrade_type = None, action = None):
     )
     params.HdfsResource(None, action="execute")
 
+    params.HdfsResource(params.livy_recovery_dir,
+                       type="directory",
+                       action="create_on_execute",
+                       owner=params.livy_user,
+                       mode=0775
+    )
+    params.HdfsResource(None, action="execute")
+
   # create livy-env.sh in etc/conf dir
   File(os.path.join(params.livy_conf, 'livy-env.sh'),
        owner=params.livy_user,

http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py
index c19f88c..b6889e4 100755
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py
@@ -207,6 +207,7 @@ if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY2,
sta
   user_group = status_params.user_group
   livy2_hdfs_user_dir = format("/user/{livy2_user}")
   livy2_server_pid_file = status_params.livy2_server_pid_file
+  livy2_recovery_dir = default("/configurations/livy2-conf/livy.server.recovery.state-store.url",
"/livy2-recovery")
 
   livy2_server_start = format("{livy2_home}/bin/livy-server start")
   livy2_server_stop = format("{livy2_home}/bin/livy-server stop")

http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py
b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py
index f2178f7..2e92509 100644
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py
@@ -40,6 +40,14 @@ def setup_livy(env, type, upgrade_type = None, action = None):
     )
     params.HdfsResource(None, action="execute")
 
+    params.HdfsResource(params.livy2_recovery_dir,
+                        type="directory",
+                        action="create_on_execute",
+                        owner=params.livy2_user,
+                        mode=0775
+       )
+    params.HdfsResource(None, action="execute")
+
   # create livy-env.sh in etc/conf dir
   File(os.path.join(params.livy2_conf, 'livy-env.sh'),
        owner=params.livy2_user,

http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/livy-conf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/livy-conf.xml
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/livy-conf.xml
index 23583c0..28eeab5 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/livy-conf.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/livy-conf.xml
@@ -78,4 +78,28 @@
      </description>
      <on-ambari-upgrade add="true"/>
   </property>
+  <property>
+    <name>livy.server.recovery.mode</name>
+    <value>recovery</value>
+    <description>
+          Recovery mode for livy, either be "off" or "recovery".
+    </description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>livy.server.recovery.state-store</name>
+    <value>filesystem</value>
+    <description>
+          Where Livy should store state for recovery.
+    </description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>livy.server.recovery.state-store.url</name>
+    <value>/livy-recovery</value>
+    <description>
+          Where Livy should store state for recovery.
+    </description>
+    <on-ambari-upgrade add="false"/>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml
index 231ef70..c8a65bd 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml
@@ -85,6 +85,30 @@
             Whether to enable HiveContext in livy interpreter
         </description>
         <on-ambari-upgrade add="false"/>
+    <property>
+    </property>
+        <name>livy.server.recovery.mode</name>
+        <value>recovery</value>
+        <description>
+            Recovery mode for livy, either be "off" or "recovery".
+        </description>
+        <on-ambari-upgrade add="false"/>
+    </property>
+    <property>
+        <name>livy.server.recovery.state-store</name>
+        <value>filesystem</value>
+        <description>
+            Where Livy should store state for recovery.
+        </description>
+        <on-ambari-upgrade add="false"/>
+    </property>
+    <property>
+        <name>livy.server.recovery.state-store.url</name>
+        <value>/livy2-recovery</value>
+        <description>
+            Where Livy should store state for recovery.
+        </description>
+        <on-ambari-upgrade add="false"/>
     </property>
 </configuration>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py b/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py
index b56474a..b9199c7 100644
--- a/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py
+++ b/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py
@@ -84,6 +84,37 @@ class TestSparkClient(RMFTestCase):
                                   dfs_type = '',
                                   hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
                                   )
+        self.assertResourceCalled('HdfsResource', '/livy-recovery',
+                                  immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+                                  security_enabled = False,
+                                  hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                                  keytab = UnknownConfigurationMock(),
+                                  default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+                                  hdfs_site = {u'a': u'b'},
+                                  kinit_path_local = '/usr/bin/kinit',
+                                  principal_name = UnknownConfigurationMock(),
+                                  user = 'hdfs',
+                                  owner = 'livy',
+                                  hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                                  type = 'directory',
+                                  action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+                                  dfs_type = '',
+                                  mode = 0775,
+                                  )
+        self.assertResourceCalled('HdfsResource', None,
+                                  immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+                                  security_enabled = False,
+                                  hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                                  keytab = UnknownConfigurationMock(),
+                                  default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+                                  hdfs_site = {u'a': u'b'},
+                                  kinit_path_local = '/usr/bin/kinit',
+                                  principal_name = UnknownConfigurationMock(),
+                                  user = 'hdfs',
+                                  action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+                                  dfs_type = '',
+                                  hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                                  )
         self.assertResourceCalled('File', '/usr/hdp/current/livy-server/conf/livy-env.sh',
                                   content = InlineTemplate(self.getConfig()['configurations']['livy-env']['content']),
                                   owner = 'livy',

http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py b/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py
index 6244d87..75aec84 100644
--- a/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py
+++ b/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py
@@ -84,6 +84,37 @@ class TestSparkClient(RMFTestCase):
                                   dfs_type = '',
                                   hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
                                   )
+        self.assertResourceCalled('HdfsResource', '/livy2-recovery',
+                                  immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+                                  security_enabled = False,
+                                  hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                                  keytab = UnknownConfigurationMock(),
+                                  default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+                                  hdfs_site = {u'a': u'b'},
+                                  kinit_path_local = '/usr/bin/kinit',
+                                  principal_name = UnknownConfigurationMock(),
+                                  user = 'hdfs',
+                                  owner = 'livy',
+                                  hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                                  type = 'directory',
+                                  action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+                                  dfs_type = '',
+                                  mode = 0775,
+                                  )
+        self.assertResourceCalled('HdfsResource', None,
+                                  immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+                                  security_enabled = False,
+                                  hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                                  keytab = UnknownConfigurationMock(),
+                                  default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+                                  hdfs_site = {u'a': u'b'},
+                                  kinit_path_local = '/usr/bin/kinit',
+                                  principal_name = UnknownConfigurationMock(),
+                                  user = 'hdfs',
+                                  action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+                                  dfs_type = '',
+                                  hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                                  )
         self.assertResourceCalled('File', '/usr/hdp/current/livy2-server/conf/livy-env.sh',
                                   content = InlineTemplate(self.getConfig()['configurations']['livy2-env']['content']),
                                   owner = 'livy',


Mime
View raw message