ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From echekans...@apache.org
Subject ambari git commit: AMBARI-20126. Add support for Spark2 upgrade from HDP-2.5 (dgrinenko via echekankiy)
Date Fri, 24 Feb 2017 09:18:51 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 043d6c06c -> 19da5823c


AMBARI-20126. Add support for Spark2 upgrade from HDP-2.5 (dgrinenko via echekankiy)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/19da5823
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/19da5823
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/19da5823

Branch: refs/heads/trunk
Commit: 19da5823c8adc8cd18048040c93f43941c0e568f
Parents: 043d6c0
Author: Eugene Chekanskiy <echekanskiy@hortonworks.com>
Authored: Fri Feb 24 11:18:33 2017 +0200
Committer: Eugene Chekanskiy <echekanskiy@hortonworks.com>
Committed: Fri Feb 24 11:18:33 2017 +0200

----------------------------------------------------------------------
 .../2.0.0/package/scripts/job_history_server.py |  2 +-
 .../2.0.0/package/scripts/livy2_server.py       |  2 +-
 .../2.0.0/package/scripts/spark_client.py       |  2 +-
 .../package/scripts/spark_thrift_server.py      |  9 ++--
 .../stacks/HDP/2.5/upgrades/config-upgrade.xml  | 11 +++++
 .../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 50 +++++++++++++++++++
 .../stacks/HDP/2.5/upgrades/upgrade-2.6.xml     | 39 +++++++++++++++
 .../HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml | 51 ++++++++++++++++++++
 .../stacks/HDP/2.6/upgrades/upgrade-2.6.xml     | 41 ++++++++++++++++
 9 files changed, 199 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/job_history_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/job_history_server.py
b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/job_history_server.py
index 154c83d..2631b49 100755
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/job_history_server.py
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/job_history_server.py
@@ -41,7 +41,7 @@ class JobHistoryServer(Script):
     
     self.install_packages(env)
     
-  def configure(self, env, upgrade_type=None):
+  def configure(self, env, upgrade_type=None, config_dir=None):
     import params
     env.set_params(params)
     

http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py
b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py
index 8c66998..cb4f5ee 100644
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_server.py
@@ -44,7 +44,7 @@ class LivyServer(Script):
 
     self.install_packages(env)
 
-  def configure(self, env, upgrade_type=None):
+  def configure(self, env, upgrade_type=None, config_dir=None):
     import params
     env.set_params(params)
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_client.py
b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_client.py
index 2c19b88..563b7e9 100755
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_client.py
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_client.py
@@ -34,7 +34,7 @@ class SparkClient(Script):
     self.install_packages(env)
     self.configure(env)
 
-  def configure(self, env, upgrade_type=None):
+  def configure(self, env, upgrade_type=None, config_dir=None):
     import params
     env.set_params(params)
     

http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_thrift_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_thrift_server.py
b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_thrift_server.py
index 426c05c..72307cb 100755
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_thrift_server.py
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_thrift_server.py
@@ -40,7 +40,7 @@ class SparkThriftServer(Script):
 
     self.install_packages(env)
 
-  def configure(self, env ,upgrade_type=None):
+  def configure(self, env, upgrade_type=None, config_dir=None):
     import params
     env.set_params(params)
     setup_spark(env, 'server', upgrade_type = upgrade_type, action = 'config')
@@ -69,10 +69,9 @@ class SparkThriftServer(Script):
     import params
 
     env.set_params(params)
-    if params.version and check_stack_feature(StackFeature.SPARK2_THRIFTSERVER, params.version):
-      Logger.info("Executing Spark2 Thrift Server Stack Upgrade pre-restart")
-      conf_select.select(params.stack_name, "spark2", params.version)
-      stack_select.select("spark2-thriftserver", params.version)
+    Logger.info("Executing Spark2 Thrift Server Stack Upgrade pre-restart")
+    conf_select.select(params.stack_name, "spark2", params.version)
+    stack_select.select("spark2-thriftserver", params.version)
       
   def get_log_folder(self):
     import params

http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
index 3069b34..da334be 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
@@ -94,6 +94,17 @@
       </component>
     </service>
 
+    <service name="SPARK2">
+      <component name="SPARK2_CLIENT">
+        <changes>
+          <definition xsi:type="configure" id="hdp_2_5_0_0_spark2_yarn_queue">
+            <type>spark2-defaults</type>
+            <set key="spark.yarn.queue" value="default" if-type="spark-defaults" if-key="spark.yarn.queue"
if-key-state="absent"/>
+          </definition>
+        </changes>
+      </component>
+    </service>
+
     <service name="TEZ">
       <component name="TEZ_CLIENT">
         <changes>

http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
index 9516b6a..d396033 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
@@ -110,6 +110,11 @@
         <component>LIVY_SERVER</component>
       </service>
 
+      <service name="SPARK2">
+        <component>SPARK2_JOBHISTORYSERVER</component>
+        <component>SPARK2_THRIFTSERVER</component>
+      </service>
+
       <service name="ZEPPELIN">
         <component>ZEPPELIN_MASTER</component>
       </service>
@@ -538,6 +543,12 @@
           <summary>Add queue customization property</summary>
         </task>
       </execute-stage>
+
+      <execute-stage service="SPARK2" component="SPARK2_CLIENT" title="Apply config changes
for Spark">
+        <task xsi:type="configure" id="hdp_2_5_0_0_spark2_yarn_queue">
+          <summary>Add queue customization property</summary>
+        </task>
+      </execute-stage>
     </group>
 
     <!--
@@ -795,6 +806,17 @@
       </service>
     </group>
 
+    <group xsi:type="restart" name="SPARK2" title="Spark2">
+      <service-check>false</service-check>
+      <skippable>true</skippable>
+      <supports-auto-skip-failure>false</supports-auto-skip-failure>
+      <parallel-scheduler/>
+      <service name="SPARK2">
+        <component>SPARK2_JOBHISTORYSERVER</component>
+        <component>SPARK2_THRIFTSERVER</component>
+      </service>
+    </group>
+
     <group xsi:type="restart" name="SPARK_CLIENTS" title="Spark Clients">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -804,6 +826,15 @@
       </service>
     </group>
 
+    <group xsi:type="restart" name="SPARK2_CLIENTS" title="Spark2 Clients">
+      <service-check>false</service-check>
+      <skippable>true</skippable>
+      <parallel-scheduler/>
+      <service name="SPARK2">
+        <component>SPARK2_CLIENT</component>
+      </service>
+    </group>
+
     <group xsi:type="restart" name="ZEPPELIN" title="Zeppelin">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -860,6 +891,7 @@
       <priority>
         <service>HIVE</service>
         <service>SPARK</service>
+        <service>SPARK2</service>
         <service>SLIDER</service>
         <service>ATLAS</service>
         <service>ZEPPELIN</service>
@@ -1275,6 +1307,24 @@
       </component>
     </service>
 
+    <service name="SPARK2">
+      <component name="SPARK2_JOBHISTORYSERVER">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+      <component name="SPARK2_THRIFTSERVER">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+      <component name="SPARK2_CLIENT">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+    </service>
+
     <service name="ZEPPELIN">
       <component name="ZEPPELIN_MASTER">
         <upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
index 04d603b..c01d11e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
@@ -259,6 +259,16 @@
       </service>
     </group>
 
+    <group name="SPARK2" title="Spark2">
+      <skippable>true</skippable>
+      <supports-auto-skip-failure>false</supports-auto-skip-failure>
+      <service-check>false</service-check>
+      <service name="SPARK2">
+        <component>SPARK2_JOBHISTORYSERVER</component>
+        <component>SPARK2_THRIFTSERVER</component>
+      </service>
+    </group>
+
     <group name="ZEPPELIN" title="Zeppelin">
       <skippable>true</skippable>
       <supports-auto-skip-failure>false</supports-auto-skip-failure>
@@ -275,6 +285,13 @@
       </service>
     </group>
 
+    <group name="SPARK2_CLIENTS" title="Spark2 Clients">
+      <skippable>true</skippable>
+      <service name="SPARK2">
+        <component>SPARK2_CLIENT</component>
+      </service>
+    </group>
+
     <group name="ATLAS" title="Atlas">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -868,6 +885,28 @@
       </component>
     </service>
 
+    <service name="SPARK2">
+      <component name="SPARK2_JOBHISTORYSERVER">
+        <upgrade>
+          <task xsi:type="restart-task" />
+        </upgrade>
+      </component>
+      <component name="SPARK2_THRIFTSERVER">
+        <upgrade>
+          <task xsi:type="restart-task" />
+        </upgrade>
+      </component>
+      <component name="SPARK2_CLIENT">
+        <pre-upgrade>
+          <task xsi:type="configure" id="hdp_2_5_0_0_spark2_yarn_queue"/>
+        </pre-upgrade>
+        <pre-downgrade/>
+        <upgrade>
+          <task xsi:type="restart-task" />
+        </upgrade>
+      </component>
+    </service>
+
     <service name="ZEPPELIN">
       <component name="ZEPPELIN_MASTER">
         <upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
index 5347f01..921e479 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
@@ -109,6 +109,12 @@
         <component>LIVY_SERVER</component>
       </service>
 
+      <service name="SPARK2">
+        <component>SPARK2_JOBHISTORYSERVER</component>
+        <component>SPARK2_THRIFTSERVER</component>
+        <component>LIVY2_SERVER</component>
+      </service>
+
       <service name="ZEPPELIN">
         <component>ZEPPELIN_MASTER</component>
       </service>
@@ -527,6 +533,18 @@
       </service>
     </group>
 
+    <group xsi:type="restart" name="SPARK2" title="Spark2">
+      <service-check>false</service-check>
+      <skippable>true</skippable>
+      <supports-auto-skip-failure>false</supports-auto-skip-failure>
+      <parallel-scheduler/>
+      <service name="SPARK2">
+        <component>SPARK2_JOBHISTORYSERVER</component>
+        <component>SPARK2_THRIFTSERVER</component>
+        <component>LIVY2_SERVER</component>
+      </service>
+    </group>
+
     <group xsi:type="restart" name="SPARK_CLIENTS" title="Spark Clients">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -536,6 +554,15 @@
       </service>
     </group>
 
+    <group xsi:type="restart" name="SPARK2_CLIENTS" title="Spark2 Clients">
+      <service-check>false</service-check>
+      <skippable>true</skippable>
+      <parallel-scheduler/>
+      <service name="SPARK2">
+        <component>SPARK2_CLIENT</component>
+      </service>
+    </group>
+
     <group xsi:type="restart" name="ZEPPELIN" title="Zeppelin">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -592,6 +619,7 @@
       <priority>
         <service>HIVE</service>
         <service>SPARK</service>
+        <service>SPARK2</service>
         <service>SLIDER</service>
         <service>ATLAS</service>
         <service>ZEPPELIN</service>
@@ -997,6 +1025,29 @@
       </component>
     </service>
 
+    <service name="SPARK2">
+      <component name="SPARK2_JOBHISTORYSERVER">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+      <component name="SPARK2_THRIFTSERVER">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+      <component name="LIVY2_SERVER">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+      <component name="SPARK2_CLIENT">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
+    </service>
+
     <service name="ZEPPELIN">
       <component name="ZEPPELIN_MASTER">
         <upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/19da5823/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
index a4719c1..8669d2b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
@@ -258,6 +258,17 @@
       </service>
     </group>
 
+    <group name="SPARK2" title="Spark2">
+      <skippable>true</skippable>
+      <supports-auto-skip-failure>false</supports-auto-skip-failure>
+      <service-check>false</service-check>
+      <service name="SPARK2">
+        <component>SPARK2_JOBHISTORYSERVER</component>
+        <component>SPARK2_THRIFTSERVER</component>
+        <component>LIVY2_SERVER</component>
+      </service>
+    </group>
+
     <group name="ZEPPELIN" title="Zeppelin">
       <skippable>true</skippable>
       <supports-auto-skip-failure>false</supports-auto-skip-failure>
@@ -274,6 +285,13 @@
       </service>
     </group>
 
+    <group name="SPARK2_CLIENTS" title="Spark2 Clients">
+      <skippable>true</skippable>
+      <service name="SPARK2">
+        <component>SPARK2_CLIENT</component>
+      </service>
+    </group>
+
     <group name="ATLAS" title="Atlas">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -779,6 +797,29 @@
       </component>
     </service>
 
+    <service name="SPARK2">
+      <component name="SPARK2_JOBHISTORYSERVER">
+        <upgrade>
+          <task xsi:type="restart-task" />
+        </upgrade>
+      </component>
+      <component name="SPARK2_THRIFTSERVER">
+        <upgrade>
+          <task xsi:type="restart-task" />
+        </upgrade>
+      </component>
+      <component name="LIVY2_SERVER">
+        <upgrade>
+          <task xsi:type="restart-task" />
+        </upgrade>
+      </component>
+      <component name="SPARK2_CLIENT">
+        <upgrade>
+          <task xsi:type="restart-task" />
+        </upgrade>
+      </component>
+    </service>
+
     <service name="ZEPPELIN">
       <component name="ZEPPELIN_MASTER">
         <upgrade>


Mime
View raw message