ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rle...@apache.org
Subject [45/50] [abbrv] ambari git commit: AMBARI-22260. Update Spark2 log4j default settings to latest. (sai.sai.shao via sshridhar).
Date Thu, 16 Nov 2017 19:46:01 GMT
AMBARI-22260. Update Spark2 log4j default settings to latest. (sai.sai.shao via sshridhar).


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4042f9fd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4042f9fd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4042f9fd

Branch: refs/heads/branch-feature-AMBARI-20859
Commit: 4042f9fde14548be77a5ed5daf45364df307c08a
Parents: 0936f46
Author: Swapan Shridhar <sshridhar@hortonworks.com>
Authored: Wed Nov 15 16:21:31 2017 -0800
Committer: Swapan Shridhar <sshridhar@hortonworks.com>
Committed: Wed Nov 15 16:21:31 2017 -0800

----------------------------------------------------------------------
 .../stacks/HDP/2.5/upgrades/config-upgrade.xml  | 23 ++++++++
 .../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 23 +++++---
 .../stacks/HDP/2.5/upgrades/upgrade-2.6.xml     | 45 +++++++++-------
 .../configuration/spark2-log4j-properties.xml   | 56 ++++++++++++++++++++
 4 files changed, 122 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/4042f9fd/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
index b1e5f91..8c2c860 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
@@ -104,14 +104,37 @@
     </service>
 
     <service name="SPARK2">
+      <component name="SPARK2_JOBHISTORYSERVER">
+        <changes>
+          <definition xsi:type="configure" id="hdp_2_5_0_0_spark2_jobhistoryserver_log4j">
+            <type>spark2-log4j-properties</type>
+            <replace key="content" find="log4j.logger.org.eclipse.jetty=WARN" replace-with="log4j.logger.org.spark_project.jetty=WARN"/>
+            <replace key="content" find="log4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR"
replace-with="log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR"/>
+          </definition>
+        </changes>
+      </component>
       <component name="SPARK2_CLIENT">
         <changes>
+          <definition xsi:type="configure" id="hdp_2_5_0_0_spark2_client_log4j">
+            <type>spark2-log4j-properties</type>
+            <replace key="content" find="log4j.logger.org.eclipse.jetty=WARN" replace-with="log4j.logger.org.spark_project.jetty=WARN"/>
+            <replace key="content" find="log4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR"
replace-with="log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR"/>
+          </definition>
           <definition xsi:type="configure" id="hdp_2_5_0_0_spark2_yarn_queue">
             <type>spark2-defaults</type>
             <set key="spark.yarn.queue" value="default" if-type="spark-defaults" if-key="spark.yarn.queue"
if-key-state="absent"/>
           </definition>
         </changes>
       </component>
+      <component name="SPARK2_THRIFTSERVER">
+        <changes>
+          <definition xsi:type="configure" id="hdp_2_5_0_0_spark2_thriftserver_log4j">
+            <type>spark2-log4j-properties</type>
+            <replace key="content" find="log4j.logger.org.eclipse.jetty=WARN" replace-with="log4j.logger.org.spark_project.jetty=WARN"/>
+            <replace key="content" find="log4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR"
replace-with="log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR"/>
+          </definition>
+        </changes>
+      </component>
     </service>
 
     <service name="TEZ">

http://git-wip-us.apache.org/repos/asf/ambari/blob/4042f9fd/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
index ace9542..9caa850 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
@@ -77,7 +77,7 @@
       <service name="FLUME">
         <component>FLUME_HANDLER</component>
       </service>
-      
+
       <service name="ACCUMULO">
         <component>ACCUMULO_TRACER</component>
         <component>ACCUMULO_GC</component>
@@ -355,7 +355,7 @@
         <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixCapacitySchedulerOrderingPolicy">
           <summary>Validate Root Queue Ordering Policy</summary>
         </task>
-      </execute-stage> 
+      </execute-stage>
 
       <!--TEZ-->
       <execute-stage service="TEZ" component="TEZ_CLIENT" title="Verify LZO codec path
for Tez">
@@ -679,6 +679,15 @@
           <summary>Add queue customization property</summary>
         </task>
       </execute-stage>
+      <execute-stage service="SPARK2" component="SPARK2_THRIFTSERVER" title="Apply config
changes for Spark2 ThriftServer log4j">
+        <task xsi:type="configure" id="hdp_2_5_0_0_spark2_thriftserver_log4j"/>
+      </execute-stage>
+      <execute-stage service="SPARK2" component="SPARK2_JOBHISTORYSERVER" title="Apply
config changes for Spark2 Job HistoryServer log4j">
+        <task xsi:type="configure" id="hdp_2_5_0_0_spark2_jobhistoryserver_log4j"/>
+      </execute-stage>
+      <execute-stage service="SPARK2" component="SPARK2_CLIENT" title="Apply config changes
for Spark2 Client log4j">
+        <task xsi:type="configure" id="hdp_2_5_0_0_spark2_client_log4j"/>
+      </execute-stage>
     </group>
 
     <!--
@@ -720,7 +729,7 @@
         </task>
       </execute-stage>
     </group>
-    
+
     <!-- Now, restart all of the services. -->
     <group xsi:type="restart" name="ZOOKEEPER" title="ZooKeeper">
       <service-check>false</service-check>
@@ -1083,7 +1092,7 @@
         <component>FLUME_HANDLER</component>
       </service>
     </group>
-    
+
     <group xsi:type="restart" name="ACCUMULO" title="Accumulo">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -1112,7 +1121,7 @@
 
     <group xsi:type="cluster" name="FINALIZE_PRE_CHECK" title="Finalize {{direction.text.proper}}
Pre-Check">
       <direction>UPGRADE</direction>
-      
+
       <execute-stage title="Check Component Versions">
         <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ComponentVersionCheckAction"
/>
       </execute-stage>
@@ -1192,7 +1201,7 @@
             <function>setup_ranger_java_patches</function>
           </task>
         </pre-upgrade>
-        
+
         <pre-downgrade copy-upgrade="true" />
 
         <upgrade>
@@ -1215,7 +1224,7 @@
             <function>configure_atlas_user_for_tagsync</function>
           </task>
         </pre-upgrade>
-        
+
         <pre-downgrade copy-upgrade="true" />
 
         <upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/4042f9fd/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
index df11ae1..4fc9b42 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
@@ -183,7 +183,7 @@
         <component>HBASE_MASTER</component>
       </service>
     </group>
-    
+
     <group name="SERVICE_CHECK_1" title="All Service Checks" xsi:type="service-check">
       <direction>UPGRADE</direction>
       <skippable>true</skippable>
@@ -199,7 +199,7 @@
         <service>LOGSEARCH</service>
       </exclude>
     </group>
-    
+
     <group name="CORE_SLAVES" title="Core Slaves" xsi:type="colocated">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -207,7 +207,7 @@
         <component>DATANODE</component>
         <component>NFS_GATEWAY</component>
       </service>
-      
+
       <service name="HBASE">
         <component>HBASE_REGIONSERVER</component>
         <component>PHOENIX_QUERY_SERVER</component>
@@ -223,7 +223,7 @@
         <message>The initial batch of {{components}} hosts have been {{direction.past}}.
You are advised to check the hosts and perform cluster/workload-specific tests against your
cluster to ensure proper operation before proceeding with {{direction.text}} of the remaining
services.</message>
       </batch>
     </group>
-    
+
     <group name="SERVICE_CHECK_2" title="All Service Checks" xsi:type="service-check">
       <direction>UPGRADE</direction>
       <skippable>true</skippable>
@@ -458,7 +458,7 @@
 
     <group xsi:type="cluster" name="FINALIZE_PRE_CHECK" title="Finalize {{direction.text.proper}}
Pre-Check">
       <direction>UPGRADE</direction>
-      
+
       <execute-stage title="Check Component Versions">
         <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ComponentVersionCheckAction"
/>
       </execute-stage>
@@ -467,7 +467,7 @@
     <group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}">
       <skippable>true</skippable>
       <supports-auto-skip-failure>false</supports-auto-skip-failure>
-      
+
       <execute-stage title="Confirm Finalize">
         <direction>UPGRADE</direction>
         <task xsi:type="manual">
@@ -491,7 +491,7 @@
           <function>finalize_rolling_upgrade</function>
         </task>
       </execute-stage>
-      
+
       <execute-stage title="Save Cluster State">
         <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FinalizeUpgradeAction">
         </task>
@@ -577,7 +577,7 @@
             <function>configure_atlas_user_for_tagsync</function>
           </task>
         </pre-upgrade>
-        
+
         <pre-downgrade copy-upgrade="true" />
 
         <upgrade>
@@ -668,7 +668,7 @@
         </upgrade>
       </component>
     </service>
-    
+
     <service name="SLIDER">
       <component name="SLIDER">
         <upgrade>
@@ -691,7 +691,7 @@
           </task>
           <task xsi:type="configure" id="hdp_2_6_0_0_mapreduce_job_queuename"/>
         </pre-upgrade>
-        
+
         <pre-downgrade copy-upgrade="true" />
 
         <upgrade>
@@ -722,11 +722,11 @@
           <task xsi:type="configure" id="yarn_site_retained_log_count" />
           <task xsi:type="configure" id="hdp_2_6_0_0_service_check_queue_name"/>
           <task xsi:type="configure" id="hdp_2_6_0_0_ats_scan_interval_default"/>
-          
+
           <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixCapacitySchedulerOrderingPolicy">
             <summary>Validate Root Queue Ordering Policy</summary>
           </task>
-          
+
         </pre-upgrade>
         <pre-downgrade />
         <upgrade>
@@ -785,9 +785,9 @@
           </task>
           <task xsi:type="configure" id="hdp_2_5_0_0_tez_queue_name"/>
         </pre-upgrade>
-        
+
         <pre-downgrade copy-upgrade="true" />
-        
+
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>
@@ -865,9 +865,9 @@
           <task xsi:type="configure" id="hdp_2_6_maint_llap_config"/>
           <task xsi:type="configure" id="llap_update_tez_shuffle_ssl_enable"/>
         </pre-upgrade>
-        
+
         <pre-downgrade />
-      
+
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>
@@ -913,9 +913,9 @@
           <task xsi:type="configure" id="hdp_2_5_0_0_rename_spark_livy_configs" />
           <task xsi:type="configure" id="hdp_2_5_0_0_add_spark_conf_dir_livy_configs"
/>
         </pre-upgrade>
-        
+
         <pre-downgrade />
-        
+
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>
@@ -933,11 +933,19 @@
 
     <service name="SPARK2">
       <component name="SPARK2_JOBHISTORYSERVER">
+        <pre-upgrade>
+          <task xsi:type="configure" id="hdp_2_5_0_0_spark2_jobhistoryserver_log4j"/>
+        </pre-upgrade>
+        <pre-downgrade />
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>
       </component>
       <component name="SPARK2_THRIFTSERVER">
+        <pre-upgrade>
+          <task xsi:type="configure" id="hdp_2_5_0_0_spark2_thriftserver_log4j"/>
+        </pre-upgrade>
+        <pre-downgrade />
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>
@@ -945,6 +953,7 @@
       <component name="SPARK2_CLIENT">
         <pre-upgrade>
           <task xsi:type="configure" id="hdp_2_5_0_0_spark2_yarn_queue"/>
+          <task xsi:type="configure" id="hdp_2_5_0_0_spark2_client_log4j"/>
         </pre-upgrade>
         <pre-downgrade/>
         <upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/4042f9fd/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-log4j-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-log4j-properties.xml
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-log4j-properties.xml
new file mode 100644
index 0000000..d7479f4
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-log4j-properties.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="false" supports_adding_forbidden="true">
+  <property>
+    <name>content</name>
+    <description>Spark2-log4j-Properties</description>
+    <value>
+# Set everything to be logged to the console
+log4j.rootCategory=INFO, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
+
+# Set the default spark-shell log level to WARN. When running the spark-shell, the
+# log level for this class is used to overwrite the root logger's log level, so that
+# the user can have different defaults for the shell and regular Spark apps.
+log4j.logger.org.apache.spark.repl.Main=WARN
+
+# Settings to quiet third party logs that are too verbose
+log4j.logger.org.spark_project.jetty=WARN
+log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR
+log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO
+log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
+log4j.logger.org.apache.parquet=ERROR
+log4j.logger.parquet=ERROR
+
+# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL
with Hive support
+log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL
+log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR
+    </value>
+    <value-attributes>
+      <type>content</type>
+      <show-property-name>false</show-property-name>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+</configuration>


Mime
View raw message