From commits-return-42559-archive-asf-public=cust-asf.ponee.io@spark.apache.org Fri May 8 17:37:01 2020 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id 14CC4180665 for ; Fri, 8 May 2020 19:37:00 +0200 (CEST) Received: (qmail 70397 invoked by uid 500); 8 May 2020 17:37:00 -0000 Mailing-List: contact commits-help@spark.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list commits@spark.apache.org Received: (qmail 70388 invoked by uid 99); 8 May 2020 17:37:00 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 08 May 2020 17:37:00 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 5A50885E29; Fri, 8 May 2020 17:37:00 +0000 (UTC) Date: Fri, 08 May 2020 17:36:54 +0000 To: "commits@spark.apache.org" Subject: [spark] branch branch-3.0 updated: [SPARK-31658][SQL] Fix SQL UI not showing write commands of AQE plan MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <158895940843.30563.12304843460716356792@gitbox.apache.org> From: lixiao@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: spark X-Git-Refname: refs/heads/branch-3.0 X-Git-Reftype: branch X-Git-Oldrev: 57b55eacaaa37b2b5aae2620fb29e046ee736b08 X-Git-Newrev: ba4392217b461d20bfd10dbc00714dbb7268d71a X-Git-Rev: ba4392217b461d20bfd10dbc00714dbb7268d71a X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. lixiao pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/branch-3.0 by this push: new ba43922 [SPARK-31658][SQL] Fix SQL UI not showing write commands of AQE plan ba43922 is described below commit ba4392217b461d20bfd10dbc00714dbb7268d71a Author: manuzhang AuthorDate: Fri May 8 10:24:13 2020 -0700 [SPARK-31658][SQL] Fix SQL UI not showing write commands of AQE plan Show write commands on SQL UI of an AQE plan Currently the leaf node of an AQE plan is always a `AdaptiveSparkPlan` which is not true when it's a child of a write command. Hence, the node of the write command as well as its metrics are not shown on the SQL UI. ![image](https://user-images.githubusercontent.com/1191767/81288918-1893f580-9098-11ea-9771-e3d0820ba806.png) ![image](https://user-images.githubusercontent.com/1191767/81289008-3a8d7800-9098-11ea-93ec-516bbaf25d2d.png) No Add UT. Closes #28474 from manuzhang/aqe-ui. Lead-authored-by: manuzhang Co-authored-by: Xiao Li Signed-off-by: gatorsmile (cherry picked from commit 77c690a7252b22c9dd8f3cb7ac32f79fd6845cad) Signed-off-by: gatorsmile --- .../execution/adaptive/AdaptiveSparkPlanExec.scala | 4 +-- .../adaptive/AdaptiveQueryExecSuite.scala | 35 ++++++++++++++++++++-- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala index cd6936b..90d1db9 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala @@ -526,8 +526,8 @@ case class AdaptiveSparkPlanExec( } else { context.session.sparkContext.listenerBus.post(SparkListenerSQLAdaptiveExecutionUpdate( executionId, - SQLExecution.getQueryExecution(executionId).toString, - SparkPlanInfo.fromSparkPlan(this))) + context.qe.toString, + SparkPlanInfo.fromSparkPlan(context.qe.executedPlan))) } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala index f30d1e9..29b9755 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala @@ -805,9 +805,11 @@ class AdaptiveQueryExecSuite test("SPARK-30953: InsertAdaptiveSparkPlan should apply AQE on child plan of write commands") { withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY.key -> "true") { - val plan = sql("CREATE TABLE t1 AS SELECT 1 col").queryExecution.executedPlan - assert(plan.isInstanceOf[DataWritingCommandExec]) - assert(plan.asInstanceOf[DataWritingCommandExec].child.isInstanceOf[AdaptiveSparkPlanExec]) + withTable("t1") { + val plan = sql("CREATE TABLE t1 AS SELECT 1 col").queryExecution.executedPlan + assert(plan.isInstanceOf[DataWritingCommandExec]) + assert(plan.asInstanceOf[DataWritingCommandExec].child.isInstanceOf[AdaptiveSparkPlanExec]) + } } } @@ -847,4 +849,31 @@ class AdaptiveQueryExecSuite } } } + + test("SPARK-31658: SQL UI should show write commands") { + withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", + SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY.key -> "true") { + withTable("t1") { + var checkDone = false + val listener = new SparkListener { + override def onOtherEvent(event: SparkListenerEvent): Unit = { + event match { + case SparkListenerSQLAdaptiveExecutionUpdate(_, _, planInfo) => + assert(planInfo.nodeName == "Execute CreateDataSourceTableAsSelectCommand") + checkDone = true + case _ => // ignore other events + } + } + } + spark.sparkContext.addSparkListener(listener) + try { + sql("CREATE TABLE t1 AS SELECT 1 col").collect() + spark.sparkContext.listenerBus.waitUntilEmpty() + assert(checkDone) + } finally { + spark.sparkContext.removeSparkListener(listener) + } + } + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org For additional commands, e-mail: commits-help@spark.apache.org