From commits-return-33410-archive-asf-public=cust-asf.ponee.io@spark.apache.org Sun Sep 16 01:20:52 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 80D86180629 for ; Sun, 16 Sep 2018 01:20:51 +0200 (CEST) Received: (qmail 81103 invoked by uid 500); 15 Sep 2018 23:20:50 -0000 Mailing-List: contact commits-help@spark.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list commits@spark.apache.org Received: (qmail 81094 invoked by uid 99); 15 Sep 2018 23:20:50 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 15 Sep 2018 23:20:50 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 874CEDFCBA; Sat, 15 Sep 2018 23:20:50 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: lixiao@apache.org To: commits@spark.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: spark git commit: [SPARK-25426][SQL] Remove the duplicate fallback logic in UnsafeProjection Date: Sat, 15 Sep 2018 23:20:50 +0000 (UTC) Repository: spark Updated Branches: refs/heads/master be454a7ce -> 5ebef33c8 [SPARK-25426][SQL] Remove the duplicate fallback logic in UnsafeProjection ## What changes were proposed in this pull request? This pr removed the duplicate fallback logic in `UnsafeProjection`. This pr comes from #22355. ## How was this patch tested? Added tests in `CodeGeneratorWithInterpretedFallbackSuite`. Closes #22417 from maropu/SPARK-25426. Authored-by: Takeshi Yamamuro Signed-off-by: gatorsmile Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5ebef33c Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5ebef33c Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5ebef33c Branch: refs/heads/master Commit: 5ebef33c85a66cdc29db2eff2343600602bbe94e Parents: be454a7 Author: Takeshi Yamamuro Authored: Sat Sep 15 16:20:45 2018 -0700 Committer: gatorsmile Committed: Sat Sep 15 16:20:45 2018 -0700 ---------------------------------------------------------------------- .../sql/catalyst/expressions/Projection.scala | 25 ++------------------ .../sql/execution/basicPhysicalOperators.scala | 3 +-- 2 files changed, 3 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/5ebef33c/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Projection.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Projection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Projection.scala index 226a4dd..5f24170 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Projection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Projection.scala @@ -17,10 +17,9 @@ package org.apache.spark.sql.catalyst.expressions -import scala.util.control.NonFatal - import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.codegen.{GenerateSafeProjection, GenerateUnsafeProjection} +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataType, StructType} /** @@ -117,7 +116,7 @@ object UnsafeProjection extends CodeGeneratorWithInterpretedFallback[Seq[Expression], UnsafeProjection] { override protected def createCodeGeneratedObject(in: Seq[Expression]): UnsafeProjection = { - GenerateUnsafeProjection.generate(in) + GenerateUnsafeProjection.generate(in, SQLConf.get.subexpressionEliminationEnabled) } override protected def createInterpretedObject(in: Seq[Expression]): UnsafeProjection = { @@ -168,26 +167,6 @@ object UnsafeProjection def create(exprs: Seq[Expression], inputSchema: Seq[Attribute]): UnsafeProjection = { create(toBoundExprs(exprs, inputSchema)) } - - /** - * Same as other create()'s but allowing enabling/disabling subexpression elimination. - * The param `subexpressionEliminationEnabled` doesn't guarantee to work. For example, - * when fallbacking to interpreted execution, it is not supported. - */ - def create( - exprs: Seq[Expression], - inputSchema: Seq[Attribute], - subexpressionEliminationEnabled: Boolean): UnsafeProjection = { - val unsafeExprs = toUnsafeExprs(toBoundExprs(exprs, inputSchema)) - try { - GenerateUnsafeProjection.generate(unsafeExprs, subexpressionEliminationEnabled) - } catch { - case NonFatal(_) => - // We should have already seen the error message in `CodeGenerator` - logWarning("Expr codegen error and falling back to interpreter mode") - InterpretedUnsafeProjection.createProjection(unsafeExprs) - } - } } /** http://git-wip-us.apache.org/repos/asf/spark/blob/5ebef33c/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala index 9434ceb..222a1b8 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala @@ -68,8 +68,7 @@ case class ProjectExec(projectList: Seq[NamedExpression], child: SparkPlan) protected override def doExecute(): RDD[InternalRow] = { child.execute().mapPartitionsWithIndexInternal { (index, iter) => - val project = UnsafeProjection.create(projectList, child.output, - subexpressionEliminationEnabled) + val project = UnsafeProjection.create(projectList, child.output) project.initialize(index) iter.map(project) } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org For additional commands, e-mail: commits-help@spark.apache.org