spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r...@apache.org
Subject spark git commit: [SPARK-16672][SQL] SQLBuilder should not raise exceptions on EXISTS queries
Date Tue, 26 Jul 2016 02:52:24 GMT
Repository: spark
Updated Branches:
  refs/heads/branch-2.0 3d3547487 -> aeb6d5c05


[SPARK-16672][SQL] SQLBuilder should not raise exceptions on EXISTS queries

## What changes were proposed in this pull request?

Currently, `SQLBuilder` raises `empty.reduceLeft` exceptions on *unoptimized* `EXISTS` queries.
We had better prevent this.
```scala
scala> sql("CREATE TABLE t1(a int)")
scala> val df = sql("select * from t1 b where exists (select * from t1 a)")
scala> new org.apache.spark.sql.catalyst.SQLBuilder(df).toSQL
java.lang.UnsupportedOperationException: empty.reduceLeft
```

## How was this patch tested?

Pass the Jenkins tests with a new test suite.

Author: Dongjoon Hyun <dongjoon@apache.org>

Closes #14307 from dongjoon-hyun/SPARK-16672.

(cherry picked from commit 8a8d26f1e27db5c2228307b1c3609b4713b9d0db)
Signed-off-by: Reynold Xin <rxin@databricks.com>


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/aeb6d5c0
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/aeb6d5c0
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/aeb6d5c0

Branch: refs/heads/branch-2.0
Commit: aeb6d5c053d4e848df0e7842a3994154df464647
Parents: 3d35474
Author: Dongjoon Hyun <dongjoon@apache.org>
Authored: Mon Jul 25 19:52:17 2016 -0700
Committer: Reynold Xin <rxin@databricks.com>
Committed: Mon Jul 25 19:52:22 2016 -0700

----------------------------------------------------------------------
 .../scala/org/apache/spark/sql/catalyst/SQLBuilder.scala  |  9 +++++++--
 sql/hive/src/test/resources/sqlgen/predicate_subquery.sql |  4 ++++
 .../apache/spark/sql/catalyst/LogicalPlanToSQLSuite.scala | 10 ++++++++++
 3 files changed, 21 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/aeb6d5c0/sql/core/src/main/scala/org/apache/spark/sql/catalyst/SQLBuilder.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/SQLBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/SQLBuilder.scala
index a8cc72f..9a02e3c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/SQLBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/SQLBuilder.scala
@@ -512,8 +512,13 @@ class SQLBuilder(logicalPlan: LogicalPlan) extends Logging {
           ScalarSubquery(rewrite, Seq.empty, exprId)
 
         case PredicateSubquery(query, conditions, false, exprId) =>
-          val plan = Project(Seq(Alias(Literal(1), "1")()),
-            Filter(conditions.reduce(And), addSubqueryIfNeeded(query)))
+          val subquery = addSubqueryIfNeeded(query)
+          val plan = if (conditions.isEmpty) {
+            subquery
+          } else {
+            Project(Seq(Alias(Literal(1), "1")()),
+              Filter(conditions.reduce(And), subquery))
+          }
           Exists(plan, exprId)
 
         case PredicateSubquery(query, conditions, true, exprId) =>

http://git-wip-us.apache.org/repos/asf/spark/blob/aeb6d5c0/sql/hive/src/test/resources/sqlgen/predicate_subquery.sql
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/resources/sqlgen/predicate_subquery.sql b/sql/hive/src/test/resources/sqlgen/predicate_subquery.sql
new file mode 100644
index 0000000..2e06b4f
--- /dev/null
+++ b/sql/hive/src/test/resources/sqlgen/predicate_subquery.sql
@@ -0,0 +1,4 @@
+-- This file is automatically generated by LogicalPlanToSQLSuite.
+select * from t1 b where exists (select * from t1 a)
+--------------------------------------------------------------------------------
+SELECT `gen_attr` AS `a` FROM (SELECT `gen_attr` FROM (SELECT `a` AS `gen_attr` FROM `default`.`t1`)
AS gen_subquery_0 WHERE EXISTS(SELECT `gen_attr` AS `a` FROM ((SELECT `gen_attr` FROM (SELECT
`a` AS `gen_attr` FROM `default`.`t1`) AS gen_subquery_0) AS gen_subquery_1) AS gen_subquery_1))
AS b

http://git-wip-us.apache.org/repos/asf/spark/blob/aeb6d5c0/sql/hive/src/test/scala/org/apache/spark/sql/catalyst/LogicalPlanToSQLSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/catalyst/LogicalPlanToSQLSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/catalyst/LogicalPlanToSQLSuite.scala
index 1f5078d..ebece38 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/catalyst/LogicalPlanToSQLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/catalyst/LogicalPlanToSQLSuite.scala
@@ -25,6 +25,7 @@ import scala.util.control.NonFatal
 import org.apache.spark.sql.Column
 import org.apache.spark.sql.catalyst.parser.ParseException
 import org.apache.spark.sql.functions._
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SQLTestUtils
 
 /**
@@ -927,6 +928,15 @@ class LogicalPlanToSQLSuite extends SQLBuilderTest with SQLTestUtils
{
     }
   }
 
+  test("predicate subquery") {
+    withTable("t1") {
+      withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "true") {
+        sql("CREATE TABLE t1(a int)")
+        checkSQL("select * from t1 b where exists (select * from t1 a)", "predicate_subquery")
+      }
+    }
+  }
+
   test("SPARK-14933 - select orc table") {
     withTable("orc_t") {
       sql("create table orc_t stored as orc as select 1 as c1, 'abc' as c2")


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message