spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hvanhov...@apache.org
Subject spark git commit: [SPARK-13721][SQL] Make GeneratorOuter unresolved.
Date Wed, 22 Feb 2017 14:42:50 GMT
Repository: spark
Updated Branches:
  refs/heads/master bf7bb4977 -> 10c566cc3


[SPARK-13721][SQL] Make GeneratorOuter unresolved.

## What changes were proposed in this pull request?

This is a small change to make GeneratorOuter always unresolved. It is mostly no-op change
but makes it more clear since GeneratorOuter shouldn't survive analysis phase.
This requires also handling in ResolveAliases rule.

## How was this patch tested?
Existing generator tests.

Author: Bogdan Raducanu <bogdan@databricks.com>
Author: Reynold Xin <rxin@databricks.com>

Closes #17026 from bogdanrdc/PR16958.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/10c566cc
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/10c566cc
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/10c566cc

Branch: refs/heads/master
Commit: 10c566cc3b5f93ddd823b1c987d9de8286cdea60
Parents: bf7bb49
Author: Bogdan Raducanu <bogdan@databricks.com>
Authored: Wed Feb 22 15:42:40 2017 +0100
Committer: Herman van Hovell <hvanhovell@databricks.com>
Committed: Wed Feb 22 15:42:40 2017 +0100

----------------------------------------------------------------------
 .../org/apache/spark/sql/catalyst/analysis/Analyzer.scala    | 2 +-
 .../apache/spark/sql/catalyst/expressions/generators.scala   | 8 +++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/10c566cc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 39a2762..c477cb4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -222,6 +222,7 @@ class Analyzer(
           expr.transformUp { case u @ UnresolvedAlias(child, optGenAliasFunc) =>
             child match {
               case ne: NamedExpression => ne
+              case go @ GeneratorOuter(g: Generator) if g.resolved => MultiAlias(go, Nil)
               case e if !e.resolved => u
               case g: Generator => MultiAlias(g, Nil)
               case c @ Cast(ne: NamedExpression, _, _) => Alias(c, ne.name)()
@@ -1665,7 +1666,6 @@ class Analyzer(
         var resolvedGenerator: Generate = null
 
         val newProjectList = projectList.flatMap {
-
           case AliasedGenerator(generator, names, outer) if generator.childrenResolved =>
             // It's a sanity check, this should not happen as the previous case will throw
             // exception earlier.

http://git-wip-us.apache.org/repos/asf/spark/blob/10c566cc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
index 1b98c30..e84796f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
@@ -173,7 +173,6 @@ case class Stack(children: Seq[Expression]) extends Generator {
     }
   }
 
-
   /**
    * Only support code generation when stack produces 50 rows or less.
    */
@@ -204,6 +203,10 @@ case class Stack(children: Seq[Expression]) extends Generator {
   }
 }
 
+/**
+ * Wrapper around another generator to specify outer behavior. This is used to implement
functions
+ * such as explode_outer. This expression gets replaced during analysis.
+ */
 case class GeneratorOuter(child: Generator) extends UnaryExpression with Generator {
   final override def eval(input: InternalRow = null): TraversableOnce[InternalRow] =
     throw new UnsupportedOperationException(s"Cannot evaluate expression: $this")
@@ -212,7 +215,10 @@ case class GeneratorOuter(child: Generator) extends UnaryExpression with
Generat
     throw new UnsupportedOperationException(s"Cannot evaluate expression: $this")
 
   override def elementSchema: StructType = child.elementSchema
+
+  override lazy val resolved: Boolean = false
 }
+
 /**
  * A base class for [[Explode]] and [[PosExplode]].
  */


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message