spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r...@apache.org
Subject spark git commit: [SPARK-14795][SQL] Remove the use of Hive's variable substitution
Date Thu, 21 Apr 2016 18:42:30 GMT
Repository: spark
Updated Branches:
  refs/heads/master 79008e6cf -> 3a21e8d5e


[SPARK-14795][SQL] Remove the use of Hive's variable substitution

## What changes were proposed in this pull request?
This patch builds on #12556 and completely removes the use of Hive's variable substitution.

## How was this patch tested?
Covered by existing tests.

Author: Reynold Xin <rxin@databricks.com>

Closes #12561 from rxin/SPARK-14795.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3a21e8d5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3a21e8d5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3a21e8d5

Branch: refs/heads/master
Commit: 3a21e8d5ed640e3f82946893e24c099aa723c169
Parents: 79008e6
Author: Reynold Xin <rxin@databricks.com>
Authored: Thu Apr 21 11:42:25 2016 -0700
Committer: Reynold Xin <rxin@databricks.com>
Committed: Thu Apr 21 11:42:25 2016 -0700

----------------------------------------------------------------------
 .../org/apache/spark/sql/hive/HiveSessionState.scala    |  3 +--
 .../apache/spark/sql/hive/execution/HiveSqlParser.scala | 12 +++++-------
 .../apache/spark/sql/hive/execution/SQLQuerySuite.scala |  4 ++--
 3 files changed, 8 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/3a21e8d5/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala
index 171def4..6f4332c 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala
@@ -21,7 +21,6 @@ import java.util.regex.Pattern
 
 import org.apache.hadoop.hive.conf.HiveConf
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars
-import org.apache.hadoop.hive.ql.parse.VariableSubstitution
 
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.analysis.Analyzer
@@ -109,7 +108,7 @@ private[hive] class HiveSessionState(ctx: SQLContext) extends SessionState(ctx)
   /**
    * Parser for HiveQl query texts.
    */
-  override lazy val sqlParser: ParserInterface = new HiveSqlParser(conf, hiveconf)
+  override lazy val sqlParser: ParserInterface = new HiveSqlParser(conf)
 
   /**
    * Planner that takes into account Hive-specific strategies.

http://git-wip-us.apache.org/repos/asf/spark/blob/3a21e8d5/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala
index 1c4cda7..989da92 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala
@@ -20,8 +20,6 @@ package org.apache.spark.sql.hive.execution
 import scala.util.Try
 
 import org.antlr.v4.runtime.Token
-import org.apache.hadoop.hive.conf.HiveConf
-import org.apache.hadoop.hive.ql.parse.VariableSubstitution
 import org.apache.hadoop.hive.serde.serdeConstants
 
 import org.apache.spark.sql.catalyst.catalog._
@@ -29,23 +27,23 @@ import org.apache.spark.sql.catalyst.parser._
 import org.apache.spark.sql.catalyst.parser.SqlBaseParser._
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.execution.SparkSqlAstBuilder
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.{SQLConf, VariableSubstitution}
 
 /**
  * Concrete parser for HiveQl statements.
  */
-class HiveSqlParser(conf: SQLConf, hiveconf: HiveConf) extends AbstractSqlParser {
+class HiveSqlParser(conf: SQLConf) extends AbstractSqlParser {
 
   val astBuilder = new HiveSqlAstBuilder(conf)
 
-  lazy val substitutor = new VariableSubstitution
+  private val substitutor = new VariableSubstitution(conf)
 
   protected override def parse[T](command: String)(toResult: SqlBaseParser => T): T =
{
-    super.parse(substitutor.substitute(hiveconf, command))(toResult)
+    super.parse(substitutor.substitute(command))(toResult)
   }
 
   protected override def nativeCommand(sqlText: String): LogicalPlan = {
-    HiveNativeCommand(substitutor.substitute(hiveconf, sqlText))
+    HiveNativeCommand(substitutor.substitute(sqlText))
   }
 }
 

http://git-wip-us.apache.org/repos/asf/spark/blob/3a21e8d5/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index b97e9fe..345ee8e 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -512,13 +512,13 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton
{
       sql("SELECT key FROM ${hiveconf:tbl} ORDER BY key, value limit 1"),
       sql("SELECT key FROM src ORDER BY key, value limit 1").collect().toSeq)
 
-    sql("set hive.variable.substitute=false") // disable the substitution
+    sql("set spark.sql.variable.substitute=false") // disable the substitution
     sql("set tbl2=src")
     intercept[Exception] {
       sql("SELECT key FROM ${hiveconf:tbl2} ORDER BY key, value limit 1").collect()
     }
 
-    sql("set hive.variable.substitute=true") // enable the substitution
+    sql("set spark.sql.variable.substitute=true") // enable the substitution
     checkAnswer(
       sql("SELECT key FROM ${hiveconf:tbl2} ORDER BY key, value limit 1"),
       sql("SELECT key FROM src ORDER BY key, value limit 1").collect().toSeq)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message