Mailing-List: contact commits-help@spark.apache.org; run by ezmlm
Precedence: bulk
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: rxin@apache.org
To: commits@spark.apache.org
Message-Id: <98a217dfe6b84c77afedbf9ee8001d55@git.apache.org>
Subject: spark git commit: [SPARK-15730][SQL] Respect the --hiveconf in the
 spark-sql command line
Date: Tue,  5 Jul 2016 23:42:52 +0000 (UTC)
archived-at: Tue, 05 Jul 2016 23:42:54 -0000

Repository: spark
Updated Branches:
  refs/heads/branch-2.0 cabee2324 -> 9c1596b6c


[SPARK-15730][SQL] Respect the --hiveconf in the spark-sql command line

## What changes were proposed in this pull request?
This PR makes spark-sql (backed by SparkSQLCLIDriver) respects confs set by hiveconf, which is what we do in previous versions. The change is that when we start SparkSQLCLIDriver, we explicitly set confs set through --hiveconf to SQLContext's conf (basically treating those confs as a SparkSQL conf).

## How was this patch tested?
A new test in CliSuite.

Closes #13542

Author: Cheng Hao <hao.cheng@intel.com>
Author: Yin Huai <yhuai@databricks.com>

Closes #14058 from yhuai/hiveConfThriftServer.

(cherry picked from commit 920cb5fe4ed0eb008cd14bf0ea45ed5b225b5022)
Signed-off-by: Reynold Xin <rxin@databricks.com>


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9c1596b6
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9c1596b6
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9c1596b6

Branch: refs/heads/branch-2.0
Commit: 9c1596b6cf5ca6d0b2529e31dc072a16db014683
Parents: cabee23
Author: Cheng Hao <hao.cheng@intel.com>
Authored: Tue Jul 5 16:42:43 2016 -0700
Committer: Reynold Xin <rxin@databricks.com>
Committed: Tue Jul 5 16:42:49 2016 -0700

----------------------------------------------------------------------
 .../spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala  |  8 ++++++++
 .../apache/spark/sql/hive/thriftserver/CliSuite.scala    | 11 +++++++++++
 2 files changed, 19 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/9c1596b6/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
----------------------------------------------------------------------
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index 7389e18..5dafec1 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -156,6 +156,14 @@ private[hive] object SparkSQLCLIDriver extends Logging {
     // Execute -i init files (always in silent mode)
     cli.processInitFiles(sessionState)
 
+    // Respect the configurations set by --hiveconf from the command line
+    // (based on Hive's CliDriver).
+    val it = sessionState.getOverriddenConfigurations.entrySet().iterator()
+    while (it.hasNext) {
+      val kv = it.next()
+      SparkSQLEnv.sqlContext.setConf(kv.getKey, kv.getValue)
+    }
+
     if (sessionState.execString != null) {
       System.exit(cli.processLine(sessionState.execString))
     }

http://git-wip-us.apache.org/repos/asf/spark/blob/9c1596b6/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
index 75535ca..d3cec11 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
@@ -91,6 +91,8 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
          |  --hiveconf ${ConfVars.METASTORECONNECTURLKEY}=$jdbcUrl
          |  --hiveconf ${ConfVars.METASTOREWAREHOUSE}=$warehousePath
          |  --hiveconf ${ConfVars.SCRATCHDIR}=$scratchDirPath
+         |  --hiveconf conf1=conftest
+         |  --hiveconf conf2=1
        """.stripMargin.split("\\s+").toSeq ++ extraArgs
     }
 
@@ -272,4 +274,13 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
       s"LIST FILE $dataFilePath;" -> "small_kv.txt"
     )
   }
+
+  test("apply hiveconf from cli command") {
+    runCliWithin(2.minute)(
+      "SET conf1;" -> "conftest",
+      "SET conf2;" -> "1",
+      "SET conf3=${hiveconf:conf1};" -> "conftest",
+      "SET conf3;" -> "conftest"
+    )
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org