spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marmb...@apache.org
Subject spark git commit: [SPARK-4908][SQL] Prevent multiple concurrent hive native commands
Date Tue, 30 Dec 2014 19:25:10 GMT
Repository: spark
Updated Branches:
  refs/heads/branch-1.2 42809db9b -> cde8a310a


[SPARK-4908][SQL] Prevent multiple concurrent hive native commands

This is just a quick fix that locks when calling `runHive`.  If we can find a way to avoid
the error without a global lock that would be better.

Author: Michael Armbrust <michael@databricks.com>

Closes #3834 from marmbrus/hiveConcurrency and squashes the following commits:

bf25300 [Michael Armbrust] prevent multiple concurrent hive native commands

(cherry picked from commit 480bd1d2edd1de06af607b0cf3ff3c0b16089add)
Signed-off-by: Michael Armbrust <michael@databricks.com>


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/cde8a310
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/cde8a310
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/cde8a310

Branch: refs/heads/branch-1.2
Commit: cde8a310a76d780b9d178218b219f85ca30a1968
Parents: 42809db
Author: Michael Armbrust <michael@databricks.com>
Authored: Tue Dec 30 11:24:46 2014 -0800
Committer: Michael Armbrust <michael@databricks.com>
Committed: Tue Dec 30 11:24:59 2014 -0800

----------------------------------------------------------------------
 .../main/scala/org/apache/spark/sql/hive/HiveContext.scala    | 2 +-
 .../org/apache/spark/sql/hive/execution/HiveQuerySuite.scala  | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/cde8a310/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
index 5d8edfc..e006d3b 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
@@ -284,7 +284,7 @@ class HiveContext(sc: SparkContext) extends SQLContext(sc) {
    * Execute the command using Hive and return the results as a sequence. Each element
    * in the sequence is one row.
    */
-  protected def runHive(cmd: String, maxRows: Int = 1000): Seq[String] = {
+  protected def runHive(cmd: String, maxRows: Int = 1000): Seq[String] = synchronized {
     try {
       val cmd_trimmed: String = cmd.trim()
       val tokens: Array[String] = cmd_trimmed.split("\\s+")

http://git-wip-us.apache.org/repos/asf/spark/blob/cde8a310/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index af45dfd..2100ae5 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -56,6 +56,13 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter {
     Locale.setDefault(originalLocale)
   }
 
+  test("SPARK-4908: concurent hive native commands") {
+    (1 to 100).par.map { _ =>
+      sql("USE default")
+      sql("SHOW TABLES")
+    }
+  }
+
   createQueryTest("constant object inspector for generic udf",
     """SELECT named_struct(
       lower("AA"), "10",


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message