spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marmb...@apache.org
Subject git commit: [SQL] Turns on in-memory columnar compression in HiveCompatibilitySuite
Date Fri, 29 Aug 2014 22:35:15 GMT
Repository: spark
Updated Branches:
  refs/heads/master dc4d577c6 -> b1eccfc88


[SQL] Turns on in-memory columnar compression in HiveCompatibilitySuite

`HiveCompatibilitySuite` already turns on in-memory columnar caching, it would be good to
also enable compression to improve test coverage.

Author: Cheng Lian <lian.cs.zju@gmail.com>

Closes #2190 from liancheng/compression-on and squashes the following commits:

88b536c [Cheng Lian] Code cleanup, narrowed field visibility
d13efd2 [Cheng Lian] Turns on in-memory columnar compression in HiveCompatibilitySuite


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b1eccfc8
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b1eccfc8
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b1eccfc8

Branch: refs/heads/master
Commit: b1eccfc88a13b937d42bbae8a0c3f09cffc5ae47
Parents: dc4d577
Author: Cheng Lian <lian.cs.zju@gmail.com>
Authored: Fri Aug 29 15:34:59 2014 -0700
Committer: Michael Armbrust <michael@databricks.com>
Committed: Fri Aug 29 15:34:59 2014 -0700

----------------------------------------------------------------------
 .../hive/execution/HiveCompatibilitySuite.scala    | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/b1eccfc8/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
index 035fd32..b589994 100644
--- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
+++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
@@ -22,6 +22,7 @@ import java.util.{Locale, TimeZone}
 
 import org.scalatest.BeforeAndAfter
 
+import org.apache.spark.sql.SQLConf
 import org.apache.spark.sql.hive.test.TestHive
 
 /**
@@ -29,29 +30,31 @@ import org.apache.spark.sql.hive.test.TestHive
  */
 class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
   // TODO: bundle in jar files... get from classpath
-  lazy val hiveQueryDir = TestHive.getHiveFile("ql" + File.separator + "src" +
-    File.separator + "test" + File.separator + "queries" + File.separator + "clientpositive")
+  private lazy val hiveQueryDir = TestHive.getHiveFile(
+    "ql/src/test/queries/clientpositive".split("/").mkString(File.separator))
 
-  var originalTimeZone: TimeZone = _
-  var originalLocale: Locale = _
+  private val originalTimeZone = TimeZone.getDefault
+  private val originalLocale = Locale.getDefault
+  private val originalUseCompression = TestHive.useCompression
 
   def testCases = hiveQueryDir.listFiles.map(f => f.getName.stripSuffix(".q") -> f)
 
   override def beforeAll() {
+    // Enable in-memory columnar caching
     TestHive.cacheTables = true
     // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
-    originalTimeZone = TimeZone.getDefault
     TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
-    
     // Add Locale setting
-    originalLocale = Locale.getDefault
     Locale.setDefault(Locale.US)
+    // Enable in-memory columnar compression
+    TestHive.setConf(SQLConf.COMPRESS_CACHED, "true")
   }
 
   override def afterAll() {
     TestHive.cacheTables = false
     TimeZone.setDefault(originalTimeZone)
     Locale.setDefault(originalLocale)
+    TestHive.setConf(SQLConf.COMPRESS_CACHED, originalUseCompression.toString)
   }
 
   /** A list of tests deemed out of scope currently and thus completely disregarded. */


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message