carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gvram...@apache.org
Subject [1/2] incubator-carbondata git commit: Fixed crabon.properties read from sqlconf first
Date Fri, 20 Jan 2017 13:03:07 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 1fb5391ff -> 556a4118e


Fixed crabon.properties read from sqlconf first


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/c2ee320b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/c2ee320b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/c2ee320b

Branch: refs/heads/master
Commit: c2ee320b746c3a872bf288d52242afa00f1cc447
Parents: 1fb5391
Author: ravipesala <ravi.pesala@gmail.com>
Authored: Thu Jan 19 19:13:51 2017 +0530
Committer: Venkata Ramana G <ramana.gollamudi@huawei.com>
Committed: Fri Jan 20 18:28:11 2017 +0530

----------------------------------------------------------------------
 .../carbondata/spark/util/CarbonScalaUtil.scala |  5 +++-
 .../spark/thriftserver/CarbonThriftServer.scala | 18 ++++++++++----
 .../apache/carbondata/spark/CarbonOption.scala  |  0
 .../spark/thriftserver/CarbonThriftServer.scala | 25 +++++++++++++-------
 4 files changed, 34 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2ee320b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index 35bd775..428df48 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -92,7 +92,10 @@ object CarbonScalaUtil {
   }
 
   def getKettleHome(sqlContext: SQLContext): String = {
-    var kettleHomePath = CarbonProperties.getInstance.getProperty("carbon.kettle.home")
+    var kettleHomePath = sqlContext.getConf("carbon.kettle.home", null)
+    if (null == kettleHomePath) {
+      kettleHomePath = CarbonProperties.getInstance.getProperty("carbon.kettle.home")
+    }
     if (null == kettleHomePath) {
       val carbonHome = System.getenv("CARBON_HOME")
       if (null != carbonHome) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2ee320b/integration/spark/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala
b/integration/spark/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala
index efedc91..b8ba9f7 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala
@@ -17,6 +17,8 @@
 
 package org.apache.carbondata.spark.thriftserver
 
+import java.io.File
+
 import org.apache.spark.{SparkConf, SparkContext}
 import org.apache.spark.sql.CarbonContext
 import org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
@@ -29,11 +31,17 @@ object CarbonThriftServer {
   def main(args: Array[String]): Unit = {
     val conf = new SparkConf()
       .setAppName("Carbon Thrift Server")
-    val sparkHome = System.getenv.get("SPARK_HOME")
-    if (null != sparkHome) {
-      conf.set("carbon.properties.filepath", sparkHome + '/' + "conf" + '/' + "carbon.properties")
-      System.setProperty("carbon.properties.filepath",
-        sparkHome + '/' + "conf" + '/' + "carbon.properties")
+    if (!conf.contains("carbon.properties.filepath")) {
+      val sparkHome = System.getenv.get("SPARK_HOME")
+      if (sparkHome != null) {
+        val file = new File(sparkHome + '/' + "conf" + '/' + "carbon.properties")
+        if (file.exists()) {
+          conf.set("carbon.properties.filepath", file.getCanonicalPath)
+          System.setProperty("carbon.properties.filepath", file.getCanonicalPath)
+        }
+      }
+    } else {
+      System.setProperty("carbon.properties.filepath", conf.get("carbon.properties.filepath"))
     }
     if (org.apache.spark.SPARK_VERSION.startsWith("1.6")) {
       conf.set("spark.sql.hive.thriftServer.singleSession", "true")

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2ee320b/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
b/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
deleted file mode 100644
index e69de29..0000000

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2ee320b/integration/spark2/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala
b/integration/spark2/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala
index 6a6ee00..aba6891 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/thriftserver/CarbonThriftServer.scala
@@ -17,11 +17,14 @@
 
 package org.apache.carbondata.spark.thriftserver
 
+import java.io.File
+
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
 
 import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.util.CarbonProperties
 
 object CarbonThriftServer {
@@ -37,16 +40,22 @@ object CarbonThriftServer {
       .appName("Carbon Thrift Server(uses CarbonSession)")
       .enableHiveSupport()
 
-    val sparkHome = System.getenv.get("SPARK_HOME")
-    if (null != sparkHome) {
-      builder.config("carbon.properties.filepath",
-        sparkHome + '/' + "conf" + '/' + "carbon.properties")
-      System.setProperty("carbon.properties.filepath",
-        sparkHome + '/' + "conf" + '/' + "carbon.properties")
+    if (!sparkConf.contains("carbon.properties.filepath")) {
+      val sparkHome = System.getenv.get("SPARK_HOME")
+      if (null != sparkHome) {
+        val file = new File(sparkHome + '/' + "conf" + '/' + "carbon.properties")
+        if (file.exists()) {
+          builder.config("carbon.properties.filepath", file.getCanonicalPath)
+          System.setProperty("carbon.properties.filepath", file.getCanonicalPath)
+        }
+      }
+    } else {
+      System.setProperty("carbon.properties.filepath", sparkConf.get("carbon.properties.filepath"))
     }
-    CarbonProperties.getInstance().addProperty("carbon.storelocation", args.head)
 
-    val spark = builder.getOrCreateCarbonSession()
+    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.STORE_LOCATION, args.head)
+
+    val spark = builder.getOrCreateCarbonSession(args.head)
     val warmUpTime = CarbonProperties.getInstance().getProperty("carbon.spark.warmUpTime",
"5000")
     try {
       Thread.sleep(Integer.parseInt(warmUpTime))


Mime
View raw message