carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] incubator-carbondata git commit: Make no kettle flow as default
Date Thu, 19 Jan 2017 14:06:29 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 3374aeb74 -> 4ad8090a3


Make no kettle flow as default

Added use_kettle property to carbon properties

Fixed style

Fixed error

Fixed comment


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/7d5a031c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/7d5a031c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/7d5a031c

Branch: refs/heads/master
Commit: 7d5a031c2b90023d0b332f64bf6088dce61b26b2
Parents: 3374aeb
Author: ravipesala <ravi.pesala@gmail.com>
Authored: Tue Jan 17 23:23:59 2017 +0530
Committer: jackylk <jacky.likun@huawei.com>
Committed: Thu Jan 19 22:05:32 2017 +0800

----------------------------------------------------------------------
 .../core/constants/CarbonCommonConstants.java         |  4 ++++
 .../carbondata/examples/util/ExampleUtils.scala       |  2 +-
 .../org/apache/carbondata/spark/CarbonOption.scala    |  2 +-
 .../carbondata/spark/CarbonDataFrameWriter.scala      |  2 +-
 .../sql/execution/command/carbonTableSchema.scala     | 14 +++++++++++---
 .../org/apache/carbondata/spark/CarbonOption.scala    |  0
 .../org/apache/spark/sql/CarbonDataFrameWriter.scala  |  2 +-
 .../sql/execution/command/carbonTableSchema.scala     | 14 +++++++++++---
 8 files changed, 30 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7d5a031c/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index c134f3b..6726d83 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1129,6 +1129,10 @@ public final class CarbonCommonConstants {
 
   public static final String LOCAL_FILE_PREFIX = "file://";
 
+  public static final String USE_KETTLE = "use_kettle";
+
+  public static final String USE_KETTLE_DEFAULT = "false";
+
   private CarbonCommonConstants() {
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7d5a031c/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
b/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
index 54d9139..41912bc 100644
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
+++ b/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
@@ -90,7 +90,7 @@ object ExampleUtils {
       .format("carbondata")
       .option("tableName", tableName)
       .option("compress", "true")
-      .option("useKettle", "false")
+      .option("use_kettle", "false")
       .option("tempCSV", "false")
       .mode(mode)
       .save()

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7d5a031c/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
index c63b43d..aa4e211 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
@@ -44,7 +44,7 @@ class CarbonOption(options: Map[String, String]) {
 
   def compress: Boolean = options.getOrElse("compress", "false").toBoolean
 
-  def useKettle: Boolean = options.getOrElse("useKettle", "true").toBoolean
+  def useKettle: Boolean = options.getOrElse("use_kettle", "false").toBoolean
 
   def bucketNumber: Int = options.getOrElse("bucketnumber", "0").toInt
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7d5a031c/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
index 7c6b171..b843f59 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
@@ -189,7 +189,7 @@ class CarbonDataFrameWriter(val dataFrame: DataFrame) {
       s"""
           LOAD DATA INPATH '$csvFolder'
           INTO TABLE ${options.dbName}.${options.tableName}
-          OPTIONS ('FILEHEADER' = '${dataFrame.columns.mkString(",")}')
+          OPTIONS ('FILEHEADER' = '${dataFrame.columns.mkString(",")}', 'USE_KETTLE' = 'true')
       """
     } else {
       s"""

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7d5a031c/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index a286086..f4eea84 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -390,11 +390,19 @@ case class LoadTable(
       val useKettle = options.get("use_kettle") match {
         case Some(value) => value.toBoolean
         case _ =>
-          val useKettleLocal = System.getProperty("use.kettle")
+          var useKettleLocal = System.getProperty("use_kettle")
+          if (useKettleLocal == null && sqlContext.sparkContext.getConf.contains("use_kettle"))
{
+            useKettleLocal = sqlContext.sparkContext.getConf.get("use_kettle")
+          }
           if (useKettleLocal == null) {
-            sqlContext.sparkContext.getConf.get("use_kettle_default", "true").toBoolean
-          } else {
+            useKettleLocal = CarbonProperties.getInstance().
+              getProperty(CarbonCommonConstants.USE_KETTLE,
+                CarbonCommonConstants.USE_KETTLE_DEFAULT)
+          }
+          try {
             useKettleLocal.toBoolean
+          } catch {
+            case e: Exception => CarbonCommonConstants.USE_KETTLE_DEFAULT.toBoolean
           }
       }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7d5a031c/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
b/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
new file mode 100644
index 0000000..e69de29

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7d5a031c/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
index 8c891c7..1c7cd3b 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
@@ -174,7 +174,7 @@ class CarbonDataFrameWriter(sqlContext: SQLContext, val dataFrame: DataFrame)
{
       s"""
           LOAD DATA INPATH '$csvFolder'
           INTO TABLE ${options.dbName}.${options.tableName}
-          OPTIONS ('FILEHEADER' = '${dataFrame.columns.mkString(",")}')
+          OPTIONS ('FILEHEADER' = '${dataFrame.columns.mkString(",")}', 'USE_KETTLE' = 'true')
       """
     } else {
       s"""

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7d5a031c/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index edc337a..c45f5a1 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -385,11 +385,19 @@ case class LoadTable(
       val useKettle = options.get("use_kettle") match {
         case Some(value) => value.toBoolean
         case _ =>
-          val useKettleLocal = System.getProperty("use.kettle")
+          var useKettleLocal = System.getProperty("use_kettle")
+          if (useKettleLocal == null && sparkSession.sparkContext.getConf.contains("use_kettle"))
{
+            useKettleLocal = sparkSession.sparkContext.getConf.get("use_kettle")
+          }
           if (useKettleLocal == null) {
-            sparkSession.sqlContext.sparkContext.getConf.get("use_kettle_default", "true").toBoolean
-          } else {
+            useKettleLocal = CarbonProperties.getInstance().
+              getProperty(CarbonCommonConstants.USE_KETTLE,
+                CarbonCommonConstants.USE_KETTLE_DEFAULT)
+          }
+          try {
             useKettleLocal.toBoolean
+          } catch {
+            case e: Exception => CarbonCommonConstants.USE_KETTLE_DEFAULT.toBoolean
           }
       }
 


Mime
View raw message