carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [11/50] carbondata git commit: fix SparkSessionExample
Date Thu, 11 May 2017 13:53:01 GMT
fix SparkSessionExample

fix comment


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9a3eeca8
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9a3eeca8
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9a3eeca8

Branch: refs/heads/branch-1.1
Commit: 9a3eeca83588a439619155a3d5f35570ed453cd8
Parents: fe730fa
Author: QiangCai <qiangcai@qq.com>
Authored: Tue Apr 25 20:51:16 2017 +0800
Committer: chenliang613 <chenliang613@huawei.com>
Committed: Thu May 4 21:18:49 2017 +0800

----------------------------------------------------------------------
 .../carbondata/examples/CarbonSessionExample.scala   |  3 ++-
 .../sql/execution/command/carbonTableSchema.scala    |  5 +++--
 .../spark/sql/CarbonDatasourceHadoopRelation.scala   | 15 ++++++++++++++-
 .../sql/execution/command/carbonTableSchema.scala    |  5 +++--
 4 files changed, 22 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a3eeca8/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
index 371b9bb..a55c39e 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
@@ -34,7 +34,8 @@ object CarbonSessionExample {
     val metastoredb = s"$rootPath/examples/spark2/target"
 
     CarbonProperties.getInstance()
-      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd HH:mm:ss")
+      .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
 
     import org.apache.spark.sql.CarbonSession._
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a3eeca8/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index ba2576a..c030321 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -691,8 +691,9 @@ private[sql] case class DropTableCommand(ifExistsSet: Boolean, databaseNameOp:
O
     val dbName = getDB.getDatabaseName(databaseNameOp, sqlContext)
     val identifier = TableIdentifier(tableName, Option(dbName))
     val carbonTableIdentifier = new CarbonTableIdentifier(dbName, tableName, "")
-    val carbonLock = CarbonLockFactory
-      .getCarbonLockObj(carbonTableIdentifier, LockUsage.DROP_TABLE_LOCK)
+    val carbonLock = CarbonLockFactory.getCarbonLockObj(carbonTableIdentifier.getDatabaseName,
+        carbonTableIdentifier.getTableName + CarbonCommonConstants.UNDERSCORE +
+        LockUsage.DROP_TABLE_LOCK)
     val storePath = CarbonEnv.get.carbonMetastore.storePath
     var isLocked = false
     try {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a3eeca8/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
index aab46af..b0044d7 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
@@ -43,7 +43,7 @@ case class CarbonDatasourceHadoopRelation(
     parameters: Map[String, String],
     tableSchema: Option[StructType],
     isSubquery: ArrayBuffer[Boolean] = new ArrayBuffer[Boolean]())
-  extends BaseRelation {
+  extends BaseRelation with InsertableRelation {
 
   lazy val absIdentifier = AbsoluteTableIdentifier.fromTablePath(paths.head)
   lazy val carbonTable = carbonRelation.tableMeta.carbonTable
@@ -76,4 +76,17 @@ case class CarbonDatasourceHadoopRelation(
   }
 
   override def sizeInBytes: Long = carbonRelation.sizeInBytes
+
+  override def insert(data: DataFrame, overwrite: Boolean): Unit = {
+    if (carbonRelation.output.size > CarbonCommonConstants.DEFAULT_MAX_NUMBER_OF_COLUMNS)
{
+      sys.error("Maximum supported column by carbon is: " +
+          CarbonCommonConstants.DEFAULT_MAX_NUMBER_OF_COLUMNS)
+    }
+    if (data.logicalPlan.output.size >= carbonRelation.output.size) {
+      LoadTableByInsert(this, data.logicalPlan).run(sparkSession)
+    } else {
+      sys.error("Cannot insert into target table because column number are different")
+    }
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a3eeca8/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index bb797ec..593de39 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -719,8 +719,9 @@ case class CarbonDropTableCommand(ifExistsSet: Boolean,
     val dbName = getDB.getDatabaseName(databaseNameOp, sparkSession)
     val identifier = TableIdentifier(tableName, Option(dbName))
     val carbonTableIdentifier = new CarbonTableIdentifier(dbName, tableName, "")
-    val carbonLock = CarbonLockFactory
-      .getCarbonLockObj(carbonTableIdentifier, LockUsage.DROP_TABLE_LOCK)
+    val carbonLock = CarbonLockFactory.getCarbonLockObj(carbonTableIdentifier.getDatabaseName,
+        carbonTableIdentifier.getTableName + CarbonCommonConstants.UNDERSCORE +
+        LockUsage.DROP_TABLE_LOCK)
     val catalog = CarbonEnv.getInstance(sparkSession).carbonMetastore
     val storePath = catalog.storePath
     var isLocked = false


Mime
View raw message