Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 37CD8200CAC for ; Thu, 11 May 2017 15:52:54 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 34862160BCF; Thu, 11 May 2017 13:52:54 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id B7711160BD0 for ; Thu, 11 May 2017 15:52:52 +0200 (CEST) Received: (qmail 13488 invoked by uid 500); 11 May 2017 13:52:52 -0000 Mailing-List: contact commits-help@carbondata.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@carbondata.apache.org Delivered-To: mailing list commits@carbondata.apache.org Received: (qmail 13368 invoked by uid 99); 11 May 2017 13:52:51 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 11 May 2017 13:52:51 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id B52A8E9645; Thu, 11 May 2017 13:52:51 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ravipesala@apache.org To: commits@carbondata.apache.org Date: Thu, 11 May 2017 13:53:01 -0000 Message-Id: In-Reply-To: <2a2c82f6952c4786a2d0e93319e2b6f7@git.apache.org> References: <2a2c82f6952c4786a2d0e93319e2b6f7@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [11/50] carbondata git commit: fix SparkSessionExample archived-at: Thu, 11 May 2017 13:52:54 -0000 fix SparkSessionExample fix comment Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9a3eeca8 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9a3eeca8 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9a3eeca8 Branch: refs/heads/branch-1.1 Commit: 9a3eeca83588a439619155a3d5f35570ed453cd8 Parents: fe730fa Author: QiangCai Authored: Tue Apr 25 20:51:16 2017 +0800 Committer: chenliang613 Committed: Thu May 4 21:18:49 2017 +0800 ---------------------------------------------------------------------- .../carbondata/examples/CarbonSessionExample.scala | 3 ++- .../sql/execution/command/carbonTableSchema.scala | 5 +++-- .../spark/sql/CarbonDatasourceHadoopRelation.scala | 15 ++++++++++++++- .../sql/execution/command/carbonTableSchema.scala | 5 +++-- 4 files changed, 22 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a3eeca8/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala ---------------------------------------------------------------------- diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala index 371b9bb..a55c39e 100644 --- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala +++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala @@ -34,7 +34,8 @@ object CarbonSessionExample { val metastoredb = s"$rootPath/examples/spark2/target" CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd") + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd HH:mm:ss") + .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd") import org.apache.spark.sql.CarbonSession._ http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a3eeca8/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala index ba2576a..c030321 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala @@ -691,8 +691,9 @@ private[sql] case class DropTableCommand(ifExistsSet: Boolean, databaseNameOp: O val dbName = getDB.getDatabaseName(databaseNameOp, sqlContext) val identifier = TableIdentifier(tableName, Option(dbName)) val carbonTableIdentifier = new CarbonTableIdentifier(dbName, tableName, "") - val carbonLock = CarbonLockFactory - .getCarbonLockObj(carbonTableIdentifier, LockUsage.DROP_TABLE_LOCK) + val carbonLock = CarbonLockFactory.getCarbonLockObj(carbonTableIdentifier.getDatabaseName, + carbonTableIdentifier.getTableName + CarbonCommonConstants.UNDERSCORE + + LockUsage.DROP_TABLE_LOCK) val storePath = CarbonEnv.get.carbonMetastore.storePath var isLocked = false try { http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a3eeca8/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala index aab46af..b0044d7 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala @@ -43,7 +43,7 @@ case class CarbonDatasourceHadoopRelation( parameters: Map[String, String], tableSchema: Option[StructType], isSubquery: ArrayBuffer[Boolean] = new ArrayBuffer[Boolean]()) - extends BaseRelation { + extends BaseRelation with InsertableRelation { lazy val absIdentifier = AbsoluteTableIdentifier.fromTablePath(paths.head) lazy val carbonTable = carbonRelation.tableMeta.carbonTable @@ -76,4 +76,17 @@ case class CarbonDatasourceHadoopRelation( } override def sizeInBytes: Long = carbonRelation.sizeInBytes + + override def insert(data: DataFrame, overwrite: Boolean): Unit = { + if (carbonRelation.output.size > CarbonCommonConstants.DEFAULT_MAX_NUMBER_OF_COLUMNS) { + sys.error("Maximum supported column by carbon is: " + + CarbonCommonConstants.DEFAULT_MAX_NUMBER_OF_COLUMNS) + } + if (data.logicalPlan.output.size >= carbonRelation.output.size) { + LoadTableByInsert(this, data.logicalPlan).run(sparkSession) + } else { + sys.error("Cannot insert into target table because column number are different") + } + } + } http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a3eeca8/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala index bb797ec..593de39 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala @@ -719,8 +719,9 @@ case class CarbonDropTableCommand(ifExistsSet: Boolean, val dbName = getDB.getDatabaseName(databaseNameOp, sparkSession) val identifier = TableIdentifier(tableName, Option(dbName)) val carbonTableIdentifier = new CarbonTableIdentifier(dbName, tableName, "") - val carbonLock = CarbonLockFactory - .getCarbonLockObj(carbonTableIdentifier, LockUsage.DROP_TABLE_LOCK) + val carbonLock = CarbonLockFactory.getCarbonLockObj(carbonTableIdentifier.getDatabaseName, + carbonTableIdentifier.getTableName + CarbonCommonConstants.UNDERSCORE + + LockUsage.DROP_TABLE_LOCK) val catalog = CarbonEnv.getInstance(sparkSession).carbonMetastore val storePath = catalog.storePath var isLocked = false