Return-Path: X-Original-To: apmail-spark-commits-archive@minotaur.apache.org Delivered-To: apmail-spark-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 1568D173AA for ; Mon, 2 Feb 2015 02:42:04 +0000 (UTC) Received: (qmail 80091 invoked by uid 500); 2 Feb 2015 02:42:04 -0000 Delivered-To: apmail-spark-commits-archive@spark.apache.org Received: (qmail 80054 invoked by uid 500); 2 Feb 2015 02:42:04 -0000 Mailing-List: contact commits-help@spark.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list commits@spark.apache.org Received: (qmail 80045 invoked by uid 99); 2 Feb 2015 02:42:04 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 02 Feb 2015 02:42:04 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 9C706E0294; Mon, 2 Feb 2015 02:42:04 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: marmbrus@apache.org To: commits@spark.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: spark git commit: [SPARK-5196][SQL] Support `comment` in Create Table Field DDL Date: Mon, 2 Feb 2015 02:42:04 +0000 (UTC) Repository: spark Updated Branches: refs/heads/master 7712ed5b1 -> 1b56f1d6b [SPARK-5196][SQL] Support `comment` in Create Table Field DDL Support `comment` in create a table field. __CREATE TEMPORARY TABLE people(name string `comment` "the name of a person")__ Author: OopsOutOfMemory Closes #3999 from OopsOutOfMemory/meta_comment and squashes the following commits: 39150d4 [OopsOutOfMemory] add comment and refine test suite Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1b56f1d6 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1b56f1d6 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1b56f1d6 Branch: refs/heads/master Commit: 1b56f1d6bb079a669ae83e70ee515373ade2a469 Parents: 7712ed5 Author: OopsOutOfMemory Authored: Sun Feb 1 18:41:49 2015 -0800 Committer: Michael Armbrust Committed: Sun Feb 1 18:41:58 2015 -0800 ---------------------------------------------------------------------- .../org/apache/spark/sql/sources/ddl.scala | 11 ++++++++--- .../spark/sql/sources/TableScanSuite.scala | 20 ++++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/1b56f1d6/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala index b4af91a..b7c721f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala @@ -27,7 +27,6 @@ import org.apache.spark.sql.execution.RunnableCommand import org.apache.spark.sql.types._ import org.apache.spark.util.Utils - /** * A parser for foreign DDL commands. */ @@ -59,6 +58,7 @@ private[sql] class DDLParser extends AbstractSparkSQLParser with Logging { protected val TABLE = Keyword("TABLE") protected val USING = Keyword("USING") protected val OPTIONS = Keyword("OPTIONS") + protected val COMMENT = Keyword("COMMENT") // Data types. protected val STRING = Keyword("STRING") @@ -111,8 +111,13 @@ private[sql] class DDLParser extends AbstractSparkSQLParser with Logging { protected lazy val pair: Parser[(String, String)] = ident ~ stringLit ^^ { case k ~ v => (k,v) } protected lazy val column: Parser[StructField] = - ident ~ dataType ^^ { case columnName ~ typ => - StructField(columnName, typ) + ident ~ dataType ~ (COMMENT ~> stringLit).? ^^ { case columnName ~ typ ~ cm => + val meta = cm match { + case Some(comment) => + new MetadataBuilder().putString(COMMENT.str.toLowerCase(), comment).build() + case None => Metadata.empty + } + StructField(columnName, typ, true, meta) } protected lazy val primitiveType: Parser[DataType] = http://git-wip-us.apache.org/repos/asf/spark/blob/1b56f1d6/sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala index b1e0919..0a4d4b6 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala @@ -344,4 +344,24 @@ class TableScanSuite extends DataSourceTest { } assert(schemaNeeded.getMessage.contains("A schema needs to be specified when using")) } + + test("SPARK-5196 schema field with comment") { + sql( + """ + |CREATE TEMPORARY TABLE student(name string comment "SN", age int comment "SA", grade int) + |USING org.apache.spark.sql.sources.AllDataTypesScanSource + |OPTIONS ( + | from '1', + | to '10' + |) + """.stripMargin) + + val planned = sql("SELECT * FROM student").queryExecution.executedPlan + val comments = planned.schema.fields.map { field => + if (field.metadata.contains("comment")) field.metadata.getString("comment") + else "NO_COMMENT" + }.mkString(",") + + assert(comments === "SN,SA,NO_COMMENT") + } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org For additional commands, e-mail: commits-help@spark.apache.org