Return-Path: X-Original-To: apmail-spark-commits-archive@minotaur.apache.org Delivered-To: apmail-spark-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 99C4E1874F for ; Mon, 4 Jan 2016 20:34:12 +0000 (UTC) Received: (qmail 83724 invoked by uid 500); 4 Jan 2016 20:34:12 -0000 Delivered-To: apmail-spark-commits-archive@spark.apache.org Received: (qmail 83687 invoked by uid 500); 4 Jan 2016 20:34:10 -0000 Mailing-List: contact commits-help@spark.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list commits@spark.apache.org Received: (qmail 83677 invoked by uid 99); 4 Jan 2016 20:34:10 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 04 Jan 2016 20:34:10 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id F0032E0547; Mon, 4 Jan 2016 20:34:09 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: marmbrus@apache.org To: commits@spark.apache.org Message-Id: <305bd8ad14794ea2857a00e4036a81bf@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: spark git commit: [SPARK-12512][SQL] support column name with dot in withColumn() Date: Mon, 4 Jan 2016 20:34:09 +0000 (UTC) Repository: spark Updated Branches: refs/heads/master 43706bf8b -> 573ac55d7 [SPARK-12512][SQL] support column name with dot in withColumn() Author: Xiu Guo Closes #10500 from xguo27/SPARK-12512. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/573ac55d Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/573ac55d Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/573ac55d Branch: refs/heads/master Commit: 573ac55d7469ea2ea7a5979b4d3eea99c98f6560 Parents: 43706bf Author: Xiu Guo Authored: Mon Jan 4 12:34:04 2016 -0800 Committer: Michael Armbrust Committed: Mon Jan 4 12:34:04 2016 -0800 ---------------------------------------------------------------------- .../scala/org/apache/spark/sql/DataFrame.scala | 32 ++++++++++++-------- .../org/apache/spark/sql/DataFrameSuite.scala | 7 +++++ 2 files changed, 27 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/573ac55d/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala index 965eaa9..0763aa4 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala @@ -1171,13 +1171,17 @@ class DataFrame private[sql]( */ def withColumn(colName: String, col: Column): DataFrame = { val resolver = sqlContext.analyzer.resolver - val replaced = schema.exists(f => resolver(f.name, colName)) - if (replaced) { - val colNames = schema.map { field => - val name = field.name - if (resolver(name, colName)) col.as(colName) else Column(name) + val output = queryExecution.analyzed.output + val shouldReplace = output.exists(f => resolver(f.name, colName)) + if (shouldReplace) { + val columns = output.map { field => + if (resolver(field.name, colName)) { + col.as(colName) + } else { + Column(field) + } } - select(colNames : _*) + select(columns : _*) } else { select(Column("*"), col.as(colName)) } @@ -1188,13 +1192,17 @@ class DataFrame private[sql]( */ private[spark] def withColumn(colName: String, col: Column, metadata: Metadata): DataFrame = { val resolver = sqlContext.analyzer.resolver - val replaced = schema.exists(f => resolver(f.name, colName)) - if (replaced) { - val colNames = schema.map { field => - val name = field.name - if (resolver(name, colName)) col.as(colName, metadata) else Column(name) + val output = queryExecution.analyzed.output + val shouldReplace = output.exists(f => resolver(f.name, colName)) + if (shouldReplace) { + val columns = output.map { field => + if (resolver(field.name, colName)) { + col.as(colName, metadata) + } else { + Column(field) + } } - select(colNames : _*) + select(columns : _*) } else { select(Column("*"), col.as(colName, metadata)) } http://git-wip-us.apache.org/repos/asf/spark/blob/573ac55d/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala index ad478b0..ab02b32 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala @@ -1221,4 +1221,11 @@ class DataFrameSuite extends QueryTest with SharedSQLContext { " _2: bigint ... 2 more fields> ... 2 more fields> ... 2 more fields]") } + + test("SPARK-12512: support `.` in column name for withColumn()") { + val df = Seq("a" -> "b").toDF("col.a", "col.b") + checkAnswer(df.select(df("*")), Row("a", "b")) + checkAnswer(df.withColumn("col.a", lit("c")), Row("c", "b")) + checkAnswer(df.withColumn("col.c", lit("c")), Row("a", "b", "c")) + } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org For additional commands, e-mail: commits-help@spark.apache.org