spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r...@apache.org
Subject spark git commit: [SPARK-12242][SQL] Add DataFrame.transform method
Date Thu, 10 Dec 2015 14:23:17 GMT
Repository: spark
Updated Branches:
  refs/heads/master 21b3d2a75 -> 76540b6df


[SPARK-12242][SQL] Add DataFrame.transform method

Author: Reynold Xin <rxin@databricks.com>

Closes #10226 from rxin/df-transform.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/76540b6d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/76540b6d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/76540b6d

Branch: refs/heads/master
Commit: 76540b6df5370b463277d3498097b2cc2d2e97a8
Parents: 21b3d2a
Author: Reynold Xin <rxin@databricks.com>
Authored: Thu Dec 10 22:23:10 2015 +0800
Committer: Reynold Xin <rxin@databricks.com>
Committed: Thu Dec 10 22:23:10 2015 +0800

----------------------------------------------------------------------
 .../src/main/scala/org/apache/spark/sql/Column.scala   |  2 +-
 .../main/scala/org/apache/spark/sql/DataFrame.scala    | 13 +++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/76540b6d/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
index d641fca..297ef22 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -84,7 +84,7 @@ class TypedColumn[-T, U](
  *   col("`a.column.with.dots`") // Escape `.` in column names.
  *   $"columnName"               // Scala short hand for a named column.
  *   expr("a + 1")               // A column that is constructed from a parsed SQL Expression.
- *   lit("1")                    // A column that produces a literal (constant) value.
+ *   lit("abc")                  // A column that produces a literal (constant) value.
  * }}}
  *
  * [[Column]] objects can be composed to form complex expressions:

http://git-wip-us.apache.org/repos/asf/spark/blob/76540b6d/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
index 243a8c8..da180a2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
@@ -1422,6 +1422,19 @@ class DataFrame private[sql](
   def first(): Row = head()
 
   /**
+   * Concise syntax for chaining custom transformations.
+   * {{{
+   *   def featurize(ds: DataFrame) = ...
+   *
+   *   df
+   *     .transform(featurize)
+   *     .transform(...)
+   * }}}
+   * @since 1.6.0
+   */
+  def transform[U](t: DataFrame => DataFrame): DataFrame = t(this)
+
+  /**
    * Returns a new RDD by applying a function to all rows of this DataFrame.
    * @group rdd
    * @since 1.3.0


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message