spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r...@apache.org
Subject spark git commit: [SPARK-7358][SQL] Move DataFrame mathfunctions into functions
Date Wed, 06 May 2015 05:56:26 GMT
Repository: spark
Updated Branches:
  refs/heads/branch-1.4 b5cd7dc1f -> 8aa6681d5


[SPARK-7358][SQL] Move DataFrame mathfunctions into functions

After a discussion on the user mailing list, it was decided to put all UDF's under `o.a.s.sql.functions`

cc rxin

Author: Burak Yavuz <brkyvz@gmail.com>

Closes #5923 from brkyvz/move-math-funcs and squashes the following commits:

a8dc3f7 [Burak Yavuz] address comments
cf7a7bb [Burak Yavuz] [SPARK-7358] Move DataFrame mathfunctions into functions

(cherry picked from commit ba2b56614d7ab1bd7409b49e9d85c248d8faa48b)
Signed-off-by: Reynold Xin <rxin@databricks.com>


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8aa6681d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8aa6681d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8aa6681d

Branch: refs/heads/branch-1.4
Commit: 8aa6681d5fecc64322bbc011afe5b169eaa44ff7
Parents: b5cd7dc
Author: Burak Yavuz <brkyvz@gmail.com>
Authored: Tue May 5 22:56:01 2015 -0700
Committer: Reynold Xin <rxin@databricks.com>
Committed: Tue May 5 22:56:23 2015 -0700

----------------------------------------------------------------------
 python/pyspark/sql/functions.py                 |  52 ++
 python/pyspark/sql/mathfunctions.py             | 101 ----
 python/pyspark/sql/tests.py                     |   2 +-
 .../expressions/ExpressionEvaluationSuite.scala |   4 +-
 .../scala/org/apache/spark/sql/functions.scala  | 485 +++++++++++++++++++
 .../org/apache/spark/sql/mathfunctions.scala    | 383 ---------------
 .../apache/spark/sql/JavaDataFrameSuite.java    |   1 -
 .../apache/spark/sql/MathExpressionsSuite.scala |   7 +-
 8 files changed, 543 insertions(+), 492 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/8aa6681d/python/pyspark/sql/functions.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index 641220a..692af86 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -51,6 +51,19 @@ def _create_function(name, doc=""):
     return _
 
 
+def _create_binary_mathfunction(name, doc=""):
+    """ Create a binary mathfunction by name"""
+    def _(col1, col2):
+        sc = SparkContext._active_spark_context
+        # users might write ints for simplicity. This would throw an error on the JVM side.
+        jc = getattr(sc._jvm.functions, name)(col1._jc if isinstance(col1, Column) else float(col1),
+                                              col2._jc if isinstance(col2, Column) else float(col2))
+        return Column(jc)
+    _.__name__ = name
+    _.__doc__ = doc
+    return _
+
+
 _functions = {
     'lit': 'Creates a :class:`Column` of literal value.',
     'col': 'Returns a :class:`Column` based on the given column name.',
@@ -63,6 +76,34 @@ _functions = {
     'sqrt': 'Computes the square root of the specified float value.',
     'abs': 'Computes the absolute value.',
 
+    # unary math functions
+    'acos': 'Computes the cosine inverse of the given value; the returned angle is in the
range' +
+            '0.0 through pi.',
+    'asin': 'Computes the sine inverse of the given value; the returned angle is in the range'
+
+            '-pi/2 through pi/2.',
+    'atan': 'Computes the tangent inverse of the given value.',
+    'cbrt': 'Computes the cube-root of the given value.',
+    'ceil': 'Computes the ceiling of the given value.',
+    'cos': 'Computes the cosine of the given value.',
+    'cosh': 'Computes the hyperbolic cosine of the given value.',
+    'exp': 'Computes the exponential of the given value.',
+    'expm1': 'Computes the exponential of the given value minus one.',
+    'floor': 'Computes the floor of the given value.',
+    'log': 'Computes the natural logarithm of the given value.',
+    'log10': 'Computes the logarithm of the given value in Base 10.',
+    'log1p': 'Computes the natural logarithm of the given value plus one.',
+    'rint': 'Returns the double value that is closest in value to the argument and' +
+            ' is equal to a mathematical integer.',
+    'signum': 'Computes the signum of the given value.',
+    'sin': 'Computes the sine of the given value.',
+    'sinh': 'Computes the hyperbolic sine of the given value.',
+    'tan': 'Computes the tangent of the given value.',
+    'tanh': 'Computes the hyperbolic tangent of the given value.',
+    'toDegrees': 'Converts an angle measured in radians to an approximately equivalent angle
' +
+             'measured in degrees.',
+    'toRadians': 'Converts an angle measured in degrees to an approximately equivalent angle
' +
+             'measured in radians.',
+
     'max': 'Aggregate function: returns the maximum value of the expression in a group.',
     'min': 'Aggregate function: returns the minimum value of the expression in a group.',
     'first': 'Aggregate function: returns the first value in a group.',
@@ -74,10 +115,21 @@ _functions = {
     'sumDistinct': 'Aggregate function: returns the sum of distinct values in the expression.',
 }
 
+# math functions that take two arguments as input
+_binary_mathfunctions = {
+    'atan2': 'Returns the angle theta from the conversion of rectangular coordinates (x,
y) to' +
+             'polar coordinates (r, theta).',
+    'hypot': 'Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.',
+    'pow': 'Returns the value of the first argument raised to the power of the second argument.'
+}
+
 for _name, _doc in _functions.items():
     globals()[_name] = _create_function(_name, _doc)
+for _name, _doc in _binary_mathfunctions.items():
+    globals()[_name] = _create_binary_mathfunction(_name, _doc)
 del _name, _doc
 __all__ += _functions.keys()
+__all__ += _binary_mathfunctions.keys()
 __all__.sort()
 
 

http://git-wip-us.apache.org/repos/asf/spark/blob/8aa6681d/python/pyspark/sql/mathfunctions.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/mathfunctions.py b/python/pyspark/sql/mathfunctions.py
deleted file mode 100644
index 7dbcab8..0000000
--- a/python/pyspark/sql/mathfunctions.py
+++ /dev/null
@@ -1,101 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""
-A collection of builtin math functions
-"""
-
-from pyspark import SparkContext
-from pyspark.sql.dataframe import Column
-
-__all__ = []
-
-
-def _create_unary_mathfunction(name, doc=""):
-    """ Create a unary mathfunction by name"""
-    def _(col):
-        sc = SparkContext._active_spark_context
-        jc = getattr(sc._jvm.mathfunctions, name)(col._jc if isinstance(col, Column) else
col)
-        return Column(jc)
-    _.__name__ = name
-    _.__doc__ = doc
-    return _
-
-
-def _create_binary_mathfunction(name, doc=""):
-    """ Create a binary mathfunction by name"""
-    def _(col1, col2):
-        sc = SparkContext._active_spark_context
-        # users might write ints for simplicity. This would throw an error on the JVM side.
-        if type(col1) is int:
-            col1 = col1 * 1.0
-        if type(col2) is int:
-            col2 = col2 * 1.0
-        jc = getattr(sc._jvm.mathfunctions, name)(col1._jc if isinstance(col1, Column) else
col1,
-                                                  col2._jc if isinstance(col2, Column) else
col2)
-        return Column(jc)
-    _.__name__ = name
-    _.__doc__ = doc
-    return _
-
-
-# math functions are found under another object therefore, they need to be handled separately
-_mathfunctions = {
-    'acos': 'Computes the cosine inverse of the given value; the returned angle is in the
range' +
-            '0.0 through pi.',
-    'asin': 'Computes the sine inverse of the given value; the returned angle is in the range'
+
-            '-pi/2 through pi/2.',
-    'atan': 'Computes the tangent inverse of the given value.',
-    'cbrt': 'Computes the cube-root of the given value.',
-    'ceil': 'Computes the ceiling of the given value.',
-    'cos': 'Computes the cosine of the given value.',
-    'cosh': 'Computes the hyperbolic cosine of the given value.',
-    'exp': 'Computes the exponential of the given value.',
-    'expm1': 'Computes the exponential of the given value minus one.',
-    'floor': 'Computes the floor of the given value.',
-    'log': 'Computes the natural logarithm of the given value.',
-    'log10': 'Computes the logarithm of the given value in Base 10.',
-    'log1p': 'Computes the natural logarithm of the given value plus one.',
-    'rint': 'Returns the double value that is closest in value to the argument and' +
-            ' is equal to a mathematical integer.',
-    'signum': 'Computes the signum of the given value.',
-    'sin': 'Computes the sine of the given value.',
-    'sinh': 'Computes the hyperbolic sine of the given value.',
-    'tan': 'Computes the tangent of the given value.',
-    'tanh': 'Computes the hyperbolic tangent of the given value.',
-    'toDeg': 'Converts an angle measured in radians to an approximately equivalent angle
' +
-             'measured in degrees.',
-    'toRad': 'Converts an angle measured in degrees to an approximately equivalent angle
' +
-             'measured in radians.'
-}
-
-# math functions that take two arguments as input
-_binary_mathfunctions = {
-    'atan2': 'Returns the angle theta from the conversion of rectangular coordinates (x,
y) to' +
-             'polar coordinates (r, theta).',
-    'hypot': 'Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.',
-    'pow': 'Returns the value of the first argument raised to the power of the second argument.'
-}
-
-for _name, _doc in _mathfunctions.items():
-    globals()[_name] = _create_unary_mathfunction(_name, _doc)
-for _name, _doc in _binary_mathfunctions.items():
-    globals()[_name] = _create_binary_mathfunction(_name, _doc)
-del _name, _doc
-__all__ += _mathfunctions.keys()
-__all__ += _binary_mathfunctions.keys()
-__all__.sort()

http://git-wip-us.apache.org/repos/asf/spark/blob/8aa6681d/python/pyspark/sql/tests.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py
index 46c4c88..b232f3a 100644
--- a/python/pyspark/sql/tests.py
+++ b/python/pyspark/sql/tests.py
@@ -416,7 +416,7 @@ class SQLTests(ReusedPySparkTestCase):
 
     def test_math_functions(self):
         df = self.sc.parallelize([Row(a=i, b=2 * i) for i in range(10)]).toDF()
-        from pyspark.sql import mathfunctions as functions
+        from pyspark.sql import functions
         import math
 
         def get_values(l):

http://git-wip-us.apache.org/repos/asf/spark/blob/8aa6681d/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala
index fa71001..faaa55a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala
@@ -1217,11 +1217,11 @@ class ExpressionEvaluationSuite extends ExpressionEvaluationBaseSuite
{
     unaryMathFunctionEvaluation(Tanh, math.tanh)
   }
 
-  test("toDeg") {
+  test("toDegrees") {
     unaryMathFunctionEvaluation(ToDegrees, math.toDegrees)
   }
 
-  test("toRad") {
+  test("toRadians") {
     unaryMathFunctionEvaluation(ToRadians, math.toRadians)
   }
 

http://git-wip-us.apache.org/repos/asf/spark/blob/8aa6681d/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index 7e28339..f2bb453 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -24,6 +24,7 @@ import org.apache.spark.annotation.Experimental
 import org.apache.spark.sql.catalyst.ScalaReflection
 import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedFunction, Star}
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.expressions.mathfuncs._
 import org.apache.spark.sql.types._
 import org.apache.spark.util.Utils
 
@@ -35,6 +36,7 @@ import org.apache.spark.util.Utils
  * @groupname agg_funcs Aggregate functions
  * @groupname sort_funcs Sorting functions
  * @groupname normal_funcs Non-aggregate functions
+ * @groupname math_funcs Math functions
  * @groupname Ungrouped Support functions for DataFrames.
  */
 @Experimental
@@ -437,6 +439,489 @@ object functions {
   def upper(e: Column): Column = Upper(e.expr)
 
   //////////////////////////////////////////////////////////////////////////////////////////////
+  // Math Functions
+  //////////////////////////////////////////////////////////////////////////////////////////////
+
+  /**
+   * Computes the cosine inverse of the given value; the returned angle is in the range
+   * 0.0 through pi.
+   *
+   * @group math_funcs
+   */
+  def acos(e: Column): Column = Acos(e.expr)
+
+  /**
+   * Computes the cosine inverse of the given column; the returned angle is in the range
+   * 0.0 through pi.
+   *
+   * @group math_funcs
+   */
+  def acos(columnName: String): Column = acos(Column(columnName))
+
+  /**
+   * Computes the sine inverse of the given value; the returned angle is in the range
+   * -pi/2 through pi/2.
+   *
+   * @group math_funcs
+   */
+  def asin(e: Column): Column = Asin(e.expr)
+
+  /**
+   * Computes the sine inverse of the given column; the returned angle is in the range
+   * -pi/2 through pi/2.
+   *
+   * @group math_funcs
+   */
+  def asin(columnName: String): Column = asin(Column(columnName))
+
+  /**
+   * Computes the tangent inverse of the given value.
+   *
+   * @group math_funcs
+   */
+  def atan(e: Column): Column = Atan(e.expr)
+
+  /**
+   * Computes the tangent inverse of the given column.
+   *
+   * @group math_funcs
+   */
+  def atan(columnName: String): Column = atan(Column(columnName))
+
+  /**
+   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+   * polar coordinates (r, theta).
+   *
+   * @group math_funcs
+   */
+  def atan2(l: Column, r: Column): Column = Atan2(l.expr, r.expr)
+
+  /**
+   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+   * polar coordinates (r, theta).
+   *
+   * @group math_funcs
+   */
+  def atan2(l: Column, rightName: String): Column = atan2(l, Column(rightName))
+
+  /**
+   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+   * polar coordinates (r, theta).
+   *
+   * @group math_funcs
+   */
+  def atan2(leftName: String, r: Column): Column = atan2(Column(leftName), r)
+
+  /**
+   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+   * polar coordinates (r, theta).
+   *
+   * @group math_funcs
+   */
+  def atan2(leftName: String, rightName: String): Column =
+    atan2(Column(leftName), Column(rightName))
+
+  /**
+   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+   * polar coordinates (r, theta).
+   *
+   * @group math_funcs
+   */
+  def atan2(l: Column, r: Double): Column = atan2(l, lit(r).expr)
+
+  /**
+   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+   * polar coordinates (r, theta).
+   *
+   * @group math_funcs
+   */
+  def atan2(leftName: String, r: Double): Column = atan2(Column(leftName), r)
+
+  /**
+   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+   * polar coordinates (r, theta).
+   *
+   * @group math_funcs
+   */
+  def atan2(l: Double, r: Column): Column = atan2(lit(l).expr, r)
+
+  /**
+   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+   * polar coordinates (r, theta).
+   *
+   * @group math_funcs
+   */
+  def atan2(l: Double, rightName: String): Column = atan2(l, Column(rightName))
+
+  /**
+   * Computes the cube-root of the given value.
+   *
+   * @group math_funcs
+   */
+  def cbrt(e: Column): Column = Cbrt(e.expr)
+
+  /**
+   * Computes the cube-root of the given column.
+   *
+   * @group math_funcs
+   */
+  def cbrt(columnName: String): Column = cbrt(Column(columnName))
+
+  /**
+   * Computes the ceiling of the given value.
+   *
+   * @group math_funcs
+   */
+  def ceil(e: Column): Column = Ceil(e.expr)
+
+  /**
+   * Computes the ceiling of the given column.
+   *
+   * @group math_funcs
+   */
+  def ceil(columnName: String): Column = ceil(Column(columnName))
+
+  /**
+   * Computes the cosine of the given value.
+   *
+   * @group math_funcs
+   */
+  def cos(e: Column): Column = Cos(e.expr)
+
+  /**
+   * Computes the cosine of the given column.
+   *
+   * @group math_funcs
+   */
+  def cos(columnName: String): Column = cos(Column(columnName))
+
+  /**
+   * Computes the hyperbolic cosine of the given value.
+   *
+   * @group math_funcs
+   */
+  def cosh(e: Column): Column = Cosh(e.expr)
+
+  /**
+   * Computes the hyperbolic cosine of the given column.
+   *
+   * @group math_funcs
+   */
+  def cosh(columnName: String): Column = cosh(Column(columnName))
+
+  /**
+   * Computes the exponential of the given value.
+   *
+   * @group math_funcs
+   */
+  def exp(e: Column): Column = Exp(e.expr)
+
+  /**
+   * Computes the exponential of the given column.
+   *
+   * @group math_funcs
+   */
+  def exp(columnName: String): Column = exp(Column(columnName))
+
+  /**
+   * Computes the exponential of the given value minus one.
+   *
+   * @group math_funcs
+   */
+  def expm1(e: Column): Column = Expm1(e.expr)
+
+  /**
+   * Computes the exponential of the given column.
+   *
+   * @group math_funcs
+   */
+  def expm1(columnName: String): Column = expm1(Column(columnName))
+
+  /**
+   * Computes the floor of the given value.
+   *
+   * @group math_funcs
+   */
+  def floor(e: Column): Column = Floor(e.expr)
+
+  /**
+   * Computes the floor of the given column.
+   *
+   * @group math_funcs
+   */
+  def floor(columnName: String): Column = floor(Column(columnName))
+
+  /**
+   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
+   *
+   * @group math_funcs
+   */
+  def hypot(l: Column, r: Column): Column = Hypot(l.expr, r.expr)
+
+  /**
+   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
+   *
+   * @group math_funcs
+   */
+  def hypot(l: Column, rightName: String): Column = hypot(l, Column(rightName))
+
+  /**
+   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
+   *
+   * @group math_funcs
+   */
+  def hypot(leftName: String, r: Column): Column = hypot(Column(leftName), r)
+
+  /**
+   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
+   *
+   * @group math_funcs
+   */
+  def hypot(leftName: String, rightName: String): Column =
+    hypot(Column(leftName), Column(rightName))
+
+  /**
+   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
+   *
+   * @group math_funcs
+   */
+  def hypot(l: Column, r: Double): Column = hypot(l, lit(r).expr)
+
+  /**
+   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
+   *
+   * @group math_funcs
+   */
+  def hypot(leftName: String, r: Double): Column = hypot(Column(leftName), r)
+
+  /**
+   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
+   *
+   * @group math_funcs
+   */
+  def hypot(l: Double, r: Column): Column = hypot(lit(l).expr, r)
+
+  /**
+   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
+   *
+   * @group math_funcs
+   */
+  def hypot(l: Double, rightName: String): Column = hypot(l, Column(rightName))
+
+  /**
+   * Computes the natural logarithm of the given value.
+   *
+   * @group math_funcs
+   */
+  def log(e: Column): Column = Log(e.expr)
+
+  /**
+   * Computes the natural logarithm of the given column.
+   *
+   * @group math_funcs
+   */
+  def log(columnName: String): Column = log(Column(columnName))
+
+  /**
+   * Computes the logarithm of the given value in Base 10.
+   *
+   * @group math_funcs
+   */
+  def log10(e: Column): Column = Log10(e.expr)
+
+  /**
+   * Computes the logarithm of the given value in Base 10.
+   *
+   * @group math_funcs
+   */
+  def log10(columnName: String): Column = log10(Column(columnName))
+
+  /**
+   * Computes the natural logarithm of the given value plus one.
+   *
+   * @group math_funcs
+   */
+  def log1p(e: Column): Column = Log1p(e.expr)
+
+  /**
+   * Computes the natural logarithm of the given column plus one.
+   *
+   * @group math_funcs
+   */
+  def log1p(columnName: String): Column = log1p(Column(columnName))
+
+  /**
+   * Returns the value of the first argument raised to the power of the second argument.
+   *
+   * @group math_funcs
+   */
+  def pow(l: Column, r: Column): Column = Pow(l.expr, r.expr)
+
+  /**
+   * Returns the value of the first argument raised to the power of the second argument.
+   *
+   * @group math_funcs
+   */
+  def pow(l: Column, rightName: String): Column = pow(l, Column(rightName))
+
+  /**
+   * Returns the value of the first argument raised to the power of the second argument.
+   *
+   * @group math_funcs
+   */
+  def pow(leftName: String, r: Column): Column = pow(Column(leftName), r)
+
+  /**
+   * Returns the value of the first argument raised to the power of the second argument.
+   *
+   * @group math_funcs
+   */
+  def pow(leftName: String, rightName: String): Column = pow(Column(leftName), Column(rightName))
+
+  /**
+   * Returns the value of the first argument raised to the power of the second argument.
+   *
+   * @group math_funcs
+   */
+  def pow(l: Column, r: Double): Column = pow(l, lit(r).expr)
+
+  /**
+   * Returns the value of the first argument raised to the power of the second argument.
+   *
+   * @group math_funcs
+   */
+  def pow(leftName: String, r: Double): Column = pow(Column(leftName), r)
+
+  /**
+   * Returns the value of the first argument raised to the power of the second argument.
+   *
+   * @group math_funcs
+   */
+  def pow(l: Double, r: Column): Column = pow(lit(l).expr, r)
+
+  /**
+   * Returns the value of the first argument raised to the power of the second argument.
+   *
+   * @group math_funcs
+   */
+  def pow(l: Double, rightName: String): Column = pow(l, Column(rightName))
+
+  /**
+   * Returns the double value that is closest in value to the argument and
+   * is equal to a mathematical integer.
+   *
+   * @group math_funcs
+   */
+  def rint(e: Column): Column = Rint(e.expr)
+
+  /**
+   * Returns the double value that is closest in value to the argument and
+   * is equal to a mathematical integer.
+   *
+   * @group math_funcs
+   */
+  def rint(columnName: String): Column = rint(Column(columnName))
+
+  /**
+   * Computes the signum of the given value.
+   *
+   * @group math_funcs
+   */
+  def signum(e: Column): Column = Signum(e.expr)
+
+  /**
+   * Computes the signum of the given column.
+   *
+   * @group math_funcs
+   */
+  def signum(columnName: String): Column = signum(Column(columnName))
+
+  /**
+   * Computes the sine of the given value.
+   *
+   * @group math_funcs
+   */
+  def sin(e: Column): Column = Sin(e.expr)
+
+  /**
+   * Computes the sine of the given column.
+   *
+   * @group math_funcs
+   */
+  def sin(columnName: String): Column = sin(Column(columnName))
+
+  /**
+   * Computes the hyperbolic sine of the given value.
+   *
+   * @group math_funcs
+   */
+  def sinh(e: Column): Column = Sinh(e.expr)
+
+  /**
+   * Computes the hyperbolic sine of the given column.
+   *
+   * @group math_funcs
+   */
+  def sinh(columnName: String): Column = sinh(Column(columnName))
+
+  /**
+   * Computes the tangent of the given value.
+   *
+   * @group math_funcs
+   */
+  def tan(e: Column): Column = Tan(e.expr)
+
+  /**
+   * Computes the tangent of the given column.
+   *
+   * @group math_funcs
+   */
+  def tan(columnName: String): Column = tan(Column(columnName))
+
+  /**
+   * Computes the hyperbolic tangent of the given value.
+   *
+   * @group math_funcs
+   */
+  def tanh(e: Column): Column = Tanh(e.expr)
+
+  /**
+   * Computes the hyperbolic tangent of the given column.
+   *
+   * @group math_funcs
+   */
+  def tanh(columnName: String): Column = tanh(Column(columnName))
+
+  /**
+   * Converts an angle measured in radians to an approximately equivalent angle measured
in degrees.
+   *
+   * @group math_funcs
+   */
+  def toDegrees(e: Column): Column = ToDegrees(e.expr)
+
+  /**
+   * Converts an angle measured in radians to an approximately equivalent angle measured
in degrees.
+   *
+   * @group math_funcs
+   */
+  def toDegrees(columnName: String): Column = toDegrees(Column(columnName))
+
+  /**
+   * Converts an angle measured in degrees to an approximately equivalent angle measured
in radians.
+   *
+   * @group math_funcs
+   */
+  def toRadians(e: Column): Column = ToRadians(e.expr)
+
+  /**
+   * Converts an angle measured in degrees to an approximately equivalent angle measured
in radians.
+   *
+   * @group math_funcs
+   */
+  def toRadians(columnName: String): Column = toRadians(Column(columnName))
+    
+
+  //////////////////////////////////////////////////////////////////////////////////////////////
   //////////////////////////////////////////////////////////////////////////////////////////////
 
   // scalastyle:off

http://git-wip-us.apache.org/repos/asf/spark/blob/8aa6681d/sql/core/src/main/scala/org/apache/spark/sql/mathfunctions.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/mathfunctions.scala b/sql/core/src/main/scala/org/apache/spark/sql/mathfunctions.scala
deleted file mode 100644
index db47480..0000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/mathfunctions.scala
+++ /dev/null
@@ -1,383 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import scala.language.implicitConversions
-
-import org.apache.spark.annotation.Experimental
-import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.expressions.mathfuncs._
-import org.apache.spark.sql.functions.lit
-
-/**
- * :: Experimental ::
- * Mathematical Functions available for [[DataFrame]].
- */
-@Experimental
-// scalastyle:off
-object mathfunctions {
-// scalastyle:on
-
-  private[this] implicit def toColumn(expr: Expression): Column = Column(expr)
-
-  /**
-   * Computes the cosine inverse of the given value; the returned angle is in the range
-   * 0.0 through pi.
-   */
-  def acos(e: Column): Column = Acos(e.expr)
-
-  /**
-   * Computes the cosine inverse of the given column; the returned angle is in the range
-   * 0.0 through pi.
-   */
-  def acos(columnName: String): Column = acos(Column(columnName))
-
-  /**
-   * Computes the sine inverse of the given value; the returned angle is in the range
-   * -pi/2 through pi/2.
-   */
-  def asin(e: Column): Column = Asin(e.expr)
-
-  /**
-   * Computes the sine inverse of the given column; the returned angle is in the range
-   * -pi/2 through pi/2.
-   */
-  def asin(columnName: String): Column = asin(Column(columnName))
-
-  /**
-   * Computes the tangent inverse of the given value.
-   */
-  def atan(e: Column): Column = Atan(e.expr)
-
-  /**
-   * Computes the tangent inverse of the given column.
-   */
-  def atan(columnName: String): Column = atan(Column(columnName))
-
-  /**
-   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
-   * polar coordinates (r, theta).
-   */
-  def atan2(l: Column, r: Column): Column = Atan2(l.expr, r.expr)
-
-  /**
-   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
-   * polar coordinates (r, theta).
-   */
-  def atan2(l: Column, rightName: String): Column = atan2(l, Column(rightName))
-
-  /**
-   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
-   * polar coordinates (r, theta).
-   */
-  def atan2(leftName: String, r: Column): Column = atan2(Column(leftName), r)
-
-  /**
-   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
-   * polar coordinates (r, theta).
-   */
-  def atan2(leftName: String, rightName: String): Column =
-    atan2(Column(leftName), Column(rightName))
-
-  /**
-   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
-   * polar coordinates (r, theta).
-   */
-  def atan2(l: Column, r: Double): Column = atan2(l, lit(r).expr)
-
-  /**
-   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
-   * polar coordinates (r, theta).=
-   */
-  def atan2(leftName: String, r: Double): Column = atan2(Column(leftName), r)
-
-  /**
-   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
-   * polar coordinates (r, theta).
-   */
-  def atan2(l: Double, r: Column): Column = atan2(lit(l).expr, r)
-
-  /**
-   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
-   * polar coordinates (r, theta).
-   */
-  def atan2(l: Double, rightName: String): Column = atan2(l, Column(rightName))
-
-  /**
-   * Computes the cube-root of the given value.
-   */
-  def cbrt(e: Column): Column = Cbrt(e.expr)
-
-  /**
-   * Computes the cube-root of the given column.
-   */
-  def cbrt(columnName: String): Column = cbrt(Column(columnName))
-
-  /**
-   * Computes the ceiling of the given value.
-   */
-  def ceil(e: Column): Column = Ceil(e.expr)
-
-  /**
-   * Computes the ceiling of the given column.
-   */
-  def ceil(columnName: String): Column = ceil(Column(columnName))
-
-  /**
-   * Computes the cosine of the given value.
-   */
-  def cos(e: Column): Column = Cos(e.expr)
-
-  /**
-   * Computes the cosine of the given column.
-   */
-  def cos(columnName: String): Column = cos(Column(columnName))
-
-  /**
-   * Computes the hyperbolic cosine of the given value.
-   */
-  def cosh(e: Column): Column = Cosh(e.expr)
-
-  /**
-   * Computes the hyperbolic cosine of the given column.
-   */
-  def cosh(columnName: String): Column = cosh(Column(columnName))
-
-  /**
-   * Computes the exponential of the given value.
-   */
-  def exp(e: Column): Column = Exp(e.expr)
-
-  /**
-   * Computes the exponential of the given column.
-   */
-  def exp(columnName: String): Column = exp(Column(columnName))
-
-  /**
-   * Computes the exponential of the given value minus one.
-   */
-  def expm1(e: Column): Column = Expm1(e.expr)
-
-  /**
-   * Computes the exponential of the given column.
-   */
-  def expm1(columnName: String): Column = expm1(Column(columnName))
-
-  /**
-   * Computes the floor of the given value.
-   */
-  def floor(e: Column): Column = Floor(e.expr)
-
-  /**
-   * Computes the floor of the given column.
-   */
-  def floor(columnName: String): Column = floor(Column(columnName))
-
-  /**
-   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
-   */
-  def hypot(l: Column, r: Column): Column = Hypot(l.expr, r.expr)
-
-  /**
-   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
-   */
-  def hypot(l: Column, rightName: String): Column = hypot(l, Column(rightName))
-
-  /**
-   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
-   */
-  def hypot(leftName: String, r: Column): Column = hypot(Column(leftName), r)
-
-  /**
-   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
-   */
-  def hypot(leftName: String, rightName: String): Column =
-    hypot(Column(leftName), Column(rightName))
-
-  /**
-   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
-   */
-  def hypot(l: Column, r: Double): Column = hypot(l, lit(r).expr)
-
-  /**
-   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
-   */
-  def hypot(leftName: String, r: Double): Column = hypot(Column(leftName), r)
-
-  /**
-   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
-   */
-  def hypot(l: Double, r: Column): Column = hypot(lit(l).expr, r)
-
-  /**
-   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
-   */
-  def hypot(l: Double, rightName: String): Column = hypot(l, Column(rightName))
-
-  /**
-   * Computes the natural logarithm of the given value.
-   */
-  def log(e: Column): Column = Log(e.expr)
-
-  /**
-   * Computes the natural logarithm of the given column.
-   */
-  def log(columnName: String): Column = log(Column(columnName))
-
-  /**
-   * Computes the logarithm of the given value in Base 10.
-   */
-  def log10(e: Column): Column = Log10(e.expr)
-
-  /**
-   * Computes the logarithm of the given value in Base 10.
-   */
-  def log10(columnName: String): Column = log10(Column(columnName))
-
-  /**
-   * Computes the natural logarithm of the given value plus one.
-   */
-  def log1p(e: Column): Column = Log1p(e.expr)
-
-  /**
-   * Computes the natural logarithm of the given column plus one.
-   */
-  def log1p(columnName: String): Column = log1p(Column(columnName))
-
-  /**
-   * Returns the value of the first argument raised to the power of the second argument.
-   */
-  def pow(l: Column, r: Column): Column = Pow(l.expr, r.expr)
-
-  /**
-   * Returns the value of the first argument raised to the power of the second argument.
-   */
-  def pow(l: Column, rightName: String): Column = pow(l, Column(rightName))
-
-  /**
-   * Returns the value of the first argument raised to the power of the second argument.
-   */
-  def pow(leftName: String, r: Column): Column = pow(Column(leftName), r)
-
-  /**
-   * Returns the value of the first argument raised to the power of the second argument.
-   */
-  def pow(leftName: String, rightName: String): Column = pow(Column(leftName), Column(rightName))
-
-  /**
-   * Returns the value of the first argument raised to the power of the second argument.
-   */
-  def pow(l: Column, r: Double): Column = pow(l, lit(r).expr)
-
-  /**
-   * Returns the value of the first argument raised to the power of the second argument.
-   */
-  def pow(leftName: String, r: Double): Column = pow(Column(leftName), r)
-
-  /**
-   * Returns the value of the first argument raised to the power of the second argument.
-   */
-  def pow(l: Double, r: Column): Column = pow(lit(l).expr, r)
-
-  /**
-   * Returns the value of the first argument raised to the power of the second argument.
-   */
-  def pow(l: Double, rightName: String): Column = pow(l, Column(rightName))
-
-  /**
-   * Returns the double value that is closest in value to the argument and
-   * is equal to a mathematical integer.
-   */
-  def rint(e: Column): Column = Rint(e.expr)
-
-  /**
-   * Returns the double value that is closest in value to the argument and
-   * is equal to a mathematical integer.
-   */
-  def rint(columnName: String): Column = rint(Column(columnName))
-
-  /**
-   * Computes the signum of the given value.
-   */
-  def signum(e: Column): Column = Signum(e.expr)
-
-  /**
-   * Computes the signum of the given column.
-   */
-  def signum(columnName: String): Column = signum(Column(columnName))
-
-  /**
-   * Computes the sine of the given value.
-   */
-  def sin(e: Column): Column = Sin(e.expr)
-
-  /**
-   * Computes the sine of the given column.
-   */
-  def sin(columnName: String): Column = sin(Column(columnName))
-
-  /**
-   * Computes the hyperbolic sine of the given value.
-   */
-  def sinh(e: Column): Column = Sinh(e.expr)
-
-  /**
-   * Computes the hyperbolic sine of the given column.
-   */
-  def sinh(columnName: String): Column = sinh(Column(columnName))
-  
-  /**
-   * Computes the tangent of the given value.
-   */
-  def tan(e: Column): Column = Tan(e.expr)
-
-  /**
-   * Computes the tangent of the given column.
-   */
-  def tan(columnName: String): Column = tan(Column(columnName))
-  
-  /**
-   * Computes the hyperbolic tangent of the given value.
-   */
-  def tanh(e: Column): Column = Tanh(e.expr)
-
-  /**
-   * Computes the hyperbolic tangent of the given column.
-   */
-  def tanh(columnName: String): Column = tanh(Column(columnName))
-
-  /**
-   * Converts an angle measured in radians to an approximately equivalent angle measured
in degrees.
-   */
-  def toDeg(e: Column): Column = ToDegrees(e.expr)
-
-  /**
-   * Converts an angle measured in radians to an approximately equivalent angle measured
in degrees.
-   */
-  def toDeg(columnName: String): Column = toDeg(Column(columnName))
-
-  /**
-   * Converts an angle measured in degrees to an approximately equivalent angle measured
in radians.
-   */
-  def toRad(e: Column): Column = ToRadians(e.expr)
-
-  /**
-   * Converts an angle measured in degrees to an approximately equivalent angle measured
in radians.
-   */
-  def toRad(columnName: String): Column = toRad(Column(columnName))
-}

http://git-wip-us.apache.org/repos/asf/spark/blob/8aa6681d/sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameSuite.java
----------------------------------------------------------------------
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameSuite.java b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameSuite.java
index 58cc8e5..72c42f4 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameSuite.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameSuite.java
@@ -39,7 +39,6 @@ import java.util.List;
 import java.util.Map;
 
 import static org.apache.spark.sql.functions.*;
-import static org.apache.spark.sql.mathfunctions.*;
 
 public class JavaDataFrameSuite {
   private transient JavaSparkContext jsc;

http://git-wip-us.apache.org/repos/asf/spark/blob/8aa6681d/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala
index 9e19bb7..c4281c4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala
@@ -19,8 +19,7 @@ package org.apache.spark.sql
 
 import java.lang.{Double => JavaDouble}
 
-import org.apache.spark.sql.functions.lit
-import org.apache.spark.sql.mathfunctions._
+import org.apache.spark.sql.functions._
 import org.apache.spark.sql.test.TestSQLContext
 import org.apache.spark.sql.test.TestSQLContext.implicits._
 
@@ -159,11 +158,11 @@ class MathExpressionsSuite extends QueryTest {
   }
 
   test("toDeg") {
-    testOneToOneMathFunction(toDeg, math.toDegrees)
+    testOneToOneMathFunction(toDegrees, math.toDegrees)
   }
 
   test("toRad") {
-    testOneToOneMathFunction(toRad, math.toRadians)
+    testOneToOneMathFunction(toRadians, math.toRadians)
   }
 
   test("cbrt") {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message