spark-reviews mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mn-mikke <...@git.apache.org>
Subject [GitHub] spark pull request #20858: [SPARK-23736][SQL] Extending the concat function ...
Date Tue, 27 Mar 2018 13:17:53 GMT
Github user mn-mikke commented on a diff in the pull request:

    https://github.com/apache/spark/pull/20858#discussion_r177419513
  
    --- Diff: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
---
    @@ -287,3 +289,152 @@ case class ArrayContains(left: Expression, right: Expression)
     
       override def prettyName: String = "array_contains"
     }
    +
    +/**
    + * Concatenates multiple arrays into one.
    + */
    +@ExpressionDescription(
    +  usage = "_FUNC_(expr, ...) - Concatenates multiple arrays into one.",
    +  examples = """
    +    Examples:
    +      > SELECT _FUNC_(array(1, 2, 3), array(4, 5), array(6));
    +       [1,2,3,4,5,6]
    +  """)
    +case class ConcatArrays(children: Seq[Expression]) extends Expression with NullSafeEvaluation
{
    +
    +  override def checkInputDataTypes(): TypeCheckResult = {
    +    val arrayCheck = checkInputDataTypesAreArrays
    +    if(arrayCheck.isFailure) arrayCheck
    +    else TypeUtils.checkForSameTypeInputExpr(children.map(_.dataType), s"function $prettyName")
    +  }
    +
    +  private def checkInputDataTypesAreArrays(): TypeCheckResult =
    +  {
    +    val mismatches = children.zipWithIndex.collect {
    +      case (child, idx) if !ArrayType.acceptsType(child.dataType) =>
    +        s"argument ${idx + 1} has to be ${ArrayType.simpleString} type, " +
    +          s"however, '${child.sql}' is of ${child.dataType.simpleString} type."
    +    }
    +
    +    if (mismatches.isEmpty) {
    +      TypeCheckResult.TypeCheckSuccess
    +    } else {
    +      TypeCheckResult.TypeCheckFailure(mismatches.mkString(" "))
    +    }
    +  }
    +
    +  override def dataType: ArrayType =
    +    children
    +      .headOption.map(_.dataType.asInstanceOf[ArrayType])
    +      .getOrElse(ArrayType.defaultConcreteType.asInstanceOf[ArrayType])
    --- End diff --
    
    Ok, changing to return type `array<string>` when no children are provided. Also
I've created the jira ticket [SPARK-23798](https://issues.apache.org/jira/browse/SPARK-23798)
since I don't see any reason why it couldn't return a default concrete type in this case.
Hope I don't miss anything.


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org


Mime
View raw message