spark-reviews mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From viirya <...@git.apache.org>
Subject [GitHub] spark pull request #21732: [SPARK-24762][SQL] Enable Option of Product encod...
Date Fri, 24 Aug 2018 04:46:54 GMT
Github user viirya commented on a diff in the pull request:

    https://github.com/apache/spark/pull/21732#discussion_r212520685
  
    --- Diff: sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/TypedAggregateExpression.scala
---
    @@ -19,25 +19,85 @@ package org.apache.spark.sql.execution.aggregate
     
     import scala.language.existentials
     
    -import org.apache.spark.sql.Encoder
    +import org.apache.spark.sql.{AnalysisException, Encoder}
     import org.apache.spark.sql.catalyst.InternalRow
    -import org.apache.spark.sql.catalyst.analysis.UnresolvedDeserializer
    -import org.apache.spark.sql.catalyst.encoders.encoderFor
    +import org.apache.spark.sql.catalyst.analysis.{GetColumnByOrdinal, UnresolvedDeserializer}
    +import org.apache.spark.sql.catalyst.encoders.{encoderFor, ExpressionEncoder}
     import org.apache.spark.sql.catalyst.expressions._
     import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateFunction, DeclarativeAggregate,
TypedImperativeAggregate}
     import org.apache.spark.sql.catalyst.expressions.codegen.GenerateSafeProjection
    -import org.apache.spark.sql.catalyst.expressions.objects.Invoke
    +import org.apache.spark.sql.catalyst.expressions.objects.{AssertNotNull, Invoke, NewInstance,
WrapOption}
     import org.apache.spark.sql.expressions.Aggregator
     import org.apache.spark.sql.types._
     import org.apache.spark.util.Utils
     
     object TypedAggregateExpression {
    +
    +  // Checks if given encoder is for `Option[Product]`.
    +  def isOptProductEncoder(encoder: ExpressionEncoder[_]): Boolean = {
    +    // Only Option[Product] is non-flat.
    +    encoder.clsTag.runtimeClass == classOf[Option[_]] && !encoder.flat
    +  }
    +
    +  /**
    +   * Flattens serializers and deserializer of given encoder. We only flatten encoder
    +   * of `Option[Product]` class.
    +   */
    +  def flattenOptProductEncoder(encoder: ExpressionEncoder[_]): ExpressionEncoder[_] =
{
    --- End diff --
    
    I will go to add some tests against this.


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org


Mime
View raw message