carbondata-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipesala <...@git.apache.org>
Subject [GitHub] carbondata pull request #1595: [CARBONDATA-1868][Spark-2.2]Carbon-Spark2.2 I...
Date Wed, 06 Dec 2017 14:31:33 GMT
Github user ravipesala commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1595#discussion_r155251303
  
    --- Diff: integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
---
    @@ -413,19 +414,35 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy
{
           relation: BaseRelation,
           predicates: Seq[Expression]): (Seq[Expression], Seq[Filter]) = {
     
    +    def isComplexAttribute(attribute: Attribute) = attribute.dataType match {
    +      case ArrayType(dataType, _) => true
    +      case StructType(_) => true
    +      case _ => false
    +    }
    +
    +    // In case of ComplexType dataTypes no filters should be pushed down. IsNotNull is
being
    +    // explicitly added by spark and pushed. That also has to be handled and pushed back
to
    +    // Spark for handling.
    +    val predicatesWithoutComplex = predicates.filter(predicate => predicate.collect
{
    +      case a: Attribute if (isComplexAttribute(a)) => a
    +    }.size == 0 )
    +
         // For conciseness, all Catalyst filter expressions of type `expressions.Expression`
below are
         // called `predicate`s, while all data source filters of type `sources.Filter` are
simply called
         // `filter`s.
     
         val translated: Seq[(Expression, Filter)] =
           for {
    -        predicate <- predicates
    +        predicate <- predicatesWithoutComplex
             filter <- translateFilter(predicate)
           } yield predicate -> filter
     
    +
    +    // val newTranslated = translated.toMap.filter()
    --- End diff --
    
    remove commented code


---

Mime
View raw message