carbondata-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jackylk <...@git.apache.org>
Subject [GitHub] carbondata pull request #1730: [CARBONDATA-1937][PARTITION] Fix partition fe...
Date Thu, 28 Dec 2017 15:13:22 GMT
Github user jackylk commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1730#discussion_r158956569
  
    --- Diff: integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
---
    @@ -405,13 +407,60 @@ object CarbonFilters {
         }
       }
     
    +  /**
    +   * Fetches partition information from hive
    +   * @param partitionFilters
    +   * @param sparkSession
    +   * @param identifier
    +   * @return
    +   */
       def getPartitions(partitionFilters: Seq[Expression],
           sparkSession: SparkSession,
           identifier: TableIdentifier): Seq[String] = {
    -    val partitions =
    -      sparkSession.sessionState.catalog.listPartitionsByFilter(identifier, partitionFilters)
    +    val partitions = {
    +      try {
    +        if (CarbonProperties.getInstance().
    +          getProperty(CarbonCommonConstants.CARBON_READ_PARTITION_HIVE_DIRECT,
    +          CarbonCommonConstants.CARBON_READ_PARTITION_HIVE_DIRECT_DEFAULT).toBoolean)
{
    +          sparkSession.sessionState.catalog.listPartitionsByFilter(identifier, partitionFilters)
    +        } else {
    +          getPartitionsAlternate(partitionFilters, sparkSession, identifier)
    +        }
    +      } catch {
    +        case e: Exception =>
    +          // Get partition information alternatively.
    +          getPartitionsAlternate(partitionFilters, sparkSession, identifier)
    +      }
    +    }
         partitions.toList.flatMap { partition =>
           partition.spec.seq.map{case (column, value) => column + "=" + value}
         }.toSet.toSeq
       }
    +
    +  /**
    +   * This is alternate way of getting partition information. It first fetches all partitions
from
    +   * hive and then apply filter instead of querying hive along with filters.
    +   * @param partitionFilters
    --- End diff --
    
    give comment for parameter


---

Mime
View raw message