spark-reviews mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From DazhuangSu <...@git.apache.org>
Subject [GitHub] spark pull request #19691: [SPARK-14922][SPARK-17732][SQL]ALTER TABLE DROP P...
Date Fri, 01 Jun 2018 12:32:45 GMT
Github user DazhuangSu commented on a diff in the pull request:

    https://github.com/apache/spark/pull/19691#discussion_r192380851
  
    --- Diff: sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala ---
    @@ -515,28 +515,66 @@ case class AlterTableRenamePartitionCommand(
      */
     case class AlterTableDropPartitionCommand(
         tableName: TableIdentifier,
    -    specs: Seq[TablePartitionSpec],
    +    partitions: Seq[(TablePartitionSpec, Seq[Expression])],
         ifExists: Boolean,
         purge: Boolean,
         retainData: Boolean)
    -  extends RunnableCommand {
    +  extends RunnableCommand with PredicateHelper {
     
       override def run(sparkSession: SparkSession): Seq[Row] = {
         val catalog = sparkSession.sessionState.catalog
         val table = catalog.getTableMetadata(tableName)
    +    val resolver = sparkSession.sessionState.conf.resolver
         DDLUtils.verifyAlterTableType(catalog, table, isView = false)
         DDLUtils.verifyPartitionProviderIsHive(sparkSession, table, "ALTER TABLE DROP PARTITION")
     
    -    val normalizedSpecs = specs.map { spec =>
    -      PartitioningUtils.normalizePartitionSpec(
    -        spec,
    -        table.partitionColumnNames,
    -        table.identifier.quotedString,
    -        sparkSession.sessionState.conf.resolver)
    +    val toDrop = partitions.flatMap { partition =>
    +      val normalizedSpecs = PartitioningUtils.normalizePartitionSpec(
    +            partition._1,
    +            table.partitionColumnNames,
    +            table.identifier.quotedString,
    +            sparkSession.sessionState.conf.resolver)
    +
    +      val partitionSet = {
    +        if (partition._2.nonEmpty) {
    +          val parts = partition._2.map { expr =>
    +            val (attrName, value) = expr match {
    +              case BinaryComparison(UnresolvedAttribute(name :: Nil), constant: Literal)
=>
    +                (name, constant.value)
    +            }
    +            if (!table.partitionColumnNames.exists(resolver(_, attrName))) {
    +              throw new AnalysisException(s"${attrName} is not a valid partition column
" +
    +                s"in table ${table.identifier.quotedString}.")
    +            }
    +            val dataType = table.partitionSchema.apply(attrName).dataType
    +            expr.withNewChildren(Seq(AttributeReference(attrName, dataType)(),
    +              Cast(Literal(value.toString), dataType)))
    --- End diff --
    
    using the parsed `constant` and if we don't cast it to partition's dataType. it will throw
an exception
    >java.lang.ClassCastException: java.lang.Integer cannot be cast to java.lang.Long
        at scala.runtime.BoxesRunTime.unboxToLong(BoxesRunTime.java:105)
        at scala.math.Ordering$Long$.compare(Ordering.scala:264)
        at scala.math.Ordering$class.gteq(Ordering.scala:91)
        at scala.math.Ordering$Long$.gteq(Ordering.scala:264)
        at org.apache.spark.sql.catalyst.expressions.GreaterThanOrEqual.nullSafeEval(predicates.scala:710)
        at org.apache.spark.sql.catalyst.expressions.BinaryExpression.eval(Expression.scala:423)
    
    for the case
     `CREATE TABLE tbl_x (a INT) PARTITIONED BY (p LONG)`
    `ALTER TABLE tbl_x DROP PARTITION (p >= 1)`
    that I mentioned above


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org


Mime
View raw message