spark-reviews mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From viirya <...@git.apache.org>
Subject [GitHub] spark pull request #19869: [SPARK-22677][SQL] cleanup whole stage codegen fo...
Date Mon, 04 Dec 2017 07:53:18 GMT
Github user viirya commented on a diff in the pull request:

    https://github.com/apache/spark/pull/19869#discussion_r154576368
  
    --- Diff: sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
---
    @@ -573,94 +574,84 @@ case class HashAggregateExec(
           enableTwoLevelHashMap(ctx)
         } else {
           sqlContext.getConf("spark.sql.codegen.aggregate.map.vectorized.enable", null) match
{
    -        case "true" => logWarning("Two level hashmap is disabled but vectorized hashmap
is " +
    -          "enabled.")
    -        case null | "" | "false" => None
    +        case "true" =>
    +          logWarning("Two level hashmap is disabled but vectorized hashmap is enabled.")
    +        case _ =>
           }
         }
    -    fastHashMapTerm = ctx.freshName("fastHashMap")
    -    val fastHashMapClassName = ctx.freshName("FastHashMap")
    -    val fastHashMapGenerator =
    -      if (isVectorizedHashMapEnabled) {
    -        new VectorizedHashMapGenerator(ctx, aggregateExpressions,
    -          fastHashMapClassName, groupingKeySchema, bufferSchema)
    -      } else {
    -        new RowBasedHashMapGenerator(ctx, aggregateExpressions,
    -          fastHashMapClassName, groupingKeySchema, bufferSchema)
    -      }
     
         val thisPlan = ctx.addReferenceObj("plan", this)
     
    -    // Create a name for iterator from vectorized HashMap
    +    // Create a name for the iterator from the fast hash map.
         val iterTermForFastHashMap = ctx.freshName("fastHashMapIter")
         if (isFastHashMapEnabled) {
    +      // Generates the fast hash map class and creates the fash hash map term.
    +      fastHashMapTerm = ctx.freshName("fastHashMap")
    +      val fastHashMapClassName = ctx.freshName("FastHashMap")
           if (isVectorizedHashMapEnabled) {
    +        val generatedMap = new VectorizedHashMapGenerator(ctx, aggregateExpressions,
    +          fastHashMapClassName, groupingKeySchema, bufferSchema).generate()
    +        ctx.addInnerClass(generatedMap)
    +
             ctx.addMutableState(fastHashMapClassName, fastHashMapTerm,
               s"$fastHashMapTerm = new $fastHashMapClassName();")
             ctx.addMutableState(
    -          "java.util.Iterator<org.apache.spark.sql.execution.vectorized.ColumnarRow>",
    +          classOf[java.util.Iterator[ColumnarRow]].getName,
    --- End diff --
    
    Is this as same as before?
    
    ```scala
    scala> classOf[java.util.Iterator[Int]].getName
    res2: String = java.util.Iterator
    ```
    



---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org


Mime
View raw message