spark-reviews mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maropu <...@git.apache.org>
Subject [GitHub] spark pull request #21608: [SPARK-24626] [SQL] Improve location size calcula...
Date Tue, 24 Jul 2018 07:44:20 GMT
Github user maropu commented on a diff in the pull request:

    https://github.com/apache/spark/pull/21608#discussion_r204654083
  
    --- Diff: sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala
---
    @@ -47,15 +48,26 @@ object CommandUtils extends Logging {
         }
       }
     
    -  def calculateTotalSize(sessionState: SessionState, catalogTable: CatalogTable): BigInt
= {
    +  def calculateTotalSize(spark: SparkSession, catalogTable: CatalogTable): BigInt = {
    +    val sessionState = spark.sessionState
         if (catalogTable.partitionColumnNames.isEmpty) {
           calculateLocationSize(sessionState, catalogTable.identifier, catalogTable.storage.locationUri)
         } else {
           // Calculate table size as a sum of the visible partitions. See SPARK-21079
           val partitions = sessionState.catalog.listPartitions(catalogTable.identifier)
    -      partitions.map { p =>
    -        calculateLocationSize(sessionState, catalogTable.identifier, p.storage.locationUri)
    -      }.sum
    +      val paths = partitions.map(x => new Path(x.storage.locationUri.get))
    +      val stagingDir = sessionState.conf.getConfString("hive.exec.stagingdir", ".hive-staging")
    +      val pathFilter = new PathFilter with Serializable {
    +        override def accept(path: Path): Boolean = {
    +          val fileName = path.getName
    +          (// Ignore metadata files starting with "_"
    +            DataSourceUtils.isDataPath(path) && !fileName.startsWith(stagingDir))
    +        }
    +      }
    +      val fileStatusSeq = InMemoryFileIndex.bulkListLeafFiles(paths,
    +        sessionState.newHadoopConf(), pathFilter,
    +        spark).flatMap(_._2)
    +      fileStatusSeq.map(_.getLen).sum
    --- End diff --
    
    How about this?
    ```
          val fileStatusSeq = InMemoryFileIndex.bulkListLeafFiles(
            paths, sessionState.newHadoopConf(), SerializablePathFilter(stagingDir), spark)
          fileStatusSeq.flatMap(_._2.map(_.getLen)).sum
    ```


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org


Mime
View raw message