carbondata-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipesala <...@git.apache.org>
Subject [GitHub] carbondata pull request #1782: [CARBONDATA-2019] Enhancement of merge index ...
Date Thu, 11 Jan 2018 10:02:03 GMT
Github user ravipesala commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1782#discussion_r160909240
  
    --- Diff: integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
---
    @@ -893,33 +893,53 @@ object CommonUtil {
     
       /**
        * Merge the carbonindex files with in the segment to carbonindexmerge file inside
same segment
    +   *
    +   * @param sparkContext
    +   * @param segmentIds
    +   * @param tablePath
    +   * @param carbonTable
    +   * @param mergeIndexProperty
    +   * @param readFileFooterFromCarbonDataFile flag to read file footer information from
carbondata
    +   *                                         file. This will used in case of upgrade
from version
    +   *                                         which do not store the blocklet info to
current version
        */
       def mergeIndexFiles(sparkContext: SparkContext,
           segmentIds: Seq[String],
           tablePath: String,
           carbonTable: CarbonTable,
    -      mergeIndexProperty: Boolean): Unit = {
    +      mergeIndexProperty: Boolean,
    +      readFileFooterFromCarbonDataFile: Boolean = false): Unit = {
         if (mergeIndexProperty) {
    -      new CarbonMergeFilesRDD(sparkContext, AbsoluteTableIdentifier.from(tablePath,
    -        carbonTable.getDatabaseName, carbonTable.getTableName).getTablePath,
    -        segmentIds).collect()
    +      new CarbonMergeFilesRDD(
    +        sparkContext,
    +        AbsoluteTableIdentifier
    +          .from(tablePath, carbonTable.getDatabaseName, carbonTable.getTableName).getTablePath,
    +        segmentIds,
    +        readFileFooterFromCarbonDataFile).collect()
    --- End diff --
    
    Use like this
    ```
    new CarbonMergeFilesRDD(
            sparkContext,
            AbsoluteTableIdentifier.from(
              tablePath, 
              carbonTable.getDatabaseName, 
              carbonTable.getTableName).getTablePath,
            segmentIds,
            readFileFooterFromCarbonDataFile).collect()
    ```


---

Mime
View raw message