carbondata-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipesala <...@git.apache.org>
Subject [GitHub] carbondata pull request #1782: [CARBONDATA-2019] Enhancement of merge index ...
Date Thu, 11 Jan 2018 10:02:32 GMT
Github user ravipesala commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1782#discussion_r160909350
  
    --- Diff: integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
---
    @@ -893,33 +893,53 @@ object CommonUtil {
     
       /**
        * Merge the carbonindex files with in the segment to carbonindexmerge file inside
same segment
    +   *
    +   * @param sparkContext
    +   * @param segmentIds
    +   * @param tablePath
    +   * @param carbonTable
    +   * @param mergeIndexProperty
    +   * @param readFileFooterFromCarbonDataFile flag to read file footer information from
carbondata
    +   *                                         file. This will used in case of upgrade
from version
    +   *                                         which do not store the blocklet info to
current version
        */
       def mergeIndexFiles(sparkContext: SparkContext,
           segmentIds: Seq[String],
           tablePath: String,
           carbonTable: CarbonTable,
    -      mergeIndexProperty: Boolean): Unit = {
    +      mergeIndexProperty: Boolean,
    +      readFileFooterFromCarbonDataFile: Boolean = false): Unit = {
         if (mergeIndexProperty) {
    -      new CarbonMergeFilesRDD(sparkContext, AbsoluteTableIdentifier.from(tablePath,
    -        carbonTable.getDatabaseName, carbonTable.getTableName).getTablePath,
    -        segmentIds).collect()
    +      new CarbonMergeFilesRDD(
    +        sparkContext,
    +        AbsoluteTableIdentifier
    +          .from(tablePath, carbonTable.getDatabaseName, carbonTable.getTableName).getTablePath,
    +        segmentIds,
    +        readFileFooterFromCarbonDataFile).collect()
         } else {
           try {
             CarbonProperties.getInstance()
               .getProperty(CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT).toBoolean
             if (CarbonProperties.getInstance().getProperty(
               CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT,
               CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT_DEFAULT).toBoolean) {
    -          new CarbonMergeFilesRDD(sparkContext, AbsoluteTableIdentifier.from(tablePath,
    -            carbonTable.getDatabaseName, carbonTable.getTableName).getTablePath,
    -            segmentIds).collect()
    +          new CarbonMergeFilesRDD(
    +            sparkContext,
    +            AbsoluteTableIdentifier
    +              .from(tablePath, carbonTable.getDatabaseName, carbonTable.getTableName).getTablePath,
    --- End diff --
    
    Please use like this 
    ``` 
    new CarbonMergeFilesRDD(
                sparkContext,
                AbsoluteTableIdentifier.from(
                  tablePath, 
                  carbonTable.getDatabaseName, 
                  carbonTable.getTableName).getTablePath,
                segmentIds,
                readFileFooterFromCarbonDataFile).collect()
    ```


---

Mime
View raw message