carbondata-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jackylk <...@git.apache.org>
Subject [GitHub] carbondata pull request #1672: [CARBONDATA-1858][PARTITION] Support querying...
Date Tue, 19 Dec 2017 01:46:09 GMT
Github user jackylk commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1672#discussion_r157646748
  
    --- Diff: integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
---
    @@ -479,22 +479,52 @@ case class CarbonLoadDataCommand(
           LogicalRDD(attributes, rdd)(sparkSession)
     
         } else {
    +      var timeStampformatString = carbonLoadModel.getTimestampformat
    +      if (timeStampformatString.isEmpty) {
    +        timeStampformatString = carbonLoadModel.getDefaultTimestampFormat
    +      }
    +      val timeStampFormat = new SimpleDateFormat(timeStampformatString)
    +      var dateFormatString = carbonLoadModel.getDateFormat
    +      if (dateFormatString.isEmpty) {
    +        dateFormatString = carbonLoadModel.getDefaultDateFormat
    +      }
    +      val dateFormat = new SimpleDateFormat(dateFormatString)
           // input data from csv files. Convert to logical plan
           CommonUtil.configureCSVInputFormat(hadoopConf, carbonLoadModel)
           hadoopConf.set(FileInputFormat.INPUT_DIR, carbonLoadModel.getFactFilePath)
           val jobConf = new JobConf(hadoopConf)
           SparkHadoopUtil.get.addCredentials(jobConf)
    -      val rdd = new NewHadoopRDD[NullWritable, StringArrayWritable](
    -        sparkSession.sparkContext,
    -        classOf[CSVInputFormat],
    -        classOf[NullWritable],
    -        classOf[StringArrayWritable],
    -        jobConf
    -      ).map(f => InternalRow.fromSeq(f._2.get().map(UTF8String.fromString)))
    -
           val attributes =
             StructType(carbonLoadModel.getCsvHeaderColumns.map(
               StructField(_, StringType))).toAttributes
    +      val rowDataTypes = attributes.map{f =>
    --- End diff --
    
    change `f` to `attribute`, add space after `map`


---

Mime
View raw message