carbondata-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipesala <...@git.apache.org>
Subject [GitHub] carbondata pull request #1672: [CARBONDATA-1858][PARTITION] Support querying...
Date Tue, 19 Dec 2017 02:46:50 GMT
Github user ravipesala commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1672#discussion_r157653428
  
    --- Diff: integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
---
    @@ -479,22 +479,52 @@ case class CarbonLoadDataCommand(
           LogicalRDD(attributes, rdd)(sparkSession)
     
         } else {
    +      var timeStampformatString = carbonLoadModel.getTimestampformat
    +      if (timeStampformatString.isEmpty) {
    +        timeStampformatString = carbonLoadModel.getDefaultTimestampFormat
    +      }
    +      val timeStampFormat = new SimpleDateFormat(timeStampformatString)
    +      var dateFormatString = carbonLoadModel.getDateFormat
    +      if (dateFormatString.isEmpty) {
    +        dateFormatString = carbonLoadModel.getDefaultDateFormat
    +      }
    +      val dateFormat = new SimpleDateFormat(dateFormatString)
           // input data from csv files. Convert to logical plan
           CommonUtil.configureCSVInputFormat(hadoopConf, carbonLoadModel)
           hadoopConf.set(FileInputFormat.INPUT_DIR, carbonLoadModel.getFactFilePath)
           val jobConf = new JobConf(hadoopConf)
           SparkHadoopUtil.get.addCredentials(jobConf)
    -      val rdd = new NewHadoopRDD[NullWritable, StringArrayWritable](
    -        sparkSession.sparkContext,
    -        classOf[CSVInputFormat],
    -        classOf[NullWritable],
    -        classOf[StringArrayWritable],
    -        jobConf
    -      ).map(f => InternalRow.fromSeq(f._2.get().map(UTF8String.fromString)))
    -
           val attributes =
             StructType(carbonLoadModel.getCsvHeaderColumns.map(
               StructField(_, StringType))).toAttributes
    +      val rowDataTypes = attributes.map{f =>
    +        relation.output.find(_.name.equalsIgnoreCase(f.name)) match {
    +          case Some(attr) => attr.dataType
    +          case _ => StringType
    +        }
    +      }
    +      val len = rowDataTypes.length
    +      val rdd =
    +        new NewHadoopRDD[NullWritable, StringArrayWritable](
    +          sparkSession.sparkContext,
    +          classOf[CSVInputFormat],
    +          classOf[NullWritable],
    +          classOf[StringArrayWritable],
    +          jobConf).map{f =>
    --- End diff --
    
    ok


---

Mime
View raw message