spark-reviews mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From HyukjinKwon <...@git.apache.org>
Subject [GitHub] spark pull request #20894: [SPARK-23786][SQL] Checking column names of csv h...
Date Wed, 16 May 2018 08:48:07 GMT
Github user HyukjinKwon commented on a diff in the pull request:

    https://github.com/apache/spark/pull/20894#discussion_r188547521
  
    --- Diff: sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala
---
    @@ -118,16 +122,62 @@ object CSVDataSource {
           TextInputCSVDataSource
         }
       }
    +
    +  def checkHeaderColumnNames(schema: StructType, columnNames: Array[String], fileName:
String,
    +      checkHeaderFlag: Boolean, caseSensitive: Boolean): Unit = {
    +    if (checkHeaderFlag && columnNames != null) {
    +      val fieldNames = schema.map(_.name).toIndexedSeq
    +      val (headerLen, schemaSize) = (columnNames.size, fieldNames.length)
    +      var error: Option[String] = None
    +
    +      if (headerLen == schemaSize) {
    +        var i = 0
    +        while (error.isEmpty && i < headerLen) {
    +          var (nameInSchema, nameInHeader) = (fieldNames(i), columnNames(i))
    +          if (caseSensitive == false) {
    +            nameInSchema = nameInSchema.toLowerCase
    +            nameInHeader = nameInHeader.toLowerCase
    +          }
    +          if (nameInHeader != nameInSchema) {
    +            error = Some(
    +              s"""|CSV file header does not contain the expected fields.
    +                  | Header: ${columnNames.mkString(", ")}
    +                  | Schema: ${fieldNames.mkString(", ")}
    +                  |Expected: ${columnNames(i)} but found: ${fieldNames(i)}
    +                  |CSV file: $fileName""".stripMargin
    +            )
    +          }
    +          i += 1
    +        }
    +      } else {
    +        error = Some(
    +          s"""|Number of column in CSV header is not equal to number of fields in the
schema:
    +              | Header length: $headerLen, schema size: $schemaSize
    +              |CSV file: $fileName""".stripMargin
    +        )
    +      }
    +
    +      error.headOption.foreach { msg =>
    +        throw new IllegalArgumentException(msg)
    +      }
    +    }
    +  }
    +
    +  def checkHeader(header: String, parser: CsvParser, schema: StructType, fileName: String,
    --- End diff --
    
    doc and style


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org


Mime
View raw message