spark-reviews mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From srowen <...@git.apache.org>
Subject [GitHub] spark pull request #20611: [SPARK-23425][SQL]Support wildcard in HDFS path f...
Date Mon, 13 Aug 2018 15:11:26 GMT
Github user srowen commented on a diff in the pull request:

    https://github.com/apache/spark/pull/20611#discussion_r209646842
  
    --- Diff: core/src/main/scala/org/apache/spark/util/Utils.scala ---
    @@ -1976,6 +1976,56 @@ private[spark] object Utils extends Logging {
         }
       }
     
    +  /**
    +   * Returns a qualified path object.
    +   * @param defaultUri default uri corresponding to the filesystem provided.
    +   * @param workingDir the working directory for the particular child path wd-relative
names.
    +   * @param path Path instance based on the path string specified by the user.
    +   * @return Returns a qualified path object
    +   */
    +  def makeQualified(defaultUri: URI, workingDir: Path, path: Path): Path = {
    +    val pathUri = {
    +      if (!path.isAbsolute()) {
    +        new Path(workingDir, path).toUri()
    +      }
    +      else {
    +        path.toUri()
    +      }
    +    }
    +    val newPaths = {
    +      if (pathUri.getScheme == null || pathUri.getAuthority == null &&
    +        defaultUri.getAuthority != null) {
    +        val scheme = {
    +          if (pathUri.getScheme == null) defaultUri.getScheme else pathUri.getScheme
    +        }
    +        val authority = {
    +          if (pathUri.getAuthority == null) {
    +            if (null == defaultUri.getAuthority) {
    +              ""
    +            }
    +            else {
    +              defaultUri.getAuthority
    +            }
    +          }
    +          else {
    +            pathUri.getAuthority
    +          }
    +        }
    +        try {
    +          var newUri = new URI(scheme, authority, pathUri.getPath, null.asInstanceOf[String],
    +            pathUri.getFragment)
    +          new Path(newUri)
    +        }
    +        catch {
    --- End diff --
    
    Same, pull this up onto the previous line


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org


Mime
View raw message