flink-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From NicoK <...@git.apache.org>
Subject [GitHub] flink pull request #4939: [FLINK-4228][yarn/s3a] fix yarn resource upload s3...
Date Mon, 06 Nov 2017 13:27:56 GMT
Github user NicoK commented on a diff in the pull request:

    https://github.com/apache/flink/pull/4939#discussion_r149077720
  
    --- Diff: flink-yarn/src/main/java/org/apache/flink/yarn/Utils.java ---
    @@ -117,27 +118,50 @@ public static void setupYarnClassPath(Configuration conf, Map<String,
String> ap
     	}
     
     	/**
    +	 * Copy a local file to a remote file system.
    +	 *
    +	 * @param fs
    +	 * 		remote filesystem
    +	 * @param appId
    +	 * 		application ID
    +	 * @param localRsrcPath
    +	 * 		path to the local file
    +	 * @param homedir
    +	 * 		remote home directory base (will be extended)
    +	 * @param relativeTargetPath
    +	 * 		relative target path of the file (will be prefixed be the full home directory we
set up)
    +	 *
     	 * @return Path to remote file (usually hdfs)
    -	 * @throws IOException
     	 */
    -	public static Path setupLocalResource(
    -			FileSystem fs,
    -			String appId, Path localRsrcPath,
    -			LocalResource appMasterJar,
    -			Path homedir) throws IOException {
    +	static Tuple2<Path, LocalResource> setupLocalResource(
    +		FileSystem fs,
    +		String appId,
    +		Path localRsrcPath,
    +		Path homedir,
    +		String relativeTargetPath) throws IOException {
    +
    +		if (new File(localRsrcPath.toUri().getPath()).isDirectory()) {
    +			throw new IllegalArgumentException("File to copy must not be a directory: " +
    +				localRsrcPath);
    +		}
     
     		// copy resource to HDFS
    -		String suffix = ".flink/" + appId + "/" + localRsrcPath.getName();
    +		String suffix = ".flink/" + appId + "/" + relativeTargetPath + "/" + localRsrcPath.getName();
     
     		Path dst = new Path(homedir, suffix);
     
     		LOG.info("Copying from " + localRsrcPath + " to " + dst);
    -		fs.copyFromLocalFile(localRsrcPath, dst);
    -		registerLocalResource(fs, dst, appMasterJar);
    -		return dst;
    +
    +		fs.copyFromLocalFile(false, true, localRsrcPath, dst);
    +
    +		// now create the resource instance
    +		LocalResource resource = Records.newRecord(LocalResource.class);
    +		registerLocalResource(fs, dst, resource);
    +		return Tuple2.of(dst, resource);
     	}
     
    -	public static void registerLocalResource(FileSystem fs, Path remoteRsrcPath, LocalResource
localResource) throws IOException {
    +	private static void registerLocalResource(
    --- End diff --
    
    oh yes - totally...


---

Mime
View raw message