Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id D30DA200B69 for ; Fri, 5 Aug 2016 19:57:44 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id D1B64160A64; Fri, 5 Aug 2016 17:57:44 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 26CAC160AAC for ; Fri, 5 Aug 2016 19:57:44 +0200 (CEST) Received: (qmail 35290 invoked by uid 500); 5 Aug 2016 17:57:43 -0000 Mailing-List: contact commits-help@flink.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@flink.apache.org Delivered-To: mailing list commits@flink.apache.org Received: (qmail 35207 invoked by uid 99); 5 Aug 2016 17:57:43 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 05 Aug 2016 17:57:43 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 19AA2EE68D; Fri, 5 Aug 2016 17:57:43 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sewen@apache.org To: commits@flink.apache.org Date: Fri, 05 Aug 2016 17:57:44 -0000 Message-Id: In-Reply-To: <8a03b1f237404ed488ebd9373db69fe3@git.apache.org> References: <8a03b1f237404ed488ebd9373db69fe3@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/4] flink git commit: [FLINK-4297] [yarn] Decode URL encoded fat jar path archived-at: Fri, 05 Aug 2016 17:57:45 -0000 [FLINK-4297] [yarn] Decode URL encoded fat jar path This solves problems with spaces and special characters in the automatically determined fat jar path which is returned URL encoded. This closes #2320 Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c7a85545 Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c7a85545 Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c7a85545 Branch: refs/heads/master Commit: c7a85545ba73e93e4a55ef8886362badaa2e2147 Parents: e629b2e Author: Maximilian Michels Authored: Mon Aug 1 15:19:15 2016 +0200 Committer: Stephan Ewen Committed: Fri Aug 5 16:03:58 2016 +0200 ---------------------------------------------------------------------- .../src/main/flink-bin/yarn-bin/yarn-session.sh | 2 +- .../apache/flink/yarn/cli/FlinkYarnSessionCli.java | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flink/blob/c7a85545/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh ---------------------------------------------------------------------- diff --git a/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh b/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh index 7c92680..502df3d 100755 --- a/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh +++ b/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh @@ -52,5 +52,5 @@ log_setting="-Dlog.file="$log" -Dlog4j.configuration=file:"$FLINK_CONF_DIR"/log4 export FLINK_CONF_DIR -$JAVA_RUN $JVM_ARGS -classpath $CC_CLASSPATH:$HADOOP_CLASSPATH:$HADOOP_CONF_DIR:$YARN_CONF_DIR $log_setting org.apache.flink.yarn.cli.FlinkYarnSessionCli -j $FLINK_LIB_DIR/flink-dist*.jar "$@" +$JAVA_RUN $JVM_ARGS -classpath "$CC_CLASSPATH:$HADOOP_CLASSPATH:$HADOOP_CONF_DIR:$YARN_CONF_DIR" $log_setting org.apache.flink.yarn.cli.FlinkYarnSessionCli -j "$FLINK_LIB_DIR"/flink-dist*.jar "$@" http://git-wip-us.apache.org/repos/asf/flink/blob/c7a85545/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java ---------------------------------------------------------------------- diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java index 3e3b640..bee6a7a 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java @@ -47,6 +47,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -253,15 +256,23 @@ public class FlinkYarnSessionCli implements CustomCommandLine Path localJarPath; if (cmd.hasOption(FLINK_JAR.getOpt())) { String userPath = cmd.getOptionValue(FLINK_JAR.getOpt()); - if(!userPath.startsWith("file://")) { + if (!userPath.startsWith("file://")) { userPath = "file://" + userPath; } localJarPath = new Path(userPath); } else { LOG.info("No path for the flink jar passed. Using the location of " + yarnClusterDescriptor.getClass() + " to locate the jar"); - localJarPath = new Path("file://" + - yarnClusterDescriptor.getClass().getProtectionDomain().getCodeSource().getLocation().getPath()); + String encodedJarPath = + yarnClusterDescriptor.getClass().getProtectionDomain().getCodeSource().getLocation().getPath(); + try { + // we have to decode the url encoded parts of the path + String decodedPath = URLDecoder.decode(encodedJarPath, Charset.defaultCharset().name()); + localJarPath = new Path(new File(decodedPath).toURI()); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("Couldn't decode the encoded Flink dist jar path: " + encodedJarPath + + " Please supply a path manually via the -" + FLINK_JAR.getOpt() + " option."); + } } yarnClusterDescriptor.setLocalJarPath(localJarPath);