flink-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From g...@apache.org
Subject [1/4] flink git commit: [FLINK-4946] [scripts] Load jar files from subdirectories of lib
Date Thu, 10 Nov 2016 17:14:10 GMT
Repository: flink
Updated Branches:
  refs/heads/master 5767167db -> 86a3dd586


[FLINK-4946] [scripts] Load jar files from subdirectories of lib

The Flink classpath is a concatenation of jar files in lib/. This commit
includes files from subdirectories of lib/ in the classpath.

This closes #2708


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/332f823b
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/332f823b
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/332f823b

Branch: refs/heads/master
Commit: 332f823bf2425b18cbd62639364c8c1daaac8ca7
Parents: 5767167
Author: Greg Hogan <code@greghogan.com>
Authored: Thu Oct 27 15:14:36 2016 -0400
Committer: Greg Hogan <code@greghogan.com>
Committed: Thu Nov 10 10:00:06 2016 -0500

----------------------------------------------------------------------
 flink-dist/src/main/flink-bin/bin/config.sh     |  8 ++---
 .../src/main/flink-bin/yarn-bin/yarn-session.sh | 19 ++++++-----
 .../yarn/AbstractYarnClusterDescriptor.java     | 34 +++++++++++++++++---
 3 files changed, 42 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/332f823b/flink-dist/src/main/flink-bin/bin/config.sh
----------------------------------------------------------------------
diff --git a/flink-dist/src/main/flink-bin/bin/config.sh b/flink-dist/src/main/flink-bin/bin/config.sh
index f7e7d58..0f24034 100755
--- a/flink-dist/src/main/flink-bin/bin/config.sh
+++ b/flink-dist/src/main/flink-bin/bin/config.sh
@@ -19,13 +19,13 @@
 
 constructFlinkClassPath() {
 
-    for jarfile in "$FLINK_LIB_DIR"/*.jar ; do
+    while read -d '' -r jarfile ; do
         if [[ $FLINK_CLASSPATH = "" ]]; then
-            FLINK_CLASSPATH=$jarfile;
+            FLINK_CLASSPATH="$jarfile";
         else
-            FLINK_CLASSPATH=$FLINK_CLASSPATH:$jarfile
+            FLINK_CLASSPATH="$FLINK_CLASSPATH":"$jarfile"
         fi
-    done
+    done < <(find "$FLINK_LIB_DIR" ! -type d -name '*.jar' -print0)
 
     echo $FLINK_CLASSPATH
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/332f823b/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh
----------------------------------------------------------------------
diff --git a/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh b/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh
index a0b531f..1755f32 100755
--- a/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh
+++ b/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh
@@ -17,8 +17,6 @@
 # limitations under the License.
 ################################################################################
 
-
-
 bin=`dirname "$0"`
 bin=`cd "$bin"; pwd`
 
@@ -35,14 +33,15 @@ JVM_ARGS="$JVM_ARGS -Xmx512m"
 # Flink CLI client
 constructCLIClientClassPath() {
 
-	for jarfile in $FLINK_LIB_DIR/*.jar ; do
-		if [[ $CC_CLASSPATH = "" ]]; then
-			CC_CLASSPATH=$jarfile;
-		else
-			CC_CLASSPATH=$CC_CLASSPATH:$jarfile
-		fi
-	done
-	echo $CC_CLASSPATH:$INTERNAL_HADOOP_CLASSPATHS
+    while read -d '' -r jarfile ; do
+        if [[ $CC_CLASSPATH = "" ]]; then
+            CC_CLASSPATH="$jarfile";
+        else
+            CC_CLASSPATH="$CC_CLASSPATH":"$jarfile"
+        fi
+    done < <(find "$FLINK_LIB_DIR" ! -type d -name '*.jar' -print0)
+
+    echo $CC_CLASSPATH:$INTERNAL_HADOOP_CLASSPATHS
 }
 
 CC_CLASSPATH=`manglePathList $(constructCLIClientClassPath)`

http://git-wip-us.apache.org/repos/asf/flink/blob/332f823b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
index 4219a9e..9d6dfdd 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
@@ -64,6 +64,10 @@ import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -80,7 +84,7 @@ import static org.apache.flink.yarn.cli.FlinkYarnSessionCli.CONFIG_FILE_LOGBACK_
 import static org.apache.flink.yarn.cli.FlinkYarnSessionCli.getDynamicProperties;
 
 /**
- * The descriptor with deployment information for spwaning or resuming a {@link YarnClusterClient}.
+ * The descriptor with deployment information for spawning or resuming a {@link YarnClusterClient}.
  */
 public abstract class AbstractYarnClusterDescriptor implements ClusterDescriptor<YarnClusterClient>
{
 	private static final Logger LOG = LoggerFactory.getLogger(YarnClusterDescriptor.class);
@@ -312,7 +316,7 @@ public abstract class AbstractYarnClusterDescriptor implements ClusterDescriptor
 		// check if required Hadoop environment variables are set. If not, warn user
 		if(System.getenv("HADOOP_CONF_DIR") == null &&
 			System.getenv("YARN_CONF_DIR") == null) {
-			LOG.warn("Neither the HADOOP_CONF_DIR nor the YARN_CONF_DIR environment variable is set."
+
+			LOG.warn("Neither the HADOOP_CONF_DIR nor the YARN_CONF_DIR environment variable is set.
" +
 				"The Flink YARN Client needs one of these to be set to properly load the Hadoop " +
 				"configuration for accessing YARN.");
 		}
@@ -638,12 +642,32 @@ public abstract class AbstractYarnClusterDescriptor implements ClusterDescriptor
 
 			localResources.put(shipFile.getName(), shipResources);
 
-			classPathBuilder.append(shipFile.getName());
 			if (shipFile.isDirectory()) {
 				// add directories to the classpath
-				classPathBuilder.append(File.separator).append("*");
+				java.nio.file.Path shipPath = shipFile.toPath();
+				final java.nio.file.Path parentPath = shipPath.getParent();
+
+				Files.walkFileTree(shipPath, new SimpleFileVisitor<java.nio.file.Path>() {
+					@Override
+					public FileVisitResult preVisitDirectory(java.nio.file.Path dir, BasicFileAttributes
attrs)
+							throws IOException {
+						super.preVisitDirectory(dir, attrs);
+
+						java.nio.file.Path relativePath = parentPath.relativize(dir);
+
+						classPathBuilder
+							.append(relativePath)
+							.append(File.separator)
+							.append("*")
+							.append(File.pathSeparator);
+
+						return FileVisitResult.CONTINUE;
+					}
+				});
+			} else {
+				// add files to the classpath
+				classPathBuilder.append(shipFile.getName()).append(File.pathSeparator);
 			}
-			classPathBuilder.append(File.pathSeparator);
 
 			envShipFileList.append(remotePath).append(",");
 		}


Mime
View raw message