flink-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rmetz...@apache.org
Subject flink git commit: [FLINK-1637] Reduce number of files in uberjar for java 6
Date Mon, 09 Mar 2015 08:47:28 GMT
Repository: flink
Updated Branches:
  refs/heads/master 7582390c1 -> 5242e9668


[FLINK-1637] Reduce number of files in uberjar for java 6

This closes #450


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/5242e966
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/5242e966
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/5242e966

Branch: refs/heads/master
Commit: 5242e9668a54cb6e74ee4f33718d35cf856b0703
Parents: 7582390
Author: Robert Metzger <rmetzger@apache.org>
Authored: Tue Mar 3 12:44:21 2015 +0100
Committer: Robert Metzger <rmetzger@apache.org>
Committed: Mon Mar 9 09:46:51 2015 +0100

----------------------------------------------------------------------
 .travis.yml                                     |  8 +++----
 flink-dist/pom.xml                              | 23 --------------------
 .../examples/java/wordcount/WordCount.java      |  3 ++-
 tools/create_release_files.sh                   |  5 +++++
 tools/deploy_to_maven.sh                        |  9 ++++----
 tools/travis_mvn_watchdog.sh                    | 19 ++++++++++++++++
 6 files changed, 35 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/5242e966/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 48d4913..3c4e10d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,13 +14,13 @@ language: java
 #See https://issues.apache.org/jira/browse/FLINK-1072
 matrix:
   include:
-    - jdk: "openjdk6"
-      env: PROFILE="-Dhadoop.profile=1"
     - jdk: "oraclejdk7"
       env: PROFILE="-Dhadoop.profile=1"
-    - jdk: "openjdk6"
+    - jdk: "openjdk6" # this will also deploy a uberjar to s3 at some point
+      env: PROFILE="-Dhadoop.profile=1"
+    - jdk: "openjdk7"
       env: PROFILE="-P!include-yarn -Dhadoop.version=2.0.0-alpha"
-    - jdk: "oraclejdk7"
+    - jdk: "openjdk6" # we must use openjdk6 here to deploy a java6 compatible uber-jar for
YARN
       env: PROFILE="-Dhadoop.version=2.2.0"
     - jdk: "oraclejdk8"
       env: PROFILE="-Dhadoop.version=2.6.0"

http://git-wip-us.apache.org/repos/asf/flink/blob/5242e966/flink-dist/pom.xml
----------------------------------------------------------------------
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index 1ac8bea..af3b100 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -115,12 +115,6 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-streaming-connectors</artifactId>
-			<version>${project.version}</version>
-		</dependency>
-
-		<dependency>
-			<groupId>org.apache.flink</groupId>
 			<artifactId>flink-streaming-examples</artifactId>
 			<version>${project.version}</version>
 		</dependency>
@@ -236,23 +230,6 @@ under the License.
 		</profile>
 
 		<profile>
-			<id>hadoop-2</id>
-			<activation>
-				<property>
-					<!-- Please do not remove the 'hadoop2' comment. See ./tools/generate_specific_pom.sh
-->
-					<!--hadoop2--><name>!hadoop.profile</name>
-				</property>
-			</activation>
-			<dependencies>
-				<dependency>
-					<groupId>org.apache.flink</groupId>
-					<artifactId>flink-hbase</artifactId>
-					<version>${project.version}</version>
-				</dependency>
-			</dependencies>
-		</profile>
-
-		<profile>
 			<id>debian-package</id>
 			<build>
 				<plugins>

http://git-wip-us.apache.org/repos/asf/flink/blob/5242e966/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/wordcount/WordCount.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/wordcount/WordCount.java
b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/wordcount/WordCount.java
index 6be5368..07aa2ce 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/wordcount/WordCount.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/wordcount/WordCount.java
@@ -53,7 +53,8 @@ public class WordCount {
 	// *************************************************************************
 	
 	public static void main(String[] args) throws Exception {
-		
+
+
 		if(!parseParameters(args)) {
 			return;
 		}

http://git-wip-us.apache.org/repos/asf/flink/blob/5242e966/tools/create_release_files.sh
----------------------------------------------------------------------
diff --git a/tools/create_release_files.sh b/tools/create_release_files.sh
index 09b4e7a..c9b1aaa 100755
--- a/tools/create_release_files.sh
+++ b/tools/create_release_files.sh
@@ -28,6 +28,11 @@
 #
 #  https://github.com/apache/spark/blob/master/dev/create-release/create-release.sh
 #
+
+# Added by rmetzger
+echo "Please make sure to use java 6 OPENJDK (not oracle) to build the release binaries"
+exit 1
+
 CURR_DIR=`pwd`
 if [[ `basename $CURR_DIR` != "tools" ]] ; then
   echo "You have to call the script from the tools/ dir"

http://git-wip-us.apache.org/repos/asf/flink/blob/5242e966/tools/deploy_to_maven.sh
----------------------------------------------------------------------
diff --git a/tools/deploy_to_maven.sh b/tools/deploy_to_maven.sh
index f4290af..1781927 100755
--- a/tools/deploy_to_maven.sh
+++ b/tools/deploy_to_maven.sh
@@ -27,9 +27,7 @@
 # 4. deploy snapshot and s3 (hadoop2 - 2.2.0)
 # 5. Nothing (hadoop2 - 2.5.1)
 
-# Changes (since travis changed the id assignment)
-# switched 2. with 3.
-# switched 5. with 6.
+
 
 echo "install lifecylce mapping fake plugin"
 git clone https://github.com/mfriedenhagen/dummy-lifecycle-mapping-plugin.git
@@ -47,9 +45,11 @@ function getVersion() {
 	fi
 	flink_home="`dirname \"$here\"`"
 	cd $flink_home
-	echo `mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version|grep
-Ev '(^\[|Download\w+:)'`
+	echo `mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version
| grep -E '^([0-9]+.[0-9]+(.[0-9]+)?(-[a-zA-Z0-9]+)?)$'`
 }
 
+pwd
+
 # this will take a while
 CURRENT_FLINK_VERSION=`getVersion`
 if [[ "$CURRENT_FLINK_VERSION" == *-SNAPSHOT ]]; then
@@ -60,6 +60,7 @@ fi
 
 echo "detected current version as: '$CURRENT_FLINK_VERSION' ; hadoop1: $CURRENT_FLINK_VERSION_HADOOP1
"
 
+
 # Check if push/commit is eligible for pushing
 echo "Job: $TRAVIS_JOB_NUMBER ; isPR: $TRAVIS_PULL_REQUEST"
 if [[ $TRAVIS_PULL_REQUEST == "false" ]] ; then

http://git-wip-us.apache.org/repos/asf/flink/blob/5242e966/tools/travis_mvn_watchdog.sh
----------------------------------------------------------------------
diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh
index e770a97..d94f977 100755
--- a/tools/travis_mvn_watchdog.sh
+++ b/tools/travis_mvn_watchdog.sh
@@ -179,5 +179,24 @@ rm $MVN_EXIT
 
 upload_artifacts_s3
 
+# Check the number of files in the uber jar and fail the build if there are too many files
(see: FLINK-1637)
+
+# since we are in flink/tools/artifacts
+# we are going back to
+cd ../../
+
+
+UBERJAR=`find . | grep uberjar | head -n 1`
+if [ -z "$UBERJAR" ] ; then
+	echo "Uberjar not found. Assuming failed build";
+else 
+	jar tf $UBERJAR | wc -l > num_files_in_uberjar
+	NUM_FILES_IN_UBERJAR=`cat num_files_in_uberjar`
+	echo "Files in uberjar: $NUM_FILES_IN_UBERJAR. Uberjar: $UBERJAR"
+	if [ "$NUM_FILES_IN_UBERJAR" -ge "65536" ] ; then
+		echo "WARN: The number of files in the uberjar ($NUM_FILES_IN_UBERJAR) exceeds the maximum
number of possible files for Java 6 (65536)"
+	fi
+fi
+
 # Exit code for Travis build success/failure
 exit $EXIT_CODE


Mime
View raw message