flink-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rmetz...@apache.org
Subject [2/2] flink git commit: [FLINK-2200] [build system] Deploy Flink 2.11 with every build (nightly)
Date Thu, 24 Sep 2015 14:13:00 GMT
[FLINK-2200] [build system] Deploy Flink 2.11 with every build (nightly)

This closes #885


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/4cadc3d6
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/4cadc3d6
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/4cadc3d6

Branch: refs/heads/master
Commit: 4cadc3d60c692edc40188fffd296cf9cad34079f
Parents: 7f07cf8
Author: Robert Metzger <rmetzger@apache.org>
Authored: Wed Sep 23 15:02:24 2015 +0200
Committer: Robert Metzger <rmetzger@apache.org>
Committed: Thu Sep 24 16:11:47 2015 +0200

----------------------------------------------------------------------
 flink-shaded-hadoop/pom.xml                     |  2 +-
 flink-staging/flink-gelly-scala/pom.xml         |  2 +-
 .../flink-language-binding/flink-python/pom.xml |  2 +-
 pom.xml                                         |  5 +
 tools/change-scala-version.sh                   | 24 +++++
 tools/deploy_to_maven.sh                        | 96 ++++++++++++--------
 6 files changed, 89 insertions(+), 42 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/4cadc3d6/flink-shaded-hadoop/pom.xml
----------------------------------------------------------------------
diff --git a/flink-shaded-hadoop/pom.xml b/flink-shaded-hadoop/pom.xml
index 6e75b25..a99f3c4 100644
--- a/flink-shaded-hadoop/pom.xml
+++ b/flink-shaded-hadoop/pom.xml
@@ -35,7 +35,7 @@ under the License.
 	<packaging>pom</packaging>
 
 	<modules>
-		<module>${shading-artifact.name}</module>
+		<module>${shading-artifact-module.name}</module>
 	</modules>
 
 	<profiles>

http://git-wip-us.apache.org/repos/asf/flink/blob/4cadc3d6/flink-staging/flink-gelly-scala/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-gelly-scala/pom.xml b/flink-staging/flink-gelly-scala/pom.xml
index 18c0516..a1f0da7 100644
--- a/flink-staging/flink-gelly-scala/pom.xml
+++ b/flink-staging/flink-gelly-scala/pom.xml
@@ -21,8 +21,8 @@ under the License.
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <artifactId>flink-staging</artifactId>
         <groupId>org.apache.flink</groupId>
+        <artifactId>flink-staging</artifactId>
         <version>0.10-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>

http://git-wip-us.apache.org/repos/asf/flink/blob/4cadc3d6/flink-staging/flink-language-binding/flink-python/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-language-binding/flink-python/pom.xml b/flink-staging/flink-language-binding/flink-python/pom.xml
index 2a11e35..8730916 100644
--- a/flink-staging/flink-language-binding/flink-python/pom.xml
+++ b/flink-staging/flink-language-binding/flink-python/pom.xml
@@ -21,8 +21,8 @@ under the License.
     <modelVersion>4.0.0</modelVersion>
 
     <parent>
-        <artifactId>flink-language-binding-parent</artifactId>
         <groupId>org.apache.flink</groupId>
+        <artifactId>flink-language-binding-parent</artifactId>
         <version>0.10-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>

http://git-wip-us.apache.org/repos/asf/flink/blob/4cadc3d6/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6603e81..6f1a137 100644
--- a/pom.xml
+++ b/pom.xml
@@ -71,7 +71,10 @@ under the License.
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+		<!-- Mutable name of the hadoop shading artifact. The module name can contain a scala
version suffix. -->
 		<shading-artifact.name>error</shading-artifact.name>
+		<!-- The shading artifact module name can be used as a module name. it does not get
a scala version suffix -->
+		<shading-artifact-module.name>error</shading-artifact-module.name>
 		<hadoop-one.version>1.2.1</hadoop-one.version>
 		<hadoop-two.version>2.3.0</hadoop-two.version>
 		<!-- Need to use a user property here because the surefire/failsafe
@@ -377,6 +380,7 @@ under the License.
 			<properties>
 				<hadoop.version>${hadoop-one.version}</hadoop.version>
 				<shading-artifact.name>flink-shaded-hadoop1</shading-artifact.name>
+				<shading-artifact-module.name>flink-shaded-hadoop1</shading-artifact-module.name>
 			</properties>
 		</profile>
 
@@ -392,6 +396,7 @@ under the License.
 			<properties>
 				<hadoop.version>${hadoop-two.version}</hadoop.version>
 				<shading-artifact.name>flink-shaded-hadoop2</shading-artifact.name>
+				<shading-artifact-module.name>flink-shaded-hadoop2</shading-artifact-module.name>
 			</properties>
 			<modules>
 				<module>flink-yarn</module>

http://git-wip-us.apache.org/repos/asf/flink/blob/4cadc3d6/tools/change-scala-version.sh
----------------------------------------------------------------------
diff --git a/tools/change-scala-version.sh b/tools/change-scala-version.sh
index ff38711..2c124a9 100755
--- a/tools/change-scala-version.sh
+++ b/tools/change-scala-version.sh
@@ -79,3 +79,27 @@ find "$BASEDIR/flink-dist" -name 'bin.xml' -not -path '*target*' -print
\
   -exec bash -c "sed_i 's/\(<source>.*flink-dist\)'$FROM_SUFFIX'/\1'$TO_SUFFIX'/g'
{}" \;
 find "$BASEDIR/flink-dist" -name 'bin.xml' -not -path '*target*' -print \
   -exec bash -c "sed_i 's/\(<include>org\.apache\.flink:flink-.*\)'$FROM_SUFFIX'<\/include>/\1'$TO_SUFFIX'<\/include>/g'
{}" \;
+
+
+if [ "$TO_VERSION" == "2.11" ]; then
+  # set the profile activation to !scala-2.11 in parent pom, so that it activates by default
+  bash -c "sed_i 's/<name>scala-2.11<\/name>/<name>!scala-2.11<\/name>/g'
$BASEDIR/pom.xml" \;
+  # set the profile activation in all sub modules to scala-2.11 (so that they are disabled
by default)
+  find $BASEDIR/flink-* -name 'pom.xml' -not -path '*target*' -print \
+    -exec bash -c "sed_i 's/<name>!scala-2.11<\/name>/<name>scala-2.11<\/name>/g'
{}" \;
+
+  # set the name of the shading artifact properly
+  bash -c "sed_i 's/\(shading-artifact.name>flink-shaded[a-z0-9\-]*\)'$FROM_SUFFIX'<\/shading-artifact.name>/\1'$TO_SUFFIX'<\/shading-artifact.name>/g'
$BASEDIR/pom.xml" \;
+fi
+
+if [ "$TO_VERSION" == "2.10" ]; then
+  # do the opposite as above
+  bash -c "sed_i 's/<name>!scala-2.11<\/name>/<name>scala-2.11<\/name>/g'
$BASEDIR/pom.xml" \;
+  # also for the other files
+  find $BASEDIR/flink-* -name 'pom.xml' -not -path '*target*' -print \
+    -exec bash -c "sed_i 's/<name>scala-2.11<\/name>/<name>!scala-2.11<\/name>/g'
{}" \;
+
+  # unset shading artifact name
+  bash -c "sed_i 's/\(shading-artifact.name>flink-shaded[a-z0-9\-]*\)'$FROM_SUFFIX'<\/shading-artifact.name>/\1'$TO_SUFFIX'<\/shading-artifact.name>/g'
$BASEDIR/pom.xml" \;
+fi
+

http://git-wip-us.apache.org/repos/asf/flink/blob/4cadc3d6/tools/deploy_to_maven.sh
----------------------------------------------------------------------
diff --git a/tools/deploy_to_maven.sh b/tools/deploy_to_maven.sh
index e51e0aa..adc1f47 100755
--- a/tools/deploy_to_maven.sh
+++ b/tools/deploy_to_maven.sh
@@ -28,12 +28,6 @@
 # 5. Nothing
 
 
-echo "install lifecylce mapping fake plugin"
-git clone https://github.com/mfriedenhagen/dummy-lifecycle-mapping-plugin.git
-cd dummy-lifecycle-mapping-plugin
-mvn -B install
-cd ..
-
 function getVersion() {
 	here="`dirname \"$0\"`"              # relative
 	here="`( cd \"$here\" && pwd )`"  # absolutized and normalized
@@ -47,57 +41,81 @@ function getVersion() {
 	echo `mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version
| grep -E '^([0-9]+.[0-9]+(.[0-9]+)?(-[a-zA-Z0-9]+)?)$'`
 }
 
-pwd
-
-# this will take a while
-CURRENT_FLINK_VERSION=`getVersion`
-if [[ "$CURRENT_FLINK_VERSION" == *-SNAPSHOT ]]; then
-	CURRENT_FLINK_VERSION_HADOOP1=${CURRENT_FLINK_VERSION/-SNAPSHOT/-hadoop1-SNAPSHOT}
-else
-	CURRENT_FLINK_VERSION_HADOOP1="$CURRENT_FLINK_VERSION-hadoop1"
-fi
+function deploy_to_s3() {
+	CURRENT_FLINK_VERSION=$1
+	HD=$2
+
+	echo "Deplyoing flink version $CURRENT_FLINK_VERSION (hadoop=$HD) to s3:"
+	mkdir flink-$CURRENT_FLINK_VERSION
+	cp -r flink-dist/target/flink-*-bin/flink-$CURRENT_FLINK_VERSION*/* flink-$CURRENT_FLINK_VERSION/
+	tar -czf flink-$CURRENT_FLINK_VERSION-bin-$HD.tgz flink-$CURRENT_FLINK_VERSION
+	travis-artifacts upload --path flink-$CURRENT_FLINK_VERSION-bin-$HD.tgz   --target-path
/ 
+	# delete files again
+	rm -rf flink-$CURRENT_FLINK_VERSION
+	rm flink-$CURRENT_FLINK_VERSION-bin-$HD.tgz
+}
 
-echo "detected current version as: '$CURRENT_FLINK_VERSION' ; hadoop1: $CURRENT_FLINK_VERSION_HADOOP1
"
+pwd
 
 
 # Check if push/commit is eligible for pushing
-echo "Job: $TRAVIS_JOB_NUMBER ; isPR: $TRAVIS_PULL_REQUEST"
-if [[ $TRAVIS_PULL_REQUEST == "false" ]] ; then
+echo "Job: $TRAVIS_JOB_NUMBER ; isPR: $TRAVIS_PULL_REQUEST ; repo slug : $TRAVIS_REPO_SLUG
"
+if [[ $TRAVIS_PULL_REQUEST == "false" ]] && [[ $TRAVIS_REPO_SLUG == "apache/flink"
]] ; then
+
+	echo "install lifecylce mapping fake plugin"
+	git clone https://github.com/mfriedenhagen/dummy-lifecycle-mapping-plugin.git
+	cd dummy-lifecycle-mapping-plugin
+	mvn -B install
+	cd ..
+
+	# this will take a while
+	CURRENT_FLINK_VERSION=`getVersion`
+	if [[ "$CURRENT_FLINK_VERSION" == *-SNAPSHOT ]]; then
+		CURRENT_FLINK_VERSION_HADOOP1=${CURRENT_FLINK_VERSION/-SNAPSHOT/-hadoop1-SNAPSHOT}
+	else
+		CURRENT_FLINK_VERSION_HADOOP1="$CURRENT_FLINK_VERSION-hadoop1"
+	fi
+
+	echo "detected current version as: '$CURRENT_FLINK_VERSION' ; hadoop1: $CURRENT_FLINK_VERSION_HADOOP1
"
 
 	#
 	# This script is called by travis to deploy our project to sonatype SNAPSHOTS.
 	# It will deploy both a hadoop v1 and a hadoop v2 (yarn) artifact
 	# 
 
-	if [[ $TRAVIS_JOB_NUMBER == *1 ]] && [[ $TRAVIS_PULL_REQUEST == "false" ]] &&
[[ $CURRENT_FLINK_VERSION == *SNAPSHOT* ]] ; then 
+	if [[ $TRAVIS_JOB_NUMBER == *1 ]] &&  [[ $CURRENT_FLINK_VERSION == *SNAPSHOT* ]]
; then 
 		# Deploy hadoop v1 to maven
 		echo "Generating poms for hadoop1"
 		./tools/generate_specific_pom.sh $CURRENT_FLINK_VERSION $CURRENT_FLINK_VERSION_HADOOP1
pom.hadoop1.xml
 		mvn -B -f pom.hadoop1.xml -Pdocs-and-source -DskipTests -Drat.ignoreErrors=true deploy
--settings deploysettings.xml; 
+
+		# deploy to s3
+		deploy_to_s3 $CURRENT_FLINK_VERSION "hadoop1"
 	fi
 
-	if [[ $TRAVIS_JOB_NUMBER == *2 ]] && [[ $TRAVIS_PULL_REQUEST == "false" ]] &&
[[ $CURRENT_FLINK_VERSION == *SNAPSHOT* ]] ; then 
+	if [[ $TRAVIS_JOB_NUMBER == *2 ]] && [[ $CURRENT_FLINK_VERSION == *SNAPSHOT* ]]
; then 
+		# the time to build and upload flink twice (scala 2.10 and scala 2.11) takes
+		# too much time. That's why we are going to do it in parallel
+		# Note that the parallel execution will cause the output to be interleaved
+		mkdir ../flink2
+		ls ../
+		cp -r . ../flink2
+		cd ../flink2
 		# deploy hadoop v2 (yarn)
-		echo "deploy standard version (hadoop2)"
-		mvn -B -DskipTests -Pdocs-and-source -Drat.ignoreErrors=true clean deploy --settings deploysettings.xml;
-	fi
+		echo "deploy standard version (hadoop2) for scala 2.10 from flink2 directory"
+		# do the hadoop2 scala 2.10 in the background
+		(mvn -B -DskipTests -Pdocs-and-source -Drat.skip=true -Drat.ignoreErrors=true clean deploy
--settings deploysettings.xml; deploy_to_s3 $CURRENT_FLINK_VERSION "hadoop2" ) &
+
+		# switch back to the regular flink directory
+		cd ../flink
+		echo "deploy hadoop2 version (standard) for scala 2.11 from flink directory"
+		./tools/change-scala-version.sh 2.11
+		mvn -B -DskipTests -Pdocs-and-source -Drat.skip=true -Drat.ignoreErrors=true clean deploy
--settings deploysettings.xml;
+
+		deploy_to_s3 $CURRENT_FLINK_VERSION "hadoop2_2.11"
 
-	if [[ $TRAVIS_JOB_NUMBER == *1 ]] || [[ $TRAVIS_JOB_NUMBER == *2 ]] ; then
-		echo "Uploading build to amazon s3. Job Number: $TRAVIS_JOB_NUMBER"
-		
-		HD="hadoop1"
-		
-		# job nr 2 is hadoop 2
-		if [[ $TRAVIS_JOB_NUMBER == *2 ]] ; then
-			HD="hadoop2"
-		fi
-
-		mkdir flink-$CURRENT_FLINK_VERSION
-		cp -r flink-dist/target/flink-*-bin/flink-$CURRENT_FLINK_VERSION*/* flink-$CURRENT_FLINK_VERSION/
-		tar -czf flink-$CURRENT_FLINK_VERSION-bin-$HD.tgz flink-$CURRENT_FLINK_VERSION
-		travis-artifacts upload --path flink-$CURRENT_FLINK_VERSION-bin-$HD.tgz   --target-path
/ 
-		echo "doing a ls -lisah:"
-		ls -lisah
+		echo "Changing back to scala 2.10"
+		./tools/change-scala-version.sh 2.10
 	fi
 fi # pull request check
 


Mime
View raw message