flink-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ches...@apache.org
Subject flink git commit: [FLINK-7047] [travis] Reorganize build profiles
Date Wed, 12 Jul 2017 19:57:38 GMT
Repository: flink
Updated Branches:
  refs/heads/master 8987df556 -> 1da93a4c7


[FLINK-7047] [travis] Reorganize build profiles


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1da93a4c
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1da93a4c
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1da93a4c

Branch: refs/heads/master
Commit: 1da93a4c73d48b6635e44214383d27cd3d31fc14
Parents: 8987df5
Author: zentol <chesnay@apache.org>
Authored: Tue Jul 11 11:20:34 2017 +0200
Committer: zentol <chesnay@apache.org>
Committed: Wed Jul 12 20:19:35 2017 +0200

----------------------------------------------------------------------
 .travis.yml                  |  48 +++++++++-----
 pom.xml                      |  35 ----------
 tools/travis_mvn_watchdog.sh | 136 ++++++++++++++++++++++++++++++++++++--
 3 files changed, 161 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/1da93a4c/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 5bfb549..0e4624f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,34 +15,46 @@ language: java
 #See https://issues.apache.org/jira/browse/FLINK-1072
 matrix:
   include:
-  # Always run test groups A and B together
     - jdk: "oraclejdk8"
-      env: PROFILE="-Dhadoop.version=2.8.0 -Pflink-fast-tests-a,include-kinesis,jdk8"
+      env:
+        - TEST="core"
+        - PROFILE="-Dhadoop.version=2.8.0 -Pjdk8"
     - jdk: "oraclejdk8"
-      env: PROFILE="-Dhadoop.version=2.8.0 -Pflink-fast-tests-b,include-kinesis,jdk8"
+      env:
+        - TEST="libraries"
+        - PROFILE="-Dhadoop.version=2.8.0 -Pjdk8"
     - jdk: "oraclejdk8"
-      env: PROFILE="-Dhadoop.version=2.8.0 -Pflink-fast-tests-c,include-kinesis,jdk8"
-
-    - jdk: "oraclejdk8"
-      env: PROFILE="-Dhadoop.version=2.7.3 -Dscala-2.10 -Pflink-fast-tests-a,include-kinesis"
+      env:
+        - TEST="connectors"
+        - PROFILE="-Dhadoop.version=2.8.0 -Pjdk8,include-kinesis"
     - jdk: "oraclejdk8"
-      env: PROFILE="-Dhadoop.version=2.7.3 -Dscala-2.10 -Pflink-fast-tests-b,include-kinesis"
+      env:
+        - TEST="tests"
+        - PROFILE="-Dhadoop.version=2.8.0 -Pjdk8"
     - jdk: "oraclejdk8"
-      env: PROFILE="-Dhadoop.version=2.7.3 -Dscala-2.10 -Pflink-fast-tests-c,include-kinesis"
-
+      env:
+        - TEST="misc"
+        - PROFILE="-Dhadoop.version=2.8.0 -Pjdk8"
     - jdk: "openjdk7"
-      env: PROFILE="-Dhadoop.version=2.6.5 -Pflink-fast-tests-a,include-kinesis"
+      env:
+        - TEST="core"
+        - PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.10"
     - jdk: "openjdk7"
-      env: PROFILE="-Dhadoop.version=2.6.5 -Pflink-fast-tests-b,include-kinesis"
-    - jdk: "openjdk7"
-      env: PROFILE="-Dhadoop.version=2.6.5 -Pflink-fast-tests-c,include-kinesis"
-
+      env:
+        - TEST="libraries"
+        - PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.10"
     - jdk: "openjdk7"
-      env: PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.10 -Pflink-fast-tests-a,include-kinesis"
+      env:
+        - TEST="connectors"
+        - PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.10 -Pinclude-kinesis"
     - jdk: "openjdk7"
-      env: PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.10 -Pflink-fast-tests-b,include-kinesis"
+      env:
+        - TEST="tests"
+        - PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.10"
     - jdk: "openjdk7"
-      env: PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.10 -Pflink-fast-tests-c,include-kinesis"
+      env:
+        - TEST="misc"
+        - PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.10"
 
 git:
   depth: 100

http://git-wip-us.apache.org/repos/asf/flink/blob/1da93a4c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3de92c7..f136594 100644
--- a/pom.xml
+++ b/pom.xml
@@ -83,8 +83,6 @@ under the License.
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-		<!-- Internal property to reduce build times on TravisCi -->
-		<flink-fast-tests-pattern>never-match-me</flink-fast-tests-pattern>
 		<hadoop.version>2.4.1</hadoop.version>
 		<!-- Need to use a user property here because the surefire
 			 forkCount is not exposed as a property. With this we can set
@@ -864,32 +862,6 @@ under the License.
 				</plugins>
 			</build>
 		</profile>
-		<!--
-			Test profile A excludes all tests starting with (A-M).
-			profile B excludes tests (N-Z).
-			This allows us to split up the test execution into separate profiles.
-		-->
-		<profile>
-			<id>flink-fast-tests-a</id>
-			<properties>
-				<!-- Allow A-I by forbidding J-Z -->
-				<flink-fast-tests-pattern>%regex[.*/[J-Z].*]</flink-fast-tests-pattern>
-			</properties>
-		</profile>
-		<profile>
-			<id>flink-fast-tests-b</id>
-			<properties>
-				<!-- Allow J-R, forbid A-I and S - Z -->
-				<flink-fast-tests-pattern>%regex[.*/[A-IS-Z].*]</flink-fast-tests-pattern>
-			</properties>
-		</profile>
-		<profile>
-			<id>flink-fast-tests-c</id>
-			<properties>
-				<!-- Allow S-Z, forbid A-R -->
-				<flink-fast-tests-pattern>%regex[.*/[A-R].*]</flink-fast-tests-pattern>
-			</properties>
-		</profile>
 	</profiles>
 
 	<build>
@@ -1099,10 +1071,6 @@ under the License.
 							<includes>
 								<include>**/*Test.*</include>
 							</includes>
-							<excludes>
-								<exclude>**/*ITCase.*</exclude>
-								<exclude>${flink-fast-tests-pattern}</exclude>
-							</excludes>
 						</configuration>
 					</execution>
 					<!--execute all the integration tests-->
@@ -1116,9 +1084,6 @@ under the License.
 							<includes>
 								<include>**/*ITCase.*</include>
 							</includes>
-							<excludes>
-								<exclude>${flink-fast-tests-pattern}</exclude>
-							</excludes>
 							<reuseForks>false</reuseForks>
 						</configuration>
 					</execution>

http://git-wip-us.apache.org/repos/asf/flink/blob/1da93a4c/tools/travis_mvn_watchdog.sh
----------------------------------------------------------------------
diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh
index b22bfe0..606492f 100755
--- a/tools/travis_mvn_watchdog.sh
+++ b/tools/travis_mvn_watchdog.sh
@@ -43,11 +43,107 @@ SLEEP_TIME=20
 
 LOG4J_PROPERTIES=${HERE}/log4j-travis.properties
 
+MODULES_CORE="\
+flink-test-utils-parent/flink-test-utils,\
+flink-contrib/flink-statebackend-rocksdb,\
+flink-clients,\
+flink-core,\
+flink-java,\
+flink-optimizer,\
+flink-runtime,\
+flink-runtime-web,\
+flink-scala,\
+flink-scala-shell,\
+flink-streaming-java,\
+flink-streaming-scala"
+
+MODULES_LIBRARIES="\
+flink-contrib/flink-storm,\
+flink-contrib/flink-storm-examples,\
+flink-libraries/flink-cep,\
+flink-libraries/flink-cep-scala,\
+flink-libraries/flink-gelly,\
+flink-libraries/flink-gelly-scala,\
+flink-libraries/flink-gelly-examples,\
+flink-libraries/flink-ml,\
+flink-libraries/flink-python,\
+flink-libraries/flink-table"
+
+MODULES_CONNECTORS="\
+flink-contrib/flink-connector-wikiedits,\
+flink-connectors/flink-avro,\
+flink-connectors/flink-hbase,\
+flink-connectors/flink-hcatalog,\
+flink-connectors/flink-hadoop-compatibility,\
+flink-connectors/flink-jdbc,\
+flink-connectors/flink-connector-cassandra,\
+flink-connectors/flink-connector-elasticsearch,\
+flink-connectors/flink-connector-elasticsearch2,\
+flink-connectors/flink-connector-elasticsearch-base,\
+flink-connectors/flink-connector-filesystem,\
+flink-connectors/flink-connector-kafka-0.8,\
+flink-connectors/flink-connector-kafka-0.9,\
+flink-connectors/flink-connector-kafka-0.10,\
+flink-connectors/flink-connector-kafka-base,\
+flink-connectors/flink-connector-nifi,\
+flink-connectors/flink-connector-rabbitmq,\
+flink-connectors/flink-connector-twitter"
+
+MODULES_TESTS="\
+flink-tests"
+
+if [[ $PROFILE == *"jdk8"* ]]; then
+	case $TEST in
+		(connectors)
+			MODULES_CONNECTORS="$MODULES_CONNECTORS,flink-connectors/flink-connector-elasticsearch5"
+		;;
+	esac
+fi
+
+if [[ $PROFILE == *"include-kinesis"* ]]; then
+	case $TEST in
+		(connectors)
+			MODULES_CONNECTORS="$MODULES_CONNECTORS,flink-connectors/flink-connector-kinesis"
+		;;
+	esac
+fi
+
+MVN_COMPILE_MODULES=""
+MVN_TEST_MODULES=""
+case $TEST in
+	(core)
+		MVN_COMPILE_MODULES="-pl $MODULES_CORE -am"
+		MVN_TEST_MODULES="-pl $MODULES_CORE"
+	;;
+	(libraries)
+		MVN_COMPILE_MODULES="-pl $MODULES_LIBRARIES -am"
+		MVN_TEST_MODULES="-pl $MODULES_LIBRARIES"
+	;;
+	(connectors)
+		MVN_COMPILE_MODULES="-pl $MODULES_CONNECTORS -am"
+		MVN_TEST_MODULES="-pl $MODULES_CONNECTORS"
+	;;
+	(tests)
+		MVN_COMPILE_MODULES="-pl $MODULES_TESTS -am"
+		MVN_TEST_MODULES="-pl $MODULES_TESTS"
+	;;
+	(misc)
+		NEGATED_CORE=\!${MODULES_CORE//,/,\!}
+		NEGATED_LIBRARIES=\!${MODULES_LIBRARIES//,/,\!}
+		NEGATED_CONNECTORS=\!${MODULES_CONNECTORS//,/,\!}
+		NEGATED_TESTS=\!${MODULES_TESTS//,/,\!}
+		# compile everything since dist needs it anyway
+		MVN_COMPILE_MODULES=""
+		MVN_TEST_MODULES="-pl $NEGATED_CORE,$NEGATED_LIBRARIES,$NEGATED_CONNECTORS,$NEGATED_TESTS"
+	;;
+esac
+
 # Maven command to run. We set the forkCount manually, because otherwise Maven sees too many
cores
 # on the Travis VMs. Set forkCountTestPackage to 1 for container-based environment (4 GiB
memory)
 # and 2 for sudo-enabled environment (7.5 GiB memory).
 MVN_LOGGING_OPTIONS="-Dlog.dir=${ARTIFACTS_DIR} -Dlog4j.configuration=file://$LOG4J_PROPERTIES
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
-MVN="mvn -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dmaven.javadoc.skip=true -B
$PROFILE $MVN_LOGGING_OPTIONS clean install"
+MVN_COMPILE="mvn -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -DskipTests -Dmaven.javadoc.skip=true
-B $PROFILE $MVN_LOGGING_OPTIONS $MVN_COMPILE_MODULES clean install"
+MVN_TEST="mvn -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dmaven.javadoc.skip=true
-B $PROFILE $MVN_LOGGING_OPTIONS $MVN_TEST_MODULES verify"
 
 MVN_PID="${ARTIFACTS_DIR}/watchdog.mvn.pid"
 MVN_EXIT="${ARTIFACTS_DIR}/watchdog.mvn.exit"
@@ -226,12 +322,35 @@ echo -en "travis_fold:end:disk_info${FOLD_ESCAPE}"
 # Make sure to be in project root
 cd $HERE/../
 
-echo "RUNNING '${MVN}'."
+# Compile modules
+
+echo "RUNNING '${MVN_COMPILE}'."
+
+# Run $MVN_COMPILE and pipe output to $MVN_OUT for the watchdog. The PID is written to $MVN_PID
to
+# allow the watchdog to kill $MVN if it is not producing any output anymore. $MVN_EXIT contains
+# the exit code. This is important for Travis' build life-cycle (success/failure).
+( $MVN_COMPILE & PID=$! ; echo $PID >&3 ; wait $PID ; echo $? >&4 ) 3>$MVN_PID
4>$MVN_EXIT | tee $MVN_OUT
+
+echo "Trying to KILL watchdog (${WD_PID})."
+
+# Make sure to kill the watchdog in any case after $MVN has completed
+( kill $WD_PID 2>&1 ) > /dev/null
+
+EXIT_CODE=$(<$MVN_EXIT)
+
+echo "MVN exited with EXIT CODE: ${EXIT_CODE}."
 
-# Run $MVN and pipe output to $MVN_OUT for the watchdog. The PID is written to $MVN_PID to
+rm $MVN_PID
+rm $MVN_EXIT
+
+# Run tests
+
+echo "RUNNING '${MVN_TEST}'."
+
+# Run $MVN_TEST and pipe output to $MVN_OUT for the watchdog. The PID is written to $MVN_PID
to
 # allow the watchdog to kill $MVN if it is not producing any output anymore. $MVN_EXIT contains
 # the exit code. This is important for Travis' build life-cycle (success/failure).
-( $MVN & PID=$! ; echo $PID >&3 ; wait $PID ; echo $? >&4 ) 3>$MVN_PID
4>$MVN_EXIT | tee $MVN_OUT
+( $MVN_TEST & PID=$! ; echo $PID >&3 ; wait $PID ; echo $? >&4 ) 3>$MVN_PID
4>$MVN_EXIT | tee $MVN_OUT
 
 echo "Trying to KILL watchdog (${WD_PID})."
 
@@ -245,7 +364,14 @@ echo "MVN exited with EXIT CODE: ${EXIT_CODE}."
 rm $MVN_PID
 rm $MVN_EXIT
 
-check_shaded_artifacts
+# Post
+
+# only misc builds flink-dist
+case $TEST in
+	(misc)
+		check_shaded_artifacts
+	;;
+esac
 
 put_yarn_logs_to_artifacts
 


Mime
View raw message