flink-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From u..@apache.org
Subject [3/3] flink git commit: [builds] Allow to upload build artifacts to S3
Date Thu, 19 Feb 2015 15:22:20 GMT
[builds] Allow to upload build artifacts to S3

This closes #407.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/6c01598a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/6c01598a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/6c01598a

Branch: refs/heads/master
Commit: 6c01598a34974b6c89df4bbb95b72d42879027f0
Parents: 6b082b5
Author: Ufuk Celebi <uce@apache.org>
Authored: Mon Feb 16 16:42:13 2015 +0100
Committer: Ufuk Celebi <uce@apache.org>
Committed: Thu Feb 19 16:21:53 2015 +0100

----------------------------------------------------------------------
 .travis.yml                                     | 24 ++++++--
 .../src/test/resources/log4j-test.properties    |  2 +-
 pom.xml                                         |  7 ++-
 tools/.gitignore                                |  1 +
 tools/log4j-travis.properties                   | 35 +++++++++++
 tools/travis_mvn_watchdog.sh                    | 61 +++++++++++++++++---
 6 files changed, 113 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/6c01598a/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 3c5dc1b..48d4913 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -47,19 +47,31 @@ env:
         - secure: "SNZkMm++fvPbjdreibc4j/XTKy7rOvGvjbvJJLQ01fVDR8ur1FGB5L/CE9tm2Aye75G8br+tJ+gf5cMX8CHL+0VrvuTk5U6flbuM08Pd0pmP64ZncmGfRFKC5qTvt24YR0u3cqxWze8RTkdduz0t8xrEdyCtb94CHs1+RNS+0HA="
         # javadocs deploy
         - secure: "f4wACodae0NCaIZtmg/JvP40G/3dE7O6+ONDMbq8P/9dnT5peRB1q7+KZxggRmFSWTMPdPIDI7YxI/rM5wZ3pUN2eX0BpjfANT58OJdgCNXN3Hr9YOa8UykD2h6b+AHpy4G89MG65m73RvNXngHTCQ8SBIfewzeMAhHdD3Fk0u8="
+        # Build artifacts like logs (variables for apache/flink repo and various commiters)
+        - secure: jJQtUL8w2bDqTF5LExDPaWmA6kizCbXhzrutiAGP9ecs79DfklDrD4yr9hSImD3i/vwupRMgI6EW0FKnBBaQU+98U2c71OX6Yk1D8r6EIhYrMOi7FVM7Nt8OIqpoYTQGg+42ol2CTHkCY7qi9ZwCpB5+VeHmsco6XrpJAF2eD4Q=
+        - secure: Mjhhzxus0WS/WDopXBmxkKY9TLGQ8lO1DQvU8fMJjExHz6MHnZW2OxPQikGCteFlp2aMgz5Y4xgzF5HHiJQ+88r5st7KSlmCNrSVSxHc1n7dQAh100viPnFf0ozoYVPsdJDvnYPHYgtUS38XqY9ba//9OZTPi/Hm3/0GKgV4g1w=
+        - secure: dQ4ttzfqjkKBNJDX07rxx0V+BDB0xBVB1R2oNwIcR3/BUEEEFRuoTdVUcEWNJ1GgXxcD4zBbNvCpgePUBFgNokKj7eWf2DYqmQTlwqeBQ9eCp4gybMRuhz4QbuUZ8Zbv+H0bPlfQI0Ne35eXrKylDc/OZspba6NxMmVzPx75cgw=
+        - secure: dfpLCkfmlGXun3riIWoNgoAfyMa8OWKBGj1HlKmyuB8Ub1q42tChK9oQiakERO+b9ucGUcscX5vwmFF2qwE2bQkxupvGND6gypZvwB6yCQ2LE8KaXRZp/CNbgl8kRIdsSc1nw8/D1bdBba5hdZZFWP84kPSdzOv+7tPdUcuYom0=
+        - secure: l7fK14ckh2yjeCYNjKkpezLWHF5jel6YFNWe/d25b8XgVIh1lbbOgWwxrZfLenQRHS1Web+mngGFWXc/e05rA2juUwd2P2YvTICJXQT5fBUrd0jqifXan6J8ntEBeMFdD0HOOwKkEJhyWqlbJXmq9DntdyaXpN7ExZEzlXb8Hu8=
+        - secure: AN8NAUhdrFvwk9j3TxFoTEHCMC7tmqXwVKF6SPJwzmSdVLiSTScQTrN7LVRthgTlr3AD23FRnLHQYiWr2xUXIC7ngaQ7o5UKQamHtIsPjJvceTFVRHcIfZs6qQCgOs47KoBDsnDP+nn4ZbgDNt+N4sMtZo/d/P2Oq3T36L0SkIw=
+        - secure: qXapBSPf9g2ids1ay890EbLvZ80CCl5KxBQK1ZnPV9PW/yyT9Hch+C8XH6z00bhvaysqHNQZ2aGbrMIu632X+YCrs6Rea53BJXnY0yqzWsXlxugx6VLI2UM/G3+PZz4djSdmYpA+5CZMkj/kPRh6f5LjQo8l2oBJq7UEjWoawoc=
+        - secure: ATJu0ZsaGNoV5EsMllmknOziwIvCi5t4v5No+F2qFD3tuKfmsGNyqOLj08EUtFmJ1d9Fn4wNXkDC7UfnLdjGUytU/fNZN3L0ffljJqrIcFX4qaxfMFpEElfbD6xe9w+D1ABqvKg2BuNw8rjXvf5imEXGnzZrBjzfarzUW3DZYx4=
+        - secure: v3adJMRxX2zYh9PYViwdsBocu2dRzCLoM4kAa4OIxwKS/LvoOQmhYytcX4VN4pzjXzRn2IebZ8go/nrzlxOVPL740RhZKHczXDMiBpiieh9l84SoTBmsmbj6u5rg3ZODmQzY3vLJ4SyU4qTzyTYZi1CkzBqm4IXyRMkixhE1/m8=
+        - secure: FmTf7OnCqD4krVZSLYMBz9yOSMUIsssGF7YGOvjTADPBywKrk4uDYT3YGRbEh11owamSFD5Xup2/dOPZP/J2DtbNWdEi5oRRjieVtPO+1uuRCI4xO1d0ht0TKMcSYFhsKmET/ol0kXwdno4lAzapo+9IRgJTMEsKZbhX0/cQ4C4=
+        - secure: XFm7b0lqDi7Opd2MlqJSrqZlwLe4MtOSlEmBMCGooUeMGjpyDTePp8KTVAejgqVmtXOFvzLykWB0mtSFQDWjJ2u9+bJgiktUJCMfqLklTDc+Gk7EWb7EI1E2f7ZBlTHRBdKCsHljuatEfMtXzitrAhOVmBGZe95xN61F8io7ktA=
+        - secure: VNoMS37cCaTivHQg1fFFBX+zn8QWqQd30/FbnZjjWJk1JgB3coJR4/mpu8vSU/NaDtsbu0yEcCHWWU1MvTAy+Ebc3GethTVkl8PyS1Z61DyfJjDl5MOuG3nvAqvITZtAbcmb1hqwOj96Jw6CdKivwYqw8OLHeKSWsMFCF0k/vO4=
+        - secure: xTnuCnUOBWyAvyqbIawvfQ8uETKaSHdIGaZR63KMTkCNk/pnY6Q3ioVy4qaM/ahJZgjrEd/31JjrfSpQdnlkY5TG4EFHq/nY5yRN8w55AzujvW9FjRKGAF5lzvzLCWFueS7kBgE6gkqaQHTMQ7W4b+q66aHs46F9kvy92AgE7Eo=
+        - secure: W3ZTeHd0WwFaBn4mjHde+iW7bgTSLL5ArQEkJUNwjxNXflbl0/1aL46qCw5aRU6JtlaWUjbuHZ5/hJKymgFtDTJafiy2IBVvWi+0kD0HBzhiHOiPx1zst27u8riFqWPoZIBZRkYG+/G0TWcfE/D4idJCtkx17lThgbZ605woZR0=
+        - secure: C1Kkwv2gqBNb9302s8tg8j+5h+V1tJGpiA5Wpj5Fz/OA8IWdgj1vNQO0+rdc+TJlSdc7aH0ggb+kayu/OgKt7JEy1bjBEZ+je7ncfN0ebp/xVlcSELIGv31yjs5hddtidi1IUW00igwN4VyyPGyPRC9s5ikrow9WUWacHDsP0Is=
 
 before_script:
    - "gem install --no-document --version 0.8.9 faraday "
    - "gem install --no-document travis-artifacts & "
 
-# we have to manually set the forkCount because maven thinks that the travis
-# machine has 32 cores
-#script: "mvn -Dflink.forkCount=2 -B $PROFILE clean install verify"
-
 # We run mvn and monitor its output. If there is no output for the specified number of seconds,
we
 # print the stack traces of all running Java processes.
 script: "./tools/travis_mvn_watchdog.sh 300"
 
 # deploy if the first job is successful; should be replaced by an after_all_success if travis
finally supports it
-after_success: 
-- "./tools/deploy_to_maven.sh"
+after_success:
+  - "./tools/deploy_to_maven.sh"

http://git-wip-us.apache.org/repos/asf/flink/blob/6c01598a/flink-runtime/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/resources/log4j-test.properties b/flink-runtime/src/test/resources/log4j-test.properties
index 7a9f6ba..e166a02 100644
--- a/flink-runtime/src/test/resources/log4j-test.properties
+++ b/flink-runtime/src/test/resources/log4j-test.properties
@@ -29,7 +29,7 @@ log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c
%x -
 # File (use 'file')
 # -----------------------------------------------------------------------------
 log4j.appender.file=org.apache.log4j.FileAppender
-log4j.appender.file.file=${log.file}
+log4j.appender.file.file=${log.dir}/{$mvn.forkNumber}.log
 log4j.appender.file.append=false
 log4j.appender.file.layout=org.apache.log4j.PatternLayout
 log4j.appender.file.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/flink/blob/6c01598a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 81e896e..1d09d02 100644
--- a/pom.xml
+++ b/pom.xml
@@ -79,6 +79,7 @@ under the License.
 			 it on the "mvn" commandline in travis. -->
 		<flink.forkCount>1.5C</flink.forkCount>
 		<flink.reuseForks>true</flink.reuseForks>
+		<log4j.configuration>log4j-test.properties</log4j.configuration>
 		<slf4j.version>1.7.7</slf4j.version>
 		<guava.version>17.0</guava.version>
 		<scala.version>2.10.4</scala.version>
@@ -1051,7 +1052,7 @@ under the License.
 						<exclude>**/scalastyle-output.xml</exclude>
 						<exclude>build-target/**</exclude>
 						<!-- Tools: watchdog -->
-						<exclude>tools/watchdog*</exclude>
+						<exclude>tools/artifacts/**</exclude>
 					</excludes>
 				</configuration>
 			</plugin>
@@ -1107,7 +1108,7 @@ under the License.
 						  the property is null when we have just the variable-->
 						<forkNumber>0${surefire.forkNumber}</forkNumber>
 					</systemPropertyVariables>
-					<argLine>-Xms256m -Xmx800m -Dlog4j.configuration=log4j-test.properties -XX:-UseGCOverheadLimit</argLine>
+					<argLine>-Xms256m -Xmx800m -Dlog4j.configuration=${log4j.configuration} -Dmvn.forkNumber=${surefire.forkNumber}
-XX:-UseGCOverheadLimit</argLine>
 				</configuration>
 			</plugin>
 			<plugin>
@@ -1120,7 +1121,7 @@ under the License.
 					<systemPropertyVariables>
 						<forkNumber>0${surefire.forkNumber}</forkNumber>
 					</systemPropertyVariables>
-					<argLine>-Xms256m -Xmx800m -Dlog4j.configuration=log4j-test.properties -XX:-UseGCOverheadLimit</argLine>
+					<argLine>-Xms256m -Xmx800m -Dlog4j.configuration=${log4j.configuration} -Dmvn.forkNumber=${surefire.forkNumber}
-XX:-UseGCOverheadLimit</argLine>
 				</configuration>
 			</plugin>
 			<plugin>

http://git-wip-us.apache.org/repos/asf/flink/blob/6c01598a/tools/.gitignore
----------------------------------------------------------------------
diff --git a/tools/.gitignore b/tools/.gitignore
index 2947abd..a7e8072 100644
--- a/tools/.gitignore
+++ b/tools/.gitignore
@@ -1,2 +1,3 @@
 _qa_workdir
 merge_pull_request.sh
+artifacts/*

http://git-wip-us.apache.org/repos/asf/flink/blob/6c01598a/tools/log4j-travis.properties
----------------------------------------------------------------------
diff --git a/tools/log4j-travis.properties b/tools/log4j-travis.properties
new file mode 100644
index 0000000..f69a673
--- /dev/null
+++ b/tools/log4j-travis.properties
@@ -0,0 +1,35 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+log4j.rootLogger=INFO, file
+
+# -----------------------------------------------------------------------------
+# Console (use 'console')
+# -----------------------------------------------------------------------------
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.layout=org.apache.flink.util.MavenForkNumberPrefixLayout
+log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+
+# -----------------------------------------------------------------------------
+# File (use 'file')
+# -----------------------------------------------------------------------------
+log4j.appender.file=org.apache.log4j.FileAppender
+log4j.appender.file.file=${log.dir}/${mvn.forkNumber}.log
+log4j.appender.file.append=true
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n

http://git-wip-us.apache.org/repos/asf/flink/blob/6c01598a/tools/travis_mvn_watchdog.sh
----------------------------------------------------------------------
diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh
index e16000c..e770a97 100755
--- a/tools/travis_mvn_watchdog.sh
+++ b/tools/travis_mvn_watchdog.sh
@@ -25,6 +25,12 @@ if [ -z "$HERE" ] ; then
 	exit 1  # fail
 fi
 
+ARTIFACTS_DIR="${HERE}/artifacts"
+
+mkdir -p $ARTIFACTS_DIR || { echo "FAILURE: cannot create log directory '${ARTIFACTS_DIR}'."
; exit 1; }
+
+echo "Build for commit ${TRAVIS_COMMIT} of ${TRAVIS_REPO_SLUG} [build ID: ${TRAVIS_BUILD_ID},
job number: $TRAVIS_JOB_NUMBER]." | tee "${ARTIFACTS_DIR}/build_info"
+
 # =============================================================================
 # CONFIG
 # =============================================================================
@@ -35,18 +41,56 @@ MAX_NO_OUTPUT=${1:-300}
 # Number of seconds to sleep before checking the output again
 SLEEP_TIME=20
 
-# Maven command to run
-MVN="mvn -Dflink.forkCount=2 -B $PROFILE clean install verify"
+LOG4J_PROPERTIES=${HERE}/log4j-travis.properties
+
+# Maven command to run. We set the forkCount manually, because otherwise Maven sees too many
cores
+# on the Travis VMs.
+MVN="mvn -Dflink.forkCount=2 -B $PROFILE -Dlog.dir=${ARTIFACTS_DIR} -Dlog4j.configuration=file://$LOG4J_PROPERTIES
clean install verify"
+
+MVN_PID="${ARTIFACTS_DIR}/watchdog.mvn.pid"
+MVN_EXIT="${ARTIFACTS_DIR}/watchdog.mvn.exit"
+MVN_OUT="${ARTIFACTS_DIR}/mvn.out"
 
-MVN_PID="${HERE}/watchdog.mvn.pid"
-MVN_EXIT="${HERE}/watchdog.mvn.exit"
-MVN_OUT="${HERE}/watchdog.mvn.out"
-TRACE_OUT="${HERE}/watchdog.trace.out"
+TRACE_OUT="${ARTIFACTS_DIR}/jps-traces.out"
+
+# E.g. travis-artifacts/apache/flink/1595/1595.1
+UPLOAD_TARGET_PATH="travis-artifacts/${TRAVIS_REPO_SLUG}/${TRAVIS_BUILD_NUMBER}/"
+# These variables are stored as secure variables in '.travis.yml', which are generated per
repo via
+# the travis command line tool.
+UPLOAD_BUCKET=$ARTIFACTS_AWS_BUCKET
+UPLOAD_ACCESS_KEY=$ARTIFACTS_AWS_ACCESS_KEY
+UPLOAD_SECRET_KEY=$ARTIFACTS_AWS_SECRET_KEY
+
+ARTIFACTS_FILE=${TRAVIS_JOB_NUMBER}.tar.gz
 
 # =============================================================================
 # FUNCTIONS
 # =============================================================================
 
+upload_artifacts_s3() {
+	echo "PRODUCED build artifacts."
+
+	ls $ARTIFACTS_DIR
+
+	if [ -n "$UPLOAD_BUCKET" ] && [ -n "$UPLOAD_ACCESS_KEY" ] && [ -n "$UPLOAD_SECRET_KEY"
]; then
+		echo "COMPRESSING build artifacts."
+
+		cd $ARTIFACTS_DIR
+		tar -zcvf $ARTIFACTS_FILE *
+
+		# Install artifacts tool
+		curl -sL https://raw.githubusercontent.com/travis-ci/artifacts/master/install | bash
+
+		PATH=$HOME/bin:$PATH
+
+		echo "UPLOADING build artifacts."
+
+		# Upload everything in $ARTIFACTS_DIR. Use relative path, otherwise the upload tool
+		# re-creates the whole directory structure from root.
+		artifacts upload --bucket $UPLOAD_BUCKET --key $UPLOAD_ACCESS_KEY --secret $UPLOAD_SECRET_KEY
--target-paths $UPLOAD_TARGET_PATH $ARTIFACTS_FILE
+	fi
+}
+
 print_stacktraces () {
 	echo "=============================================================================="
 	echo "The following Java processes are running (JPS)"
@@ -114,7 +158,7 @@ echo "STARTED watchdog (${WD_PID})."
 # Make sure to be in project root
 cd $HERE/../
 
-echo "RUNNING ${MVN} command."
+echo "RUNNING '${MVN}'."
 
 # Run $MVN and pipe output to $MVN_OUT for the watchdog. The PID is written to $MVN_PID to
 # allow the watchdog to kill $MVN if it is not producing any output anymore. $MVN_EXIT contains
@@ -133,4 +177,7 @@ echo "MVN exited with EXIT CODE: ${EXIT_CODE}."
 rm $MVN_PID
 rm $MVN_EXIT
 
+upload_artifacts_s3
+
+# Exit code for Travis build success/failure
 exit $EXIT_CODE


Mime
View raw message