flink-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rmetz...@apache.org
Subject [32/39] git commit: Create release scripts, add support for "cdh4" build.
Date Sat, 09 Aug 2014 12:40:03 GMT
Create release scripts, add support for "cdh4" build.


Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/ecb829ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/ecb829ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/ecb829ed

Branch: refs/heads/travis_test
Commit: ecb829ed47544242953db8c1806338445c40713d
Parents: e4c3754
Author: Robert Metzger <rmetzger@apache.org>
Authored: Wed Jul 30 19:57:21 2014 +0200
Committer: Robert Metzger <rmetzger@apache.org>
Committed: Wed Aug 6 11:43:05 2014 +0200

----------------------------------------------------------------------
 DISCLAIMER                                      |  11 ++
 flink-addons/flink-hadoop-compatibility/pom.xml |  23 ++++
 .../mapred/wrapper/HadoopDummyReporter.java     |   3 +-
 flink-clients/pom.xml                           |   6 +
 flink-dist/pom.xml                              |  24 +++-
 flink-dist/src/main/assemblies/bin.xml          |   5 +-
 flink-dist/src/main/assemblies/yarn.xml         |   9 +-
 flink-runtime/pom.xml                           |  25 ++++
 flink-test-utils/pom.xml                        |  12 ++
 flink-tests/pom.xml                             |  12 ++
 pom.xml                                         |   1 +
 tools/create_release_files.sh                   | 134 +++++++++++++++++++
 tools/how_to_release.md                         |  16 +++
 tools/prepare_release.sh                        |  23 ++++
 14 files changed, 291 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/DISCLAIMER
----------------------------------------------------------------------
diff --git a/DISCLAIMER b/DISCLAIMER
new file mode 100644
index 0000000..48a1a19
--- /dev/null
+++ b/DISCLAIMER
@@ -0,0 +1,11 @@
+Apache Flink is an effort undergoing incubation at the Apache Software
+Foundation (ASF), sponsored by the Apache Incubator PMC.
+
+Incubation is required of all newly accepted projects until a further review
+indicates that the infrastructure, communications, and decision making process
+have stabilized in a manner consistent with other successful ASF projects.
+
+While incubation status is not necessarily a reflection of the completeness
+or stability of the code, it does indicate that the project has yet to be
+fully endorsed by the ASF.
+

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-addons/flink-hadoop-compatibility/pom.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/pom.xml b/flink-addons/flink-hadoop-compatibility/pom.xml
index 746b782..52c2917 100644
--- a/flink-addons/flink-hadoop-compatibility/pom.xml
+++ b/flink-addons/flink-hadoop-compatibility/pom.xml
@@ -78,5 +78,28 @@ under the License.
 				</dependency>
 			</dependencies>
 		</profile>
+			<profile>
+			<id>cdh4</id>
+			<!-- 
+				See flink-runtime/pom.xml for details
+			-->
+			<dependencies>
+				<dependency>
+					<groupId>org.apache.hadoop</groupId>
+					<artifactId>hadoop-mapreduce-client-core</artifactId>
+					<version>${hadoop.cdh4.hadoop.version}</version>
+				</dependency>
+				<dependency>
+					<groupId>org.apache.hadoop</groupId>
+					<artifactId>hadoop-hdfs</artifactId>
+					<version>${hadoop.cdh4.hadoop.version}</version>
+				</dependency>
+				<dependency>
+					<groupId>org.apache.hadoop</groupId>
+					<artifactId>hadoop-common</artifactId>
+					<version>${hadoop.cdh4.hadoop.version}</version>
+				</dependency>
+			</dependencies>
+		</profile>
 	</profiles>
 </project>

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java
b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java
index 87a6014..ac2b4b6 100644
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java
+++ b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java
@@ -62,8 +62,7 @@ public class HadoopDummyReporter implements Reporter {
 	public InputSplit getInputSplit() throws UnsupportedOperationException {
 		return null;
 	}
-
-	@Override
+	// There should be an @Override, but some CDH4 dependency does not contain this method
 	public float getProgress() {
 		return 0;
 	}

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-clients/pom.xml
----------------------------------------------------------------------
diff --git a/flink-clients/pom.xml b/flink-clients/pom.xml
index 7456209..b035c19 100644
--- a/flink-clients/pom.xml
+++ b/flink-clients/pom.xml
@@ -95,6 +95,12 @@ under the License.
 			<version>2.4</version>
 			<scope>compile</scope>
 		</dependency>
+		<dependency>
+			<groupId>commons-lang</groupId>
+			<artifactId>commons-lang</artifactId>
+			<version>2.4</version>
+			<scope>compile</scope>
+		</dependency>
 	</dependencies>
 
 	<!-- More information on this:

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-dist/pom.xml
----------------------------------------------------------------------
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index 5672fde..8c46149 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -132,8 +132,9 @@ under the License.
 						<artifactId>maven-assembly-plugin</artifactId>
 						<version>2.4</version>
 						<executions>
+							<!--  Uber-jar -->
 							<execution>
-								<id>single-jar</id>
+								<id>uber-jar</id>
 								<phase>package</phase>
 								<goals>
 									<goal>single</goal>
@@ -154,8 +155,23 @@ under the License.
 									</archive>
 									<descriptors>
 										<descriptor>src/main/assemblies/yarn-uberjar.xml</descriptor>
+									</descriptors>
+								</configuration>
+							</execution>
+							
+							<!-- yarn bin directory -->
+							<execution>
+								<id>yarn-bin</id>
+								<phase>package</phase>
+								<goals>
+									<goal>single</goal>
+								</goals>
+								<configuration>
+									<descriptors>
 										<descriptor>src/main/assemblies/yarn.xml</descriptor>
 									</descriptors>
+									<finalName>flink-${project.version}-bin</finalName>
+									<appendAssemblyId>false</appendAssemblyId>
 								</configuration>
 							</execution>
 						</executions>
@@ -321,13 +337,15 @@ under the License.
 		</profile>
 	</profiles>
 	
+	
+
 	<build>
 		<plugins>
 			<plugin>
 				<artifactId>maven-assembly-plugin</artifactId>
 				<executions>
 					<execution>
-						<id>generate-package</id>
+						<id>bin</id>
 						<phase>package</phase>
 						<goals>
 							<goal>single</goal>
@@ -336,6 +354,8 @@ under the License.
 							<descriptors>
 								<descriptor>src/main/assemblies/bin.xml</descriptor>
 							</descriptors>
+							<finalName>flink-${project.version}-bin</finalName>
+							<appendAssemblyId>false</appendAssemblyId>
 						</configuration>
 					</execution>
 				</executions>

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-dist/src/main/assemblies/bin.xml
----------------------------------------------------------------------
diff --git a/flink-dist/src/main/assemblies/bin.xml b/flink-dist/src/main/assemblies/bin.xml
index cc5850a..6ade3be 100644
--- a/flink-dist/src/main/assemblies/bin.xml
+++ b/flink-dist/src/main/assemblies/bin.xml
@@ -23,10 +23,6 @@ under the License.
 	<id>bin</id>
 	<formats>
 		<format>dir</format>
-		<!--
-		<format>tar.gz</format>
-		<format>zip</format>
-		-->
 	</formats>
 
 	<includeBaseDirectory>true</includeBaseDirectory>
@@ -114,6 +110,7 @@ under the License.
 				<include>*.txt</include>
 				<include>LICENSE*</include>
 				<include>NOTICE*</include>
+				<include>DISCLAIMER*</include>
 			</includes>
 		</fileSet>
 

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-dist/src/main/assemblies/yarn.xml
----------------------------------------------------------------------
diff --git a/flink-dist/src/main/assemblies/yarn.xml b/flink-dist/src/main/assemblies/yarn.xml
index 2fa19cb..f262ab0 100644
--- a/flink-dist/src/main/assemblies/yarn.xml
+++ b/flink-dist/src/main/assemblies/yarn.xml
@@ -23,11 +23,7 @@ under the License.
 	xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0
http://maven.apache.org/xsd/assembly-1.1.0.xsd">
 	<id>yarn</id>
 	<formats>
-		<format>tar.gz</format>
-		<!--
-		<format>tar.gz</format>
-		<format>zip</format>
-		-->
+		<format>dir</format>
 	</formats>
 
 	<includeBaseDirectory>true</includeBaseDirectory>
@@ -68,6 +64,9 @@ under the License.
 			<includes>
 				<include>*.txt</include>
 				<include>*.properties</include>
+				<include>DISCLAIMER*</include>
+				<include>LICENSE*</include>
+				<include>NOTICE*</include>
 			</includes>
 		</fileSet>
 		

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-runtime/pom.xml
----------------------------------------------------------------------
diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
index 7bb49f4..7b22708 100644
--- a/flink-runtime/pom.xml
+++ b/flink-runtime/pom.xml
@@ -217,5 +217,30 @@ under the License.
 				</dependency>
 			</dependencies>
 		</profile>
+
+		<profile>
+			<id>cdh4</id>
+			<!-- 
+
+			It seems http://www.cloudera.com/content/cloudera-content/cloudera-docs/CDH4/4.2.0/CDH4-Installation-Guide/cdh4ig_topic_31.html
+			that this particular CDH4 version has already separate packages for hdfs and common.
+
+			To activate the cdh4 profile, use the -Pcdh4 flag. (You probably want -Pcdh4,vendor-repos)
+			You also HAVE TO set the -Dhadoop.cdh4.hadoop.version property (for example to: 2.0.0-cdh4.2.0)
+
+			-->
+			<dependencies>
+				<dependency>
+					<groupId>org.apache.hadoop</groupId>
+					<artifactId>hadoop-hdfs</artifactId>
+					<version>${hadoop.cdh4.hadoop.version}</version>
+				</dependency>
+				<dependency>
+					<groupId>org.apache.hadoop</groupId>
+					<artifactId>hadoop-common</artifactId>
+					<version>${hadoop.cdh4.hadoop.version}</version>
+				</dependency>
+			</dependencies>
+		</profile>
 	</profiles>
 </project>

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-test-utils/pom.xml
----------------------------------------------------------------------
diff --git a/flink-test-utils/pom.xml b/flink-test-utils/pom.xml
index 1a0fe1b..a9d21ed 100644
--- a/flink-test-utils/pom.xml
+++ b/flink-test-utils/pom.xml
@@ -66,4 +66,16 @@ under the License.
 			<version>4.11</version>
 		</dependency>
 	</dependencies>
+	<profiles>
+		<profile>
+			<id>cdh4</id>
+			<dependencies>
+				<dependency>
+					<groupId>org.apache.hadoop</groupId>
+					<artifactId>hadoop-common</artifactId>
+					<version>${hadoop.cdh4.hadoop.version}</version>
+				</dependency>
+			</dependencies>
+		</profile>
+	</profiles>
 </project>

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-tests/pom.xml
----------------------------------------------------------------------
diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml
index 950e78e..b4b11ea 100644
--- a/flink-tests/pom.xml
+++ b/flink-tests/pom.xml
@@ -236,4 +236,16 @@ under the License.
 			</plugins>
 		</pluginManagement>
 	</build>
+	<profiles>
+		<profile>
+			<id>cdh4</id>
+			<dependencies>
+				<dependency>
+					<groupId>org.apache.hadoop</groupId>
+					<artifactId>hadoop-common</artifactId>
+					<version>${hadoop.cdh4.hadoop.version}</version>
+				</dependency>
+			</dependencies>
+		</profile>
+	</profiles>
 </project>

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8058539..a22c7e7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -460,6 +460,7 @@ under the License.
 						<exclude>**/target/**</exclude>
 						<!-- Documentation -->
 						<exclude>**/docs/**</exclude>
+						<exclude>**/tools/*.md</exclude>
 					</excludes>
 				</configuration>
 			</plugin>

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/tools/create_release_files.sh
----------------------------------------------------------------------
diff --git a/tools/create_release_files.sh b/tools/create_release_files.sh
new file mode 100755
index 0000000..aa8875d
--- /dev/null
+++ b/tools/create_release_files.sh
@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Quick-and-dirty automation of making maven and binary releases. Not robust at all.
+# Publishes releases to Maven and packages/copies binary release artifacts.
+# Expects to be run in a totally empty directory.
+#
+
+#
+#  NOTE: The code in this file is based on code from the Apache Spark
+#  project, licensed under the Apache License v 2.0
+#
+#  https://github.com/apache/spark/blob/master/dev/create-release/create-release.sh
+#
+CURR_DIR=`pwd`
+if [[ `basename $CURR_DIR` != "tools" ]] ; then
+  echo "You have to call the script from the tools/ dir"
+  exit 1
+fi
+
+GIT_USERNAME=${GIT_USERNAME:-pwendell}
+GIT_PASSWORD=${GIT_PASSWORD:-XXX}
+GPG_PASSPHRASE=${GPG_PASSPHRASE:-XXX}
+GPG_KEY=${GPG_KEY:-XXX}
+GIT_BRANCH=${GIT_BRANCH:-branch-1.0}
+RELEASE_VERSION=${NEW_VERSION}
+USER_NAME=${USER_NAME:-pwendell}
+MVN=${MVN:-mvn}
+
+set -e
+
+# create source package
+
+git clone https://github.com/rmetzger/incubator-flink.git flink
+cd flink
+git checkout -b release-0.6 origin/release-0.6
+rm .gitignore
+rm .travis.yml
+rm deploysettings.xml
+rm CHANGELOG
+rm -rf .git
+cd ..
+
+
+cp -r flink flink-$RELEASE_VERSION
+tar cvzf flink-$RELEASE_VERSION.tgz flink-$RELEASE_VERSION
+echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY --passphrase-fd 0 --armour --output
flink-$RELEASE_VERSION.tgz.asc \
+  --detach-sig flink-$RELEASE_VERSION.tgz
+echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY --passphrase-fd 0 --print-md MD5
flink-$RELEASE_VERSION.tgz > \
+  flink-$RELEASE_VERSION.tgz.md5
+echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY --passphrase-fd 0 --print-md SHA512
flink-$RELEASE_VERSION.tgz > \
+  flink-$RELEASE_VERSION.tgz.sha
+rm -rf flink-$RELEASE_VERSION
+
+
+make_binary_release() {
+  NAME=$1
+  FLAGS=$2
+  cp -r flink flink-$RELEASE_VERSION-bin-$NAME
+  
+  cd flink-$RELEASE_VERSION-bin-$NAME
+  # make distribution
+  $MVN clean package $FLAGS -DskipTests
+  cd flink-dist/target/flink-$RELEASE_VERSION-bin/
+  tar cvzf flink-$RELEASE_VERSION-bin-$NAME.tgz flink-$RELEASE_VERSION
+  if [ -d "flink-yarn-$RELEASE_VERSION" ] ; then
+    tar cvzf flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz flink-yarn-$RELEASE_VERSION
+  fi
+  cp flink-*.tgz ../../../../
+  cd ../../../../
+  rm -rf flink-$RELEASE_VERSION
+
+  # Sign md5 and sha the tgz
+  echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY \
+    --passphrase-fd 0 --armour \
+    --output flink-$RELEASE_VERSION-bin-$NAME.tgz.asc \
+    --detach-sig flink-$RELEASE_VERSION-bin-$NAME.tgz
+  echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY \
+    --passphrase-fd 0 --print-md \
+    MD5 flink-$RELEASE_VERSION-bin-$NAME.tgz > \
+    flink-$RELEASE_VERSION-bin-$NAME.tgz.md5
+  echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY \
+    --passphrase-fd 0 --print-md \
+    SHA512 flink-$RELEASE_VERSION-bin-$NAME.tgz > \
+    flink-$RELEASE_VERSION-bin-$NAME.tgz.sha
+
+  if [ -f "flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz" ] ; then
+    echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY \
+    --passphrase-fd 0 --armour \
+    --output flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz.asc \
+    --detach-sig flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz
+    echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY \
+      --passphrase-fd 0 --print-md \
+      MD5 flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz > \
+      flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz.md5
+    echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY \
+      --passphrase-fd 0 --print-md \
+      SHA512 flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz > \
+      flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz.sha
+  fi
+}
+
+make_binary_release "hadoop1" ""
+make_binary_release "cdh4" "-Pvendor-repos,cdh4 -Dhadoop.version=2.0.0-mr1-cdh4.2.0 -Dhadoop.cdh4.hadoop.version=2.0.0-cdh4.2.0"
+# make_binary_release "cdh5" "-Dhadoop.profile=2 -Pvendor-repos -Dhadoop.version=2.3.0+cdh5.1.0+795"
+make_binary_release "hadoop2" "-Dhadoop.profile=2"
+# make_binary_release "mapr4" "-Dhadoop.profile=2 -Pvendor-repos -Dhadoop.version=2.3.0-mapr-4.0.0-FCS"
+# make_binary_release "hdp21" "-Dhadoop.profile=2 -Pvendor-repos -Dhadoop.version=2.4.0.2.1.3.0-563"
+
+
+# Copy data
+echo "Copying release tarballs"
+folder=flink-$RELEASE_VERSION
+ssh $USER_NAME@people.apache.org \
+  mkdir /home/$USER_NAME/public_html/$folder
+scp flink-* \
+  $USER_NAME@people.apache.org:/home/$USER_NAME/public_html/$folder/
+

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/tools/how_to_release.md
----------------------------------------------------------------------
diff --git a/tools/how_to_release.md b/tools/how_to_release.md
new file mode 100644
index 0000000..65cc3b7
--- /dev/null
+++ b/tools/how_to_release.md
@@ -0,0 +1,16 @@
+# How to release a new version of Apache Flink
+
+
+You need to have the following environment variables set:
+
+export OLD_VERSION=0.6-SNAPSHOT
+export NEW_VERSION=0.6
+export GPG_KEY=XX
+export GPG_PASSPHRASE=XX
+
+./prepare_release_files.sh
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/tools/prepare_release.sh
----------------------------------------------------------------------
diff --git a/tools/prepare_release.sh b/tools/prepare_release.sh
new file mode 100644
index 0000000..17354b9
--- /dev/null
+++ b/tools/prepare_release.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+echo "Updating version from $OLD_VERSION to $NEW_VERSION"
+
+find .. -name 'pom.xml' -type f -exec sed -i 's#<version>$OLD_VERSION</version>#<version>$NEW_VERSION</version>#'
{} \;
+


Mime
View raw message