flink-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tzuli...@apache.org
Subject [1/2] flink git commit: [FLINK-8362][elasticsearch] shade all dependencies
Date Thu, 08 Feb 2018 16:21:43 GMT
Repository: flink
Updated Branches:
  refs/heads/release-1.4 5f9e367be -> 33ebc85c2


[FLINK-8362][elasticsearch] shade all dependencies


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/0c53e798
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/0c53e798
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/0c53e798

Branch: refs/heads/release-1.4
Commit: 0c53e798cb25bacdde241d4b4696108dbae29117
Parents: 5f9e367
Author: Nico Kruber <nico@data-artisans.com>
Authored: Thu Jan 4 21:09:09 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tzulitai@apache.org>
Committed: Thu Feb 8 17:21:15 2018 +0100

----------------------------------------------------------------------
 docs/dev/connectors/elasticsearch.md            |  33 +----
 .../flink-connector-elasticsearch/pom.xml       | 100 +++++++++++++++
 .../flink-connector-elasticsearch2/pom.xml      | 125 ++++++++++++++++++
 .../flink-connector-elasticsearch5/pom.xml      | 126 +++++++++++++++++++
 tools/travis_mvn_watchdog.sh                    |  29 +++++
 5 files changed, 382 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/0c53e798/docs/dev/connectors/elasticsearch.md
----------------------------------------------------------------------
diff --git a/docs/dev/connectors/elasticsearch.md b/docs/dev/connectors/elasticsearch.md
index 8774fcb..52d1b58 100644
--- a/docs/dev/connectors/elasticsearch.md
+++ b/docs/dev/connectors/elasticsearch.md
@@ -440,36 +440,7 @@ For the execution of your Flink program, it is recommended to build a
 so-called uber-jar (executable jar) containing all your dependencies
 (see [here]({{site.baseurl}}/dev/linking.html) for further information).
 
-However, when an uber-jar containing an Elasticsearch sink is executed,
-an `IllegalArgumentException` may occur, which is caused by conflicting
-files of Elasticsearch and it's dependencies in `META-INF/services`:
-
-```
-IllegalArgumentException[An SPI class of type org.apache.lucene.codecs.PostingsFormat with
name 'Lucene50' does not exist.  You need to add the corresponding JAR file supporting this
SPI to your classpath.  The current classpath supports the following names: [es090, completion090,
XBloomFilter]]
-```
-
-If the uber-jar is built using Maven, this issue can be avoided by
-adding the following to the Maven POM file in the plugins section:
-
-~~~xml
-<plugin>
-    <groupId>org.apache.maven.plugins</groupId>
-    <artifactId>maven-shade-plugin</artifactId>
-    <version>2.4.3</version>
-    <executions>
-        <execution>
-            <phase>package</phase>
-            <goals>
-                <goal>shade</goal>
-            </goals>
-            <configuration>
-                <transformers>
-                    <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
-                </transformers>
-            </configuration>
-        </execution>
-    </executions>
-</plugin>
-~~~
+Alternatively, you can put the connector's jar file into Flink's `lib/` folder to make it
available
+system-wide, i.e. for all job being run.
 
 {% top %}

http://git-wip-us.apache.org/repos/asf/flink/blob/0c53e798/flink-connectors/flink-connector-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-elasticsearch/pom.xml b/flink-connectors/flink-connector-elasticsearch/pom.xml
index 7785a77..b6b876a 100644
--- a/flink-connectors/flink-connector-elasticsearch/pom.xml
+++ b/flink-connectors/flink-connector-elasticsearch/pom.xml
@@ -93,6 +93,106 @@ under the License.
 					<rerunFailingTestsCount>3</rerunFailingTestsCount>
 				</configuration>
 			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-shade-plugin</artifactId>
+				<executions>
+					<execution>
+						<id>shade-flink</id>
+						<phase>package</phase>
+						<goals>
+							<goal>shade</goal>
+						</goals>
+						<configuration>
+							<shadeTestJar>false</shadeTestJar>
+							<artifactSet>
+								<includes>
+									<include>*:*</include>
+								</includes>
+							</artifactSet>
+							<relocations>
+								<!-- elasticsearch 1.x already shades this but forgets the service file -->
+								<relocation>
+									<pattern>com.fasterxml.jackson</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch.shaded.org.elasticsearch.common.jackson</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>com.spatial4j</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch.shaded.com.spatial4j</shadedPattern>
+								</relocation>
+
+								<!-- relocate everything from the flink-connector-elasticsearch base project -->
+								<relocation>
+									<pattern>org.apache.flink.streaming.connectors.elasticsearch.</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch.shaded.org.apache.flink.streaming.connectors.elasticsearch.</shadedPattern>
+									<excludes>
+										<!-- keep this project's classes as they are -->
+										<exclude>org.apache.flink.streaming.connectors.elasticsearch.Elasticsearch1ApiCallBridge</exclude>
+										<exclude>org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSink</exclude>
+										<exclude>org.apache.flink.streaming.connectors.elasticsearch.IndexRequestBuilder</exclude>
+										<exclude>org.apache.flink.streaming.connectors.elasticsearch.IndexRequestBuilderWrapperFunction</exclude>
+									</excludes>
+								</relocation>
+								<relocation>
+									<pattern>org.apache</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch.shaded.org.apache</shadedPattern>
+									<excludes>
+										<!-- keep flink classes as they are (exceptions as above) -->
+										<exclude>org.apache.flink.**</exclude>
+										<exclude>org.apache.log4j.**</exclude> <!-- provided -->
+									</excludes>
+								</relocation>
+
+								<relocation>
+									<pattern>org.antlr</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch.shaded.org.antlr</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>org.elasticsearch</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch.shaded.org.elasticsearch</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>org.joda</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch.shaded.org.joda</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>org.tartarus</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch.shaded.org.tartarus</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>org.yaml</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch.shaded.org.yaml</shadedPattern>
+								</relocation>
+
+							</relocations>
+							<filters>
+								<filter>
+									<artifact>*</artifact>
+									<excludes>
+										<exclude>log4j.properties</exclude>
+										<exclude>config/favicon.ico</exclude>
+										<exclude>mozilla/**</exclude>
+										<exclude>META-INF/maven/com*/**</exclude>
+										<exclude>META-INF/maven/io*/**</exclude>
+										<exclude>META-INF/maven/joda*/**</exclude>
+										<exclude>META-INF/maven/net*/**</exclude>
+										<exclude>META-INF/maven/org.an*/**</exclude>
+										<exclude>META-INF/maven/org.apache.h*/**</exclude>
+										<exclude>META-INF/maven/org.apache.commons/**</exclude>
+										<exclude>META-INF/maven/org.apache.flink/flink-connector-elasticsearch-base*/**</exclude>
+										<exclude>META-INF/maven/org.apache.flink/force-shading/**</exclude>
+										<exclude>META-INF/maven/org.apache.logging*/**</exclude>
+										<exclude>META-INF/maven/org.e*/**</exclude>
+										<exclude>META-INF/maven/org.h*/**</exclude>
+										<exclude>META-INF/maven/org.j*/**</exclude>
+										<exclude>META-INF/maven/org.y*/**</exclude>
+									</excludes>
+								</filter>
+							</filters>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
 		</plugins>
 	</build>
 

http://git-wip-us.apache.org/repos/asf/flink/blob/0c53e798/flink-connectors/flink-connector-elasticsearch2/pom.xml
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-elasticsearch2/pom.xml b/flink-connectors/flink-connector-elasticsearch2/pom.xml
index 1f342bc..1d6ce04 100644
--- a/flink-connectors/flink-connector-elasticsearch2/pom.xml
+++ b/flink-connectors/flink-connector-elasticsearch2/pom.xml
@@ -91,4 +91,129 @@ under the License.
 
 	</dependencies>
 
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-shade-plugin</artifactId>
+				<executions>
+					<execution>
+						<id>shade-flink</id>
+						<phase>package</phase>
+						<goals>
+							<goal>shade</goal>
+						</goals>
+						<configuration>
+							<shadeTestJar>false</shadeTestJar>
+							<artifactSet>
+								<includes>
+									<include>*:*</include>
+								</includes>
+							</artifactSet>
+							<relocations>
+								<relocation>
+									<pattern>com.carrotsearch</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.com.carrotsearch</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>com.fasterxml</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.com.fasterxml</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>com.google</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.com.google</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>com.ning</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.com.ning</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>com.spatial4j</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.com.spatial4j</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>com.tdunning</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.com.tdunning</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>com.twitter</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.com.twitter</shadedPattern>
+								</relocation>
+
+								<!-- relocate everything from the flink-connector-elasticsearch base project -->
+								<relocation>
+									<pattern>org.apache.flink.streaming.connectors.elasticsearch</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.org.apache.flink.streaming.connectors.elasticsearch</shadedPattern>
+									<excludes>
+										<!-- keep this project's classes as they are -->
+										<exclude>org.apache.flink.streaming.connectors.elasticsearch2.**</exclude>
+									</excludes>
+								</relocation>
+								<relocation>
+									<pattern>org.apache</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.org.apache</shadedPattern>
+									<excludes>
+										<!-- keep flink classes as they are (exceptions as above) -->
+										<exclude>org.apache.flink.**</exclude>
+										<exclude>org.apache.log4j.**</exclude> <!-- provided -->
+									</excludes>
+								</relocation>
+
+								<relocation>
+									<pattern>org.elasticsearch</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.org.elasticsearch</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>org.jboss</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.org.jboss</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>org.joda</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.org.joda</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>org.HdrHistogram</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.org.HdrHistogram</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>org.tartarus</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.org.tartarus</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>org.yaml</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.org.yaml</shadedPattern>
+								</relocation>
+
+							</relocations>
+							<filters>
+								<filter>
+									<artifact>*</artifact>
+									<excludes>
+										<exclude>log4j.properties</exclude>
+										<exclude>config/favicon.ico</exclude>
+										<exclude>mozilla/**</exclude>
+										<exclude>META-INF/maven/com*/**</exclude>
+										<exclude>META-INF/maven/io*/**</exclude>
+										<exclude>META-INF/maven/joda*/**</exclude>
+										<exclude>META-INF/maven/net*/**</exclude>
+										<exclude>META-INF/maven/org.an*/**</exclude>
+										<exclude>META-INF/maven/org.apache.h*/**</exclude>
+										<exclude>META-INF/maven/org.apache.commons/**</exclude>
+										<exclude>META-INF/maven/org.apache.flink/flink-connector-elasticsearch-base*/**</exclude>
+										<exclude>META-INF/maven/org.apache.flink/force-shading/**</exclude>
+										<exclude>META-INF/maven/org.apache.logging*/**</exclude>
+										<exclude>META-INF/maven/org.e*/**</exclude>
+										<exclude>META-INF/maven/org.h*/**</exclude>
+										<exclude>META-INF/maven/org.j*/**</exclude>
+										<exclude>META-INF/maven/org.y*/**</exclude>
+									</excludes>
+								</filter>
+							</filters>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+	</build>
+
 </project>

http://git-wip-us.apache.org/repos/asf/flink/blob/0c53e798/flink-connectors/flink-connector-elasticsearch5/pom.xml
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-elasticsearch5/pom.xml b/flink-connectors/flink-connector-elasticsearch5/pom.xml
index 72a0d18..82d578b 100644
--- a/flink-connectors/flink-connector-elasticsearch5/pom.xml
+++ b/flink-connectors/flink-connector-elasticsearch5/pom.xml
@@ -156,6 +156,132 @@ under the License.
 					</classpathDependencyExcludes>
 				</configuration>
 			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-shade-plugin</artifactId>
+				<executions>
+					<execution>
+						<id>shade-flink</id>
+						<phase>package</phase>
+						<goals>
+							<goal>shade</goal>
+						</goals>
+						<configuration>
+							<shadeTestJar>false</shadeTestJar>
+							<artifactSet>
+								<includes>
+									<include>*:*</include>
+								</includes>
+							</artifactSet>
+							<relocations>
+								<relocation>
+									<pattern>com.carrotsearch</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.com.carrotsearch</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>com.fasterxml</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.com.fasterxml</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>com.github</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.com.github</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>com.sun</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.com.sun</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>com.github</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.com.github</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>com.tdunning</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.com.tdunning</shadedPattern>
+								</relocation>
+
+								<relocation>
+									<pattern>io.netty</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.io.netty</shadedPattern>
+								</relocation>
+
+								<relocation>
+									<pattern>joptsimple</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.joptsimple</shadedPattern>
+								</relocation>
+
+								<!-- relocate everything from the flink-connector-elasticsearch base project -->
+								<relocation>
+									<pattern>org.apache.flink.streaming.connectors.elasticsearch</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.org.apache.flink.streaming.connectors.elasticsearch</shadedPattern>
+									<excludes>
+										<!-- keep this project's classes as they are -->
+										<exclude>org.apache.flink.streaming.connectors.elasticsearch5.**</exclude>
+									</excludes>
+								</relocation>
+								<relocation>
+									<pattern>org.apache</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.org.apache</shadedPattern>
+									<excludes>
+										<!-- keep flink classes as they are (exceptions as above) -->
+										<exclude>org.apache.flink.**</exclude>
+										<exclude>org.apache.log4j.**</exclude> <!-- provided -->
+									</excludes>
+								</relocation>
+
+								<relocation>
+									<pattern>org.elasticsearch</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.org.elasticsearch</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>org.HdrHistogram</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.org.HdrHistogram</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>org.jboss</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.org.jboss</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>org.joda</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.org.joda</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>org.tartarus</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.org.tartarus</shadedPattern>
+								</relocation>
+								<relocation>
+									<pattern>org.yaml</pattern>
+									<shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.org.yaml</shadedPattern>
+								</relocation>
+
+							</relocations>
+							<filters>
+								<filter>
+									<artifact>*</artifact>
+									<excludes>
+										<exclude>log4j.properties</exclude>
+										<exclude>config/favicon.ico</exclude>
+										<exclude>mozilla/**</exclude>
+										<exclude>META-INF/maven/com*/**</exclude>
+										<exclude>META-INF/maven/io*/**</exclude>
+										<exclude>META-INF/maven/joda*/**</exclude>
+										<exclude>META-INF/maven/net*/**</exclude>
+										<exclude>META-INF/maven/org.an*/**</exclude>
+										<exclude>META-INF/maven/org.apache.h*/**</exclude>
+										<exclude>META-INF/maven/org.apache.commons/**</exclude>
+										<exclude>META-INF/maven/org.apache.flink/flink-connector-elasticsearch-base*/**</exclude>
+										<exclude>META-INF/maven/org.apache.flink/force-shading/**</exclude>
+										<exclude>META-INF/maven/org.apache.logging*/**</exclude>
+										<exclude>META-INF/maven/org.e*/**</exclude>
+										<exclude>META-INF/maven/org.h*/**</exclude>
+										<exclude>META-INF/maven/org.j*/**</exclude>
+										<exclude>META-INF/maven/org.y*/**</exclude>
+									</excludes>
+								</filter>
+							</filters>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
 		</plugins>
 	</build>
 

http://git-wip-us.apache.org/repos/asf/flink/blob/0c53e798/tools/travis_mvn_watchdog.sh
----------------------------------------------------------------------
diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh
index 7c27392..4fb7be9 100755
--- a/tools/travis_mvn_watchdog.sh
+++ b/tools/travis_mvn_watchdog.sh
@@ -447,6 +447,32 @@ check_shaded_artifacts_s3_fs() {
 	return 0
 }
 
+# Check the elasticsearch connectors' fat jars for illegal or missing artifacts
+check_shaded_artifacts_connector_elasticsearch() {
+	VARIANT=$1
+	find flink-connectors/flink-connector-elasticsearch${VARIANT}/target/flink-connector-elasticsearch${VARIANT}*.jar
! -name "*-tests.jar" -exec jar tf {} \; > allClasses
+
+	UNSHADED_CLASSES=`cat allClasses | grep -v -e '^META-INF' -e '^assets' -e "^org/apache/flink/streaming/connectors/elasticsearch${VARIANT}/"
| grep '\.class$'`
+	if [ "$?" == "0" ]; then
+		echo "=============================================================================="
+		echo "Detected unshaded dependencies in flink-connector-elasticsearch${VARIANT}'s fat jar:"
+		echo "${UNSHADED_CLASSES}"
+		echo "=============================================================================="
+		return 1
+	fi
+
+	UNSHADED_SERVICES=`cat allClasses | grep '^META-INF/services/' | grep -v -e '^META-INF/services/org\.apache\.flink\.core\.fs\.FileSystemFactory$'
-e "^META-INF/services/org\.apache\.flink\.fs\.s3${VARIANT}\.shaded" -e '^META-INF/services/'`
+	if [ "$?" == "0" ]; then
+		echo "=============================================================================="
+		echo "Detected unshaded service files in flink-connector-elasticsearch${VARIANT}'s fat
jar:"
+		echo "${UNSHADED_SERVICES}"
+		echo "=============================================================================="
+		return 1
+	fi
+
+	return 0
+}
+
 # =============================================================================
 # WATCHDOG
 # =============================================================================
@@ -526,6 +552,9 @@ case $TEST in
 			check_shaded_artifacts_s3_fs hadoop
 			EXIT_CODE=$(($EXIT_CODE+$?))
 			check_shaded_artifacts_s3_fs presto
+			check_shaded_artifacts_connector_elasticsearch ""
+			check_shaded_artifacts_connector_elasticsearch 2
+			check_shaded_artifacts_connector_elasticsearch 5
 			EXIT_CODE=$(($EXIT_CODE+$?))
 		else
 			echo "=============================================================================="


Mime
View raw message