flink-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rmetz...@apache.org
Subject [2/4] flink git commit: [FLINK-1392] Add Kryo serializer for Protobuf
Date Mon, 09 Feb 2015 13:49:12 GMT
[FLINK-1392] Add Kryo serializer for Protobuf

Conflicts:
	flink-java/pom.xml
	flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java

Conflicts:
	flink-shaded/pom.xml
	pom.xml


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c4c3db50
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c4c3db50
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c4c3db50

Branch: refs/heads/release-0.8
Commit: c4c3db50a7aa6a7349189f547ee04c1373df5578
Parents: 84c4998
Author: Robert Metzger <rmetzger@apache.org>
Authored: Tue Jan 13 10:21:29 2015 +0100
Committer: Robert Metzger <rmetzger@apache.org>
Committed: Mon Feb 9 14:48:34 2015 +0100

----------------------------------------------------------------------
 flink-java/pom.xml                              | 33 ++++++++++++++++++++
 .../java/typeutils/runtime/KryoSerializer.java  | 14 ++++++---
 flink-shaded/pom.xml                            |  5 ++-
 pom.xml                                         |  6 +++-
 4 files changed, 52 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/c4c3db50/flink-java/pom.xml
----------------------------------------------------------------------
diff --git a/flink-java/pom.xml b/flink-java/pom.xml
index 4a4bbfa..cc5664b 100644
--- a/flink-java/pom.xml
+++ b/flink-java/pom.xml
@@ -69,6 +69,39 @@ under the License.
 			<version>0.5.1</version>
 		</dependency>
 
+		<dependency>
+			<groupId>com.twitter</groupId>
+			<artifactId>chill-protobuf</artifactId>
+			<version>0.5.1</version>
+		</dependency>
+		<!-- We need protobuf for chill-protobuf -->
+		<dependency>
+			<groupId>com.google.protobuf</groupId>
+			<artifactId>protobuf-java</artifactId>
+			<version>2.5.0</version>
+		</dependency>
+
+		<dependency>
+			<groupId>com.twitter</groupId>
+			<artifactId>chill-thrift</artifactId>
+			<version>0.5.1</version>
+		</dependency>
+		<!-- libthrift is required by chill-thrift -->
+		<dependency>
+			<groupId>org.apache.thrift</groupId>
+			<artifactId>libthrift</artifactId>
+			<version>0.6.1</version>
+			<exclusions>
+				<exclusion>
+					<groupId>javax.servlet</groupId>
+					<artifactId>servlet-api</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.apache.httpcomponents</groupId>
+					<artifactId>httpclient</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
 
 		<!--  guava needs to be in "provided" scope, to make sure it is not included into the
jars by the shading -->
 		<dependency>

http://git-wip-us.apache.org/repos/asf/flink/blob/c4c3db50/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java
b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java
index f7a90a5..da09242 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java
@@ -26,15 +26,15 @@ import com.esotericsoftware.kryo.io.Input;
 import com.esotericsoftware.kryo.io.Output;
 import com.esotericsoftware.kryo.serializers.CollectionSerializer;
 import com.esotericsoftware.kryo.serializers.JavaSerializer;
+import com.google.protobuf.Message;
 import com.twitter.chill.ScalaKryoInstantiator;
 
-import org.apache.avro.Schema;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.specific.SpecificRecordBase;
+import com.twitter.chill.protobuf.ProtobufSerializer;
+import com.twitter.chill.thrift.TBaseSerializer;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
-import scala.reflect.ClassTag;
+import org.apache.thrift.TBase;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -216,6 +216,12 @@ public class KryoSerializer<T> extends TypeSerializer<T>
{
 			// Throwable and all subclasses should be serialized via java serialization
 			kryo.addDefaultSerializer(Throwable.class, new JavaSerializer());
 
+			// add serializers for popular other serialization frameworks
+			// Google Protobuf (FLINK-1392)
+			this.kryo.addDefaultSerializer(Message.class, ProtobufSerializer.class);
+			// thrift
+			this.kryo.addDefaultSerializer(TBase.class, TBaseSerializer.class);
+
 			// If the type we have to serialize as a GenricType is implementing SpecificRecordBase,
 			// we have to register the avro serializer
 			// This rule only applies if users explicitly use the GenericTypeInformation for the avro
types

http://git-wip-us.apache.org/repos/asf/flink/blob/c4c3db50/flink-shaded/pom.xml
----------------------------------------------------------------------
diff --git a/flink-shaded/pom.xml b/flink-shaded/pom.xml
index 4dbae5d..51191c7 100644
--- a/flink-shaded/pom.xml
+++ b/flink-shaded/pom.xml
@@ -25,7 +25,7 @@ under the License.
 	<parent>
 		<groupId>org.apache.flink</groupId>
 		<artifactId>flink-parent</artifactId>
-		<version>0.8-SNAPSHOT</version>
+		<version>0.9-SNAPSHOT</version>
 		<relativePath>..</relativePath>
 	</parent>
 
@@ -71,6 +71,9 @@ under the License.
 						<relocation>
 							<pattern>com.google</pattern>
 							<shadedPattern>org.apache.flink.shaded.com.google</shadedPattern>
+							<excludes>
+								<exclude>com.google.protobuf.**</exclude>
+							</excludes>
 						</relocation>
 					</relocations>
 				</configuration>

http://git-wip-us.apache.org/repos/asf/flink/blob/c4c3db50/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e59f340..84690b2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -277,7 +277,8 @@ under the License.
 					</exclusion>
 				</exclusions>
 			</dependency>
-			<!-- YARN -->
+
+			<!-- Hadoop 2 Dependencies -->
 			<dependency>
 				<groupId>org.apache.hadoop</groupId>
 				<artifactId>hadoop-common</artifactId>
@@ -853,6 +854,9 @@ under the License.
 						<relocation>
 							<pattern>com.google</pattern>
 							<shadedPattern>org.apache.flink.shaded.com.google</shadedPattern>
+							<excludes>
+								<exclude>com.google.protobuf.**</exclude>
+							</excludes>
 						</relocation>
 					</relocations>
 				</configuration>


Mime
View raw message