From commits-return-14638-archive-asf-public=cust-asf.ponee.io@pulsar.incubator.apache.org Wed Sep 19 05:17:26 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 35BE8180672 for ; Wed, 19 Sep 2018 05:17:25 +0200 (CEST) Received: (qmail 84883 invoked by uid 500); 19 Sep 2018 03:17:24 -0000 Mailing-List: contact commits-help@pulsar.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@pulsar.incubator.apache.org Delivered-To: mailing list commits@pulsar.incubator.apache.org Received: (qmail 84873 invoked by uid 99); 19 Sep 2018 03:17:24 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 19 Sep 2018 03:17:24 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 9483E82C36; Wed, 19 Sep 2018 03:17:23 +0000 (UTC) Date: Wed, 19 Sep 2018 03:17:23 +0000 To: "commits@pulsar.apache.org" Subject: [incubator-pulsar] branch master updated: fix behavior of JSONSchema for derived classes (#2577) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <153732704343.5216.12193270906084863542@gitbox.apache.org> From: sanjeevrk@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: incubator-pulsar X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 49fc5e508a996cfe59949effbcaf0abfa46028ce X-Git-Newrev: 701f3ea7e3541f61f064167043de04e8a2db5c0f X-Git-Rev: 701f3ea7e3541f61f064167043de04e8a2db5c0f X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. sanjeevrk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 701f3ea fix behavior of JSONSchema for derived classes (#2577) 701f3ea is described below commit 701f3ea7e3541f61f064167043de04e8a2db5c0f Author: Boyang Jerry Peng AuthorDate: Tue Sep 18 20:17:17 2018 -0700 fix behavior of JSONSchema for derived classes (#2577) * fix behavior of JSONSchema for derived classes * adding pom changes * fix for nested classes * adding to test * removing debug log * improving tests --- pulsar-client-schema/pom.xml | 6 ++ .../pulsar/client/impl/schema/JSONSchema.java | 41 ++++++++++--- .../pulsar/client/schema/JSONSchemaTest.java | 67 ++++++++++++++++++++-- .../pulsar/client/schema/SchemaTestUtils.java | 42 +++++++++++--- 4 files changed, 138 insertions(+), 18 deletions(-) diff --git a/pulsar-client-schema/pom.xml b/pulsar-client-schema/pom.xml index 477627e..5e72f6b 100644 --- a/pulsar-client-schema/pom.xml +++ b/pulsar-client-schema/pom.xml @@ -70,6 +70,12 @@ jackson-module-jsonSchema + + com.google.code.gson + gson + ${gson.version} + + org.apache.pulsar diff --git a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java index 5465f8c..d46c84b 100644 --- a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java +++ b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java @@ -22,28 +22,53 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.module.jsonSchema.JsonSchema; import com.fasterxml.jackson.module.jsonSchema.JsonSchemaGenerator; +import com.google.gson.ExclusionStrategy; +import com.google.gson.FieldAttributes; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.reflect.ReflectData; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SchemaSerializationException; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; -import java.io.IOException; import java.util.Collections; +import java.util.HashSet; import java.util.Map; +import java.util.Set; +@Slf4j public class JSONSchema implements Schema{ private final org.apache.avro.Schema schema; private final SchemaInfo schemaInfo; - private final ObjectMapper objectMapper; + private final Gson gson; private final Class pojo; private Map properties; private JSONSchema(Class pojo, Map properties) { this.pojo = pojo; this.properties = properties; - this.objectMapper = new ObjectMapper(); + this.gson = new GsonBuilder().addSerializationExclusionStrategy(new ExclusionStrategy() { + Set classes = new HashSet<>(); + + @Override + public boolean shouldSkipField(FieldAttributes f) { + boolean skip = !(f.getDeclaringClass().equals(pojo) + || classes.contains(f.getDeclaringClass().getName()) + || f.getDeclaringClass().isAssignableFrom(pojo)); + if (!skip) { + classes.add(f.getDeclaredClass().getName()); + } + return skip; + } + + @Override + public boolean shouldSkipClass(Class clazz) { + return false; + } + }).create(); this.schema = ReflectData.AllowNull.get().getSchema(pojo); this.schemaInfo = new SchemaInfo(); @@ -55,9 +80,10 @@ public class JSONSchema implements Schema{ @Override public byte[] encode(T message) throws SchemaSerializationException { + try { - return objectMapper.writeValueAsBytes(message); - } catch (JsonProcessingException e) { + return this.gson.toJson(message).getBytes(); + } catch (RuntimeException e) { throw new SchemaSerializationException(e); } } @@ -65,8 +91,8 @@ public class JSONSchema implements Schema{ @Override public T decode(byte[] bytes) { try { - return objectMapper.readValue(new String(bytes), pojo); - } catch (IOException e) { + return this.gson.fromJson(new String(bytes), this.pojo); + } catch (RuntimeException e) { throw new RuntimeException(new SchemaSerializationException(e)); } } @@ -85,6 +111,7 @@ public class JSONSchema implements Schema{ public SchemaInfo getBackwardsCompatibleJsonSchemaInfo() { SchemaInfo backwardsCompatibleSchemaInfo; try { + ObjectMapper objectMapper = new ObjectMapper(); JsonSchemaGenerator schemaGen = new JsonSchemaGenerator(objectMapper); JsonSchema jsonBackwardsCompatibileSchema = schemaGen.generateSchema(pojo); backwardsCompatibleSchemaInfo = new SchemaInfo(); diff --git a/pulsar-client-schema/src/test/java/org/apache/pulsar/client/schema/JSONSchemaTest.java b/pulsar-client-schema/src/test/java/org/apache/pulsar/client/schema/JSONSchemaTest.java index 7a677f4..c7b16bf 100644 --- a/pulsar-client-schema/src/test/java/org/apache/pulsar/client/schema/JSONSchemaTest.java +++ b/pulsar-client-schema/src/test/java/org/apache/pulsar/client/schema/JSONSchemaTest.java @@ -18,17 +18,20 @@ */ package org.apache.pulsar.client.schema; -import static org.apache.pulsar.client.schema.SchemaTestUtils.FOO_FIELDS; -import static org.apache.pulsar.client.schema.SchemaTestUtils.SCHEMA_JSON; - +import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; import org.apache.pulsar.client.impl.schema.JSONSchema; import org.apache.pulsar.client.schema.SchemaTestUtils.Bar; +import org.apache.pulsar.client.schema.SchemaTestUtils.DerivedFoo; import org.apache.pulsar.client.schema.SchemaTestUtils.Foo; import org.apache.pulsar.common.schema.SchemaType; import org.testng.Assert; import org.testng.annotations.Test; +import static org.apache.pulsar.client.schema.SchemaTestUtils.FOO_FIELDS; +import static org.apache.pulsar.client.schema.SchemaTestUtils.SCHEMA_JSON; + +@Slf4j public class JSONSchemaTest { @Test @@ -37,6 +40,7 @@ public class JSONSchemaTest { Assert.assertEquals(jsonSchema.getSchemaInfo().getType(), SchemaType.JSON); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(jsonSchema.getSchemaInfo().getSchema()); + log.info("schemaJson: {}", schemaJson); Assert.assertEquals(schemaJson, SCHEMA_JSON); Schema schema = parser.parse(schemaJson); @@ -54,10 +58,14 @@ public class JSONSchemaTest { public void testEncodeAndDecode() { JSONSchema jsonSchema = JSONSchema.of(Foo.class, null); + Bar bar = new Bar(); + bar.setField1(true); + Foo foo1 = new Foo(); foo1.setField1("foo1"); foo1.setField2("bar1"); - foo1.setField4(new Bar()); + foo1.setField4(bar); + foo1.setColor(SchemaTestUtils.Color.BLUE); Foo foo2 = new Foo(); foo2.setField1("foo2"); @@ -75,4 +83,55 @@ public class JSONSchemaTest { Assert.assertEquals(object1, foo1); Assert.assertEquals(object2, foo2); } + + @Test + public void testCorrectPolymorphism() { + + Bar bar = new Bar(); + bar.setField1(true); + + DerivedFoo derivedFoo = new DerivedFoo(); + derivedFoo.setField1("foo1"); + derivedFoo.setField2("bar2"); + derivedFoo.setField3(4); + derivedFoo.setField4(bar); + derivedFoo.setField5("derived1"); + derivedFoo.setField6(2); + + Foo foo = new Foo(); + foo.setField1("foo1"); + foo.setField2("bar2"); + foo.setField3(4); + foo.setField4(bar); + + SchemaTestUtils.DerivedDerivedFoo derivedDerivedFoo = new SchemaTestUtils.DerivedDerivedFoo(); + derivedDerivedFoo.setField1("foo1"); + derivedDerivedFoo.setField2("bar2"); + derivedDerivedFoo.setField3(4); + derivedDerivedFoo.setField4(bar); + derivedDerivedFoo.setField5("derived1"); + derivedDerivedFoo.setField6(2); + derivedDerivedFoo.setFoo2(foo); + derivedDerivedFoo.setDerivedFoo(derivedFoo); + + // schema for base class + JSONSchema baseJsonSchema = JSONSchema.of(Foo.class); + Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(foo)), foo); + Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedFoo)), foo); + Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedDerivedFoo)), foo); + + // schema for derived class + JSONSchema derivedJsonSchema = JSONSchema.of(DerivedFoo.class); + Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedFoo)), derivedFoo); + log.info("derivedJsonSchema.encode(derivedDerivedFoo)): {}", derivedJsonSchema.decode(derivedJsonSchema.encode(derivedDerivedFoo))); + Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedDerivedFoo)), derivedFoo); + + //schema for derived derived class + JSONSchema derivedDerivedJsonSchema + = JSONSchema.of(SchemaTestUtils.DerivedDerivedFoo.class); + Assert.assertEquals(derivedDerivedJsonSchema.decode(derivedDerivedJsonSchema.encode(derivedDerivedFoo)), derivedDerivedFoo); + + + + } } diff --git a/pulsar-client-schema/src/test/java/org/apache/pulsar/client/schema/SchemaTestUtils.java b/pulsar-client-schema/src/test/java/org/apache/pulsar/client/schema/SchemaTestUtils.java index 52d80eb..123b81c 100644 --- a/pulsar-client-schema/src/test/java/org/apache/pulsar/client/schema/SchemaTestUtils.java +++ b/pulsar-client-schema/src/test/java/org/apache/pulsar/client/schema/SchemaTestUtils.java @@ -35,6 +35,7 @@ public class SchemaTestUtils { private String field2; private int field3; private Bar field4; + private Color color; } @Data @@ -44,18 +45,45 @@ public class SchemaTestUtils { private boolean field1; } - public static final String SCHEMA_JSON = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache" + - ".pulsar.client" + - ".schema.SchemaTestUtils$\",\"fields\":[{\"name\":\"field1\",\"type\":[\"null\",\"string\"]," + - "\"default\":null},{\"name\":\"field2\",\"type\":[\"null\",\"string\"],\"default\":null}," + - "{\"name\":\"field3\",\"type\":\"int\"},{\"name\":\"field4\",\"type\":[\"null\",{\"type\":\"record\"," + - "\"name\":\"Bar\",\"fields\":[{\"name\":\"field1\",\"type\":\"boolean\"}]}],\"default\":null}]}"; + @Data + @ToString + @EqualsAndHashCode + public static class DerivedFoo extends Foo { + private String field5; + private int field6; + private Foo foo; + } + + public enum Color { + RED, + BLUE + } + + @Data + @ToString + @EqualsAndHashCode + public static class DerivedDerivedFoo extends DerivedFoo { + private String field7; + private int field8; + private DerivedFoo derivedFoo; + private Foo foo2; + } + + public static final String SCHEMA_JSON + = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.schema" + + ".SchemaTestUtils$\",\"fields\":[{\"name\":\"field1\",\"type\":[\"null\",\"string\"],\"default\":null}," + + "{\"name\":\"field2\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field3\"," + + "\"type\":\"int\"},{\"name\":\"field4\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"Bar\"," + + "\"fields\":[{\"name\":\"field1\",\"type\":\"boolean\"}]}],\"default\":null},{\"name\":\"color\"," + + "\"type\":[\"null\",{\"type\":\"enum\",\"name\":\"Color\",\"symbols\":[\"RED\",\"BLUE\"]}]," + + "\"default\":null}]}"; public static String[] FOO_FIELDS = { "field1", "field2", "field3", - "field4" + "field4", + "color" }; }