From commits-return-26511-archive-asf-public=cust-asf.ponee.io@geode.apache.org Wed Apr 11 02:15:46 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 4388418064C for ; Wed, 11 Apr 2018 02:15:45 +0200 (CEST) Received: (qmail 42305 invoked by uid 500); 11 Apr 2018 00:15:44 -0000 Mailing-List: contact commits-help@geode.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@geode.apache.org Delivered-To: mailing list commits@geode.apache.org Received: (qmail 42296 invoked by uid 99); 11 Apr 2018 00:15:44 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 11 Apr 2018 00:15:44 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 4787782FDC; Wed, 11 Apr 2018 00:15:43 +0000 (UTC) Date: Wed, 11 Apr 2018 00:15:43 +0000 To: "commits@geode.apache.org" Subject: [geode] branch feature/transcoding_experiments updated: Handle nested objects/ArrayLists in ProtobufStructSerializer MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <152340574275.22913.6277761712025168921@gitbox.apache.org> From: upthewaterspout@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: geode X-Git-Refname: refs/heads/feature/transcoding_experiments X-Git-Reftype: branch X-Git-Oldrev: 68b65862dc44062ca687a26d0cb53394324f4c32 X-Git-Newrev: 1c7d1376b0b20cd4169157923bfb12b0994f427e X-Git-Rev: 1c7d1376b0b20cd4169157923bfb12b0994f427e X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. upthewaterspout pushed a commit to branch feature/transcoding_experiments in repository https://gitbox.apache.org/repos/asf/geode.git The following commit(s) were added to refs/heads/feature/transcoding_experiments by this push: new 1c7d137 Handle nested objects/ArrayLists in ProtobufStructSerializer 1c7d137 is described below commit 1c7d1376b0b20cd4169157923bfb12b0994f427e Author: Dan Smith AuthorDate: Tue Apr 10 17:13:07 2018 -0700 Handle nested objects/ArrayLists in ProtobufStructSerializer Adding support for nested objects and lists. Currently, only PdxInstances with nested PdxInstance or ArrayList fields are supported, other object types are not supported. --- .../serialization/ProtobufStructSerializer.java | 150 +++++++++++++-------- .../serialization/PdxInstanceFactoryMock.java | 35 ----- .../serialization/PdxInstanceGenerator.java | 62 +++++++-- .../ProtobufStructSerializerTest.java | 104 +++++++++++--- 4 files changed, 232 insertions(+), 119 deletions(-) diff --git a/geode-protobuf/src/main/java/org/apache/geode/protocol/serialization/ProtobufStructSerializer.java b/geode-protobuf/src/main/java/org/apache/geode/protocol/serialization/ProtobufStructSerializer.java index c9a0ba7..d653936 100644 --- a/geode-protobuf/src/main/java/org/apache/geode/protocol/serialization/ProtobufStructSerializer.java +++ b/geode-protobuf/src/main/java/org/apache/geode/protocol/serialization/ProtobufStructSerializer.java @@ -15,13 +15,18 @@ package org.apache.geode.protocol.serialization; import java.io.IOException; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import com.google.protobuf.ByteString; +import com.google.protobuf.NullValue; import com.google.protobuf.UnsafeByteOperations; import org.apache.geode.cache.Cache; import org.apache.geode.internal.protocol.protobuf.v1.BasicTypes; +import org.apache.geode.internal.protocol.protobuf.v1.ListValue; +import org.apache.geode.internal.protocol.protobuf.v1.ProtobufSerializationService; import org.apache.geode.internal.protocol.protobuf.v1.Struct; import org.apache.geode.internal.protocol.protobuf.v1.Value; import org.apache.geode.pdx.PdxInstance; @@ -33,85 +38,114 @@ public class ProtobufStructSerializer implements ValueSerializer { @Override public ByteString serialize(Object object) throws IOException { + return serializeStruct(object).toByteString(); + } + + private Struct serializeStruct(Object object) { + PdxInstance pdxInstance = (PdxInstance) object; Struct.Builder structBuilder = Struct.newBuilder(); for (String fieldName : pdxInstance.getFieldNames()) { Object value = pdxInstance.getField(fieldName); - Value.Builder builder = Value.newBuilder(); + Value serialized = serializeValue(value); + structBuilder.putFields(fieldName, serialized); + } + + return structBuilder.build(); + } + + private Value serializeValue(Object value) { + Value.Builder builder = Value.newBuilder(); + if (value instanceof String) { + builder.setEncodedValue( + BasicTypes.EncodedValue.newBuilder().setStringResult((String) value).build()); + } else if (value instanceof Boolean) { + builder.setEncodedValue( + BasicTypes.EncodedValue.newBuilder().setBooleanResult((Boolean) value).build()); + } else if (value instanceof Integer) { + builder.setEncodedValue( + BasicTypes.EncodedValue.newBuilder().setIntResult((Integer) value).build()); + } else if (value instanceof Byte) { + builder.setEncodedValue( + BasicTypes.EncodedValue.newBuilder().setByteResult((Byte) value).build()); + } else if (value instanceof Long) { + builder.setEncodedValue( + BasicTypes.EncodedValue.newBuilder().setLongResult((Long) value).build()); + } else if (value instanceof byte[]) { + builder.setEncodedValue(BasicTypes.EncodedValue.newBuilder() + .setBinaryResult(UnsafeByteOperations.unsafeWrap((byte[]) value)).build()); + } else if (value instanceof PdxInstance) { + builder.setStructValue(serializeStruct(value)); + } else if (value instanceof List) { + List values = + ((List) value).stream().map(this::serializeValue).collect(Collectors.toList()); + builder.setListValue(ListValue.newBuilder().addAllValues(values).build()); + } else if (value == null) { + builder.setEncodedValue( + BasicTypes.EncodedValue.newBuilder().setNullResult(NullValue.NULL_VALUE).build()); + } else { + throw new IllegalStateException( + "Don't know how to translate object of type " + value.getClass() + ": " + value); + } + return builder.build(); + } + + @Override + public Object deserialize(ByteString bytes) throws IOException, ClassNotFoundException { + Struct struct = Struct.parseFrom(bytes); + return deserialize(struct); + } + + private Object deserialize(Struct struct) { + PdxInstanceFactory pdxInstanceFactory = cache.createPdxInstanceFactory(PROTOBUF_STRUCT); + + for (Map.Entry field : struct.getFieldsMap().entrySet()) { + String fieldName = field.getKey(); + Object value = deserialize(field.getValue()); if (value instanceof String) { - builder.setEncodedValue( - BasicTypes.EncodedValue.newBuilder().setStringResult((String) value).build()); + pdxInstanceFactory.writeString(fieldName, (String) value); } else if (value instanceof Boolean) { - builder.setEncodedValue( - BasicTypes.EncodedValue.newBuilder().setBooleanResult((Boolean) value).build()); + pdxInstanceFactory.writeBoolean(fieldName, (Boolean) value); } else if (value instanceof Integer) { - builder.setEncodedValue( - BasicTypes.EncodedValue.newBuilder().setIntResult((Integer) value).build()); + pdxInstanceFactory.writeInt(fieldName, (Integer) value); } else if (value instanceof Byte) { - builder.setEncodedValue( - BasicTypes.EncodedValue.newBuilder().setByteResult((Byte) value).build()); + pdxInstanceFactory.writeByte(fieldName, (Byte) value); } else if (value instanceof Long) { - builder.setEncodedValue( - BasicTypes.EncodedValue.newBuilder().setLongResult((Long) value).build()); + pdxInstanceFactory.writeLong(fieldName, (Long) value); } else if (value instanceof byte[]) { - builder.setEncodedValue(BasicTypes.EncodedValue.newBuilder() - .setBinaryResult(UnsafeByteOperations.unsafeWrap((byte[]) value)).build()); + pdxInstanceFactory.writeByteArray(fieldName, (byte[]) value); + } else if (value instanceof PdxInstance) { + pdxInstanceFactory.writeObject(fieldName, value); + } else if (value instanceof List) { + pdxInstanceFactory.writeObject(fieldName, value); + } else if (value == null) { + pdxInstanceFactory.writeObject(fieldName, null); } else { throw new IllegalStateException( "Don't know how to translate object of type " + value.getClass() + ": " + value); } - structBuilder.putFields(fieldName, builder.build()); } - return structBuilder.build().toByteString(); + return pdxInstanceFactory.create(); } - @Override - public Object deserialize(ByteString bytes) throws IOException, ClassNotFoundException { - Struct struct = Struct.parseFrom(bytes); - PdxInstanceFactory pdxInstanceFactory = cache.createPdxInstanceFactory(PROTOBUF_STRUCT); + private List deserialize(List listValue) { + return listValue.stream().map(this::deserialize).collect(Collectors.toList()); + } - for (Map.Entry field : struct.getFieldsMap().entrySet()) { - String fieldName = field.getKey(); - Value value = field.getValue(); - switch (value.getKindCase()) { - case ENCODEDVALUE: - switch (value.getEncodedValue().getValueCase()) { - case STRINGRESULT: - pdxInstanceFactory.writeString(fieldName, value.getEncodedValue().getStringResult()); - break; - case BOOLEANRESULT: - pdxInstanceFactory.writeBoolean(fieldName, - value.getEncodedValue().getBooleanResult()); - break; - case INTRESULT: - pdxInstanceFactory.writeInt(fieldName, value.getEncodedValue().getIntResult()); - break; - case BYTERESULT: - pdxInstanceFactory.writeByte(fieldName, - (byte) value.getEncodedValue().getByteResult()); - break; - case LONGRESULT: - pdxInstanceFactory.writeLong(fieldName, value.getEncodedValue().getLongResult()); - break; - case BINARYRESULT: - pdxInstanceFactory.writeByteArray(fieldName, - value.getEncodedValue().getBinaryResult().toByteArray()); - break; - default: - throw new IllegalStateException("Don't know how to translate object of type " - + value.getEncodedValue().getValueCase() + ": " + value); - } - break; - - default: - throw new IllegalStateException( - "Don't know how to translate object of type " + value.getKindCase() + ": " + value); - } + private Object deserialize(Value value) { + switch (value.getKindCase()) { + case ENCODEDVALUE: + return new ProtobufSerializationService().decode(value.getEncodedValue()); + case STRUCTVALUE: + return deserialize(value.getStructValue()); + case LISTVALUE: + return deserialize(value.getListValue().getValuesList()); + default: + throw new IllegalStateException( + "Don't know how to translate object of type " + value.getKindCase() + ": " + value); } - - return pdxInstanceFactory.create(); } @Override diff --git a/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/PdxInstanceFactoryMock.java b/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/PdxInstanceFactoryMock.java deleted file mode 100644 index 32d477e..0000000 --- a/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/PdxInstanceFactoryMock.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.geode.protocol.serialization; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import org.apache.geode.internal.cache.InternalCache; -import org.apache.geode.pdx.PdxInstanceFactory; -import org.apache.geode.pdx.internal.PdxInstanceFactoryImpl; -import org.apache.geode.pdx.internal.TypeRegistration; -import org.apache.geode.pdx.internal.TypeRegistry; - -public class PdxInstanceFactoryMock { - - public static final PdxInstanceFactory createMockFactory(String className) { - TypeRegistration registration = mock(TypeRegistration.class); - InternalCache cache = mock(InternalCache.class); - when(cache.getPdxRegistry()).thenReturn(new TypeRegistry(registration)); - - return PdxInstanceFactoryImpl.newCreator(ProtobufStructSerializer.PROTOBUF_STRUCT, true, cache); - } -} diff --git a/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/PdxInstanceGenerator.java b/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/PdxInstanceGenerator.java index 0985e7e..5f13f6f 100644 --- a/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/PdxInstanceGenerator.java +++ b/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/PdxInstanceGenerator.java @@ -24,9 +24,12 @@ import java.lang.annotation.Retention; import java.lang.annotation.Target; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; @@ -35,12 +38,35 @@ import com.pholser.junit.quickcheck.generator.Generator; import com.pholser.junit.quickcheck.generator.GeneratorConfiguration; import com.pholser.junit.quickcheck.random.SourceOfRandomness; +import org.apache.geode.cache.CacheFactory; import org.apache.geode.pdx.PdxInstance; import org.apache.geode.pdx.PdxInstanceFactory; public class PdxInstanceGenerator extends Generator { + private static final Map, Method> supportedTypes; + public static final GenerationStatus.Key DEPTH = new GenerationStatus.Key("depth", Integer.class); + + static { + HashMap, Method> types = new HashMap<>(); + Method[] methods = PdxInstanceFactory.class.getDeclaredMethods(); + for (Method method : methods) { + if (method.getName().startsWith("write") && method.getParameterTypes().length == 2) { + Class type = method.getParameterTypes()[1]; + + if (type == Object.class) { + types.put(PdxInstance.class, method); + types.put(ArrayList.class, method); + } else { + types.put(type, method); + } + } + } + + supportedTypes = Collections.unmodifiableMap(types); + + } private Set> allowedFieldTypes; private String className = "NO_CLASS"; @@ -55,14 +81,31 @@ public class PdxInstanceGenerator extends Generator { Map, Method> writeMethods = getAllowedWriteMethods(); int numFields = random.nextInt(0, 20); - PdxInstanceFactory factory = PdxInstanceFactoryMock.createMockFactory(className); + PdxInstanceFactory factory = CacheFactory.getAnyInstance().createPdxInstanceFactory(className); Set fieldNames = new HashSet<>(gen().type(String.class).times(numFields).generate(random, status)); for (String fieldName : fieldNames) { Map.Entry, Method> writeMethod = random.choose(writeMethods.entrySet()); Class type = writeMethod.getKey(); Method method = writeMethod.getValue(); - Object value = gen().type(type).generate(random, status); + Object value = null; + if (type == PdxInstance.class) { + int depth = (int) status.valueOf(DEPTH).orElse(0); + if (depth < status.size()) { + status.setValue(DEPTH, depth + 1); + value = generate(random, status); + } + } else if (type == ArrayList.class) { + int depth = (int) status.valueOf(DEPTH).orElse(0); + if (depth < status.size()) { + status.setValue(DEPTH, depth + 1); + ArrayList list = new ArrayList<>(); + list.add((PdxInstance) generate(random, status)); + value = list; + } + } else { + value = gen().type(type).generate(random, status); + } try { method.invoke(factory, fieldName, value); } catch (IllegalAccessException | InvocationTargetException e) { @@ -74,16 +117,13 @@ public class PdxInstanceGenerator extends Generator { } private Map, Method> getAllowedWriteMethods() { - final Map, Method> writeMethods = new HashMap<>(); + final Map, Method> writeMethods = new HashMap<>(supportedTypes); + writeMethods.keySet().retainAll(allowedFieldTypes); - Method[] methods = PdxInstanceFactory.class.getDeclaredMethods(); - for (Method method : methods) { - if (method.getName().startsWith("write") && method.getParameterTypes().length == 2 - && (allowedFieldTypes == null - || allowedFieldTypes.contains(method.getParameterTypes()[1]))) { - Class type = method.getParameterTypes()[1]; - writeMethods.put(type, method); - } + if (writeMethods.size() != allowedFieldTypes.size()) { + HashSet> classes = new HashSet<>(allowedFieldTypes); + classes.removeAll(supportedTypes.keySet()); + throw new IllegalStateException("Cannot generate value of types " + classes); } return writeMethods; diff --git a/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/ProtobufStructSerializerTest.java b/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/ProtobufStructSerializerTest.java index 294f235..2f1401e 100644 --- a/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/ProtobufStructSerializerTest.java +++ b/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/ProtobufStructSerializerTest.java @@ -21,6 +21,9 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import com.google.protobuf.ByteString; import com.pholser.junit.quickcheck.From; @@ -28,14 +31,18 @@ import com.pholser.junit.quickcheck.Property; import com.pholser.junit.quickcheck.When; import com.pholser.junit.quickcheck.runner.JUnitQuickcheck; import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.apache.geode.cache.Cache; import org.apache.geode.cache.CacheFactory; +import org.apache.geode.distributed.ConfigurationProperties; import org.apache.geode.internal.protocol.protobuf.v1.BasicTypes; +import org.apache.geode.internal.protocol.protobuf.v1.ListValue; import org.apache.geode.internal.protocol.protobuf.v1.Struct; import org.apache.geode.internal.protocol.protobuf.v1.Value; import org.apache.geode.pdx.PdxInstance; @@ -47,50 +54,117 @@ import org.apache.geode.test.junit.categories.UnitTest; public class ProtobufStructSerializerTest { private ProtobufStructSerializer serializer; - private Cache cache; + private static Cache cache; + + @BeforeClass + public static void createCache() { + cache = new CacheFactory().set(ConfigurationProperties.LOG_LEVEL, "error") + .setPdxReadSerialized(true).create(); + } @Before public void createSerializer() { - cache = mock(Cache.class); - when(cache.createPdxInstanceFactory(any())) - .then(invocation -> PdxInstanceFactoryMock.createMockFactory(invocation.getArgument(0))); serializer = new ProtobufStructSerializer(); serializer.init(cache); } - @After - public void tearDown() { + @AfterClass + public static void tearDown() { cache.close(); } @Test public void testDeserialize() throws IOException, ClassNotFoundException { - Struct struct = Struct.newBuilder() - .putFields("field1", Value.newBuilder() - .setEncodedValue(BasicTypes.EncodedValue.newBuilder().setStringResult("value")).build()) - .build(); + Struct struct = structWithStringField(); ByteString bytes = struct.toByteString(); PdxInstance value = (PdxInstance) serializer.deserialize(bytes); assertEquals("value", value.getField("field1")); } + private Struct structWithStringField() { + return Struct.newBuilder() + .putFields("field1", Value.newBuilder() + .setEncodedValue(BasicTypes.EncodedValue.newBuilder().setStringResult("value")).build()) + .build(); + } + @Test public void testSerialize() throws IOException, ClassNotFoundException { - PdxInstance value = cache.createPdxInstanceFactory(ProtobufStructSerializer.PROTOBUF_STRUCT) - .writeString("field1", "value").create(); + PdxInstance value = pdxWithStringField(); ByteString bytes = serializer.serialize(value); Struct struct = Struct.parseFrom(bytes); assertEquals("value", struct.getFieldsMap().get("field1").getEncodedValue().getStringResult()); } - @Property(trials = 100) + private PdxInstance pdxWithStringField() { + return cache.createPdxInstanceFactory(ProtobufStructSerializer.PROTOBUF_STRUCT) + .writeString("field1", "value").create(); + } + + @Test + public void canSerializeWithNestedPdxInstance() throws IOException, ClassNotFoundException { + PdxInstance value = cache.createPdxInstanceFactory(ProtobufStructSerializer.PROTOBUF_STRUCT) + .writeObject("field1", pdxWithStringField()).create(); + ByteString bytes = serializer.serialize(value); + Struct struct = Struct.parseFrom(bytes); + + assertEquals("value", struct.getFieldsMap().get("field1").getStructValue().getFieldsMap() + .get("field1").getEncodedValue().getStringResult()); + } + + @Test + public void canSerializeWithNestedList() throws IOException, ClassNotFoundException { + ArrayList list = new ArrayList<>(); + list.add(pdxWithStringField()); + PdxInstance value = cache.createPdxInstanceFactory(ProtobufStructSerializer.PROTOBUF_STRUCT) + .writeObject("field2", list).create(); + ByteString bytes = serializer.serialize(value); + Struct struct = Struct.parseFrom(bytes); + + assertEquals(Struct.newBuilder() + .putFields("field2", + Value.newBuilder() + .setListValue(ListValue.newBuilder() + .addValues(Value.newBuilder().setStructValue(structWithStringField()))) + .build()) + .build(), struct); + } + + @Test + public void canDeserializeWithNestedStruct() throws IOException, ClassNotFoundException { + Struct.Builder builder = Struct.newBuilder(); + builder.putFields("field1", Value.newBuilder().setStructValue(structWithStringField()).build()); + ByteString bytes = builder.build().toByteString(); + PdxInstance value = (PdxInstance) serializer.deserialize(bytes); + + PdxInstance nested = (PdxInstance) value.getField("field1"); + assertEquals("value", nested.getField("field1")); + } + + @Test + public void canDeserializeWithNestedList() throws IOException, ClassNotFoundException { + Struct.Builder builder = Struct.newBuilder(); + builder.putFields("field1", + Value.newBuilder() + .setListValue(ListValue.newBuilder().addValues(Value.newBuilder() + .setEncodedValue(BasicTypes.EncodedValue.newBuilder().setStringResult("value")))) + .build()); + ByteString bytes = builder.build().toByteString(); + PdxInstance value = (PdxInstance) serializer.deserialize(bytes); + + List nested = (List) value.getField("field1"); + assertEquals(Arrays.asList("value"), nested); + } + + + @Property(trials = 10) public void testSymmetry( @When( seed = 793351614853016898L) @PdxInstanceGenerator.ClassName(ProtobufStructSerializer.PROTOBUF_STRUCT) @PdxInstanceGenerator.FieldTypes({ - String.class, int.class, long.class, byte.class, - byte[].class}) @From(PdxInstanceGenerator.class) PdxInstance original) + String.class, int.class, long.class, byte.class, byte[].class, PdxInstance.class, + ArrayList.class}) @From(PdxInstanceGenerator.class) PdxInstance original) throws IOException, ClassNotFoundException { ByteString bytes = serializer.serialize(original); Struct struct = Struct.parseFrom(bytes); -- To stop receiving notification emails like this one, please contact upthewaterspout@apache.org.