avro-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Zoltan Farkas (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (AVRO-1667) Parser symbol tree flattening is broken for recursive schemas
Date Mon, 07 Mar 2016 20:31:40 GMT

    [ https://issues.apache.org/jira/browse/AVRO-1667?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15183646#comment-15183646
] 

Zoltan Farkas commented on AVRO-1667:
-------------------------------------

I extended, the unit test to complicate the recursiveness, an I am unable to break my original
implementation.

I like your implementation since it does not mutate the fixups... however I believe in my
implementation I was mutating the existing fixups, since the old ones were not valid fixups
anymore?
I still have to re-understand this....

In any case here is the lastest unit test code, I would really like to test and catch the
issue my original implementation had:

{code}

package org.apache.avro.io.parsing;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Set;
import junit.framework.Assert;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.io.BinaryDecoder;
import org.apache.avro.io.BinaryEncoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.EncoderFactory;
import org.junit.Test;


public class SymbolTest {


    private static final String SCHEMA = "{\"type\":\"record\",\"name\":\"SampleNode\","
            + "\"namespace\":\"org.spf4j.ssdump2.avro\",\n" +
" \"fields\":[\n" +
"    {\"name\":\"count\",\"type\":\"int\",\"default\":0},\n" +
"    {\"name\":\"otherNode\",\"type\":[\"null\",\"SampleNode\"], \"default\" : null},\n" +
"    {\"name\":\"subNodes\",\"type\":\n" +
"       {\"type\":\"array\",\"items\":{\n" +
"           \"type\":\"record\",\"name\":\"SamplePair\",\n" +
"           \"fields\":[\n" +
"              {\"name\":\"method\",\"type\":\n" +
"                  {\"type\":\"record\",\"name\":\"Method\",\n" +
"                  \"fields\":[\n" +
"                     {\"name\":\"declaringClass\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},\n"
+
"                     {\"name\":\"methodName\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}}\n"
+
"                  ]}},\n" +
"              {\"name\":\"node\",\"type\":\"SampleNode\"},"
            + "{\"name\":\"otherNode\",\"type\":\"SampleNode\"}]}}}"
            + ",{\"name\":\"subNodes2\",\"type\":\n" +
"       {\"type\":\"array\",\"items\": \"SampleNode\"}}"
           + ",{\"name\":\"subNodes3\",\"type\":\n" +
"       {\"type\":\"array\",\"items\": \"SampleNode\"}} ]}";


    @Test
    public void testValidSymbolTree() throws IOException {

        Schema schema = new Schema.Parser().parse(SCHEMA);

        Symbol root = Symbol.root(new ResolvingGrammarGenerator()
                .generate(schema, schema, new HashMap<ValidatingGrammarGenerator.LitS,
Symbol>()));
        validateNonNull(root, new HashSet<Symbol>());

        Schema samplePairSchema = schema.getField("subNodes").schema().getElementType();
        Schema methodSchema = samplePairSchema.getField("method").schema();

        GenericData.Record method = new GenericData.Record(methodSchema);
        method.put("methodName", "m1");
        method.put("declaringClass", "c1");

        GenericData.Record samplePair = new GenericData.Record(samplePairSchema);
        samplePair.put("method", method);

        GenericData.Record sampleNode1 = new GenericData.Record(schema);
        sampleNode1.put("subNodes", Collections.EMPTY_LIST);
        sampleNode1.put("subNodes2", Collections.EMPTY_LIST);
        sampleNode1.put("subNodes3", Collections.EMPTY_LIST);
        sampleNode1.put("count", 0);
        sampleNode1.put("otherNode", null);

        samplePair.put("node", sampleNode1);
        samplePair.put("otherNode", sampleNode1);

        GenericData.Record sampleNode2 = new GenericData.Record(schema);
        sampleNode2.put("subNodes", Arrays.asList(samplePair));
        sampleNode2.put("subNodes2", Arrays.asList(sampleNode1));
        sampleNode2.put("subNodes3", Arrays.asList(sampleNode1));
        sampleNode2.put("count", 0);
        sampleNode2.put("otherNode", null);

        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        GenericDatumWriter writer = new GenericDatumWriter(schema);
        BinaryEncoder directBinaryEncoder = new EncoderFactory().directBinaryEncoder(bos,
null);
        writer.write(sampleNode2, directBinaryEncoder);

        ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
        GenericDatumReader reader = new GenericDatumReader(schema);
        BinaryDecoder binaryDecoder = new DecoderFactory().binaryDecoder(bis, null);
        Object read = reader.read(null, binaryDecoder);

        System.out.println(read);
        Assert.assertEquals(read, sampleNode2);


    }

    private static void validateNonNull(final Symbol symb, Set<Symbol> seen) {
        if (seen.contains(symb)) {
            return;
        } else {
            seen.add(symb);
        }
        if (symb.production != null) {
            for (Symbol s : symb.production) {
                if (s == null) {
                    Assert.fail("invalid parsing tree should not contain nulls");
                }
                if (s.kind != Symbol.Kind.ROOT) {
                    validateNonNull(s, seen);;
                }
            }
        }
    }

}

{code}

> Parser symbol tree flattening is broken for recursive schemas
> -------------------------------------------------------------
>
>                 Key: AVRO-1667
>                 URL: https://issues.apache.org/jira/browse/AVRO-1667
>             Project: Avro
>          Issue Type: Bug
>    Affects Versions: 1.7.7
>            Reporter: Zoltan Farkas
>         Attachments: AVRO-1667.2.patch, avro-1667.patch
>
>
> Here is a unit test to reproduce:
> {noformat}
> package org.apache.avro.io.parsing;
> import java.io.IOException;
> import java.util.HashMap;
> import java.util.HashSet;
> import java.util.Set;
> import junit.framework.Assert;
> import org.apache.avro.Schema;
> import org.junit.Test;
> public class SymbolTest {
>     private static final String SCHEMA = "{\"type\":\"record\",\"name\":\"SampleNode\","
>             + "\"namespace\":\"org.spf4j.ssdump2.avro\",\n" +
> " \"fields\":[\n" +
> "    {\"name\":\"count\",\"type\":\"int\",\"default\":0},\n" +
> "    {\"name\":\"subNodes\",\"type\":\n" +
> "       {\"type\":\"array\",\"items\":{\n" +
> "           \"type\":\"record\",\"name\":\"SamplePair\",\n" +
> "           \"fields\":[\n" +
> "              {\"name\":\"method\",\"type\":\n" +
> "                  {\"type\":\"record\",\"name\":\"Method\",\n" +
> "                  \"fields\":[\n" +
> "                     {\"name\":\"declaringClass\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},\n"
+
> "                     {\"name\":\"methodName\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}}\n"
+
> "                  ]}},\n" +
> "              {\"name\":\"node\",\"type\":\"SampleNode\"}]}}}]}";
>     @Test
>     public void testSomeMethod() throws IOException {
>         Schema schema = new Schema.Parser().parse(SCHEMA);
>         Symbol root = Symbol.root(new ResolvingGrammarGenerator()
>                 .generate(schema, schema, new HashMap<ValidatingGrammarGenerator.LitS,
Symbol>()));
>         validateNonNull(root, new HashSet<Symbol>());
>     }
>     private static void validateNonNull(final Symbol symb, Set<Symbol> seen) {
>         if (seen.contains(symb)) {
>             return;
>         } else {
>             seen.add(symb);
>         }
>         if (symb.production != null) {
>             for (Symbol s : symb.production) {
>                 if (s == null) {
>                     Assert.fail("invalid parsing tree should not contain nulls");
>                 }
>                 if (s.kind != Symbol.Kind.ROOT) {
>                     validateNonNull(s, seen);;
>                 }
>             }
>         }
>     }
> }
> {noformat}



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Mime
View raw message