Return-Path: Delivered-To: apmail-hadoop-avro-commits-archive@minotaur.apache.org Received: (qmail 28623 invoked from network); 22 Jul 2009 20:20:20 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 22 Jul 2009 20:20:20 -0000 Received: (qmail 80054 invoked by uid 500); 22 Jul 2009 20:21:26 -0000 Delivered-To: apmail-hadoop-avro-commits-archive@hadoop.apache.org Received: (qmail 80031 invoked by uid 500); 22 Jul 2009 20:21:26 -0000 Mailing-List: contact avro-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: avro-dev@hadoop.apache.org Delivered-To: mailing list avro-commits@hadoop.apache.org Received: (qmail 80021 invoked by uid 99); 22 Jul 2009 20:21:25 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 22 Jul 2009 20:21:25 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 22 Jul 2009 20:21:19 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 311DE238888F; Wed, 22 Jul 2009 20:20:57 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r796868 [2/2] - in /hadoop/avro/trunk: ./ src/java/org/apache/avro/generic/ src/java/org/apache/avro/io/ src/java/org/apache/avro/io/doc-files/ src/java/org/apache/avro/io/parsing/ src/java/org/apache/avro/io/parsing/doc-files/ src/java/org... Date: Wed, 22 Jul 2009 20:20:56 -0000 To: avro-commits@hadoop.apache.org From: cutting@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20090722202057.311DE238888F@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Added: hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/Symbol.java URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/Symbol.java?rev=796868&view=auto ============================================================================== --- hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/Symbol.java (added) +++ hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/Symbol.java Wed Jul 22 20:20:55 2009 @@ -0,0 +1,382 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.io.parsing; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +import org.codehaus.jackson.JsonEncoding; +import org.codehaus.jackson.JsonFactory; +import org.codehaus.jackson.JsonGenerator; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.ObjectMapper; + +/** + * Symbol is the base of all symbols (terminals and non-terminals) of + * the grammar. + */ +public abstract class Symbol { + /* + * The type of symbol. + */ + public enum Kind { + /** terminal symbols which have no productions */ + TERMINAL, + /** Start symbol for some grammar */ + ROOT, + /** non-terminal symbol which is a sequence of one or more other symbols */ + SEQUENCE, + /** non-termial to represent the contents of an array or map */ + REPEATER, + /** non-terminal to represent the union */ + ALTERNATIVE, + /** non-terminal action symbol which are automatically consumed */ + IMPLICIT_ACTION, + /** non-terminal action symbol which is explicitly consumed */ + EXPLICIT_ACTION + }; + + /// The kind of this symbol. + public final Kind kind; + + /** + * The production for this symbol. If this symbol is a terminal + * this is null. Otherwise this holds the the sequence of + * the symbols that forms the production for this symbol. The + * sequence is in the reverse order of production. This is useful + * for easy copying onto parsing stack. + * + * Please note that this is a final. So the production for a symbol + * should be known before that symbol is constructed. This requirement + * cannot be met for those symbols which are recursive (e.g. a record that + * holds union a branch of which is the record itself). To resolve this + * problem, we initialize the symbol with an array of nulls. Later we + * fill the symbols. Not clean, but works. The other option is to not have + * this field a final. But keeping it final and thus keeping symbol immutable + * gives some confort. See various generators how we generate records. + */ + public final Symbol[] production; + /** + * Constructs a new symbol of the given kind kind. + */ + protected Symbol(Kind kind) { + this(kind, null); + } + + + protected Symbol(Kind kind, Symbol[] production) { + this.production = production; + this.kind = kind; + } + + /** + * A convenience method to construct a root symbol. + */ + static Symbol root(Symbol... symbols) { + return new Root(symbols); + } + /** + * A convenience method to construct a sequence. + * @param production The constituent symbols of the sequence. + */ + static Symbol seq(Symbol... production) { + return new Sequence(production); + } + + /** + * A convenience method to construct a repeater. + * @param symsToRepeat The symbols to repeat in the repeater. + */ + static Symbol repeat(Symbol endSymbol, Symbol... symsToRepeat) { + return new Repeater(endSymbol, symsToRepeat); + } + + /** + * A convenience method to construct a union. + */ + static Symbol alt(Symbol[] symbols, String[] labels) { + return new Alternative(symbols, labels); + } + + /** + * A convenience method to construct an ErrorAction. + * @param e + * @return + */ + static Symbol error(String e) { + return new ErrorAction(e); + } + + /** + * A convenience method to construct a ResolvingAction. + * @param w The writer symbol + * @param r The reader symbol + */ + static Symbol resolve(Symbol w, Symbol r) { + return new ResolvingAction(w, r); + } + + private static class Terminal extends Symbol { + private final String printName; + public Terminal(String printName) { + super(Kind.TERMINAL); + this.printName = printName; + } + public String toString() { return printName; } + } + + private static class ImplicitAction extends Symbol { + private ImplicitAction() { + super(Kind.IMPLICIT_ACTION); + } + } + + protected static class Root extends Symbol { + private Root(Symbol... symbols) { + super(Kind.ROOT, makeProduction(symbols)); + production[0] = this; + } + + private static Symbol[] makeProduction(Symbol[] symbols) { + Symbol[] result = new Symbol[symbols.length + 1]; + System.arraycopy(symbols, 0, result, 1, symbols.length); + return result; + } + } + protected static class Sequence extends Symbol implements Iterable { + private Sequence(Symbol[] productions) { + super(Kind.SEQUENCE, productions); + } + + public Symbol get(int index) { + return production[index]; + } + + public int size() { + return production.length; + } + + public Iterator iterator() { + return new Iterator() { + private int pos = production.length; + + public boolean hasNext() { + return 0 < pos; + } + + public Symbol next() { + if (0 < pos) { + return production[--pos]; + } else { + throw new NoSuchElementException(); + } + } + + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + } + + public static class Repeater extends Symbol { + public final Symbol end; + + private Repeater(Symbol end, Symbol... sequenceToRepeat) { + super(Kind.REPEATER, makeProduction(sequenceToRepeat)); + this.end = end; + production[0] = this; + } + + private static Symbol[] makeProduction(Symbol[] p) { + Symbol[] result = new Symbol[p.length + 1]; + System.arraycopy(p, 0, result, 1, p.length); + return result; + } + } + + public static class Alternative extends Symbol { + public final Symbol[] symbols; + public final String[] labels; + private Alternative(Symbol[] symbols, String[] labels) { + super(Kind.ALTERNATIVE); + this.symbols = symbols; + this.labels = labels; + } + + public Symbol getSymbol(int index) { + return symbols[index]; + } + + public String getLabel(int index) { + return labels[index]; + } + + public int size() { + return symbols.length; + } + + public int findLabel(String label) { + if (label != null) { + for (int i = 0; i < labels.length; i++) { + if (label.equals(labels[i])) { + return i; + } + } + } + return -1; + } + } + + public static class ErrorAction extends ImplicitAction { + public final String msg; + private ErrorAction(String msg) { + this.msg = msg; + } + } + + public static class IntCheckAction extends Symbol { + public final int size; + public IntCheckAction(int size) { + super(Kind.EXPLICIT_ACTION); + this.size = size; + } + } + + public static class EnumAdjustAction extends IntCheckAction { + public final Object[] adjustments; + public EnumAdjustAction(int rsymCount, Object[] adjustments) { + super(rsymCount); + this.adjustments = adjustments; + } + } + + public static class WriterUnionAction extends ImplicitAction { + } + + public static class ResolvingAction extends ImplicitAction { + public final Symbol writer; + public final Symbol reader; + private ResolvingAction(Symbol writer, Symbol reader) { + this.writer = writer; + this.reader = reader; + } + } + + public static class SkipAction extends ImplicitAction { + public final Symbol symToSkip; + public SkipAction(Symbol symToSkip) { + this.symToSkip = symToSkip; + } + } + + public static class FieldAdjustAction extends ImplicitAction { + public final int rindex; + public final String fname; + public FieldAdjustAction(int rindex, String fname) { + this.rindex = rindex; + this.fname = fname; + } + } + + public static class DefaultStartAction extends ImplicitAction { + public final Symbol root; + public final byte[] contents; + public DefaultStartAction(Symbol root, JsonNode defaultValue) + throws IOException { + this.root = root; + ByteArrayOutputStream os = new ByteArrayOutputStream(); + JsonGenerator g = new JsonFactory().createJsonGenerator(os, + JsonEncoding.UTF8); + new ObjectMapper().writeTree(g, defaultValue); + g.flush(); + this.contents = os.toByteArray(); + } + } + + public static class UnionAdjustAction extends ImplicitAction { + public final int rindex; + public final Symbol symToParse; + public UnionAdjustAction(int rindex, Symbol symToParse) { + this.rindex = rindex; + this.symToParse = symToParse; + } + } + + /** For JSON. */ + public static class EnumLabelsAction extends IntCheckAction { + public final List symbols; + public EnumLabelsAction(List symbols) { + super(symbols.size()); + this.symbols = symbols; + } + + public String getLabel(int n) { + return symbols.get(n); + } + + public int findLabel(String l) { + if (l != null) { + for (int i = 0; i < symbols.size(); i++) { + if (l.equals(symbols.get(i))) { + return i; + } + } + } + return -1; + } + } + + /** + * The terminal symbols for the grammar. + */ + public static final Symbol NULL = new Symbol.Terminal("null"); + public static final Symbol BOOLEAN = new Symbol.Terminal("boolean"); + public static final Symbol INT = new Symbol.Terminal("int"); + public static final Symbol LONG = new Symbol.Terminal("long"); + public static final Symbol FLOAT = new Symbol.Terminal("float"); + public static final Symbol DOUBLE = new Symbol.Terminal("double"); + public static final Symbol STRING = new Symbol.Terminal("string"); + public static final Symbol BYTES = new Symbol.Terminal("bytes"); + public static final Symbol FIXED = new Symbol.Terminal("fixed"); + public static final Symbol ENUM = new Symbol.Terminal("enum"); + public static final Symbol UNION = new Symbol.Terminal("union"); + + public static final Symbol ARRAY_START = new Symbol.Terminal("array-start"); + public static final Symbol ARRAY_END = new Symbol.Terminal("array-end"); + public static final Symbol MAP_START = new Symbol.Terminal("map-start"); + public static final Symbol MAP_END = new Symbol.Terminal("map-end"); + public static final Symbol END = new Symbol.Terminal("end"); + public static final Symbol ITEM_END = new Symbol.Terminal("item-end"); + + /* a pseudo terminal used by parsers */ + public static final Symbol CONTINUE = new Symbol.Terminal("continue"); + public static final Symbol FIELD_ACTION = + new Symbol.Terminal("field-action"); + + public static final Symbol RECORD_START = new ImplicitAction(); + public static final Symbol RECORD_END = new ImplicitAction(); + public static final Symbol UNION_END = new ImplicitAction(); + + public static final Symbol DEFAULT_END_ACTION = new ImplicitAction(); + public static final Symbol MAP_KEY_MARKER = + new Symbol.Terminal("map-key-marker"); +} Added: hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java?rev=796868&view=auto ============================================================================== --- hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java (added) +++ hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java Wed Jul 22 20:20:55 2009 @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.io.parsing; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.avro.Schema; +import org.apache.avro.Schema.Field; + +/** + * The class that generates validating grammar. + */ +public class ValidatingGrammarGenerator { + /** + * Returns the non-terminal that is the start symbol + * for the grammar for the given schema sc. + */ + public Symbol generate(Schema schema) { + return Symbol.root(generate(schema, new HashMap())); + } + + /** + * Returns the non-terminal that is the start symbol + * for the grammar for the given schema sc. If there is already an entry + * for the given schema in the given map seen then + * that entry is returned. Otherwise a new symbol is generated and + * an entry is inserted into the map. + * @param sc The schema for which the start symbol is required + * @param seen A map of schema to symbol mapping done so far. + * @return The start symbol for the schema + */ + public Symbol generate(Schema sc, Map seen) { + switch (sc.getType()) { + case NULL: + return Symbol.NULL; + case BOOLEAN: + return Symbol.BOOLEAN; + case INT: + return Symbol.INT; + case LONG: + return Symbol.LONG; + case FLOAT: + return Symbol.FLOAT; + case DOUBLE: + return Symbol.DOUBLE; + case STRING: + return Symbol.STRING; + case BYTES: + return Symbol.BYTES; + case FIXED: + return Symbol.seq(new Symbol.IntCheckAction(sc.getFixedSize()), + Symbol.FIXED); + case ENUM: + return Symbol.seq(new Symbol.IntCheckAction(sc.getEnumSymbols().size()), + Symbol.ENUM); + case ARRAY: + return Symbol.seq(Symbol.ARRAY_END, + Symbol.repeat(Symbol.ARRAY_END, generate(sc.getElementType(), seen)), + Symbol.ARRAY_START); + case MAP: + return Symbol.seq(Symbol.MAP_END, + Symbol.repeat(Symbol.MAP_END, + generate(sc.getValueType(), seen), Symbol.STRING), + Symbol.MAP_START); + case RECORD: { + LitS wsc = new LitS(sc); + Symbol rresult = seen.get(wsc); + if (rresult == null) { + Symbol[] production = new Symbol[sc.getFields().size()]; + + /** + * We construct a symbol without filling the array. Please see + * {@link Symbol#production} for the reason. + */ + rresult = Symbol.seq(production); + seen.put(wsc, rresult); + + int i = production.length; + for (Field f : sc.getFields().values()) { + production[--i] = generate(f.schema(), seen); + } + } + return rresult; + } + case UNION: + List subs = sc.getTypes(); + Symbol[] symbols = new Symbol[subs.size()]; + String[] labels = new String[subs.size()]; + + int i = 0; + for (Schema b : sc.getTypes()) { + symbols[i] = generate(b, seen); + labels[i] = b.getType().name(); + i++; + } + return Symbol.seq(Symbol.alt(symbols, labels), Symbol.UNION); + + default: + throw new RuntimeException("Unexpected schema type"); + } + } + + /** A wrapper around Schema that does "==" equality. */ + static class LitS { + public final Schema actual; + public LitS(Schema actual) { this.actual = actual; } + + /** + * Two LitS are equal if and only if their underlying schema is + * the same (not merely equal). + */ + public boolean equals(Object o) { + if (! (o instanceof LitS)) return false; + return actual == ((LitS)o).actual; + } + + public int hashCode() { + return actual.hashCode(); + } + } +} Added: hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/package.html URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/package.html?rev=796868&view=auto ============================================================================== --- hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/package.html (added) +++ hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/package.html Wed Jul 22 20:20:55 2009 @@ -0,0 +1,40 @@ + + + + + +Implementation of Avro schemas as LL(1) grammars. + +

See +parser documentation for details on how +this is achieved. + +

+The classes in this package are used by +{@link org.apache.avro.io.ValidatingEncoder}, +{@link org.apache.avro.io.ValidatingDecoder}, +{@link org.apache.avro.io.ResolvingDecoder}, +{@link org.apache.avro.io.JsonEncoder} and +{@link org.apache.avro.io.JsonDecoder}, + +

+Unless one plans to generate a variation of the grammar or use a grammar, +one not need to understand these classes. + + + Modified: hadoop/avro/trunk/src/java/org/apache/avro/reflect/ReflectDatumWriter.java URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/java/org/apache/avro/reflect/ReflectDatumWriter.java?rev=796868&r1=796867&r2=796868&view=diff ============================================================================== --- hadoop/avro/trunk/src/java/org/apache/avro/reflect/ReflectDatumWriter.java (original) +++ hadoop/avro/trunk/src/java/org/apache/avro/reflect/ReflectDatumWriter.java Wed Jul 22 20:20:55 2009 @@ -45,7 +45,7 @@ protected void writeEnum(Schema schema, Object datum, Encoder out) throws IOException { - out.writeInt(((Enum)datum).ordinal()); + out.writeEnum(((Enum)datum).ordinal()); } protected boolean isEnum(Object datum) { Modified: hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java?rev=796868&r1=796867&r2=796868&view=diff ============================================================================== --- hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java (original) +++ hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java Wed Jul 22 20:20:55 2009 @@ -36,6 +36,8 @@ import org.apache.avro.io.Encoder; import org.apache.avro.io.BinaryDecoder; import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.JsonDecoder; +import org.apache.avro.io.JsonEncoder; import org.apache.avro.util.Utf8; import org.testng.annotations.Test; @@ -186,29 +188,47 @@ assertTrue("Datum does not validate against schema "+datum, GenericData.validate(schema, datum)); - checkSerialization(schema, datum, - new GenericDatumWriter(), new GenericDatumReader()); + checkBinary(schema, datum, + new GenericDatumWriter(), + new GenericDatumReader()); + checkJson(schema, datum, + new GenericDatumWriter(), + new GenericDatumReader()); } } - private static void checkSerialization(Schema schema, Object datum, - DatumWriter writer, - DatumReader reader) + private static void checkBinary(Schema schema, Object datum, + DatumWriter writer, + DatumReader reader) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); writer.setSchema(schema); writer.write(datum, new BinaryEncoder(out)); - byte[] data = out.toByteArray(); - // System.out.println("length = "+data.length); reader.setSchema(schema); Object decoded = reader.read(null, new BinaryDecoder(new ByteArrayInputStream(data))); - // System.out.println(GenericData.toString(datum)); - // System.out.println(GenericData.toString(decoded)); + assertEquals("Decoded data does not match.", datum, decoded); + } + + private static void checkJson(Schema schema, Object datum, + DatumWriter writer, + DatumReader reader) + throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + Encoder encoder = new JsonEncoder(schema, out); + writer.setSchema(schema); + writer.write(datum, encoder); + encoder.flush(); + byte[] data = out.toByteArray(); + + reader.setSchema(schema); + Object decoded = + reader.read(null, new JsonDecoder(schema, new ByteArrayInputStream(data))); + assertEquals("Decoded data does not match.", datum, decoded); } Modified: hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestResolvingIO.java URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestResolvingIO.java?rev=796868&r1=796867&r2=796868&view=diff ============================================================================== --- hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestResolvingIO.java (original) +++ hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestResolvingIO.java Wed Jul 22 20:20:55 2009 @@ -19,268 +19,176 @@ import java.io.ByteArrayInputStream; import java.io.IOException; -import java.util.Vector; +import java.io.InputStream; +import java.util.Iterator; import org.apache.avro.Schema; +import org.apache.avro.io.TestValidatingIO.Encoding; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; public class TestResolvingIO { - @Test(dataProvider="data") - public void testRead(String jsonWriterSchema, + @Test(dataProvider="data1") + public void test_identical(Encoding encoding, + int skipLevel, String jsonWriterSchema, String writerCalls, - String jsonReaderSchema, - String readerCalls) throws IOException { - test(jsonWriterSchema, writerCalls, jsonReaderSchema, readerCalls, -1); + String jsonReaderSchema, String readerCalls) + throws IOException { + performTest(encoding, skipLevel, jsonWriterSchema, writerCalls, + jsonReaderSchema, readerCalls); } - - @Test(dataProvider="data") - public void testSkip_0(String jsonWriterSchema, + + private static final int COUNT = 10; + + @Test(dataProvider="data2") + public void test_compatible(Encoding encoding, + int skipLevel, String jsonWriterSchema, String writerCalls, - String jsonReaderSchema, - String readerCalls) throws IOException { - test(jsonWriterSchema, writerCalls, jsonReaderSchema, readerCalls, 0); + String jsonReaderSchema, String readerCalls) + throws IOException { + performTest(encoding, skipLevel, jsonWriterSchema, writerCalls, + jsonReaderSchema, readerCalls); } - @Test(dataProvider="data") - public void testSkip_1(String jsonWriterSchema, - String writerCalls, - String jsonReaderSchema, - String readerCalls) throws IOException { - test(jsonWriterSchema, writerCalls, jsonReaderSchema, readerCalls, 1); + @Test(dataProvider="data3") + public void test_resolving(Encoding encoding, int skipLevel, + String jsonWriterSchema, String writerCalls, + Object[] writerValues, + String jsonReaderSchema, String readerCalls, Object[] readerValues) + throws IOException { + Schema writerSchema = Schema.parse(jsonWriterSchema); + byte[] bytes = TestValidatingIO.make(writerSchema, writerCalls, + writerValues, Encoding.BINARY); + Schema readerSchema = Schema.parse(jsonReaderSchema); + check(writerSchema, readerSchema, bytes, readerCalls, + readerValues, + Encoding.BINARY, skipLevel); } - @Test(dataProvider="data") - public void testSkip_2(String jsonWriterSchema, + private void performTest(Encoding encoding, + int skipLevel, String jsonWriterSchema, String writerCalls, - String jsonReaderSchema, - String readerCalls) throws IOException { - test(jsonWriterSchema, writerCalls, jsonReaderSchema, readerCalls, 2); - } - - private void test(String jsonWriterSchema, String writerCalls, - String jsonReaderSchema, String readerCalls, int skipLevel) - throws IOException { - for (int i = 0; i < 10; i++) { - testOnce(jsonWriterSchema, writerCalls, jsonReaderSchema, - readerCalls, skipLevel); + String jsonReaderSchema, String readerCalls) + throws IOException { + for (int i = 0; i < COUNT; i++) { + testOnce(jsonWriterSchema, writerCalls, + jsonReaderSchema, readerCalls, encoding, skipLevel); } } private void testOnce(String jsonWriterSchema, - String writerCalls, + String writerCalls, String jsonReaderSchema, String readerCalls, + Encoding encoding, int skipLevel) throws IOException { - Vector values = new Vector(); + Object[] values = TestValidatingIO.randomValues(writerCalls); Schema writerSchema = Schema.parse(jsonWriterSchema); byte[] bytes = TestValidatingIO.make(writerSchema, writerCalls, - values, false); + values, encoding); Schema readerSchema = Schema.parse(jsonReaderSchema); - // dump(bytes); - check(writerSchema, readerSchema, bytes, readerCalls, values.toArray(), - skipLevel); + check(writerSchema, readerSchema, bytes, readerCalls, + values, + encoding, skipLevel); } private static void check(Schema wsc, Schema rsc, byte[] bytes, - String calls, Object[] values, int skipLevel) throws IOException { - // dump(bytes); - Decoder bvi= new BinaryDecoder(new ByteArrayInputStream(bytes)); + String calls, Object[] values, Encoding encoding, + int skipLevel) + throws IOException { + // TestValidatingIO.dump(bytes); + // System.out.println(new String(bytes, "UTF-8")); + InputStream in = new ByteArrayInputStream(bytes); + Decoder bvi = null; + switch (encoding) { + case BINARY: + case BLOCKING_BINARY: + bvi = new BinaryDecoder(in); + break; + case JSON: + bvi = new JsonDecoder(wsc, in); + break; + } Decoder vi = new ResolvingDecoder(wsc, rsc, bvi); TestValidatingIO.check(vi, calls, values, skipLevel); } @DataProvider - public static Object[][] data() { - /** - * The first argument is a schema. - * The second one is a sequence of (single character) mnemonics: - * N null - * B boolean - * I int - * L long - * F float - * D double - * S string - * b bytes - * [ Start array - * ] End array - * { Start map - * } End map - * s start item - * 0-9 branch id for union - */ + public static Iterator data1() { + return TestValidatingIO.cartesian(encodings, skipLevels, + TestValidatingIO.paste(TestValidatingIO.testSchemas(), + TestValidatingIO.testSchemas())); + } + + @DataProvider + public static Iterator data2() { + return TestValidatingIO.cartesian(encodings, skipLevels, testSchemas()); + } + + @DataProvider + public static Iterator data3() { + return TestValidatingIO.cartesian(encodings, skipLevels, + dataForResolvingTests()); + } + + private static Object[][] encodings = new Object[][] { { Encoding.BINARY } }; + private static Object[][] skipLevels = + new Object[][] { { -1 }, { 0 }, { 1 }, { 2 } }; + private static Object[][] testSchemas() { + // The mnemonics are the same as {@link TestValidatingIO#testSchemas} return new Object[][] { - { "\"null\"", "N", "\"null\"", "N" }, - { "\"boolean\"", "B", "\"boolean\"", "B" }, - { "\"int\"", "I", "\"int\"", "I" }, - { "\"int\"", "I", "\"long\"", "L" }, - // { "\"int\"", "I", "\"float\"", "F" }, // These promotions make sense? + // { "\"int\"", "I", "\"float\"", "F" }, // makes sense? { "\"int\"", "I", "\"double\"", "D" }, - { "\"long\"", "L", "\"long\"", "L" }, // { "\"long\"", "L", "\"float\"", "F" }, // And this? { "\"long\"", "L", "\"double\"", "D" }, - { "\"float\"", "F", "\"float\"", "F" }, { "\"float\"", "F", "\"double\"", "D" }, - { "\"double\"", "D", "\"double\"", "D" }, { "\"double\"", "D", "\"long\"", "L" }, - { "\"string\"", "S", "\"string\"", "S" }, - { "\"bytes\"", "b", "\"bytes\"", "b" }, - { "{\"type\":\"fixed\", \"name\":\"fi\", \"size\": 0}", "f0", - "{\"type\":\"fixed\", \"name\":\"fi\", \"size\": 0}", "f0" }, - { "{\"type\":\"fixed\", \"name\":\"fi\", \"size\": 10}", "f10", - "{\"type\":\"fixed\", \"name\":\"fi\", \"size\": 10}", "f10" }, - { "{\"type\":\"enum\", \"name\":\"en\", \"symbols\":[\"v1\", \"v2\"]}", - "e2", - "{\"type\":\"enum\", \"name\":\"en\", \"symbols\":[\"v1\", \"v2\"]}", - "e2"}, - { "{\"type\":\"array\", \"items\": \"boolean\"}", "[]", - "{\"type\":\"array\", \"items\": \"boolean\"}", "[]" }, - { "{\"type\":\"array\", \"items\": \"int\"}", "[]", - "{\"type\":\"array\", \"items\": \"int\"}", "[]" }, { "{\"type\":\"array\", \"items\": \"int\"}", "[]", - "{\"type\":\"array\", \"items\": \"long\"}", "[]" }, + "{\"type\":\"array\", \"items\": \"long\"}", "[]", }, { "{\"type\":\"array\", \"items\": \"int\"}", "[]", "{\"type\":\"array\", \"items\": \"double\"}", "[]" }, { "{\"type\":\"array\", \"items\": \"long\"}", "[]", - "{\"type\":\"array\", \"items\": \"long\"}", "[]" }, - { "{\"type\":\"array\", \"items\": \"long\"}", "[]", "{\"type\":\"array\", \"items\": \"double\"}", "[]" }, { "{\"type\":\"array\", \"items\": \"float\"}", "[]", - "{\"type\":\"array\", \"items\": \"float\"}", "[]" }, - { "{\"type\":\"array\", \"items\": \"float\"}", "[]", "{\"type\":\"array\", \"items\": \"double\"}", "[]" }, - { "{\"type\":\"array\", \"items\": \"double\"}", "[]", - "{\"type\":\"array\", \"items\": \"double\"}", "[]", }, - { "{\"type\":\"array\", \"items\": \"string\"}", "[]", - "{\"type\":\"array\", \"items\": \"string\"}", "[]" }, - { "{\"type\":\"array\", \"items\": \"bytes\"}", "[]", - "{\"type\":\"array\", \"items\": \"bytes\"}", "[]" }, - { "{\"type\":\"array\", \"items\": \"boolean\"}", "[c1sB]", - "{\"type\":\"array\", \"items\": \"boolean\"}", "[c1sB]" }, - { "{\"type\":\"array\", \"items\": \"int\"}", "[c1sI]", - "{\"type\":\"array\", \"items\": \"int\"}", "[c1sI]" }, { "{\"type\":\"array\", \"items\": \"int\"}", "[c1sI]", "{\"type\":\"array\", \"items\": \"long\"}", "[c1sL]" }, { "{\"type\":\"array\", \"items\": \"int\"}", "[c1sI]", "{\"type\":\"array\", \"items\": \"double\"}", "[c1sD]" }, { "{\"type\":\"array\", \"items\": \"long\"}", "[c1sL]", - "{\"type\":\"array\", \"items\": \"long\"}", "[c1sL]" }, - { "{\"type\":\"array\", \"items\": \"long\"}", "[c1sL]", "{\"type\":\"array\", \"items\": \"double\"}", "[c1sD]" }, { "{\"type\":\"array\", \"items\": \"float\"}", "[c1sF]", - "{\"type\":\"array\", \"items\": \"float\"}", "[c1sF]" }, - { "{\"type\":\"array\", \"items\": \"float\"}", "[c1sF]", "{\"type\":\"array\", \"items\": \"double\"}", "[c1sD]" }, - { "{\"type\":\"array\", \"items\": \"double\"}", "[c1sD]", - "{\"type\":\"array\", \"items\": \"double\"}", "[c1sD]" }, - { "{\"type\":\"array\", \"items\": \"string\"}", "[c1sS]", - "{\"type\":\"array\", \"items\": \"string\"}", "[c1sS]" }, - { "{\"type\":\"array\", \"items\": \"bytes\"}", "[c1sb]", - "{\"type\":\"array\", \"items\": \"bytes\"}", "[c1sb]" }, - { "{\"type\":\"map\", \"values\": \"boolean\"}", "{}", - "{\"type\":\"map\", \"values\": \"boolean\"}", "{}" }, - { "{\"type\":\"map\", \"values\": \"int\"}", "{}", - "{\"type\":\"map\", \"values\": \"int\"}", "{}" }, { "{\"type\":\"map\", \"values\": \"int\"}", "{}", "{\"type\":\"map\", \"values\": \"long\"}", "{}" }, { "{\"type\":\"map\", \"values\": \"int\"}", "{}", "{\"type\":\"map\", \"values\": \"double\"}", "{}" }, { "{\"type\":\"map\", \"values\": \"long\"}", "{}", - "{\"type\":\"map\", \"values\": \"long\"}", "{}" }, - { "{\"type\":\"map\", \"values\": \"long\"}", "{}", "{\"type\":\"map\", \"values\": \"double\"}", "{}" }, { "{\"type\":\"map\", \"values\": \"float\"}", "{}", - "{\"type\":\"map\", \"values\": \"float\"}", "{}" }, - { "{\"type\":\"map\", \"values\": \"float\"}", "{}", - "{\"type\":\"map\", \"values\": \"double\"}", "{}" }, - { "{\"type\":\"map\", \"values\": \"double\"}", "{}", "{\"type\":\"map\", \"values\": \"double\"}", "{}" }, - { "{\"type\":\"map\", \"values\": \"string\"}", "{}", - "{\"type\":\"map\", \"values\": \"string\"}", "{}" }, - { "{\"type\":\"map\", \"values\": \"bytes\"}", "{}", - "{\"type\":\"map\", \"values\": \"bytes\"}", "{}" }, - { "{\"type\":\"map\", \"values\": " - + "{\"type\":\"array\", \"items\":\"int\"}}", "{}", - "{\"type\":\"map\", \"values\": " - + "{\"type\":\"array\", \"items\":\"int\"}}", "{}" }, - - { "{\"type\":\"map\", \"values\": \"boolean\"}", "{c1sSB}", - "{\"type\":\"map\", \"values\": \"boolean\"}", "{c1sSB}" }, - { "{\"type\":\"map\", \"values\": \"int\"}", "{c1sSI}", - "{\"type\":\"map\", \"values\": \"int\"}", "{c1sSI}" }, - { "{\"type\":\"map\", \"values\": \"int\"}", "{c1sSI}", - "{\"type\":\"map\", \"values\": \"long\"}", "{c1sSL}" }, - { "{\"type\":\"map\", \"values\": \"int\"}", "{c1sSI}", - "{\"type\":\"map\", \"values\": \"double\"}", "{c1sSD}" }, - { "{\"type\":\"map\", \"values\": \"long\"}", "{c1sSL}", - "{\"type\":\"map\", \"values\": \"long\"}", "{c1sSL}" }, - { "{\"type\":\"map\", \"values\": \"long\"}", "{c1sSL}", - "{\"type\":\"map\", \"values\": \"double\"}", "{c1sSD}" }, - { "{\"type\":\"map\", \"values\": \"float\"}", "{c1sSF}", - "{\"type\":\"map\", \"values\": \"float\"}", "{c1sSF}" }, - { "{\"type\":\"map\", \"values\": \"float\"}", "{c1sSF}", - "{\"type\":\"map\", \"values\": \"double\"}", "{c1sSD}" }, - { "{\"type\":\"map\", \"values\": \"double\"}", "{c1sSD}", - "{\"type\":\"map\", \"values\": \"double\"}", "{c1sSD}" }, - { "{\"type\":\"map\", \"values\": \"string\"}", "{c1sSS}", - "{\"type\":\"map\", \"values\": \"string\"}", "{c1sSS}" }, - { "{\"type\":\"map\", \"values\": \"bytes\"}", "{c1sSb}", - "{\"type\":\"map\", \"values\": \"bytes\"}", "{c1sSb}" }, - { "{\"type\":\"map\", \"values\": " - + "{\"type\":\"array\", \"items\":\"int\"}}", "{c1sS[c3sIsIsI]}", - "{\"type\":\"map\", \"values\": " - + "{\"type\":\"array\", \"items\":\"int\"}}", "{c1sS[c3sIsIsI]}" }, - { "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f\", \"type\":\"boolean\"}]}", "B", - "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f\", \"type\":\"boolean\"}]}", "B" }, - { "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f\", \"type\":\"int\"}]}", "I", - "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f\", \"type\":\"int\"}]}", "I"}, - { "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f\", \"type\":\"int\"}]}", "I", - "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f\", \"type\":\"long\"}]}", "L"}, + { "{\"type\":\"map\", \"values\": \"int\"}", "{c1sK5I}", + "{\"type\":\"map\", \"values\": \"long\"}", "{c1sK5L}" }, + { "{\"type\":\"map\", \"values\": \"int\"}", "{c1sK5I}", + "{\"type\":\"map\", \"values\": \"double\"}", "{c1sK5D}" }, + { "{\"type\":\"map\", \"values\": \"long\"}", "{c1sK5L}", + "{\"type\":\"map\", \"values\": \"double\"}", "{c1sK5D}" }, + { "{\"type\":\"map\", \"values\": \"float\"}", "{c1sK5F}", + "{\"type\":\"map\", \"values\": \"double\"}", "{c1sK5D}" }, + { "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" + "{\"name\":\"f\", \"type\":\"int\"}]}", "I", "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f\", \"type\":\"double\"}]}", "D"}, - { "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f\", \"type\":\"long\"}]}", "L", - "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" + "{\"name\":\"f\", \"type\":\"long\"}]}", "L" }, { "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f\", \"type\":\"float\"}]}", "F", - "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f\", \"type\":\"float\"}]}", "F"}, - { "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f\", \"type\":\"double\"}]}", "D", - "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f\", \"type\":\"double\"}]}", "D"}, - { "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f\", \"type\":\"string\"}]}", "S", - "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f\", \"type\":\"string\"}]}", "S"}, - { "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f\", \"type\":\"bytes\"}]}", "b", + + "{\"name\":\"f\", \"type\":\"int\"}]}", "I", "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f\", \"type\":\"bytes\"}]}", "b"}, + + "{\"name\":\"f\", \"type\":\"double\"}]}", "D" }, - // multi-field record - { "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f1\", \"type\":\"int\"}," - + "{\"name\":\"f2\", \"type\":\"double\"}," - + "{\"name\":\"f3\", \"type\":\"string\"}]}", "IDS", - "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f1\", \"type\":\"int\"}," - + "{\"name\":\"f2\", \"type\":\"double\"}," - + "{\"name\":\"f3\", \"type\":\"string\"}]}", "IDS" }, // multi-field record with promotions { "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" + "{\"name\":\"f0\", \"type\":\"boolean\"}," @@ -293,68 +201,66 @@ + "{\"name\":\"f2\", \"type\":\"double\"}," + "{\"name\":\"f3\", \"type\":\"string\"}]}", "BLDS" }, - // record with array - { "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f1\", \"type\":\"long\"}," - + "{\"name\":\"f2\", " - + "\"type\":{\"type\":\"array\", \"items\":\"int\"}}]}", "L[c1sI]", - "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f1\", \"type\":\"long\"}," - + "{\"name\":\"f2\", " - + "\"type\":{\"type\":\"array\", \"items\":\"int\"}}]}", "L[c1sI]" }, + { "[\"int\"]", "U0I", + "[\"long\"]", "U0L" }, + { "[\"int\"]", "U0I", + "[\"double\"]", "U0D" }, + { "[\"long\"]", "U0L", + "[\"double\"]", "U0D" }, + { "[\"float\"]", "U0F", + "[\"double\"]", "U0D" }, + + { "\"int\"", "I", "[\"int\"]", "U0I" }, + + { "[\"int\"]", "U0I", "\"int\"", "I" }, + { "[\"int\"]", "U0I", "\"long\"", "L" }, + + { "[\"boolean\", \"int\"]", "U1I", + "[\"boolean\", \"long\"]", "U1L" }, + { "[\"boolean\", \"int\"]", "U1I", + "[\"long\", \"boolean\"]", "U0L" }, + }; + } - // record with map + private static Object[][] dataForResolvingTests() { + // The mnemonics are the same as {@link TestValidatingIO#testSchemas} + return new Object[][] { + // Reordered fields { "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f1\", \"type\":\"long\"}," - + "{\"name\":\"f2\", " - + "\"type\":{\"type\":\"map\", \"values\":\"int\"}}]}", "L{c1sSI}", + + "{\"name\":\"f1\", \"type\":\"int\"}," + + "{\"name\":\"f2\", \"type\":\"string\"}]}", "IS10", + new Object[] { 10, "hello" }, "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" - + "{\"name\":\"f1\", \"type\":\"long\"}," - + "{\"name\":\"f2\", " - + "\"type\":{\"type\":\"map\", \"values\":\"int\"}}]}", "L{c1sSI}" }, - - { "[\"boolean\"]", "u0B", "[\"boolean\"]", "u0B" }, - { "[\"int\"]", "u0I", "[\"int\"]", "u0I" }, - { "[\"int\"]", "u0I", "[\"long\"]", "u0L" }, - { "[\"int\"]", "u0I", "[\"double\"]", "u0D" }, - { "[\"long\"]", "u0L", "[\"long\"]", "u0L" }, - { "[\"long\"]", "u0L", "[\"double\"]", "u0D" }, - { "[\"float\"]", "u0F", "[\"float\"]", "u0F" }, - { "[\"float\"]", "u0F", "[\"double\"]", "u0D" }, - { "[\"double\"]", "u0D", "[\"double\"]", "u0D" }, - { "[\"string\"]", "u0S", "[\"string\"]", "u0S" }, - { "[\"bytes\"]", "u0b", "[\"bytes\"]", "u0b" }, - - { "\"int\"", "I", "[\"int\"]", "u0I" }, - - { "[\"int\"]", "u0I", "\"int\"", "I" }, - { "[\"int\"]", "u0I", "\"long\"", "I" }, - - { "[\"null\", \"int\"]", "u0N", "[\"null\", \"int\"]", "u0N" }, - { "[\"boolean\", \"int\"]", "u0B", "[\"boolean\", \"int\"]", "u0B" }, - { "[\"boolean\", \"int\"]", "u1I", "[\"boolean\", \"int\"]", "u1I" }, - { "[\"boolean\", \"int\"]", "u1I", "[\"int\", \"boolean\"]", "u0I" }, - { "[\"boolean\", \"int\"]", "u1I", "[\"boolean\", \"long\"]", "u1L" }, - { "[\"boolean\", \"int\"]", "u1I", "[\"long\", \"boolean\"]", "u0L" }, - { "[\"boolean\", {\"type\":\"array\", \"items\":\"int\"} ]", "u0B", - "[\"boolean\", {\"type\":\"array\", \"items\":\"int\"} ]", "u0B" }, - { "[\"boolean\", {\"type\":\"array\", \"items\":\"int\"} ]", "u1[c1sI]", - "[\"boolean\", {\"type\":\"array\", \"items\":\"int\"} ]", "u1[c1sI]" }, - - // TODO: test more complicated types such as arrays of records + + "{\"name\":\"f2\", \"type\":\"string\" }," + + "{\"name\":\"f1\", \"type\":\"long\"}]}", "LS10", + new Object[] { 10L, "hello" } }, + + // Default values + { "{\"type\":\"record\",\"name\":\"r\",\"fields\":[]}", "", + new Object[] { }, + "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" + + "{\"name\":\"f\", \"type\":\"int\", \"default\": 100}]}", "I", + new Object[] { 100 } }, + { "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" + + "{\"name\":\"f2\", \"type\":\"int\"}]}", "I", + new Object[] { 10 }, + "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" + + "{\"name\":\"f1\", \"type\":\"int\", \"default\": 101}," + + "{\"name\":\"f2\", \"type\":\"int\"}]}", "II", + new Object[] { 10, 101 } }, + { "{\"type\":\"record\",\"name\":\"outer\",\"fields\":[" + + "{\"name\": \"g1\", " + + "\"type\":{\"type\":\"record\",\"name\":\"inner\",\"fields\":[" + + "{\"name\":\"f2\", \"type\":\"int\"}]}}, " + + "{\"name\": \"g2\", \"type\": \"long\"}]}", "IL", + new Object[] { 10, 11L }, + "{\"type\":\"record\",\"name\":\"outer\",\"fields\":[" + + "{\"name\": \"g1\", " + + "\"type\":{\"type\":\"record\",\"name\":\"inner\",\"fields\":[" + + "{\"name\":\"f1\", \"type\":\"int\", \"default\": 101}," + + "{\"name\":\"f2\", \"type\":\"int\"}]}}, " + + "{\"name\": \"g2\", \"type\": \"long\"}]}}", "IIL", + new Object[] { 10, 101, 11L } }, }; } - - protected static void dump(byte[] bb) { - int col = 0; - for (byte b : bb) { - if (col % 16 == 0) { - System.out.println(); - } - col++; - System.out.print(Integer.toHexString(b & 0xff) + " "); - } - System.out.println(); - } - } Modified: hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestValidatingIO.java URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestValidatingIO.java?rev=796868&r1=796867&r2=796868&view=diff ============================================================================== --- hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestValidatingIO.java (original) +++ hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestValidatingIO.java Wed Jul 22 20:20:55 2009 @@ -20,7 +20,9 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; +import java.util.Iterator; import java.util.Random; import java.util.Vector; @@ -32,146 +34,139 @@ public class TestValidatingIO { - @Test(dataProvider="data") - public void testRead(String jsonSchema, String calls) throws IOException { - test(jsonSchema, calls, -1, false); - } - - @Test(dataProvider="data") - public void testSkip_0(String jsonSchema, String calls) throws IOException { - test(jsonSchema, calls, 0, false); - } - - @Test(dataProvider="data") - public void testSkip_1(String jsonSchema, String calls) throws IOException { - test(jsonSchema, calls, 1, false); - } - - @Test(dataProvider="data") - public void testSkip_2(String jsonSchema, String calls) throws IOException { - test(jsonSchema, calls, 2, false); - } - - @Test(dataProvider="data") - public void testRead_blocking(String jsonSchema, String calls) - throws IOException { - test(jsonSchema, calls, -1, true); - } - - @Test(dataProvider="data") - public void testSkip_0_blocking(String jsonSchema, String calls) - throws IOException { - test(jsonSchema, calls, 0, true); - } + enum Encoding { + BINARY, + BLOCKING_BINARY, + JSON, + }; - @Test(dataProvider="data") - public void testSkip_1_blocking(String jsonSchema, String calls) - throws IOException { - test(jsonSchema, calls, 1, true); - } + private static int COUNT = 1; @Test(dataProvider="data") - public void testSkip_2_blocking(String jsonSchema, String calls) - throws IOException { - test(jsonSchema, calls, 2, true); - } - - private void test(String jsonSchema, String calls, int skipLevel, - boolean useBlocking) - throws IOException { - for (int i = 0; i < 10; i++) { - testOnce(Schema.parse(jsonSchema), calls, skipLevel, useBlocking); + public void test(Encoding encoding, int skipLevel, + String jsonSchema, String calls) throws IOException { + for (int i = 0; i < COUNT; i++) { + testOnce(Schema.parse(jsonSchema), calls, skipLevel, encoding); } } - - private void testOnce(Schema schema, String calls, int skipLevel, - boolean useBlocking) + + private void testOnce(Schema schema, String calls, + int skipLevel, + Encoding encoding) throws IOException { - Vector values = new Vector(); - byte[] bytes = make(schema, calls, values, useBlocking); - check(schema, bytes, calls, values.toArray(), skipLevel); + Object[] values = randomValues(calls); + byte[] bytes = make(schema, calls, values, encoding); + check(schema, bytes, calls, values, skipLevel, encoding); } public static byte[] make(Schema sc, String calls, - Vector values, boolean useBlocking) throws IOException { + Object[] values, Encoding encoding) throws IOException { ByteArrayOutputStream ba = new ByteArrayOutputStream(); - Encoder bvo = useBlocking ? new BlockingBinaryEncoder(ba) : - new BinaryEncoder(ba); + Encoder bvo = null; + switch (encoding) { + case BINARY: + bvo = new BinaryEncoder(ba); + break; + case BLOCKING_BINARY: + bvo = new BlockingBinaryEncoder(ba); + break; + case JSON: + bvo = new JsonEncoder(sc, ba); + break; + } + Encoder vo = new ValidatingEncoder(sc, bvo); generate(vo, calls, values); vo.flush(); return ba.toByteArray(); } + public static class InputScanner { + private final char[] chars; + private int cpos = 0; + + public InputScanner(char[] chars) { + this.chars = chars; + } + + public boolean next() { + if (cpos < chars.length) { + cpos++; + } + return cpos != chars.length; + } + + public char cur() { + return chars[cpos]; + } + + public boolean isDone() { + return cpos == chars.length; + } + } public static void generate(Encoder vw, String calls, - Vector values) throws IOException { - char[] cc = calls.toCharArray(); - Random r = new Random(); - for (int i = 0; i < cc.length; i++) { - char c = cc[i]; + Object[] values) throws IOException { + InputScanner cs = new InputScanner(calls.toCharArray()); + int p = 0; + while (! cs.isDone()) { + char c = cs.cur(); + cs.next(); switch (c) { case 'N': vw.writeNull(); break; case 'B': - boolean b = r.nextBoolean(); - values.add(b); + boolean b = (Boolean) values[p++]; vw.writeBoolean(b); break; case 'I': - int ii = r.nextInt(); - values.add(ii); + int ii = (Integer) values[p++]; vw.writeInt(ii); break; case 'L': - long l = r.nextLong(); - values.add(l); + long l = (Long) values[p++]; vw.writeLong(l); break; case 'F': - float f = r.nextFloat(); - values.add(f); + float f = (Float) values[p++]; vw.writeFloat(f); break; case 'D': - double d = r.nextDouble(); - values.add(d); + double d = (Double) values[p++]; vw.writeDouble(d); break; case 'S': { - IntPair ip = extractInt(cc, i); - String s = nextString(r, ip.first); - values.add(s); + extractInt(cs); + String s = (String) values[p++]; vw.writeString(new Utf8(s)); - i = ip.second; break; } + case 'K': + { + extractInt(cs); + String s = (String) values[p++]; + vw.writeString(s); + break; + } case 'b': { - IntPair ip = extractInt(cc, i); - byte[] bb = nextBytes(r, ip.first); - values.add(bb); + extractInt(cs); + byte[] bb = (byte[]) values[p++]; vw.writeBytes(bb); - i = ip.second; break; } case 'f': { - IntPair ip = extractInt(cc, i); - byte[] bb = nextBytes(r, ip.first); - values.add(bb); + extractInt(cs); + byte[] bb = (byte[]) values[p++]; vw.writeFixed(bb); - i = ip.second; break; } case 'e': { - IntPair ip = extractInt(cc, i); - int e = r.nextInt(ip.first); - values.add(e); + int e = extractInt(cs); vw.writeEnum(e); - i = ip.second; break; } case '[': @@ -187,20 +182,14 @@ vw.writeMapEnd(); break; case 'c': - { - IntPair ip = extractInt(cc, i); - vw.setItemCount(ip.first); - i = ip.second; - } + vw.setItemCount(extractInt(cs)); break; case 's': vw.startItem(); break; - case 'u': + case 'U': { - IntPair ip = extractInt(cc, i); - vw.writeIndex(ip.first); - i = ip.second; + vw.writeIndex(extractInt(cs)); break; } default: @@ -210,25 +199,70 @@ } } - private static IntPair extractInt(char[] cc, int i) { - int r = 0; - i++; - while (i < cc.length && Character.isDigit(cc[i])) { - r = r * 10 + cc[i] - '0'; - i++; + public static Object[] randomValues(String calls) { + Random r = new Random(); + InputScanner cs = new InputScanner(calls.toCharArray()); + Vector result = new Vector(); + while (! cs.isDone()) { + char c = cs.cur(); + cs.next(); + switch (c) { + case 'N': + break; + case 'B': + result.add(r.nextBoolean()); + break; + case 'I': + result.add(r.nextInt()); + break; + case 'L': + result.add(r.nextLong()); + break; + case 'F': + result.add(r.nextFloat()); + break; + case 'D': + result.add(r.nextDouble()); + break; + case 'S': + case 'K': + result.add(nextString(r, extractInt(cs))); + break; + case 'b': + case 'f': + result.add(nextBytes(r, extractInt(cs))); + break; + case 'e': + case 'c': + case 'U': + extractInt(cs); + case '[': + case ']': + case '{': + case '}': + case 's': + break; + default: + Assert.fail(); + break; + } } - return new IntPair(r, i - 1); + return result.toArray(); } - private static class IntPair { - public final int first; - public final int second; - - public IntPair(int first, int second) { - this.first = first; - this.second = second; + private static int extractInt(InputScanner sc) { + int r = 0; + while (! sc.isDone()) { + if (Character.isDigit(sc.cur())) { + r = r * 10 + sc.cur() - '0'; + sc.next(); + } else { + break; + } } + return r; } + private static byte[] nextBytes(Random r, int length) { byte[] bb = new byte[length]; r.nextBytes(bb); @@ -243,24 +277,36 @@ return new String(cc); } - private static void check(Schema sc, byte[] bytes, - String calls, Object[] values, final int skipLevel) throws IOException { + private static void check(Schema sc, byte[] bytes, String calls, + Object[] values, final int skipLevel, Encoding encoding) + throws IOException { // dump(bytes); - Decoder bvi= new BinaryDecoder(new ByteArrayInputStream(bytes)); + // System.out.println(new String(bytes, "UTF-8")); + InputStream in = new ByteArrayInputStream(bytes); + Decoder bvi = null; + switch (encoding) { + case BINARY: + case BLOCKING_BINARY: + bvi = new BinaryDecoder(in); + break; + case JSON: + bvi = new JsonDecoder(sc, in); + } Decoder vi = new ValidatingDecoder(sc, bvi); check(vi, calls, values, skipLevel); } - public static void check(Decoder vi, String calls, Object[] values, - final int skipLevel) throws IOException { - char[] cc = calls.toCharArray(); + public static void check(Decoder vi, String calls, + Object[] values, final int skipLevel) throws IOException { + InputScanner cs = new InputScanner(calls.toCharArray()); int p = 0; int level = 0; long[] counts = new long[100]; boolean[] isArray = new boolean[100]; boolean[] isEmpty = new boolean[100]; - for (int i = 0; i < cc.length; i++) { - final char c = cc[i]; + while (! cs.isDone()) { + final char c = cs.cur(); + cs.next(); switch (c) { case 'N': vi.readNull(); @@ -278,7 +324,7 @@ Assert.assertEquals(vi.readLong(), l); break; case 'F': - float f = ((Float) values[p++]).floatValue(); + float f = floatValue(values[p++]); Assert.assertEquals(vi.readFloat(), f, Math.abs(f / 1000)); break; case 'D': @@ -286,7 +332,17 @@ Assert.assertEquals(vi.readDouble(), d, Math.abs(d / 1000)); break; case 'S': - i = extractInt(cc, i).second; + extractInt(cs); + if (level == skipLevel) { + vi.skipString(); + p++; + } else { + String s = (String) values[p++]; + Assert.assertEquals(vi.readString(null), new Utf8(s)); + } + break; + case 'K': + extractInt(cs); if (level == skipLevel) { vi.skipString(); p++; @@ -296,7 +352,7 @@ } break; case 'b': - i = extractInt(cc, i).second; + extractInt(cs); if (level == skipLevel) { vi.skipBytes(); p++; @@ -311,14 +367,13 @@ break; case 'f': { - IntPair ip = extractInt(cc, i); - i = ip.second; + int len = extractInt(cs); if (level == skipLevel) { - vi.skipFixed(ip.first); + vi.skipFixed(len); p++; } else { byte[] bb = (byte[]) values[p++]; - byte[] actBytes = new byte[ip.first]; + byte[] actBytes = new byte[len]; vi.readFixed(actBytes); Assert.assertEquals(actBytes, bb); } @@ -326,22 +381,17 @@ break; case 'e': { - IntPair ip = extractInt(cc, i); - i = ip.second; + int e = extractInt(cs); if (level == skipLevel) { vi.readEnum(); - p++; } else { - int e = ((Integer)(values[p++])).intValue(); Assert.assertEquals(vi.readEnum(), e); } } break; case '[': if (level == skipLevel) { - IntPair ip = skip(cc, i, vi); - i = ip.second; - p += ip.first; + p += skip(cs, vi, true); break; } else { level++; @@ -352,9 +402,7 @@ } case '{': if (level == skipLevel) { - IntPair ip = skip(cc, i, vi); - i = ip.second; - p += ip.first; + p += skip(cs, vi, false); break; } else { level++; @@ -388,13 +436,12 @@ counts[level]--; continue; case 'c': - i = extractInt(cc, i).second; + extractInt(cs); continue; - case 'u': + case 'U': { - IntPair ip = extractInt(cc, i); - Assert.assertEquals(ip.first, vi.readIndex()); - i = ip.second; + int idx = extractInt(cs); + Assert.assertEquals(idx, vi.readIndex()); continue; } default: @@ -404,6 +451,12 @@ Assert.assertEquals(p, values.length); } + private static float floatValue(Object object) { + return (object instanceof Integer) ? ((Integer) object).floatValue() : + (object instanceof Long) ? ((Long) object).floatValue() : + ((Float) object).floatValue(); + } + private static double doubleValue(Object object) { return (object instanceof Double) ? ((Double) object).doubleValue() : (object instanceof Float) ? ((Float) object).doubleValue() : @@ -417,26 +470,28 @@ ((Integer) object).longValue(); } - private static IntPair skip(char[] cc, int i, Decoder vi) + private static int skip(InputScanner cs, Decoder vi, boolean isArray) throws IOException { - char end = cc[i] == '[' ? ']' : '}'; - if (end == ']') { + final char end = isArray ? ']' : '}'; + if (isArray) { Assert.assertEquals(vi.skipArray(), 0); - } else { + } else if (end == '}'){ Assert.assertEquals(vi.skipMap(), 0); } int level = 0; int p = 0; - while (++i < cc.length) { - switch (cc[i]) { + while (! cs.isDone()) { + char c = cs.cur(); + cs.next(); + switch (c) { case '[': case '{': ++level; break; case ']': case '}': - if (cc[i] == end && level == 0) { - return new IntPair(p, i); + if (c == end && level == 0) { + return p; } level--; break; @@ -446,6 +501,7 @@ case 'F': case 'D': case 'S': + case 'K': case 'b': case 'f': case 'e': @@ -453,11 +509,93 @@ break; } } - return null; + throw new RuntimeException("Don't know how to skip"); } @DataProvider - public static Object[][] data() { + public static Iterator data() { + return cartesian(encodings, skipLevels, testSchemas()); + } + + private static Object[][] encodings = new Object[][] { + { Encoding.BINARY }, { Encoding.BLOCKING_BINARY }, + { Encoding.JSON } + }; + + private static Object[][] skipLevels = new Object[][] { + { -1 }, { 0 }, { 1 }, { 2 }, + }; + + /** + * Returns the Cartesian product of input sequences. + */ + public static Iterator cartesian(final Object[][]... values) { + return new Iterator() { + private int[] pos = new int[values.length]; + @Override + public boolean hasNext() { + return pos[0] < values[0].length; + } + + @Override + public Object[] next() { + Object[][] v = new Object[values.length][]; + for (int i = 0; i < v.length; i++) { + v[i] = values[i][pos[i]]; + } + for (int i = v.length - 1; i >= 0; i--) { + if (++pos[i] == values[i].length) { + if (i != 0) { + pos[i] = 0; + } + } else { + break; + } + } + return concat(v); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + + /** + * Concatenates the input sequences in order and forms a longer sequence. + */ + public static Object[] concat(Object[]... oo) { + int l = 0; + for (Object[] o : oo) { + l += o.length; + } + Object[] result = new Object[l]; + l = 0; + for (Object[] o : oo) { + System.arraycopy(o, 0, result, l, o.length); + l += o.length; + } + return result; + } + + /** + * Pastes incoming tables to form a wider table. All incoming tables + * should be of same height. + */ + static Object[][] paste(Object[][]... in) { + Object[][] result = new Object[in[0].length][]; + Object[][] cc = new Object[in.length][]; + for (int i = 0; i < result.length; i++) { + for (int j = 0; j < cc.length; j++) { + cc[j] = in[j][i]; + } + result[i] = concat(cc); + } + return result; + } + + public static Object[][] testSchemas() { /** * The first argument is a schema. * The second one is a sequence of (single character) mnemonics: @@ -467,10 +605,13 @@ * L long * F float * D double - * S followed by integer - string and its size - * b followed by integer - bytes and size - * u followed by integer - union and the index to choose + * K followed by integer - key-name (and its length) in a map + * S followed by integer - string and its length + * b followed by integer - bytes and length + * f followed by integer - fixed and length * c Number of items to follow in an array/map. + * U followed by integer - Union and its branch + * e followed by integer - Enum and its value * [ Start array * ] End array * { Start map @@ -491,15 +632,15 @@ { "{\"type\":\"fixed\", \"name\":\"fi\", \"size\": 0}", "f0" }, { "{\"type\":\"fixed\", \"name\":\"fi\", \"size\": 10}", "f10" }, { "{\"type\":\"enum\", \"name\":\"en\", \"symbols\":[\"v1\", \"v2\"]}", - "e2" }, + "e1" }, - { "{\"type\":\"array\", \"items\": \"boolean\"}", "[]" }, - { "{\"type\":\"array\", \"items\": \"int\"}", "[]" }, - { "{\"type\":\"array\", \"items\": \"long\"}", "[]" }, - { "{\"type\":\"array\", \"items\": \"float\"}", "[]" }, - { "{\"type\":\"array\", \"items\": \"double\"}", "[]" }, - { "{\"type\":\"array\", \"items\": \"string\"}", "[]" }, - { "{\"type\":\"array\", \"items\": \"bytes\"}", "[]" }, + { "{\"type\":\"array\", \"items\": \"boolean\"}", "[]", }, + { "{\"type\":\"array\", \"items\": \"int\"}", "[]", }, + { "{\"type\":\"array\", \"items\": \"long\"}", "[]", }, + { "{\"type\":\"array\", \"items\": \"float\"}", "[]", }, + { "{\"type\":\"array\", \"items\": \"double\"}", "[]", }, + { "{\"type\":\"array\", \"items\": \"string\"}", "[]", }, + { "{\"type\":\"array\", \"items\": \"bytes\"}", "[]", }, { "{\"type\":\"array\", \"items\":{\"type\":\"fixed\", " + "\"name\":\"fi\", \"size\": 10}}", "[]" }, @@ -525,18 +666,18 @@ { "{\"type\":\"map\", \"values\": " + "{\"type\":\"array\", \"items\":\"int\"}}", "{}" }, - { "{\"type\":\"map\", \"values\": \"boolean\"}", "{c1sSB}" }, - { "{\"type\":\"map\", \"values\": \"int\"}", "{c1sS5I}" }, - { "{\"type\":\"map\", \"values\": \"long\"}", "{c1sS5L}" }, - { "{\"type\":\"map\", \"values\": \"float\"}", "{c1sS5F}" }, - { "{\"type\":\"map\", \"values\": \"double\"}", "{c1sS5D}" }, - { "{\"type\":\"map\", \"values\": \"string\"}", "{c1sS5S10}" }, - { "{\"type\":\"map\", \"values\": \"bytes\"}", "{c1sS5b10}" }, + { "{\"type\":\"map\", \"values\": \"boolean\"}", "{c1sK5B}" }, + { "{\"type\":\"map\", \"values\": \"int\"}", "{c1sK5I}" }, + { "{\"type\":\"map\", \"values\": \"long\"}", "{c1sK5L}" }, + { "{\"type\":\"map\", \"values\": \"float\"}", "{c1sK5F}" }, + { "{\"type\":\"map\", \"values\": \"double\"}", "{c1sK5D}" }, + { "{\"type\":\"map\", \"values\": \"string\"}", "{c1sK5S10}" }, + { "{\"type\":\"map\", \"values\": \"bytes\"}", "{c1sK5b10}" }, { "{\"type\":\"map\", \"values\": " - + "{\"type\":\"array\", \"items\":\"int\"}}", "{c1sS[c3sIsIsI]}" }, + + "{\"type\":\"array\", \"items\":\"int\"}}", "{c1sK5[c3sIsIsI]}" }, { "{\"type\":\"map\", \"values\": \"boolean\"}", - "{c1sSBc2sSBsSB}" }, + "{c1sK5Bc2sK5BsK5B}" }, { "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" + "{\"name\":\"f\", \"type\":\"boolean\"}]}", "B" }, @@ -568,7 +709,7 @@ + "{\"name\":\"f6\", \"type\":\"string\"}," + "{\"name\":\"f7\", \"type\":\"bytes\"}]}", "NBILFDS10b25" }, - + // record of records { "{\"type\":\"record\",\"name\":\"outer\",\"fields\":[" + "{\"name\":\"f1\", \"type\":{\"type\":\"record\", " @@ -582,20 +723,23 @@ { "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" + "{\"name\":\"f1\", \"type\":\"long\"}," + "{\"name\":\"f2\", " - + "\"type\":{\"type\":\"array\", \"items\":\"int\"}}]}", "L[c1sI]" }, + + "\"type\":{\"type\":\"array\", \"items\":\"int\"}}]}", + "L[c1sI]" }, // record with map { "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" + "{\"name\":\"f1\", \"type\":\"long\"}," + "{\"name\":\"f2\", " - + "\"type\":{\"type\":\"map\", \"values\":\"int\"}}]}", "L{c1sS5I}" }, - + + "\"type\":{\"type\":\"map\", \"values\":\"int\"}}]}", + "L{c1sK5I}" }, + // array of records { "{\"type\":\"array\", \"items\":" + "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" + "{\"name\":\"f1\", \"type\":\"long\"}," + "{\"name\":\"f2\", \"type\":\"null\"}]}}", "[c2sLNsLN]" }, + { "{\"type\":\"array\", \"items\":" + "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" + "{\"name\":\"f1\", \"type\":\"long\"}," @@ -607,32 +751,60 @@ + "{\"name\":\"f1\", \"type\":\"long\"}," + "{\"name\":\"f2\", " + "\"type\":{\"type\":\"map\", \"values\":\"int\"}}]}}", - "[c2sL{c1sS5I}sL{c2sS5IsS5I}]" }, + "[c2sL{c1sK5I}sL{c2sK5IsK5I}]" }, { "{\"type\":\"array\", \"items\":" + "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" + "{\"name\":\"f1\", \"type\":\"long\"}," + "{\"name\":\"f2\", " + "\"type\":[\"null\", \"int\"]}]}}", - "[c2sLu0NsLu1I]" }, + "[c2sLU0NsLU1I]" }, - { "[\"boolean\"]", "u0B" }, - { "[\"int\"]", "u0I" }, - { "[\"long\"]", "u0L" }, - { "[\"float\"]", "u0F" }, - { "[\"double\"]", "u0D" }, - { "[\"string\"]", "u0S10" }, - { "[\"bytes\"]", "u0b10" }, - - { "[\"null\", \"int\"]", "u0N" }, - { "[\"boolean\", \"int\"]", "u0B" }, - { "[\"boolean\", \"int\"]", "u1I" }, - { "[\"boolean\", {\"type\":\"array\", \"items\":\"int\"} ]", "u0B" }, + { "[\"boolean\"]", "U0B" }, + { "[\"int\"]", "U0I" }, + { "[\"long\"]", "U0L" }, + { "[\"float\"]", "U0F" }, + { "[\"double\"]", "U0D" }, + { "[\"string\"]", "U0S10" }, + { "[\"bytes\"]", "U0b10" }, + + { "[\"null\", \"int\"]", "U0N" }, + { "[\"boolean\", \"int\"]", "U0B" }, + { "[\"boolean\", \"int\"]", "U1I" }, { "[\"boolean\", {\"type\":\"array\", \"items\":\"int\"} ]", - "u1[c1sI]" }, + "U0B" }, + + { "[\"boolean\", {\"type\":\"array\", \"items\":\"int\"} ]", + "U1[c1sI]" }, + + // Recursion + { "{\"type\": \"record\", \"name\": \"Node\", \"fields\": [" + + "{\"name\":\"label\", \"type\":\"string\"}," + + "{\"name\":\"children\", \"type\":" + + "{\"type\": \"array\", \"items\": \"Node\" }}]}", + "S10[c1sS10[]]" }, + + { "{\"type\": \"record\", \"name\": \"Lisp\", \"fields\": [" + + "{\"name\":\"value\", \"type\":[\"null\", \"string\"," + + "{\"type\": \"record\", \"name\": \"Cons\", \"fields\": [" + + "{\"name\":\"car\", \"type\":\"Lisp\"}," + + "{\"name\":\"cdr\", \"type\":\"Lisp\"}]}]}]}", + "U0N"}, + { "{\"type\": \"record\", \"name\": \"Lisp\", \"fields\": [" + + "{\"name\":\"value\", \"type\":[\"null\", \"string\"," + + "{\"type\": \"record\", \"name\": \"Cons\", \"fields\": [" + + "{\"name\":\"car\", \"type\":\"Lisp\"}," + + "{\"name\":\"cdr\", \"type\":\"Lisp\"}]}]}]}", + "U1S10"}, + { "{\"type\": \"record\", \"name\": \"Lisp\", \"fields\": [" + + "{\"name\":\"value\", \"type\":[\"null\", \"string\"," + + "{\"type\": \"record\", \"name\": \"Cons\", \"fields\": [" + + "{\"name\":\"car\", \"type\":\"Lisp\"}," + + "{\"name\":\"cdr\", \"type\":\"Lisp\"}]}]}]}", + "U2U1S10U0N"}, }; } - protected static void dump(byte[] bb) { + static void dump(byte[] bb) { int col = 0; for (byte b : bb) { if (col % 16 == 0) {