avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r796868 [2/2] - in /hadoop/avro/trunk: ./ src/java/org/apache/avro/generic/ src/java/org/apache/avro/io/ src/java/org/apache/avro/io/doc-files/ src/java/org/apache/avro/io/parsing/ src/java/org/apache/avro/io/parsing/doc-files/ src/java/org...
Date Wed, 22 Jul 2009 20:20:56 GMT
Added: hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/Symbol.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/Symbol.java?rev=796868&view=auto
==============================================================================
--- hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/Symbol.java (added)
+++ hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/Symbol.java Wed Jul 22 20:20:55 2009
@@ -0,0 +1,382 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.avro.io.parsing;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import org.codehaus.jackson.JsonEncoding;
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonGenerator;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+
+/**
+ * Symbol is the base of all symbols (terminals and non-terminals) of
+ * the grammar.
+ */
+public abstract class Symbol {
+  /*
+   * The type of symbol.
+   */
+  public enum Kind {
+    /** terminal symbols which have no productions */
+    TERMINAL,
+    /** Start symbol for some grammar */
+    ROOT,
+    /** non-terminal symbol which is a sequence of one or more other symbols */
+    SEQUENCE,
+    /** non-termial to represent the contents of an array or map */
+    REPEATER,
+    /** non-terminal to represent the union */
+    ALTERNATIVE,
+    /** non-terminal action symbol which are automatically consumed */
+    IMPLICIT_ACTION,
+    /** non-terminal action symbol which is explicitly consumed */
+    EXPLICIT_ACTION
+  };
+
+  /// The kind of this symbol.
+  public final Kind kind;
+
+  /**
+   * The production for this symbol. If this symbol is a terminal
+   * this is <tt>null</tt>. Otherwise this holds the the sequence of
+   * the symbols that forms the production for this symbol. The
+   * sequence is in the reverse order of production. This is useful
+   * for easy copying onto parsing stack.
+   * 
+   * Please note that this is a final. So the production for a symbol
+   * should be known before that symbol is constructed. This requirement
+   * cannot be met for those symbols which are recursive (e.g. a record that
+   * holds union a branch of which is the record itself). To resolve this
+   * problem, we initialize the symbol with an array of nulls. Later we
+   * fill the symbols. Not clean, but works. The other option is to not have
+   * this field a final. But keeping it final and thus keeping symbol immutable
+   * gives some confort. See various generators how we generate records.
+   */
+  public final Symbol[] production;
+  /**
+   * Constructs a new symbol of the given kind <tt>kind</tt>.
+   */
+  protected Symbol(Kind kind) {
+    this(kind, null);
+  }
+    
+    
+  protected Symbol(Kind kind, Symbol[] production) {
+    this.production = production;
+    this.kind = kind;
+  }
+
+  /**
+   * A convenience method to construct a root symbol.
+   */
+  static Symbol root(Symbol... symbols) {
+    return new Root(symbols);
+  }
+  /**
+   * A convenience method to construct a sequence.
+   * @param production  The constituent symbols of the sequence.
+   */
+  static Symbol seq(Symbol... production) {
+    return new Sequence(production);
+  }
+
+  /**
+   * A convenience method to construct a repeater.
+   * @param symsToRepeat The symbols to repeat in the repeater.
+   */
+  static Symbol repeat(Symbol endSymbol, Symbol... symsToRepeat) {
+    return new Repeater(endSymbol, symsToRepeat);
+  }
+
+  /**
+   *  A convenience method to construct a union.
+   */
+  static Symbol alt(Symbol[] symbols, String[] labels) {
+    return new Alternative(symbols, labels);
+  }
+
+  /**
+   * A convenience method to construct an ErrorAction.
+   * @param e
+   * @return
+   */
+  static Symbol error(String e) {
+    return new ErrorAction(e);
+  }
+  
+  /**
+   * A convenience method to construct a ResolvingAction.
+   * @param w The writer symbol
+   * @param r The reader symbol
+   */
+  static Symbol resolve(Symbol w, Symbol r) {
+    return new ResolvingAction(w, r);
+  }
+  
+  private static class Terminal extends Symbol {
+    private final String printName;
+    public Terminal(String printName) {
+      super(Kind.TERMINAL);
+      this.printName = printName;
+    }
+    public String toString() { return printName; }
+  }
+
+  private static class ImplicitAction extends Symbol {
+    private ImplicitAction() {
+      super(Kind.IMPLICIT_ACTION);
+    }
+  }
+  
+  protected static class Root extends Symbol {
+    private Root(Symbol... symbols) {
+      super(Kind.ROOT, makeProduction(symbols));
+      production[0] = this;
+    }
+
+    private static Symbol[] makeProduction(Symbol[] symbols) {
+      Symbol[] result = new Symbol[symbols.length + 1];
+      System.arraycopy(symbols, 0, result, 1, symbols.length);
+      return result;
+    }
+  }
+  protected static class Sequence extends Symbol implements Iterable<Symbol> {
+    private Sequence(Symbol[] productions) {
+      super(Kind.SEQUENCE, productions);
+    }
+
+    public Symbol get(int index) {
+      return production[index];
+    }
+    
+    public int size() {
+      return production.length;
+    }
+    
+    public Iterator<Symbol> iterator() {
+      return new Iterator<Symbol>() {
+        private int pos = production.length;
+        
+        public boolean hasNext() {
+          return 0 < pos;
+        }
+        
+        public Symbol next() {
+          if (0 < pos) {
+            return production[--pos];
+          } else {
+            throw new NoSuchElementException();
+          }
+        }
+        
+        public void remove() {
+          throw new UnsupportedOperationException();
+        }
+      };
+    }
+  }
+
+  public static class Repeater extends Symbol {
+    public final Symbol end;
+   
+    private Repeater(Symbol end, Symbol... sequenceToRepeat) {
+      super(Kind.REPEATER, makeProduction(sequenceToRepeat));
+      this.end = end;
+      production[0] = this;
+    }
+    
+    private static Symbol[] makeProduction(Symbol[] p) {
+      Symbol[] result = new Symbol[p.length + 1];
+      System.arraycopy(p, 0, result, 1, p.length);
+      return result;
+    }
+  }
+    
+  public static class Alternative extends Symbol {
+    public final Symbol[] symbols;
+    public final String[] labels;
+    private Alternative(Symbol[] symbols, String[] labels) {
+      super(Kind.ALTERNATIVE);
+      this.symbols = symbols;
+      this.labels = labels;
+    }
+    
+    public Symbol getSymbol(int index) {
+      return symbols[index];
+    }
+    
+    public String getLabel(int index) {
+      return labels[index];
+    }
+    
+    public int size() {
+      return symbols.length;
+    }
+
+    public int findLabel(String label) {
+      if (label != null) {
+        for (int i = 0; i < labels.length; i++) {
+          if (label.equals(labels[i])) {
+            return i;
+          }
+        }
+      }
+      return -1;
+    }
+  }
+
+  public static class ErrorAction extends ImplicitAction {
+    public final String msg;
+    private ErrorAction(String msg) {
+      this.msg = msg;
+    }
+  }
+
+  public static class IntCheckAction extends Symbol {
+    public final int size;
+    public IntCheckAction(int size) {
+      super(Kind.EXPLICIT_ACTION);
+      this.size = size;
+    }
+  }
+
+  public static class EnumAdjustAction extends IntCheckAction {
+    public final Object[] adjustments;
+    public EnumAdjustAction(int rsymCount, Object[] adjustments) {
+      super(rsymCount);
+      this.adjustments = adjustments;
+    }
+  }
+
+  public static class WriterUnionAction extends ImplicitAction {
+  }
+
+  public static class ResolvingAction extends ImplicitAction {
+    public final Symbol writer;
+    public final Symbol reader;
+    private ResolvingAction(Symbol writer, Symbol reader) {
+      this.writer = writer;
+      this.reader = reader;
+    }
+  }
+  
+  public static class SkipAction extends ImplicitAction {
+    public final Symbol symToSkip;
+    public SkipAction(Symbol symToSkip) {
+      this.symToSkip = symToSkip;
+    }
+  }
+
+  public static class FieldAdjustAction extends ImplicitAction {
+    public final int rindex;
+    public final String fname;
+    public FieldAdjustAction(int rindex, String fname) {
+      this.rindex = rindex;
+      this.fname = fname;
+    }
+  }
+  
+  public static class DefaultStartAction extends ImplicitAction {
+    public final Symbol root;
+    public final byte[] contents;
+    public DefaultStartAction(Symbol root, JsonNode defaultValue)
+      throws IOException {
+      this.root = root;
+      ByteArrayOutputStream os = new ByteArrayOutputStream();
+      JsonGenerator g = new JsonFactory().createJsonGenerator(os,
+          JsonEncoding.UTF8);
+      new ObjectMapper().writeTree(g, defaultValue);
+      g.flush();
+      this.contents = os.toByteArray();
+    }
+  }
+
+  public static class UnionAdjustAction extends ImplicitAction {
+    public final int rindex;
+    public final Symbol symToParse;
+    public UnionAdjustAction(int rindex, Symbol symToParse) {
+      this.rindex = rindex;
+      this.symToParse = symToParse;
+    }
+  }
+
+  /** For JSON. */
+  public static class EnumLabelsAction extends IntCheckAction {
+    public final List<String> symbols;
+    public EnumLabelsAction(List<String> symbols) {
+      super(symbols.size());
+      this.symbols = symbols;
+    }
+    
+    public String getLabel(int n) {
+      return symbols.get(n);
+    }
+    
+    public int findLabel(String l) {
+      if (l != null) {
+        for (int i = 0; i < symbols.size(); i++) {
+          if (l.equals(symbols.get(i))) {
+            return i;
+          }
+        }
+      }
+      return -1;
+    }
+  }
+
+  /**
+   * The terminal symbols for the grammar.
+   */
+  public static final Symbol NULL = new Symbol.Terminal("null");
+  public static final Symbol BOOLEAN = new Symbol.Terminal("boolean");
+  public static final Symbol INT = new Symbol.Terminal("int");
+  public static final Symbol LONG = new Symbol.Terminal("long");
+  public static final Symbol FLOAT = new Symbol.Terminal("float");
+  public static final Symbol DOUBLE = new Symbol.Terminal("double");
+  public static final Symbol STRING = new Symbol.Terminal("string");
+  public static final Symbol BYTES = new Symbol.Terminal("bytes");
+  public static final Symbol FIXED = new Symbol.Terminal("fixed");
+  public static final Symbol ENUM = new Symbol.Terminal("enum");
+  public static final Symbol UNION = new Symbol.Terminal("union");
+
+  public static final Symbol ARRAY_START = new Symbol.Terminal("array-start");
+  public static final Symbol ARRAY_END = new Symbol.Terminal("array-end");
+  public static final Symbol MAP_START = new Symbol.Terminal("map-start");
+  public static final Symbol MAP_END = new Symbol.Terminal("map-end");
+  public static final Symbol END = new Symbol.Terminal("end");
+  public static final Symbol ITEM_END = new Symbol.Terminal("item-end");
+
+  /* a pseudo terminal used by parsers */
+  public static final Symbol CONTINUE = new Symbol.Terminal("continue");
+  public static final Symbol FIELD_ACTION =
+    new Symbol.Terminal("field-action");
+
+  public static final Symbol RECORD_START = new ImplicitAction();
+  public static final Symbol RECORD_END = new ImplicitAction();
+  public static final Symbol UNION_END = new ImplicitAction();
+  
+  public static final Symbol DEFAULT_END_ACTION = new ImplicitAction();
+  public static final Symbol MAP_KEY_MARKER =
+    new Symbol.Terminal("map-key-marker");
+}

Added: hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java?rev=796868&view=auto
==============================================================================
--- hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java (added)
+++ hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java Wed Jul 22 20:20:55 2009
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.avro.io.parsing;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.avro.Schema;
+import org.apache.avro.Schema.Field;
+
+/**
+ * The class that generates validating grammar.
+ */
+public class ValidatingGrammarGenerator {
+  /**
+   * Returns the non-terminal that is the start symbol
+   * for the grammar for the given schema <tt>sc</tt>.
+   */
+  public Symbol generate(Schema schema) {
+    return Symbol.root(generate(schema, new HashMap<LitS, Symbol>()));
+  }
+
+  /**
+   * Returns the non-terminal that is the start symbol
+   * for the grammar for the given schema <tt>sc</tt>. If there is already an entry
+   * for the given schema in the given map <tt>seen</tt> then
+   * that entry is returned. Otherwise a new symbol is generated and
+   * an entry is inserted into the map.
+   * @param sc    The schema for which the start symbol is required
+   * @param seen  A map of schema to symbol mapping done so far.
+   * @return      The start symbol for the schema
+   */
+  public Symbol generate(Schema sc, Map<LitS, Symbol> seen) {
+    switch (sc.getType()) {
+    case NULL:
+      return Symbol.NULL;
+    case BOOLEAN:
+      return Symbol.BOOLEAN;
+    case INT:
+      return Symbol.INT;
+    case LONG:
+      return Symbol.LONG;
+    case FLOAT:
+      return Symbol.FLOAT;
+    case DOUBLE:
+      return Symbol.DOUBLE;
+    case STRING:
+      return Symbol.STRING;
+    case BYTES:
+      return Symbol.BYTES;
+    case FIXED:
+      return Symbol.seq(new Symbol.IntCheckAction(sc.getFixedSize()),
+          Symbol.FIXED);
+    case ENUM:
+      return Symbol.seq(new Symbol.IntCheckAction(sc.getEnumSymbols().size()),
+          Symbol.ENUM);
+    case ARRAY:
+      return Symbol.seq(Symbol.ARRAY_END,
+          Symbol.repeat(Symbol.ARRAY_END, generate(sc.getElementType(), seen)),
+          Symbol.ARRAY_START);
+    case MAP:
+      return Symbol.seq(Symbol.MAP_END,
+          Symbol.repeat(Symbol.MAP_END,
+              generate(sc.getValueType(), seen), Symbol.STRING),
+          Symbol.MAP_START);
+    case RECORD: {
+      LitS wsc = new LitS(sc);
+      Symbol rresult = seen.get(wsc);
+      if (rresult == null) {
+        Symbol[] production = new Symbol[sc.getFields().size()];
+
+        /**
+         * We construct a symbol without filling the array. Please see
+         * {@link Symbol#production} for the reason.
+         */
+        rresult = Symbol.seq(production);
+        seen.put(wsc, rresult);
+
+        int i = production.length;
+        for (Field f : sc.getFields().values()) {
+          production[--i] = generate(f.schema(), seen);
+        }
+      }
+      return rresult;
+    }
+    case UNION:
+      List<Schema> subs = sc.getTypes();
+      Symbol[] symbols = new Symbol[subs.size()];
+      String[] labels = new String[subs.size()];
+      
+      int i = 0;
+      for (Schema b : sc.getTypes()) {
+        symbols[i] = generate(b, seen);
+        labels[i] = b.getType().name();
+        i++;
+      }
+      return Symbol.seq(Symbol.alt(symbols, labels), Symbol.UNION);
+
+    default:
+      throw new RuntimeException("Unexpected schema type");
+    }
+  }
+
+  /** A wrapper around Schema that does "==" equality. */
+  static class LitS {
+    public final Schema actual;
+    public LitS(Schema actual) { this.actual = actual; }
+    
+    /**
+     * Two LitS are equal if and only if their underlying schema is
+     * the same (not merely equal).
+     */
+    public boolean equals(Object o) {
+      if (! (o instanceof LitS)) return false;
+      return actual == ((LitS)o).actual;
+    }
+    
+    public int hashCode() {
+      return actual.hashCode();
+    }
+  }
+}

Added: hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/package.html
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/package.html?rev=796868&view=auto
==============================================================================
--- hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/package.html (added)
+++ hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/package.html Wed Jul 22 20:20:55 2009
@@ -0,0 +1,40 @@
+<html>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<body>
+Implementation of Avro schemas as LL(1) grammars.
+
+<p>See
+<a href="doc-files/parsing.html">parser documentation</a> for details on how
+this is achieved.
+
+<p>
+The classes in this package are used by
+{@link org.apache.avro.io.ValidatingEncoder},
+{@link org.apache.avro.io.ValidatingDecoder},
+{@link org.apache.avro.io.ResolvingDecoder},
+{@link org.apache.avro.io.JsonEncoder} and
+{@link org.apache.avro.io.JsonDecoder},
+
+<p>
+Unless one plans to generate a variation of the grammar or use a grammar,
+one not need to understand these classes.
+
+</body>
+</html>

Modified: hadoop/avro/trunk/src/java/org/apache/avro/reflect/ReflectDatumWriter.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/java/org/apache/avro/reflect/ReflectDatumWriter.java?rev=796868&r1=796867&r2=796868&view=diff
==============================================================================
--- hadoop/avro/trunk/src/java/org/apache/avro/reflect/ReflectDatumWriter.java (original)
+++ hadoop/avro/trunk/src/java/org/apache/avro/reflect/ReflectDatumWriter.java Wed Jul 22 20:20:55 2009
@@ -45,7 +45,7 @@
   
   protected void writeEnum(Schema schema, Object datum, Encoder out)
     throws IOException {
-    out.writeInt(((Enum)datum).ordinal());
+    out.writeEnum(((Enum)datum).ordinal());
   }
 
   protected boolean isEnum(Object datum) {

Modified: hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java?rev=796868&r1=796867&r2=796868&view=diff
==============================================================================
--- hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java (original)
+++ hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java Wed Jul 22 20:20:55 2009
@@ -36,6 +36,8 @@
 import org.apache.avro.io.Encoder;
 import org.apache.avro.io.BinaryDecoder;
 import org.apache.avro.io.BinaryEncoder;
+import org.apache.avro.io.JsonDecoder;
+import org.apache.avro.io.JsonEncoder;
 import org.apache.avro.util.Utf8;
 import org.testng.annotations.Test;
 
@@ -186,29 +188,47 @@
       assertTrue("Datum does not validate against schema "+datum,
                  GenericData.validate(schema, datum));
 
-      checkSerialization(schema, datum,
-                         new GenericDatumWriter<Object>(), new GenericDatumReader<Object>());
+      checkBinary(schema, datum,
+                  new GenericDatumWriter<Object>(),
+                  new GenericDatumReader<Object>());
+      checkJson(schema, datum,
+                  new GenericDatumWriter<Object>(),
+                  new GenericDatumReader<Object>());
     }
   }
 
-  private static void checkSerialization(Schema schema, Object datum,
-                                         DatumWriter<Object> writer,
-                                         DatumReader<Object> reader)
+  private static void checkBinary(Schema schema, Object datum,
+                                  DatumWriter<Object> writer,
+                                  DatumReader<Object> reader)
     throws IOException {
     ByteArrayOutputStream out = new ByteArrayOutputStream();
     writer.setSchema(schema);
     writer.write(datum, new BinaryEncoder(out));
-      
     byte[] data = out.toByteArray();
-    // System.out.println("length = "+data.length);
 
     reader.setSchema(schema);
         
     Object decoded =
       reader.read(null, new BinaryDecoder(new ByteArrayInputStream(data)));
       
-    // System.out.println(GenericData.toString(datum));
-    // System.out.println(GenericData.toString(decoded));
+    assertEquals("Decoded data does not match.", datum, decoded);
+  }
+
+  private static void checkJson(Schema schema, Object datum,
+                                DatumWriter<Object> writer,
+                                DatumReader<Object> reader)
+    throws IOException {
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    Encoder encoder = new JsonEncoder(schema, out);
+    writer.setSchema(schema);
+    writer.write(datum, encoder);
+    encoder.flush();
+    byte[] data = out.toByteArray();
+
+    reader.setSchema(schema);
+    Object decoded =
+      reader.read(null, new JsonDecoder(schema, new ByteArrayInputStream(data)));
+      
     assertEquals("Decoded data does not match.", datum, decoded);
   }
 

Modified: hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestResolvingIO.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestResolvingIO.java?rev=796868&r1=796867&r2=796868&view=diff
==============================================================================
--- hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestResolvingIO.java (original)
+++ hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestResolvingIO.java Wed Jul 22 20:20:55 2009
@@ -19,268 +19,176 @@
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
-import java.util.Vector;
+import java.io.InputStream;
+import java.util.Iterator;
 
 import org.apache.avro.Schema;
+import org.apache.avro.io.TestValidatingIO.Encoding;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 public class TestResolvingIO {
-  @Test(dataProvider="data")
-  public void testRead(String jsonWriterSchema,
+  @Test(dataProvider="data1")
+  public void test_identical(Encoding encoding,
+      int skipLevel, String jsonWriterSchema,
       String writerCalls, 
-      String jsonReaderSchema,
-      String readerCalls) throws IOException {
-    test(jsonWriterSchema, writerCalls, jsonReaderSchema, readerCalls, -1);
+      String jsonReaderSchema, String readerCalls)
+  throws IOException {
+    performTest(encoding, skipLevel, jsonWriterSchema, writerCalls,
+        jsonReaderSchema, readerCalls);
   }
-
-  @Test(dataProvider="data")
-  public void testSkip_0(String jsonWriterSchema,
+  
+  private static final int COUNT = 10;
+  
+  @Test(dataProvider="data2")
+  public void test_compatible(Encoding encoding,
+      int skipLevel, String jsonWriterSchema,
       String writerCalls, 
-      String jsonReaderSchema,
-      String readerCalls) throws IOException {
-    test(jsonWriterSchema, writerCalls, jsonReaderSchema, readerCalls, 0);
+      String jsonReaderSchema, String readerCalls)
+  throws IOException {
+    performTest(encoding, skipLevel, jsonWriterSchema, writerCalls,
+        jsonReaderSchema, readerCalls);
   }
 
-  @Test(dataProvider="data")
-  public void testSkip_1(String jsonWriterSchema,
-      String writerCalls, 
-      String jsonReaderSchema,
-      String readerCalls) throws IOException {
-    test(jsonWriterSchema, writerCalls, jsonReaderSchema, readerCalls, 1);
+  @Test(dataProvider="data3")
+  public void test_resolving(Encoding encoding, int skipLevel,
+      String jsonWriterSchema, String writerCalls,
+      Object[] writerValues,
+      String jsonReaderSchema, String readerCalls, Object[] readerValues)
+    throws IOException {
+    Schema writerSchema = Schema.parse(jsonWriterSchema);
+    byte[] bytes = TestValidatingIO.make(writerSchema, writerCalls,
+        writerValues, Encoding.BINARY);
+    Schema readerSchema = Schema.parse(jsonReaderSchema);
+    check(writerSchema, readerSchema, bytes, readerCalls,
+        readerValues,
+        Encoding.BINARY, skipLevel);
   }
 
-  @Test(dataProvider="data")
-  public void testSkip_2(String jsonWriterSchema,
+  private void performTest(Encoding encoding,
+      int skipLevel, String jsonWriterSchema,
       String writerCalls, 
-      String jsonReaderSchema,
-      String readerCalls) throws IOException {
-    test(jsonWriterSchema, writerCalls, jsonReaderSchema, readerCalls, 2);
-  }
-
-  private void test(String jsonWriterSchema, String writerCalls,
-      String jsonReaderSchema, String readerCalls, int skipLevel)
-      throws IOException {
-    for (int i = 0; i < 10; i++) {
-      testOnce(jsonWriterSchema, writerCalls, jsonReaderSchema,
-          readerCalls, skipLevel);
+      String jsonReaderSchema, String readerCalls)
+  throws IOException {
+    for (int i = 0; i < COUNT; i++) {
+      testOnce(jsonWriterSchema, writerCalls,
+          jsonReaderSchema, readerCalls, encoding, skipLevel);
     }
   }
   
   private void testOnce(String jsonWriterSchema,
-      String writerCalls, 
+      String writerCalls,
       String jsonReaderSchema,
       String readerCalls,
+      Encoding encoding,
       int skipLevel) throws IOException {
-    Vector<Object> values = new Vector<Object>();
+    Object[] values = TestValidatingIO.randomValues(writerCalls);
     Schema writerSchema = Schema.parse(jsonWriterSchema);
     byte[] bytes = TestValidatingIO.make(writerSchema, writerCalls,
-        values, false);
+        values, encoding);
     Schema readerSchema = Schema.parse(jsonReaderSchema);
-    // dump(bytes);
-    check(writerSchema, readerSchema, bytes, readerCalls, values.toArray(),
-        skipLevel);
+    check(writerSchema, readerSchema, bytes, readerCalls,
+        values,
+        encoding, skipLevel);
   }
 
   private static void check(Schema wsc, Schema rsc, byte[] bytes,
-      String calls, Object[] values, int skipLevel) throws IOException {
-    // dump(bytes);
-    Decoder bvi= new BinaryDecoder(new ByteArrayInputStream(bytes));
+      String calls, Object[] values, Encoding encoding,
+      int skipLevel)
+      throws IOException {
+    // TestValidatingIO.dump(bytes);
+    // System.out.println(new String(bytes, "UTF-8"));
+    InputStream in = new ByteArrayInputStream(bytes);
+    Decoder bvi = null;
+    switch (encoding) {
+    case BINARY:
+    case BLOCKING_BINARY:
+      bvi = new BinaryDecoder(in);
+      break;
+    case JSON:
+      bvi = new JsonDecoder(wsc, in);
+      break;
+    }
     Decoder vi = new ResolvingDecoder(wsc, rsc, bvi);
     TestValidatingIO.check(vi, calls, values, skipLevel);
   }
   
   @DataProvider
-  public static Object[][] data() {
-    /**
-     * The first argument is a schema.
-     * The second one is a sequence of (single character) mnemonics:
-     * N  null
-     * B  boolean
-     * I  int
-     * L  long
-     * F  float
-     * D  double
-     * S  string
-     * b  bytes
-     * [  Start array
-     * ]  End array
-     * {  Start map
-     * }  End map
-     * s  start item
-     * 0-9  branch id for union
-     */
+  public static Iterator<Object[]> data1() {
+    return TestValidatingIO.cartesian(encodings, skipLevels,
+        TestValidatingIO.paste(TestValidatingIO.testSchemas(),
+            TestValidatingIO.testSchemas()));
+  }
+  
+  @DataProvider
+  public static Iterator<Object[]> data2() {
+    return TestValidatingIO.cartesian(encodings, skipLevels, testSchemas());
+  }
+  
+  @DataProvider
+  public static Iterator<Object[]> data3() {
+    return TestValidatingIO.cartesian(encodings, skipLevels,
+        dataForResolvingTests());
+  }
+
+  private static Object[][] encodings = new Object[][] { { Encoding.BINARY } };
+  private static Object[][] skipLevels =
+    new Object[][] { { -1 }, { 0 }, { 1 }, { 2 }  };
+  private static Object[][] testSchemas() {
+    // The mnemonics are the same as {@link TestValidatingIO#testSchemas}
     return new Object[][] {
-        { "\"null\"", "N", "\"null\"", "N" },
-        { "\"boolean\"", "B", "\"boolean\"", "B" },
-        { "\"int\"", "I", "\"int\"", "I" },
-        { "\"int\"", "I", "\"long\"", "L" },
-        // { "\"int\"", "I", "\"float\"", "F" },  // These promotions make sense?
+        // { "\"int\"", "I", "\"float\"", "F" }, // makes sense?
         { "\"int\"", "I", "\"double\"", "D" },
-        { "\"long\"", "L", "\"long\"", "L" },
         // { "\"long\"", "L", "\"float\"", "F" }, // And this?
         { "\"long\"", "L", "\"double\"", "D" },
-        { "\"float\"", "F", "\"float\"", "F" },
         { "\"float\"", "F", "\"double\"", "D" },
-        { "\"double\"", "D", "\"double\"", "D" },
         { "\"double\"", "D", "\"long\"", "L" },
-        { "\"string\"", "S", "\"string\"", "S" },
-        { "\"bytes\"", "b", "\"bytes\"", "b" },
-        { "{\"type\":\"fixed\", \"name\":\"fi\", \"size\": 0}", "f0",
-          "{\"type\":\"fixed\", \"name\":\"fi\", \"size\": 0}", "f0" },
-        { "{\"type\":\"fixed\", \"name\":\"fi\", \"size\": 10}", "f10",
-          "{\"type\":\"fixed\", \"name\":\"fi\", \"size\": 10}", "f10" },
-        { "{\"type\":\"enum\", \"name\":\"en\", \"symbols\":[\"v1\", \"v2\"]}",
-            "e2",
-          "{\"type\":\"enum\", \"name\":\"en\", \"symbols\":[\"v1\", \"v2\"]}",
-            "e2"},
 
-        { "{\"type\":\"array\", \"items\": \"boolean\"}", "[]",
-          "{\"type\":\"array\", \"items\": \"boolean\"}", "[]" },
-        { "{\"type\":\"array\", \"items\": \"int\"}", "[]",
-          "{\"type\":\"array\", \"items\": \"int\"}", "[]" },
         { "{\"type\":\"array\", \"items\": \"int\"}", "[]",
-          "{\"type\":\"array\", \"items\": \"long\"}", "[]" },
+          "{\"type\":\"array\", \"items\": \"long\"}", "[]", },
         { "{\"type\":\"array\", \"items\": \"int\"}", "[]",
           "{\"type\":\"array\", \"items\": \"double\"}", "[]" },
         { "{\"type\":\"array\", \"items\": \"long\"}", "[]",
-          "{\"type\":\"array\", \"items\": \"long\"}", "[]" },
-        { "{\"type\":\"array\", \"items\": \"long\"}", "[]",
           "{\"type\":\"array\", \"items\": \"double\"}", "[]" },
         { "{\"type\":\"array\", \"items\": \"float\"}", "[]",
-          "{\"type\":\"array\", \"items\": \"float\"}", "[]" },
-        { "{\"type\":\"array\", \"items\": \"float\"}", "[]",
           "{\"type\":\"array\", \"items\": \"double\"}", "[]" },
-        { "{\"type\":\"array\", \"items\": \"double\"}", "[]",
-          "{\"type\":\"array\", \"items\": \"double\"}", "[]", },
-        { "{\"type\":\"array\", \"items\": \"string\"}", "[]",
-          "{\"type\":\"array\", \"items\": \"string\"}", "[]" },
-        { "{\"type\":\"array\", \"items\": \"bytes\"}", "[]",
-          "{\"type\":\"array\", \"items\": \"bytes\"}", "[]" },
 
-        { "{\"type\":\"array\", \"items\": \"boolean\"}", "[c1sB]",
-            "{\"type\":\"array\", \"items\": \"boolean\"}", "[c1sB]" },
-        { "{\"type\":\"array\", \"items\": \"int\"}", "[c1sI]",
-          "{\"type\":\"array\", \"items\": \"int\"}", "[c1sI]" },
         { "{\"type\":\"array\", \"items\": \"int\"}", "[c1sI]",
           "{\"type\":\"array\", \"items\": \"long\"}", "[c1sL]" },
         { "{\"type\":\"array\", \"items\": \"int\"}", "[c1sI]",
           "{\"type\":\"array\", \"items\": \"double\"}", "[c1sD]" },
         { "{\"type\":\"array\", \"items\": \"long\"}", "[c1sL]",
-            "{\"type\":\"array\", \"items\": \"long\"}", "[c1sL]" },
-        { "{\"type\":\"array\", \"items\": \"long\"}", "[c1sL]",
           "{\"type\":\"array\", \"items\": \"double\"}", "[c1sD]" },
         { "{\"type\":\"array\", \"items\": \"float\"}", "[c1sF]",
-            "{\"type\":\"array\", \"items\": \"float\"}", "[c1sF]" },
-        { "{\"type\":\"array\", \"items\": \"float\"}", "[c1sF]",
           "{\"type\":\"array\", \"items\": \"double\"}", "[c1sD]" },
-        { "{\"type\":\"array\", \"items\": \"double\"}", "[c1sD]",
-            "{\"type\":\"array\", \"items\": \"double\"}", "[c1sD]" },
-        { "{\"type\":\"array\", \"items\": \"string\"}", "[c1sS]",
-            "{\"type\":\"array\", \"items\": \"string\"}", "[c1sS]" },
-        { "{\"type\":\"array\", \"items\": \"bytes\"}", "[c1sb]",
-            "{\"type\":\"array\", \"items\": \"bytes\"}", "[c1sb]" },
 
-        { "{\"type\":\"map\", \"values\": \"boolean\"}", "{}",
-          "{\"type\":\"map\", \"values\": \"boolean\"}", "{}" },
-        { "{\"type\":\"map\", \"values\": \"int\"}", "{}",
-          "{\"type\":\"map\", \"values\": \"int\"}", "{}" },
         { "{\"type\":\"map\", \"values\": \"int\"}", "{}",
           "{\"type\":\"map\", \"values\": \"long\"}", "{}" },
         { "{\"type\":\"map\", \"values\": \"int\"}", "{}",
           "{\"type\":\"map\", \"values\": \"double\"}", "{}" },
         { "{\"type\":\"map\", \"values\": \"long\"}", "{}",
-          "{\"type\":\"map\", \"values\": \"long\"}", "{}" },
-        { "{\"type\":\"map\", \"values\": \"long\"}", "{}",
           "{\"type\":\"map\", \"values\": \"double\"}", "{}" },
         { "{\"type\":\"map\", \"values\": \"float\"}", "{}",
-          "{\"type\":\"map\", \"values\": \"float\"}", "{}" },
-        { "{\"type\":\"map\", \"values\": \"float\"}", "{}",
-          "{\"type\":\"map\", \"values\": \"double\"}", "{}" },
-        { "{\"type\":\"map\", \"values\": \"double\"}", "{}",
           "{\"type\":\"map\", \"values\": \"double\"}", "{}" },
-        { "{\"type\":\"map\", \"values\": \"string\"}", "{}",
-          "{\"type\":\"map\", \"values\": \"string\"}", "{}" },
-        { "{\"type\":\"map\", \"values\": \"bytes\"}", "{}",
-          "{\"type\":\"map\", \"values\": \"bytes\"}", "{}" },
-        { "{\"type\":\"map\", \"values\": "
-          + "{\"type\":\"array\", \"items\":\"int\"}}", "{}",
-          "{\"type\":\"map\", \"values\": "
-            + "{\"type\":\"array\", \"items\":\"int\"}}", "{}" },
-
-        { "{\"type\":\"map\", \"values\": \"boolean\"}", "{c1sSB}",
-          "{\"type\":\"map\", \"values\": \"boolean\"}", "{c1sSB}" },
-        { "{\"type\":\"map\", \"values\": \"int\"}", "{c1sSI}",
-          "{\"type\":\"map\", \"values\": \"int\"}", "{c1sSI}" },
-        { "{\"type\":\"map\", \"values\": \"int\"}", "{c1sSI}",
-          "{\"type\":\"map\", \"values\": \"long\"}", "{c1sSL}" },
-        { "{\"type\":\"map\", \"values\": \"int\"}", "{c1sSI}",
-          "{\"type\":\"map\", \"values\": \"double\"}", "{c1sSD}" },
-        { "{\"type\":\"map\", \"values\": \"long\"}", "{c1sSL}",
-          "{\"type\":\"map\", \"values\": \"long\"}", "{c1sSL}" },
-        { "{\"type\":\"map\", \"values\": \"long\"}", "{c1sSL}",
-          "{\"type\":\"map\", \"values\": \"double\"}", "{c1sSD}" },
-        { "{\"type\":\"map\", \"values\": \"float\"}", "{c1sSF}",
-          "{\"type\":\"map\", \"values\": \"float\"}", "{c1sSF}" },
-        { "{\"type\":\"map\", \"values\": \"float\"}", "{c1sSF}",
-          "{\"type\":\"map\", \"values\": \"double\"}", "{c1sSD}" },
-        { "{\"type\":\"map\", \"values\": \"double\"}", "{c1sSD}",
-          "{\"type\":\"map\", \"values\": \"double\"}", "{c1sSD}" },
-        { "{\"type\":\"map\", \"values\": \"string\"}", "{c1sSS}",
-          "{\"type\":\"map\", \"values\": \"string\"}", "{c1sSS}" },
-        { "{\"type\":\"map\", \"values\": \"bytes\"}", "{c1sSb}",
-          "{\"type\":\"map\", \"values\": \"bytes\"}", "{c1sSb}" },
-        { "{\"type\":\"map\", \"values\": "
-          + "{\"type\":\"array\", \"items\":\"int\"}}", "{c1sS[c3sIsIsI]}",
-          "{\"type\":\"map\", \"values\": "
-          + "{\"type\":\"array\", \"items\":\"int\"}}", "{c1sS[c3sIsIsI]}" },
 
-        { "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f\", \"type\":\"boolean\"}]}", "B",
-          "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f\", \"type\":\"boolean\"}]}", "B" },
-        { "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f\", \"type\":\"int\"}]}", "I",
-          "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f\", \"type\":\"int\"}]}", "I"},
-        { "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f\", \"type\":\"int\"}]}", "I",
-          "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f\", \"type\":\"long\"}]}", "L"},
+        { "{\"type\":\"map\", \"values\": \"int\"}", "{c1sK5I}",
+          "{\"type\":\"map\", \"values\": \"long\"}", "{c1sK5L}" },
+        { "{\"type\":\"map\", \"values\": \"int\"}", "{c1sK5I}",
+          "{\"type\":\"map\", \"values\": \"double\"}", "{c1sK5D}" },
+        { "{\"type\":\"map\", \"values\": \"long\"}", "{c1sK5L}",
+          "{\"type\":\"map\", \"values\": \"double\"}", "{c1sK5D}" },
+        { "{\"type\":\"map\", \"values\": \"float\"}", "{c1sK5F}",
+          "{\"type\":\"map\", \"values\": \"double\"}", "{c1sK5D}" },
+
         { "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
           + "{\"name\":\"f\", \"type\":\"int\"}]}", "I",
           "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f\", \"type\":\"double\"}]}", "D"},
-        { "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f\", \"type\":\"long\"}]}", "L",
-          "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
           + "{\"name\":\"f\", \"type\":\"long\"}]}", "L" },
         { "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f\", \"type\":\"float\"}]}", "F",
-          "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f\", \"type\":\"float\"}]}", "F"},
-        { "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f\", \"type\":\"double\"}]}", "D",
-          "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f\", \"type\":\"double\"}]}", "D"},
-        { "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f\", \"type\":\"string\"}]}", "S",
-          "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f\", \"type\":\"string\"}]}", "S"},
-        { "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f\", \"type\":\"bytes\"}]}", "b",
+          + "{\"name\":\"f\", \"type\":\"int\"}]}", "I",
           "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f\", \"type\":\"bytes\"}]}", "b"},
+          + "{\"name\":\"f\", \"type\":\"double\"}]}", "D" },
 
-        // multi-field record
-        { "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f1\", \"type\":\"int\"},"
-          + "{\"name\":\"f2\", \"type\":\"double\"},"
-          + "{\"name\":\"f3\", \"type\":\"string\"}]}", "IDS",
-          "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f1\", \"type\":\"int\"},"
-          + "{\"name\":\"f2\", \"type\":\"double\"},"
-          + "{\"name\":\"f3\", \"type\":\"string\"}]}", "IDS" },
         // multi-field record with promotions
         { "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
           + "{\"name\":\"f0\", \"type\":\"boolean\"},"
@@ -293,68 +201,66 @@
           + "{\"name\":\"f2\", \"type\":\"double\"},"
           + "{\"name\":\"f3\", \"type\":\"string\"}]}", "BLDS" },
 
-        // record with array
-        { "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f1\", \"type\":\"long\"},"
-          + "{\"name\":\"f2\", "
-          + "\"type\":{\"type\":\"array\", \"items\":\"int\"}}]}", "L[c1sI]",
-          "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f1\", \"type\":\"long\"},"
-          + "{\"name\":\"f2\", "
-          + "\"type\":{\"type\":\"array\", \"items\":\"int\"}}]}", "L[c1sI]" },
+        { "[\"int\"]", "U0I",
+              "[\"long\"]", "U0L" },
+        { "[\"int\"]", "U0I",
+              "[\"double\"]", "U0D" },
+        { "[\"long\"]", "U0L",
+              "[\"double\"]", "U0D" },
+        { "[\"float\"]", "U0F",
+              "[\"double\"]", "U0D" },
+
+        { "\"int\"", "I", "[\"int\"]", "U0I" },
+
+        { "[\"int\"]", "U0I", "\"int\"", "I" },
+        { "[\"int\"]", "U0I", "\"long\"", "L" },
+
+        { "[\"boolean\", \"int\"]", "U1I",
+              "[\"boolean\", \"long\"]", "U1L" },
+        { "[\"boolean\", \"int\"]", "U1I",
+              "[\"long\", \"boolean\"]", "U0L" },
+    };
+  }
 
-        // record with map
+  private static Object[][] dataForResolvingTests() {
+    // The mnemonics are the same as {@link TestValidatingIO#testSchemas}
+    return new Object[][] {
+        // Reordered fields
         { "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f1\", \"type\":\"long\"},"
-          + "{\"name\":\"f2\", "
-          + "\"type\":{\"type\":\"map\", \"values\":\"int\"}}]}", "L{c1sSI}",
+          + "{\"name\":\"f1\", \"type\":\"int\"},"
+          + "{\"name\":\"f2\", \"type\":\"string\"}]}", "IS10",
+          new Object[] { 10, "hello" },
           "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
-          + "{\"name\":\"f1\", \"type\":\"long\"},"
-          + "{\"name\":\"f2\", "
-          + "\"type\":{\"type\":\"map\", \"values\":\"int\"}}]}", "L{c1sSI}" },
-
-        { "[\"boolean\"]", "u0B", "[\"boolean\"]", "u0B" },
-        { "[\"int\"]", "u0I", "[\"int\"]", "u0I" },
-        { "[\"int\"]", "u0I", "[\"long\"]", "u0L" },
-        { "[\"int\"]", "u0I", "[\"double\"]", "u0D" },
-        { "[\"long\"]", "u0L", "[\"long\"]", "u0L" },
-        { "[\"long\"]", "u0L", "[\"double\"]", "u0D" },
-        { "[\"float\"]", "u0F", "[\"float\"]", "u0F" },
-        { "[\"float\"]", "u0F", "[\"double\"]", "u0D" },
-        { "[\"double\"]", "u0D", "[\"double\"]", "u0D" },
-        { "[\"string\"]", "u0S", "[\"string\"]", "u0S" },
-        { "[\"bytes\"]", "u0b", "[\"bytes\"]", "u0b" },
-
-        { "\"int\"", "I", "[\"int\"]", "u0I" },
-
-        { "[\"int\"]", "u0I", "\"int\"", "I" },
-        { "[\"int\"]", "u0I", "\"long\"", "I" },
-
-        { "[\"null\", \"int\"]", "u0N", "[\"null\", \"int\"]", "u0N" },
-        { "[\"boolean\", \"int\"]", "u0B", "[\"boolean\", \"int\"]", "u0B" },
-        { "[\"boolean\", \"int\"]", "u1I", "[\"boolean\", \"int\"]", "u1I" },
-        { "[\"boolean\", \"int\"]", "u1I", "[\"int\", \"boolean\"]", "u0I" },
-        { "[\"boolean\", \"int\"]", "u1I", "[\"boolean\", \"long\"]", "u1L" },
-        { "[\"boolean\", \"int\"]", "u1I", "[\"long\", \"boolean\"]", "u0L" },
-        { "[\"boolean\", {\"type\":\"array\", \"items\":\"int\"} ]", "u0B",
-          "[\"boolean\", {\"type\":\"array\", \"items\":\"int\"} ]", "u0B" },
-        { "[\"boolean\", {\"type\":\"array\", \"items\":\"int\"} ]", "u1[c1sI]",
-          "[\"boolean\", {\"type\":\"array\", \"items\":\"int\"} ]", "u1[c1sI]" },
-
-          // TODO: test more complicated types such as arrays of records
+          + "{\"name\":\"f2\", \"type\":\"string\" },"
+          + "{\"name\":\"f1\", \"type\":\"long\"}]}", "LS10",
+          new Object[] { 10L, "hello" } },
+        
+        // Default values
+        { "{\"type\":\"record\",\"name\":\"r\",\"fields\":[]}", "",
+          new Object[] { },
+          "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
+          + "{\"name\":\"f\", \"type\":\"int\", \"default\": 100}]}", "I",
+          new Object[] { 100 } },
+        { "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
+            + "{\"name\":\"f2\", \"type\":\"int\"}]}", "I",
+          new Object[] { 10 },
+          "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
+          + "{\"name\":\"f1\", \"type\":\"int\", \"default\": 101},"
+          + "{\"name\":\"f2\", \"type\":\"int\"}]}", "II",
+          new Object[] { 10, 101 } },
+        { "{\"type\":\"record\",\"name\":\"outer\",\"fields\":["
+            + "{\"name\": \"g1\", " +
+            		"\"type\":{\"type\":\"record\",\"name\":\"inner\",\"fields\":["
+                + "{\"name\":\"f2\", \"type\":\"int\"}]}}, "
+            + "{\"name\": \"g2\", \"type\": \"long\"}]}", "IL",
+          new Object[] { 10, 11L },
+          "{\"type\":\"record\",\"name\":\"outer\",\"fields\":["
+            + "{\"name\": \"g1\", " +
+            		"\"type\":{\"type\":\"record\",\"name\":\"inner\",\"fields\":["
+                + "{\"name\":\"f1\", \"type\":\"int\", \"default\": 101},"
+                + "{\"name\":\"f2\", \"type\":\"int\"}]}}, "
+          + "{\"name\": \"g2\", \"type\": \"long\"}]}}", "IIL",
+          new Object[] { 10, 101, 11L } },
     };
   }
-  
-  protected static void dump(byte[] bb) {
-    int col = 0;
-    for (byte b : bb) {
-      if (col % 16 == 0) {
-        System.out.println();
-      }
-      col++;
-      System.out.print(Integer.toHexString(b & 0xff) + " ");
-    }
-    System.out.println();
-  }
-
 }

Modified: hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestValidatingIO.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestValidatingIO.java?rev=796868&r1=796867&r2=796868&view=diff
==============================================================================
--- hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestValidatingIO.java (original)
+++ hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestValidatingIO.java Wed Jul 22 20:20:55 2009
@@ -20,7 +20,9 @@
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.ByteBuffer;
+import java.util.Iterator;
 import java.util.Random;
 import java.util.Vector;
 
@@ -32,146 +34,139 @@
 
 
 public class TestValidatingIO {
-  @Test(dataProvider="data")
-  public void testRead(String jsonSchema, String calls) throws IOException {
-    test(jsonSchema, calls, -1, false);
-  }
-
-  @Test(dataProvider="data")
-  public void testSkip_0(String jsonSchema, String calls) throws IOException {
-    test(jsonSchema, calls, 0, false);
-  }
-  
-  @Test(dataProvider="data")
-  public void testSkip_1(String jsonSchema, String calls) throws IOException {
-    test(jsonSchema, calls, 1, false);
-  }
-  
-  @Test(dataProvider="data")
-  public void testSkip_2(String jsonSchema, String calls) throws IOException {
-    test(jsonSchema, calls, 2, false);
-  }
-  
-  @Test(dataProvider="data")
-  public void testRead_blocking(String jsonSchema, String calls)
-    throws IOException {
-    test(jsonSchema, calls, -1, true);
-  }
-
-  @Test(dataProvider="data")
-  public void testSkip_0_blocking(String jsonSchema, String calls)
-    throws IOException {
-    test(jsonSchema, calls, 0, true);
-  }
+  enum Encoding {
+    BINARY,
+    BLOCKING_BINARY,
+    JSON,
+  };
   
-  @Test(dataProvider="data")
-  public void testSkip_1_blocking(String jsonSchema, String calls)
-    throws IOException {
-    test(jsonSchema, calls, 1, true);
-  }
+  private static int COUNT = 1;
   
   @Test(dataProvider="data")
-  public void testSkip_2_blocking(String jsonSchema, String calls)
-    throws IOException {
-    test(jsonSchema, calls, 2, true);
-  }
-  
-  private void test(String jsonSchema, String calls, int skipLevel,
-      boolean useBlocking)
-    throws IOException {
-    for (int i = 0; i < 10; i++) {
-      testOnce(Schema.parse(jsonSchema), calls, skipLevel, useBlocking);
+  public void test(Encoding encoding, int skipLevel,
+      String jsonSchema, String calls) throws IOException {
+    for (int i = 0; i < COUNT; i++) {
+      testOnce(Schema.parse(jsonSchema), calls, skipLevel, encoding);
     }
   }
-  
-  private void testOnce(Schema schema, String calls, int skipLevel,
-      boolean useBlocking)
+
+  private void testOnce(Schema schema, String calls,
+      int skipLevel,
+      Encoding encoding)
     throws IOException {
-    Vector<Object> values = new Vector<Object>();
-    byte[] bytes = make(schema, calls, values, useBlocking);
-    check(schema, bytes, calls, values.toArray(), skipLevel);
+    Object[] values = randomValues(calls);
+    byte[] bytes = make(schema, calls, values, encoding);
+    check(schema, bytes, calls, values, skipLevel, encoding);
   }
 
   public static byte[] make(Schema sc, String calls,
-      Vector<Object> values, boolean useBlocking) throws IOException {
+      Object[] values, Encoding encoding) throws IOException {
     ByteArrayOutputStream ba = new ByteArrayOutputStream();
-    Encoder bvo = useBlocking ? new BlockingBinaryEncoder(ba) :
-        new BinaryEncoder(ba);
+    Encoder bvo = null;
+    switch (encoding) {
+    case BINARY:
+      bvo = new BinaryEncoder(ba);
+      break;
+    case BLOCKING_BINARY:
+      bvo = new BlockingBinaryEncoder(ba);
+      break;
+    case JSON:
+      bvo = new JsonEncoder(sc, ba);
+      break;
+    }
+        
     Encoder vo = new ValidatingEncoder(sc, bvo);
     generate(vo, calls, values);
     vo.flush();
     return ba.toByteArray();
   }
 
+  public static class InputScanner {
+    private final char[] chars;
+    private int cpos = 0;
+    
+    public InputScanner(char[] chars) {
+      this.chars = chars;
+    }
+    
+    public boolean next() {
+      if (cpos < chars.length) {
+        cpos++;
+      }
+      return cpos != chars.length;
+    }
+    
+    public char cur() {
+      return chars[cpos];
+    }
+    
+    public boolean isDone() {
+      return cpos == chars.length;
+    }
+  }
   public static void generate(Encoder vw, String calls,
-      Vector<Object> values) throws IOException {
-    char[] cc = calls.toCharArray();
-    Random r = new Random();
-    for (int i = 0; i < cc.length; i++) {
-      char c = cc[i];
+      Object[] values) throws IOException {
+    InputScanner cs = new InputScanner(calls.toCharArray());
+    int p = 0;
+    while (! cs.isDone()) {
+      char c = cs.cur();
+      cs.next();
       switch (c) {
       case 'N':
         vw.writeNull();
         break;
       case 'B':
-        boolean b = r.nextBoolean();
-        values.add(b);
+        boolean b = (Boolean) values[p++];
         vw.writeBoolean(b);
         break;
       case 'I':
-        int ii = r.nextInt();
-        values.add(ii);
+        int ii = (Integer) values[p++];
         vw.writeInt(ii);
         break;
       case 'L':
-        long l = r.nextLong();
-        values.add(l);
+        long l = (Long) values[p++];
         vw.writeLong(l);
         break;
       case 'F':
-        float f = r.nextFloat();
-        values.add(f);
+        float f = (Float) values[p++];
         vw.writeFloat(f);
         break;
       case 'D':
-        double d = r.nextDouble();
-        values.add(d);
+        double d = (Double) values[p++];
         vw.writeDouble(d);
         break;
       case 'S':
         {
-          IntPair ip = extractInt(cc, i);
-          String s = nextString(r, ip.first);
-          values.add(s);
+          extractInt(cs);
+          String s = (String) values[p++];
           vw.writeString(new Utf8(s));
-          i = ip.second;
           break;
         }
+      case 'K':
+      {
+        extractInt(cs);
+        String s = (String) values[p++];
+        vw.writeString(s);
+        break;
+      }
       case 'b':
         {
-          IntPair ip = extractInt(cc, i);
-          byte[] bb = nextBytes(r, ip.first);
-          values.add(bb);
+          extractInt(cs);
+          byte[] bb = (byte[]) values[p++];
           vw.writeBytes(bb);
-          i = ip.second;
           break;
         }
       case 'f':
         {
-          IntPair ip = extractInt(cc, i);
-          byte[] bb = nextBytes(r, ip.first);
-          values.add(bb);
+          extractInt(cs);
+          byte[] bb = (byte[]) values[p++];
           vw.writeFixed(bb);
-          i = ip.second;
           break;
         }
       case 'e':
         {
-          IntPair ip = extractInt(cc, i);
-          int e = r.nextInt(ip.first);
-          values.add(e);
+          int e = extractInt(cs);
           vw.writeEnum(e);
-          i = ip.second;
           break;
         }
       case '[':
@@ -187,20 +182,14 @@
         vw.writeMapEnd();
         break;
       case 'c':
-        {
-          IntPair ip = extractInt(cc, i);
-          vw.setItemCount(ip.first);
-          i = ip.second;
-        }
+        vw.setItemCount(extractInt(cs));
         break;
       case 's':
         vw.startItem();
         break;
-      case 'u':
+      case 'U':
         {
-          IntPair ip = extractInt(cc, i);
-          vw.writeIndex(ip.first);
-          i = ip.second;
+          vw.writeIndex(extractInt(cs));
           break;
         }
       default:
@@ -210,25 +199,70 @@
     }
   }
 
-  private static IntPair extractInt(char[] cc, int i) {
-    int r = 0;
-    i++;
-    while (i < cc.length && Character.isDigit(cc[i])) {
-      r = r * 10 + cc[i] - '0';
-      i++;
+  public static Object[] randomValues(String calls) {
+    Random r = new Random();
+    InputScanner cs = new InputScanner(calls.toCharArray());
+    Vector<Object> result = new Vector<Object>();
+    while (! cs.isDone()) {
+      char c = cs.cur();
+      cs.next();
+      switch (c) {
+      case 'N':
+        break;
+      case 'B':
+        result.add(r.nextBoolean());
+        break;
+      case 'I':
+        result.add(r.nextInt());
+        break;
+      case 'L':
+        result.add(r.nextLong());
+        break;
+      case 'F':
+        result.add(r.nextFloat());
+        break;
+      case 'D':
+        result.add(r.nextDouble());
+        break;
+      case 'S':
+      case 'K':
+        result.add(nextString(r, extractInt(cs)));
+        break;
+      case 'b':
+      case 'f':
+        result.add(nextBytes(r, extractInt(cs)));
+        break;
+      case 'e':
+      case 'c':
+      case 'U':
+        extractInt(cs);
+      case '[':
+      case ']':
+      case '{':
+      case '}':
+      case 's':
+        break;
+      default:
+        Assert.fail();
+        break;
+      }
     }
-    return new IntPair(r, i - 1);
+    return result.toArray();
   }
 
-  private static class IntPair {
-    public final int first;
-    public final int second;
-    
-    public IntPair(int first, int second) {
-      this.first = first;
-      this.second = second;
+  private static int extractInt(InputScanner sc) {
+    int r = 0;
+    while (! sc.isDone()) {
+      if (Character.isDigit(sc.cur())) {
+        r = r * 10 + sc.cur() - '0';
+        sc.next();
+      } else {
+        break;
+      }
     }
+    return r;
   }
+
   private static byte[] nextBytes(Random r, int length) {
     byte[] bb = new byte[length];
     r.nextBytes(bb);
@@ -243,24 +277,36 @@
     return new String(cc);
   }
 
-  private static void check(Schema sc, byte[] bytes,
-      String calls, Object[] values, final int skipLevel) throws IOException {
+  private static void check(Schema sc, byte[] bytes, String calls,
+      Object[] values, final int skipLevel, Encoding encoding)
+    throws IOException {
     // dump(bytes);
-    Decoder bvi= new BinaryDecoder(new ByteArrayInputStream(bytes));
+    // System.out.println(new String(bytes, "UTF-8"));
+    InputStream in = new ByteArrayInputStream(bytes);
+    Decoder bvi = null;
+    switch (encoding) {
+    case BINARY:
+    case BLOCKING_BINARY:
+      bvi = new BinaryDecoder(in);
+      break;
+    case JSON:
+      bvi = new JsonDecoder(sc, in);
+    }
     Decoder vi = new ValidatingDecoder(sc, bvi);
     check(vi, calls, values, skipLevel);
   }
   
-  public static void check(Decoder vi, String calls, Object[] values,
-      final int skipLevel) throws IOException {
-    char[] cc = calls.toCharArray();
+  public static void check(Decoder vi, String calls,
+      Object[] values, final int skipLevel) throws IOException {
+    InputScanner cs = new InputScanner(calls.toCharArray());
     int p = 0;
     int level = 0;
     long[] counts = new long[100];
     boolean[] isArray = new boolean[100];
     boolean[] isEmpty = new boolean[100];
-    for (int i = 0; i < cc.length; i++) {
-      final char c = cc[i];
+    while (! cs.isDone()) {
+      final char c = cs.cur();
+      cs.next();
       switch (c) {
       case 'N':
         vi.readNull();
@@ -278,7 +324,7 @@
         Assert.assertEquals(vi.readLong(), l);
         break;
       case 'F':
-        float f = ((Float) values[p++]).floatValue(); 
+        float f = floatValue(values[p++]); 
         Assert.assertEquals(vi.readFloat(), f, Math.abs(f / 1000));
         break;
       case 'D':
@@ -286,7 +332,17 @@
         Assert.assertEquals(vi.readDouble(), d, Math.abs(d / 1000));
         break;
       case 'S':
-        i = extractInt(cc, i).second;
+        extractInt(cs);
+        if (level == skipLevel) {
+          vi.skipString();
+          p++;
+        } else {
+          String s = (String) values[p++]; 
+          Assert.assertEquals(vi.readString(null), new Utf8(s));
+        }
+        break;
+      case 'K':
+        extractInt(cs);
         if (level == skipLevel) {
           vi.skipString();
           p++;
@@ -296,7 +352,7 @@
         }
         break;
       case 'b':
-        i = extractInt(cc, i).second;
+        extractInt(cs);
         if (level == skipLevel) {
           vi.skipBytes();
           p++;
@@ -311,14 +367,13 @@
         break;
       case 'f':
         {
-          IntPair ip = extractInt(cc, i);
-          i = ip.second;
+          int len = extractInt(cs);
           if (level == skipLevel) {
-            vi.skipFixed(ip.first);
+            vi.skipFixed(len);
             p++;
           } else {
             byte[] bb = (byte[]) values[p++];
-            byte[] actBytes = new byte[ip.first];
+            byte[] actBytes = new byte[len];
             vi.readFixed(actBytes);
             Assert.assertEquals(actBytes, bb);
           }
@@ -326,22 +381,17 @@
         break;
       case 'e':
       {
-        IntPair ip = extractInt(cc, i);
-        i = ip.second;
+        int e = extractInt(cs);
         if (level == skipLevel) {
           vi.readEnum();
-          p++;
         } else {
-          int e = ((Integer)(values[p++])).intValue();
           Assert.assertEquals(vi.readEnum(), e);
         }
       }
       break;
       case '[':
         if (level == skipLevel) {
-          IntPair ip = skip(cc, i, vi);
-          i = ip.second;
-          p += ip.first;
+          p += skip(cs, vi, true);
           break;
         } else {
           level++;
@@ -352,9 +402,7 @@
         }
       case '{':
         if (level == skipLevel) {
-          IntPair ip = skip(cc, i, vi);
-          i = ip.second;
-          p += ip.first;
+          p += skip(cs, vi, false);
           break;
         } else {
           level++;
@@ -388,13 +436,12 @@
         counts[level]--;
         continue;
       case 'c':
-        i = extractInt(cc, i).second;
+        extractInt(cs);
         continue;
-      case 'u':
+      case 'U':
         {
-          IntPair ip = extractInt(cc, i);
-          Assert.assertEquals(ip.first, vi.readIndex());
-          i = ip.second;
+          int idx = extractInt(cs);
+          Assert.assertEquals(idx, vi.readIndex());
           continue;
         }
       default:
@@ -404,6 +451,12 @@
     Assert.assertEquals(p, values.length);
   }
   
+  private static float floatValue(Object object) {
+    return (object instanceof Integer) ? ((Integer) object).floatValue() :
+      (object instanceof Long) ? ((Long) object).floatValue() :
+      ((Float) object).floatValue();
+  }
+
   private static double doubleValue(Object object) {
     return (object instanceof Double) ? ((Double) object).doubleValue() :
       (object instanceof Float) ? ((Float) object).doubleValue() :
@@ -417,26 +470,28 @@
       ((Integer) object).longValue();
   }
 
-  private static IntPair skip(char[] cc, int i, Decoder vi)
+  private static int skip(InputScanner cs, Decoder vi, boolean isArray)
     throws IOException {
-    char end = cc[i] == '[' ? ']' : '}';
-    if (end == ']') {
+    final char end = isArray ? ']' : '}';
+    if (isArray) {
       Assert.assertEquals(vi.skipArray(), 0);
-    } else {
+    } else if (end == '}'){
       Assert.assertEquals(vi.skipMap(), 0);
     }
     int level = 0;
     int p = 0;
-    while (++i < cc.length) {
-      switch (cc[i]) {
+    while (! cs.isDone()) {
+      char c = cs.cur();
+      cs.next();
+      switch (c) {
       case '[':
       case '{':
         ++level;
         break;
       case ']':
       case '}':
-        if (cc[i] == end && level == 0) {
-          return new IntPair(p, i);
+        if (c == end && level == 0) {
+          return p;
         }
         level--;
         break;
@@ -446,6 +501,7 @@
       case 'F':
       case 'D':
       case 'S':
+      case 'K':
       case 'b':
       case 'f':
       case 'e':
@@ -453,11 +509,93 @@
         break;
       }
     }
-    return null;
+    throw new RuntimeException("Don't know how to skip");
   }
 
   @DataProvider
-  public static Object[][] data() {
+  public static Iterator<Object[]> data() {
+    return cartesian(encodings, skipLevels, testSchemas());
+  }
+  
+  private static Object[][] encodings = new Object[][] {
+      { Encoding.BINARY }, { Encoding.BLOCKING_BINARY },
+      { Encoding.JSON }
+    }; 
+
+  private static Object[][] skipLevels = new Object[][] {
+      { -1 }, { 0 }, { 1 }, { 2 },
+  };
+  
+  /**
+   * Returns the Cartesian product of input sequences.
+   */
+  public static Iterator<Object[]> cartesian(final Object[][]... values) {
+    return new Iterator<Object[]>() {
+      private int[] pos = new int[values.length];
+      @Override
+      public boolean hasNext() {
+        return pos[0] < values[0].length;
+      }
+
+      @Override
+      public Object[] next() {
+        Object[][] v = new Object[values.length][];
+        for (int i = 0; i < v.length; i++) {
+          v[i] = values[i][pos[i]];
+        }
+        for (int i = v.length - 1; i >= 0; i--) {
+          if (++pos[i] == values[i].length) {
+            if (i != 0) {
+              pos[i] = 0;
+            }
+          } else {
+            break;
+          }
+        }
+        return concat(v);
+      }
+
+      @Override
+      public void remove() {
+        throw new UnsupportedOperationException();
+      }
+    };
+  }
+  
+  /**
+   * Concatenates the input sequences in order and forms a longer sequence.
+   */
+  public static Object[] concat(Object[]... oo) {
+    int l = 0;
+    for (Object[] o : oo) {
+      l += o.length;
+    }
+    Object[] result = new Object[l];
+    l = 0;
+    for (Object[] o : oo) {
+      System.arraycopy(o, 0, result, l, o.length);
+      l += o.length;
+    }
+    return result;
+  }
+
+  /**
+   * Pastes incoming tables to form a wider table. All incoming tables
+   * should be of same height.
+   */
+  static Object[][] paste(Object[][]... in) {
+    Object[][] result = new Object[in[0].length][];
+    Object[][] cc = new Object[in.length][]; 
+    for (int i = 0; i < result.length; i++) {
+      for (int j = 0; j < cc.length; j++) {
+        cc[j] = in[j][i];
+      }
+      result[i] = concat(cc);
+    }
+    return result;
+  }
+
+  public static Object[][] testSchemas() {
     /**
      * The first argument is a schema.
      * The second one is a sequence of (single character) mnemonics:
@@ -467,10 +605,13 @@
      * L  long
      * F  float
      * D  double
-     * S followed by integer - string and its size
-     * b followed by integer - bytes and size
-     * u followed by integer - union and the index to choose
+     * K followed by integer - key-name (and its length) in a map
+     * S followed by integer - string and its length
+     * b followed by integer - bytes and length
+     * f followed by integer - fixed and length
      * c  Number of items to follow in an array/map.
+     * U followed by integer - Union and its branch
+     * e followed by integer - Enum and its value
      * [  Start array
      * ]  End array
      * {  Start map
@@ -491,15 +632,15 @@
         { "{\"type\":\"fixed\", \"name\":\"fi\", \"size\": 0}", "f0" },
         { "{\"type\":\"fixed\", \"name\":\"fi\", \"size\": 10}", "f10" },
         { "{\"type\":\"enum\", \"name\":\"en\", \"symbols\":[\"v1\", \"v2\"]}",
-            "e2" },
+            "e1" },
 
-        { "{\"type\":\"array\", \"items\": \"boolean\"}", "[]" },
-        { "{\"type\":\"array\", \"items\": \"int\"}", "[]" },
-        { "{\"type\":\"array\", \"items\": \"long\"}", "[]" },
-        { "{\"type\":\"array\", \"items\": \"float\"}", "[]" },
-        { "{\"type\":\"array\", \"items\": \"double\"}", "[]" },
-        { "{\"type\":\"array\", \"items\": \"string\"}", "[]" },
-        { "{\"type\":\"array\", \"items\": \"bytes\"}", "[]" },
+        { "{\"type\":\"array\", \"items\": \"boolean\"}", "[]", },
+        { "{\"type\":\"array\", \"items\": \"int\"}", "[]", },
+        { "{\"type\":\"array\", \"items\": \"long\"}", "[]", },
+        { "{\"type\":\"array\", \"items\": \"float\"}", "[]", },
+        { "{\"type\":\"array\", \"items\": \"double\"}", "[]", },
+        { "{\"type\":\"array\", \"items\": \"string\"}", "[]", },
+        { "{\"type\":\"array\", \"items\": \"bytes\"}", "[]", },
         { "{\"type\":\"array\", \"items\":{\"type\":\"fixed\", "
           + "\"name\":\"fi\", \"size\": 10}}", "[]" },
 
@@ -525,18 +666,18 @@
         { "{\"type\":\"map\", \"values\": "
           + "{\"type\":\"array\", \"items\":\"int\"}}", "{}" },
 
-        { "{\"type\":\"map\", \"values\": \"boolean\"}", "{c1sSB}" },
-        { "{\"type\":\"map\", \"values\": \"int\"}", "{c1sS5I}" },
-        { "{\"type\":\"map\", \"values\": \"long\"}", "{c1sS5L}" },
-        { "{\"type\":\"map\", \"values\": \"float\"}", "{c1sS5F}" },
-        { "{\"type\":\"map\", \"values\": \"double\"}", "{c1sS5D}" },
-        { "{\"type\":\"map\", \"values\": \"string\"}", "{c1sS5S10}" },
-        { "{\"type\":\"map\", \"values\": \"bytes\"}", "{c1sS5b10}" },
+        { "{\"type\":\"map\", \"values\": \"boolean\"}", "{c1sK5B}" },
+        { "{\"type\":\"map\", \"values\": \"int\"}", "{c1sK5I}" },
+        { "{\"type\":\"map\", \"values\": \"long\"}", "{c1sK5L}" },
+        { "{\"type\":\"map\", \"values\": \"float\"}", "{c1sK5F}" },
+        { "{\"type\":\"map\", \"values\": \"double\"}", "{c1sK5D}" },
+        { "{\"type\":\"map\", \"values\": \"string\"}", "{c1sK5S10}" },
+        { "{\"type\":\"map\", \"values\": \"bytes\"}", "{c1sK5b10}" },
         { "{\"type\":\"map\", \"values\": "
-          + "{\"type\":\"array\", \"items\":\"int\"}}", "{c1sS[c3sIsIsI]}" },
+          + "{\"type\":\"array\", \"items\":\"int\"}}", "{c1sK5[c3sIsIsI]}" },
 
         { "{\"type\":\"map\", \"values\": \"boolean\"}",
-            "{c1sSBc2sSBsSB}" },
+            "{c1sK5Bc2sK5BsK5B}" },
 
         { "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
           + "{\"name\":\"f\", \"type\":\"boolean\"}]}", "B" },
@@ -568,7 +709,7 @@
           + "{\"name\":\"f6\", \"type\":\"string\"},"
           + "{\"name\":\"f7\", \"type\":\"bytes\"}]}",
             "NBILFDS10b25" },
-
+        
         // record of records
         { "{\"type\":\"record\",\"name\":\"outer\",\"fields\":["
           + "{\"name\":\"f1\", \"type\":{\"type\":\"record\", "
@@ -582,20 +723,23 @@
         { "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
           + "{\"name\":\"f1\", \"type\":\"long\"},"
           + "{\"name\":\"f2\", "
-          + "\"type\":{\"type\":\"array\", \"items\":\"int\"}}]}", "L[c1sI]" },
+          + "\"type\":{\"type\":\"array\", \"items\":\"int\"}}]}",
+          "L[c1sI]" },
 
         // record with map
         { "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
           + "{\"name\":\"f1\", \"type\":\"long\"},"
           + "{\"name\":\"f2\", "
-          + "\"type\":{\"type\":\"map\", \"values\":\"int\"}}]}", "L{c1sS5I}" },
-          
+          + "\"type\":{\"type\":\"map\", \"values\":\"int\"}}]}",
+          "L{c1sK5I}" },
+
         // array of records
         { "{\"type\":\"array\", \"items\":"
             + "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
           + "{\"name\":\"f1\", \"type\":\"long\"},"
           + "{\"name\":\"f2\", \"type\":\"null\"}]}}",
             "[c2sLNsLN]" },
+
         { "{\"type\":\"array\", \"items\":"
             + "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
           + "{\"name\":\"f1\", \"type\":\"long\"},"
@@ -607,32 +751,60 @@
           + "{\"name\":\"f1\", \"type\":\"long\"},"
           + "{\"name\":\"f2\", "
           + "\"type\":{\"type\":\"map\", \"values\":\"int\"}}]}}",
-            "[c2sL{c1sS5I}sL{c2sS5IsS5I}]" },
+            "[c2sL{c1sK5I}sL{c2sK5IsK5I}]" },
         { "{\"type\":\"array\", \"items\":"
             + "{\"type\":\"record\",\"name\":\"r\",\"fields\":["
           + "{\"name\":\"f1\", \"type\":\"long\"},"
           + "{\"name\":\"f2\", "
           + "\"type\":[\"null\", \"int\"]}]}}",
-            "[c2sLu0NsLu1I]" },
+            "[c2sLU0NsLU1I]" },
 
-        { "[\"boolean\"]", "u0B" },
-        { "[\"int\"]", "u0I" },
-        { "[\"long\"]", "u0L" },
-        { "[\"float\"]", "u0F" },
-        { "[\"double\"]", "u0D" },
-        { "[\"string\"]", "u0S10" },
-        { "[\"bytes\"]", "u0b10" },
-
-        { "[\"null\", \"int\"]", "u0N" },
-        { "[\"boolean\", \"int\"]", "u0B" },
-        { "[\"boolean\", \"int\"]", "u1I" },
-        { "[\"boolean\", {\"type\":\"array\", \"items\":\"int\"} ]", "u0B" },
+        { "[\"boolean\"]", "U0B" },
+        { "[\"int\"]", "U0I" },
+        { "[\"long\"]", "U0L" },
+        { "[\"float\"]", "U0F" },
+        { "[\"double\"]", "U0D" },
+        { "[\"string\"]", "U0S10" },
+        { "[\"bytes\"]", "U0b10" },
+
+        { "[\"null\", \"int\"]", "U0N" },
+        { "[\"boolean\", \"int\"]", "U0B" },
+        { "[\"boolean\", \"int\"]", "U1I" },
         { "[\"boolean\", {\"type\":\"array\", \"items\":\"int\"} ]",
-            "u1[c1sI]" },
+          "U0B" },
+
+        { "[\"boolean\", {\"type\":\"array\", \"items\":\"int\"} ]",
+            "U1[c1sI]" },
+          
+        // Recursion
+        { "{\"type\": \"record\", \"name\": \"Node\", \"fields\": ["
+          + "{\"name\":\"label\", \"type\":\"string\"},"
+          + "{\"name\":\"children\", \"type\":"
+          + "{\"type\": \"array\", \"items\": \"Node\" }}]}",
+          "S10[c1sS10[]]" },
+          
+        { "{\"type\": \"record\", \"name\": \"Lisp\", \"fields\": ["
+          + "{\"name\":\"value\", \"type\":[\"null\", \"string\","
+          + "{\"type\": \"record\", \"name\": \"Cons\", \"fields\": ["
+          + "{\"name\":\"car\", \"type\":\"Lisp\"},"
+          + "{\"name\":\"cdr\", \"type\":\"Lisp\"}]}]}]}",
+          "U0N"},
+        { "{\"type\": \"record\", \"name\": \"Lisp\", \"fields\": ["
+          + "{\"name\":\"value\", \"type\":[\"null\", \"string\","
+          + "{\"type\": \"record\", \"name\": \"Cons\", \"fields\": ["
+          + "{\"name\":\"car\", \"type\":\"Lisp\"},"
+          + "{\"name\":\"cdr\", \"type\":\"Lisp\"}]}]}]}",
+          "U1S10"},
+        { "{\"type\": \"record\", \"name\": \"Lisp\", \"fields\": ["
+          + "{\"name\":\"value\", \"type\":[\"null\", \"string\","
+          + "{\"type\": \"record\", \"name\": \"Cons\", \"fields\": ["
+          + "{\"name\":\"car\", \"type\":\"Lisp\"},"
+          + "{\"name\":\"cdr\", \"type\":\"Lisp\"}]}]}]}",
+          "U2U1S10U0N"},
     };
   }
   
-  protected static void dump(byte[] bb) {
+  static void dump(byte[] bb) {
     int col = 0;
     for (byte b : bb) {
       if (col % 16 == 0) {



Mime
View raw message