avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r805549 - in /hadoop/avro/trunk: ./ src/java/org/apache/avro/ src/java/org/apache/avro/io/ src/java/org/apache/avro/io/parsing/ src/test/java/org/apache/avro/
Date Tue, 18 Aug 2009 19:43:47 GMT
Author: cutting
Date: Tue Aug 18 19:43:46 2009
New Revision: 805549

URL: http://svn.apache.org/viewvc?rev=805549&view=rev
Log:
AVRO-90.  Fix Java's JSON codec to correctly encode unions.

Modified:
    hadoop/avro/trunk/CHANGES.txt
    hadoop/avro/trunk/build.xml
    hadoop/avro/trunk/src/java/org/apache/avro/Schema.java
    hadoop/avro/trunk/src/java/org/apache/avro/io/JsonDecoder.java
    hadoop/avro/trunk/src/java/org/apache/avro/io/JsonEncoder.java
    hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ResolvingGrammarGenerator.java
    hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java
    hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java

Modified: hadoop/avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/CHANGES.txt?rev=805549&r1=805548&r2=805549&view=diff
==============================================================================
--- hadoop/avro/trunk/CHANGES.txt (original)
+++ hadoop/avro/trunk/CHANGES.txt Tue Aug 18 19:43:46 2009
@@ -46,6 +46,8 @@
     change test-java build target to fail on javadoc warnings.
     (Thiruvalluvan M. G. and cutting)
 
+    AVRO-90. Fix Java's JSON codec to correctly encode unions. (cutting)
+
 Avro 1.0.0 -- 9 July 2009
 
   INCOMPATIBLE CHANGES

Modified: hadoop/avro/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/build.xml?rev=805549&r1=805548&r2=805549&view=diff
==============================================================================
--- hadoop/avro/trunk/build.xml (original)
+++ hadoop/avro/trunk/build.xml Tue Aug 18 19:43:46 2009
@@ -281,7 +281,7 @@
           <fileset dir="@{files.location}" includes="**/${testcase}.java"/>
         </batchtest>
       </junit>
-      <antcall target="checkfailure" />
+      <fail if="tests.failed">Tests Failed!</fail>
     </sequential>
   </macrodef>
 
@@ -354,11 +354,6 @@
     <test-runner files.location="${test.java.classes}" tests.pattern="**/TestDataFile$InteropTest.class"
/>
   </target>
 
-  <target name="checkfailure" if="tests.failed">
-    <touch file="${test.build.dir}/testsfailed"/>
-    <fail unless="continueOnFailure">Tests failed!</fail>
-  </target>
-
   <target name="test-interop-data-py" depends="generate-test-data" 
     description="Run python data file interoperability tests">
     <taskdef name="py-test" classname="org.pyant.tasks.PythonTestTask">

Modified: hadoop/avro/trunk/src/java/org/apache/avro/Schema.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/java/org/apache/avro/Schema.java?rev=805549&r1=805548&r2=805549&view=diff
==============================================================================
--- hadoop/avro/trunk/src/java/org/apache/avro/Schema.java (original)
+++ hadoop/avro/trunk/src/java/org/apache/avro/Schema.java Tue Aug 18 19:43:46 2009
@@ -66,9 +66,12 @@
   }
 
   /** The type of a schema. */
-  public enum Type
-  { RECORD, ENUM, ARRAY, MAP, UNION, FIXED, STRING, BYTES,
-      INT, LONG, FLOAT, DOUBLE, BOOLEAN, NULL };
+  public enum Type {
+    RECORD, ENUM, ARRAY, MAP, UNION, FIXED, STRING, BYTES,
+      INT, LONG, FLOAT, DOUBLE, BOOLEAN, NULL;
+    private String name;
+    private Type() { this.name = this.name().toLowerCase(); }
+  };
 
   private final Type type;
 
@@ -156,10 +159,9 @@
     throw new AvroRuntimeException("Not an enum: "+this);
   }    
 
-  /** If this is a record, enum or fixed, returns its name, if any. */
-  public String getName() {
-    throw new AvroRuntimeException("Not a named type: "+this);
-  }
+  /** If this is a record, enum or fixed, returns its name, otherwise the name
+   * of the primitive type. */
+  public String getName() { return type.name; }
 
   /** If this is a record, enum or fixed, returns its namespace, if any. */
   public String getNamespace() {
@@ -204,7 +206,9 @@
     }
   }
 
-  abstract void toJson(Names names, JsonGenerator gen) throws IOException;
+  void toJson(Names names, JsonGenerator gen) throws IOException {
+    gen.writeString(getName());
+  }
 
   void fieldsToJson(Names names, JsonGenerator gen) throws IOException {
     throw new AvroRuntimeException("Not a record: "+this);
@@ -518,58 +522,34 @@
 
   private static class StringSchema extends Schema {
     public StringSchema() { super(Type.STRING); }
-    void toJson(Names names, JsonGenerator gen) throws IOException {
-      gen.writeString("string");
-    }
   }
 
   private static class BytesSchema extends Schema {
     public BytesSchema() { super(Type.BYTES); }
-    void toJson(Names names, JsonGenerator gen) throws IOException {
-      gen.writeString("bytes");
-    }
   }
 
   private static class IntSchema extends Schema {
     public IntSchema() { super(Type.INT); }
-    void toJson(Names names, JsonGenerator gen) throws IOException {
-      gen.writeString("int");
-    }
   }
 
   private static class LongSchema extends Schema {
     public LongSchema() { super(Type.LONG); }
-    void toJson(Names names, JsonGenerator gen) throws IOException {
-      gen.writeString("long");
-    }
   }
 
   private static class FloatSchema extends Schema {
     public FloatSchema() { super(Type.FLOAT); }
-    void toJson(Names names, JsonGenerator gen) throws IOException {
-      gen.writeString("float");
-    }
   }
 
   private static class DoubleSchema extends Schema {
     public DoubleSchema() { super(Type.DOUBLE); }
-    void toJson(Names names, JsonGenerator gen) throws IOException {
-      gen.writeString("double");
-    }
   }
 
   private static class BooleanSchema extends Schema {
     public BooleanSchema() { super(Type.BOOLEAN); }
-    void toJson(Names names, JsonGenerator gen) throws IOException {
-      gen.writeString("boolean");
-    }
   }
   
   private static class NullSchema extends Schema {
     public NullSchema() { super(Type.NULL); }
-    void toJson(Names names, JsonGenerator gen) throws IOException {
-      gen.writeString("null");
-    }
   }
   
   private static final StringSchema  STRING_SCHEMA =  new StringSchema();

Modified: hadoop/avro/trunk/src/java/org/apache/avro/io/JsonDecoder.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/java/org/apache/avro/io/JsonDecoder.java?rev=805549&r1=805548&r2=805549&view=diff
==============================================================================
--- hadoop/avro/trunk/src/java/org/apache/avro/io/JsonDecoder.java (original)
+++ hadoop/avro/trunk/src/java/org/apache/avro/io/JsonDecoder.java Tue Aug 18 19:43:46 2009
@@ -338,20 +338,23 @@
   public int readIndex() throws IOException {
     parser.advance(Symbol.UNION);
     Symbol.Alternative a = (Symbol.Alternative) parser.popSymbol();
-    if (in.getCurrentToken() == JsonToken.START_OBJECT &&
-      in.nextToken() == JsonToken.FIELD_NAME) {
-      String label = in.getText();
+    
+    String label;
+    if (in.getCurrentToken() == JsonToken.VALUE_NULL) {
+      label = "null";
+    } else if (in.getCurrentToken() == JsonToken.START_OBJECT &&
+               in.nextToken() == JsonToken.FIELD_NAME) {
+      label = in.getText();
       in.nextToken();
-      int n = a.findLabel(label);
-      if (n < 0) {
-        throw new AvroTypeException("Unknown union branch " + label);
-      }
       parser.pushSymbol(Symbol.UNION_END);
-      parser.pushSymbol(a.getSymbol(n));
-      return n;
     } else {
       throw error("start-union");
     }
+    int n = a.findLabel(label);
+    if (n < 0)
+      throw new AvroTypeException("Unknown union branch " + label);
+    parser.pushSymbol(a.getSymbol(n));
+    return n;
   }
 
   public Symbol doAction(Symbol input, Symbol top) throws IOException {

Modified: hadoop/avro/trunk/src/java/org/apache/avro/io/JsonEncoder.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/java/org/apache/avro/io/JsonEncoder.java?rev=805549&r1=805548&r2=805549&view=diff
==============================================================================
--- hadoop/avro/trunk/src/java/org/apache/avro/io/JsonEncoder.java (original)
+++ hadoop/avro/trunk/src/java/org/apache/avro/io/JsonEncoder.java Tue Aug 18 19:43:46 2009
@@ -214,10 +214,13 @@
   public void writeIndex(int unionIndex) throws IOException {
     parser.advance(Symbol.UNION);
     Symbol.Alternative top = (Symbol.Alternative) parser.popSymbol();
-    out.writeStartObject();
-    out.writeFieldName(top.getLabel(unionIndex));
-    parser.pushSymbol(Symbol.UNION_END);
-    parser.pushSymbol(top.getSymbol(unionIndex));
+    Symbol symbol = top.getSymbol(unionIndex);
+    if (symbol != Symbol.NULL) {
+      out.writeStartObject();
+      out.writeFieldName(top.getLabel(unionIndex));
+      parser.pushSymbol(Symbol.UNION_END);
+    }
+    parser.pushSymbol(symbol);
   }
 
   @Override

Modified: hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ResolvingGrammarGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ResolvingGrammarGenerator.java?rev=805549&r1=805548&r2=805549&view=diff
==============================================================================
--- hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ResolvingGrammarGenerator.java (original)
+++ hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ResolvingGrammarGenerator.java Tue
Aug 18 19:43:46 2009
@@ -180,7 +180,7 @@
     int i = 0;
     for (Schema w : alts) {
       symbols[i] = generate(w, reader, seen);
-      labels[i] = w.getType().name();
+      labels[i] = w.getName();
       i++;
     }
     return Symbol.seq(Symbol.alt(symbols, labels),

Modified: hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java?rev=805549&r1=805548&r2=805549&view=diff
==============================================================================
--- hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java
(original)
+++ hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java
Tue Aug 18 19:43:46 2009
@@ -107,7 +107,7 @@
       int i = 0;
       for (Schema b : sc.getTypes()) {
         symbols[i] = generate(b, seen);
-        labels[i] = b.getType().name();
+        labels[i] = b.getName();
         i++;
       }
       return Symbol.seq(Symbol.alt(symbols, labels), Symbol.UNION);

Modified: hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java?rev=805549&r1=805548&r2=805549&view=diff
==============================================================================
--- hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java (original)
+++ hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java Tue Aug 18 19:43:46 2009
@@ -157,6 +157,22 @@
   public void testUnion() throws Exception {
     check("[\"string\", \"long\"]", false);
     checkDefault("[\"double\", \"long\"]", "1.1", new Double(1.1));
+
+    // check union json
+    String record = "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[]}";
+    String fixed = "{\"type\":\"fixed\",\"name\":\"Bar\",\"size\": 1}";
+    String enu = "{\"type\":\"enum\",\"name\":\"Baz\",\"symbols\": [\"X\"]}";
+    Schema union = Schema.parse("[\"null\",\"string\","
+                                +record+","+ enu+","+fixed+"]");
+    checkJson(union, null, "null");
+    checkJson(union, new Utf8("foo"), "{\"string\":\"foo\"}");
+    checkJson(union,
+              new GenericData.Record(Schema.parse(record)),
+              "{\"Foo\":{}}");
+    checkJson(union,
+              new GenericData.Fixed(new byte[]{(byte)'a'}),
+              "{\"Bar\":\"a\"}");
+    checkJson(union, "X", "{\"Baz\":\"X\"}");
   }
 
   private static void check(String schemaJson, String defaultJson,
@@ -232,6 +248,27 @@
     assertEquals("Decoded data does not match.", datum, decoded);
   }
 
+  private static void checkJson(Schema schema, Object datum,
+                                String json) throws Exception {
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    Encoder encoder = new JsonEncoder(schema, out);
+    DatumWriter<Object> writer = new GenericDatumWriter<Object>();
+    writer.setSchema(schema);
+    writer.write(datum, encoder);
+    encoder.flush();
+    byte[] data = out.toByteArray();
+
+    String encoded = new String(data, "UTF-8");
+    assertEquals("Encoded data does not match.", json, encoded);
+
+    DatumReader<Object> reader = new GenericDatumReader<Object>();
+    reader.setSchema(schema);
+    Object decoded =
+      reader.read(null, new JsonDecoder(schema,new ByteArrayInputStream(data)));
+      
+    assertEquals("Decoded data does not match.", datum, decoded);
+  }
+
   private static final Schema ACTUAL =            // an empty record schema
     Schema.parse("{\"type\":\"record\", \"name\":\"Foo\", \"fields\":[]}");
 



Mime
View raw message