avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r886927 - in /hadoop/avro/trunk: CHANGES.txt src/java/org/apache/avro/Schema.java src/test/java/org/apache/avro/TestSchema.java
Date Thu, 03 Dec 2009 21:32:38 GMT
Author: cutting
Date: Thu Dec  3 21:32:37 2009
New Revision: 886927

URL: http://svn.apache.org/viewvc?rev=886927&view=rev
Log:
AVRO-242.  In Java, add support for extensible, string-valued properties.

Modified:
    hadoop/avro/trunk/CHANGES.txt
    hadoop/avro/trunk/src/java/org/apache/avro/Schema.java
    hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java

Modified: hadoop/avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/CHANGES.txt?rev=886927&r1=886926&r2=886927&view=diff
==============================================================================
--- hadoop/avro/trunk/CHANGES.txt (original)
+++ hadoop/avro/trunk/CHANGES.txt Thu Dec  3 21:32:37 2009
@@ -115,6 +115,9 @@
     AVRO-240. In Python, if simplejson is not available, try using
     2.6's built-in json module. (Jeff Hammerbacher via cutting)
 
+    AVRO-242. In Java, add support for extensible string-valued
+    properties to schemas.  (cutting)
+
   OPTIMIZATIONS
 
     AVRO-172. More efficient schema processing (massie)

Modified: hadoop/avro/trunk/src/java/org/apache/avro/Schema.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/java/org/apache/avro/Schema.java?rev=886927&r1=886926&r2=886927&view=diff
==============================================================================
--- hadoop/avro/trunk/src/java/org/apache/avro/Schema.java (original)
+++ hadoop/avro/trunk/src/java/org/apache/avro/Schema.java Thu Dec  3 21:32:37 2009
@@ -21,6 +21,8 @@
 import java.io.IOException;
 import java.io.StringReader;
 import java.io.StringWriter;
+import java.util.Collections;
+import java.util.Iterator;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -74,24 +76,47 @@
   };
 
   private final Type type;
+  Map<String,String> props = new HashMap<String,String>(1);
 
   Schema(Type type) { this.type = type; }
 
   /** Create a schema for a primitive type. */
   public static Schema create(Type type) {
     switch (type) {
-    case STRING:  return STRING_SCHEMA;
-    case BYTES:   return BYTES_SCHEMA;
-    case INT:     return INT_SCHEMA ;
-    case LONG:    return LONG_SCHEMA ;
-    case FLOAT:   return FLOAT_SCHEMA;
-    case DOUBLE:  return DOUBLE_SCHEMA;
-    case BOOLEAN: return BOOLEAN_SCHEMA;
-    case NULL:    return NULL_SCHEMA;
+    case STRING:  return new StringSchema();
+    case BYTES:   return new BytesSchema();
+    case INT:     return new IntSchema();
+    case LONG:    return new LongSchema();
+    case FLOAT:   return new FloatSchema();
+    case DOUBLE:  return new DoubleSchema();
+    case BOOLEAN: return new BooleanSchema();
+    case NULL:    return new NullSchema();
     default: throw new AvroRuntimeException("Can't create a: "+type);
     }
   }
 
+  private static final Set<String> RESERVED_PROPS = new HashSet<String>();
+  static {
+    Collections.addAll(RESERVED_PROPS,
+                       "fields", "items", "name", "namespace",
+                       "size", "symbols", "values", "type");
+  }
+
+  /** Return the value of the named property in this schema. */
+  public synchronized String getProp(String name) {
+    return props.get(name);
+  }
+
+  /** Set the value of the named property in this schema. */
+  public synchronized void setProp(String name, String value) {
+    if (RESERVED_PROPS.contains(name))
+      throw new AvroRuntimeException("Can't set a reserved property: "+name);
+    if (value == null)
+      props.remove(name);
+    else
+      props.put(name, value);
+  }
+
   /** Create an anonymous record schema. */
   public static Schema createRecord(LinkedHashMap<String,Field> fields) {
     Schema result = createRecord(null, null, false);
@@ -220,19 +245,33 @@
   }
 
   void toJson(Names names, JsonGenerator gen) throws IOException {
-    gen.writeString(getName());
+    if (props.size() == 0) {                      // no props defined
+      gen.writeString(getName());                 // just write name
+    } else {
+      gen.writeStartObject();
+      gen.writeStringField("type", getName());
+      writeProps(gen);
+      gen.writeEndObject();
+    }
+  }
+
+  void writeProps(JsonGenerator gen) throws IOException {
+    for (Map.Entry<String,String> e : props.entrySet())
+      gen.writeStringField(e.getKey(), e.getValue());
   }
 
   void fieldsToJson(Names names, JsonGenerator gen) throws IOException {
     throw new AvroRuntimeException("Not a record: "+this);
   }
 
-
   public boolean equals(Object o) {
     if (o == this) return true;
-    return o instanceof Schema && type.equals(((Schema)o).type);
+    if (!(o instanceof Schema)) return false;
+    Schema that = (Schema)o;
+    if (!(this.type == that.type)) return false;
+    return props.equals(that.props);
   }
-  public int hashCode() { return getType().hashCode(); }
+  public int hashCode() { return getType().hashCode() + props.hashCode(); }
 
   /** A field within a record. */
   public static class Field {
@@ -344,7 +383,9 @@
     public boolean equalNames(NamedSchema that) {
       return this.name.equals(that.name);
     }
-    public int hashCode() { return getType().hashCode() + name.hashCode(); }
+    public int hashCode() {
+      return getType().hashCode() + name.hashCode() + props.hashCode();
+    }
   }
 
   private static class SeenPair {
@@ -399,7 +440,7 @@
       if (!(o instanceof RecordSchema)) return false;
       RecordSchema that = (RecordSchema)o;
       if (!equalNames(that)) return false;
-      if (!(o instanceof RecordSchema)) return false;
+      if (!props.equals(that.props)) return false;
       Set seen = SEEN_EQUALS.get();
       SeenPair here = new SeenPair(this, o);
       if (seen.contains(here)) return true;       // prevent stack overflow
@@ -427,6 +468,7 @@
       writeName(names, gen);
       gen.writeFieldName("fields");
       fieldsToJson(names, gen);
+      writeProps(gen);
       gen.writeEndObject();
     }
 
@@ -466,7 +508,9 @@
       if (o == this) return true;
       if (!(o instanceof EnumSchema)) return false;
       EnumSchema that = (EnumSchema)o;
-      return equalNames(that) && symbols.equals(that.symbols);
+      return equalNames(that)
+        && symbols.equals(that.symbols)
+        && props.equals(that.props);
     }
     public int hashCode() { return super.hashCode() + symbols.hashCode(); }
     void toJson(Names names, JsonGenerator gen) throws IOException {
@@ -478,6 +522,7 @@
       for (String symbol : symbols)
         gen.writeString(symbol);
       gen.writeEndArray();
+      writeProps(gen);
       gen.writeEndObject();
     }
   }
@@ -491,15 +536,19 @@
     public Schema getElementType() { return elementType; }
     public boolean equals(Object o) {
       if (o == this) return true;
-      return o instanceof ArraySchema
-        && elementType.equals(((ArraySchema)o).elementType);
+      if (!(o instanceof ArraySchema)) return false;
+      ArraySchema that = (ArraySchema)o;
+      return elementType.equals(that.elementType) && props.equals(that.props);
+    }
+    public int hashCode() {
+      return getType().hashCode() + elementType.hashCode() + props.hashCode();
     }
-    public int hashCode() {return getType().hashCode()+elementType.hashCode();}
     void toJson(Names names, JsonGenerator gen) throws IOException {
       gen.writeStartObject();
       gen.writeStringField("type", "array");
       gen.writeFieldName("items");
       elementType.toJson(names, gen);
+      writeProps(gen);
       gen.writeEndObject();
     }
   }
@@ -513,17 +562,19 @@
     public Schema getValueType() { return valueType; }
     public boolean equals(Object o) {
       if (o == this) return true;
-      return o instanceof MapSchema
-        && valueType.equals(((MapSchema)o).valueType);
+      if (!(o instanceof MapSchema)) return false;
+      MapSchema that = (MapSchema)o;
+      return valueType.equals(that.valueType) && props.equals(that.props);
     }
     public int hashCode() {
-      return getType().hashCode()+valueType.hashCode();
+      return getType().hashCode() + valueType.hashCode() + props.hashCode();
     }
     void toJson(Names names, JsonGenerator gen) throws IOException {
       gen.writeStartObject();
       gen.writeStringField("type", "map");
       gen.writeFieldName("values");
       valueType.toJson(names, gen);
+      writeProps(gen);
       gen.writeEndObject();
     }
   }
@@ -552,9 +603,16 @@
     public List<Schema> getTypes() { return types; }
     public boolean equals(Object o) {
       if (o == this) return true;
-      return o instanceof UnionSchema && types.equals(((UnionSchema)o).types);
+      if (!(o instanceof UnionSchema)) return false;
+      UnionSchema that = (UnionSchema)o;
+      return types.equals(that.types) && props.equals(that.props);
+    }
+    public int hashCode() {
+      return getType().hashCode() + types.hashCode() + props.hashCode();
+    }
+    public void setProp(String name, String value) {
+      throw new AvroRuntimeException("Can't set properties on a union: "+this);
     }
-    public int hashCode() {return getType().hashCode()+types.hashCode();}
     void toJson(Names names, JsonGenerator gen) throws IOException {
       gen.writeStartArray();
       for (Schema type : types)
@@ -576,7 +634,7 @@
       if (o == this) return true;
       if (!(o instanceof FixedSchema)) return false;
       FixedSchema that = (FixedSchema)o;
-      return equalNames(that) && size == that.size;
+      return equalNames(that) && size == that.size && props.equals(that.props);
     }
     public int hashCode() { return super.hashCode() + size; }
     void toJson(Names names, JsonGenerator gen) throws IOException {
@@ -585,6 +643,7 @@
       gen.writeStringField("type", "fixed");
       writeName(names, gen);
       gen.writeNumberField("size", size);
+      writeProps(gen);
       gen.writeEndObject();
     }
   }
@@ -620,15 +679,6 @@
   private static class NullSchema extends Schema {
     public NullSchema() { super(Type.NULL); }
   }
-  
-  private static final StringSchema  STRING_SCHEMA =  new StringSchema();
-  private static final BytesSchema   BYTES_SCHEMA =   new BytesSchema();
-  private static final IntSchema     INT_SCHEMA =     new IntSchema();
-  private static final LongSchema    LONG_SCHEMA =    new LongSchema();
-  private static final FloatSchema   FLOAT_SCHEMA =   new FloatSchema();
-  private static final DoubleSchema  DOUBLE_SCHEMA =  new DoubleSchema();
-  private static final BooleanSchema BOOLEAN_SCHEMA = new BooleanSchema();
-  private static final NullSchema    NULL_SCHEMA =    new NullSchema();
 
   public static Schema parse(File file) throws IOException {
     JsonParser parser = FACTORY.createJsonParser(file);
@@ -644,16 +694,16 @@
     return parse(parseJson(jsonSchema), new Names());
   }
 
-  static final Map<String,Schema> PRIMITIVES = new HashMap<String,Schema>();
+  static final Map<String,Type> PRIMITIVES = new HashMap<String,Type>();
   static {
-    PRIMITIVES.put("string",  STRING_SCHEMA);
-    PRIMITIVES.put("bytes",   BYTES_SCHEMA);
-    PRIMITIVES.put("int",     INT_SCHEMA);
-    PRIMITIVES.put("long",    LONG_SCHEMA);
-    PRIMITIVES.put("float",   FLOAT_SCHEMA);
-    PRIMITIVES.put("double",  DOUBLE_SCHEMA);
-    PRIMITIVES.put("boolean", BOOLEAN_SCHEMA);
-    PRIMITIVES.put("null",    NULL_SCHEMA);
+    PRIMITIVES.put("string",  Type.STRING);
+    PRIMITIVES.put("bytes",   Type.BYTES);
+    PRIMITIVES.put("int",     Type.INT);
+    PRIMITIVES.put("long",    Type.LONG);
+    PRIMITIVES.put("float",   Type.FLOAT);
+    PRIMITIVES.put("double",  Type.DOUBLE);
+    PRIMITIVES.put("boolean", Type.BOOLEAN);
+    PRIMITIVES.put("null",    Type.NULL);
   }
 
   static class Names extends LinkedHashMap<Name, Schema> {
@@ -669,8 +719,8 @@
     public Schema get(Object o) {
       Name name;
       if (o instanceof String) {
-        Schema primitive = PRIMITIVES.get((String)o);
-        if (primitive != null) return primitive;
+        Type primitive = PRIMITIVES.get((String)o);
+        if (primitive != null) return Schema.create(primitive);
         name = new Name((String)o, space);
       } else {
         name = (Name)o;
@@ -699,6 +749,7 @@
         throw new SchemaParseException("Undefined name: "+schema);
       return result;
     } else if (schema.isObject()) {
+      Schema result;
       String type = getRequiredText(schema, "type", "No type");
       String name = null, space = null;
       if (type.equals("record") || type.equals("error")
@@ -710,10 +761,11 @@
         if (names.space() == null && space != null)
           names.space(space);                     // set default namespace
       }
-      if (type.equals("record") || type.equals("error")) { // record
+      if (PRIMITIVES.containsKey(type)) {         // primitive
+        result = create(PRIMITIVES.get(type));
+      } else if (type.equals("record") || type.equals("error")) { // record
         LinkedHashMap<String,Field> fields = new LinkedHashMap<String,Field>();
-        RecordSchema result =
-          new RecordSchema(name, space, type.equals("error"));
+        result = new RecordSchema(name, space, type.equals("error"));
         if (name != null) names.add(result);
         JsonNode fieldsNode = schema.get("fields");
         if (fieldsNode == null || !fieldsNode.isArray())
@@ -734,7 +786,6 @@
                      new Field(fieldSchema, field.get("default"), order));
         }
         result.setFields(fields);
-        return result;
       } else if (type.equals("enum")) {           // enum
         JsonNode symbolsNode = schema.get("symbols");
         if (symbolsNode == null || !symbolsNode.isArray())
@@ -742,28 +793,33 @@
         List<String> symbols = new ArrayList<String>();
         for (JsonNode n : symbolsNode)
           symbols.add(n.getTextValue());
-        Schema result = new EnumSchema(name, space, symbols);
+        result = new EnumSchema(name, space, symbols);
         if (name != null) names.add(result);
-        return result;
       } else if (type.equals("array")) {          // array
         JsonNode itemsNode = schema.get("items");
         if (itemsNode == null)
           throw new SchemaParseException("Array has no items type: "+schema);
-        return new ArraySchema(parse(itemsNode, names));
+        result = new ArraySchema(parse(itemsNode, names));
       } else if (type.equals("map")) {            // map
         JsonNode valuesNode = schema.get("values");
         if (valuesNode == null)
           throw new SchemaParseException("Map has no values type: "+schema);
-        return new MapSchema(parse(valuesNode, names));
+        result = new MapSchema(parse(valuesNode, names));
       } else if (type.equals("fixed")) {          // fixed
         JsonNode sizeNode = schema.get("size");
         if (sizeNode == null || !sizeNode.isInt())
           throw new SchemaParseException("Invalid or no size: "+schema);
-        Schema result = new FixedSchema(name, space, sizeNode.getIntValue());
+        result = new FixedSchema(name, space, sizeNode.getIntValue());
         if (name != null) names.add(result);
-        return result;
       } else
-        throw new SchemaParseException("Type not yet supported: "+type);
+        throw new SchemaParseException("Type not supported: "+type);
+      Iterator<String> i = schema.getFieldNames();
+      while (i.hasNext()) {                       // add properties
+        String prop = i.next();
+        if (!RESERVED_PROPS.contains(prop))       // ignore reserved
+          result.setProp(prop, schema.get(prop).getTextValue());
+      }
+      return result;
     } else if (schema.isArray()) {                // union
       List<Schema> types = new ArrayList<Schema>(schema.size());
       for (JsonNode typeNode : schema)

Modified: hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java?rev=886927&r1=886926&r2=886927&view=diff
==============================================================================
--- hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java (original)
+++ hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java Thu Dec  3 21:32:37 2009
@@ -20,6 +20,7 @@
 import org.junit.Test;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.fail;
 
 import java.io.ByteArrayInputStream;
@@ -191,6 +192,13 @@
     checkJson(union, "X", "{\"Baz\":\"X\"}");
   }
 
+  @Test
+  public void testComplexProp() throws Exception {
+    String json = "{\"type\":\"null\", \"foo\": [0]}";
+    Schema s = Schema.parse(json);
+    assertEquals(null, s.getProp("foo"));
+  }
+
   private static void checkParseError(String json) {
     try {
       Schema schema = Schema.parse(json);
@@ -214,9 +222,8 @@
   private static void check(String jsonSchema, boolean induce)
     throws Exception {
     Schema schema = Schema.parse(jsonSchema);
-    //System.out.println(schema);
+    checkProp(schema);
     for (Object datum : new RandomData(schema, COUNT)) {
-      //System.out.println(GenericData.get().toString(datum));
 
       if (induce) {
         Schema induced = GenericData.get().induce(datum);
@@ -239,6 +246,19 @@
     }
   }
 
+  private static void checkProp(Schema s0) throws Exception {
+    if(s0.getType().equals(Schema.Type.UNION)) return; // unions have no props
+    assertEquals(null, s0.getProp("foo"));
+    Schema s1 = Schema.parse(s0.toString());
+    s1.setProp("foo", "bar");
+    assertEquals("bar", s1.getProp("foo"));
+    assertFalse(s0.equals(s1));
+    Schema s2 = Schema.parse(s1.toString());
+    assertEquals("bar", s2.getProp("foo"));
+    assertEquals(s1, s2);
+    assertFalse(s0.equals(s2));
+  }
+
   private static void checkBinary(Schema schema, Object datum,
                                   DatumWriter<Object> writer,
                                   DatumReader<Object> reader)



Mime
View raw message