avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r1458349 - in /avro/trunk: CHANGES.txt lang/ruby/lib/avro/protocol.rb lang/ruby/lib/avro/schema.rb lang/ruby/test/test_protocol.rb lang/ruby/test/test_schema.rb
Date Tue, 19 Mar 2013 15:44:37 GMT
Author: cutting
Date: Tue Mar 19 15:44:37 2013
New Revision: 1458349

URL: http://svn.apache.org/r1458349
Log:
AVRO-1272. Ruby: Improve schema namespace handling.  Contributed by Martin Kleppmann.

Added:
    avro/trunk/lang/ruby/test/test_schema.rb   (with props)
Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/lang/ruby/lib/avro/protocol.rb
    avro/trunk/lang/ruby/lib/avro/schema.rb
    avro/trunk/lang/ruby/test/test_protocol.rb

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1458349&r1=1458348&r2=1458349&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Tue Mar 19 15:44:37 2013
@@ -10,6 +10,9 @@ Trunk (not yet released)
 
     AVRO-1267. Java: Permit dashes in IDL property names. (cutting)
 
+    AVRO-1272. Ruby: Improve schema namespace handling.
+    (Martin Kleppmann via cutting)
+
   BUG FIXES
 
 Avro 1.7.4 (22 February 2012)

Modified: avro/trunk/lang/ruby/lib/avro/protocol.rb
URL: http://svn.apache.org/viewvc/avro/trunk/lang/ruby/lib/avro/protocol.rb?rev=1458349&r1=1458348&r2=1458349&view=diff
==============================================================================
--- avro/trunk/lang/ruby/lib/avro/protocol.rb (original)
+++ avro/trunk/lang/ruby/lib/avro/protocol.rb Tue Mar 19 15:44:37 2013
@@ -71,7 +71,7 @@ module Avro
       types.collect do |type|
         # FIXME adding type.name to type_names is not defined in the
         # spec. Possible bug in the python impl and the spec.
-        type_object = Schema.real_parse(type, type_names)
+        type_object = Schema.real_parse(type, type_names, namespace)
         unless VALID_TYPE_SCHEMA_TYPES_SYM.include?(type_object.type_sym)
           msg = "Type #{type} not an enum, record, fixed or error."
           raise ProtocolParseError, msg
@@ -92,47 +92,42 @@ module Avro
         request  = body['request']
         response = body['response']
         errors   = body['errors']
-        message_objects[name] = Message.new(name, request, response, errors, names)
+        message_objects[name] = Message.new(name, request, response, errors, names, namespace)
       end
       message_objects
     end
 
     protected
-    def to_avro
+    def to_avro(names=Set.new)
       hsh = {'protocol' => name}
       hsh['namespace'] = namespace if namespace
-      hsh['types'] = types.map{|t| t.to_avro } if types
+      hsh['types'] = types.map{|t| t.to_avro(names) } if types
 
       if messages
-        hsh['messages'] = messages.collect_hash{|k,t| [k, t.to_avro] }
+        hsh['messages'] = messages.collect_hash{|k,t| [k, t.to_avro(names)] }
       end
 
       hsh
     end
 
     class Message
-      attr_reader :name, :response_from_names, :request, :response, :errors
-      def initialize(name, request, response, errors=nil, names=nil)
-        @name = name
-        @response_from_names = false
+      attr_reader :name, :request, :response, :errors, :default_namespace
 
+      def initialize(name, request, response, errors=nil, names=nil, default_namespace=nil)
+        @name = name
+        @default_namespace = default_namespace
         @request = parse_request(request, names)
         @response = parse_response(response, names)
         @errors = parse_errors(errors, names) if errors
       end
 
-      def to_avro
-        hsh = {'request' => request.to_avro}
-        if response_from_names
-          hsh['response'] = response.fullname
-        else
-          hsh['response'] = response.to_avro
+      def to_avro(names=Set.new)
+        {
+          'request' => request.to_avro(names),
+          'response' => response.to_avro(names)
+        }.tap do |hash|
+          hash['errors'] = errors.to_avro(names) if errors
         end
-
-        if errors
-          hsh['errors'] = errors.to_avro
-        end
-        hsh
       end
 
       def to_s
@@ -143,23 +138,23 @@ module Avro
         unless request.is_a?(Array)
           raise ProtocolParseError, "Request property not an Array: #{request.inspect}"
         end
-        Schema::RecordSchema.new(nil, nil, request, names, :request)
+        Schema::RecordSchema.new(nil, default_namespace, request, names, :request)
       end
 
       def parse_response(response, names)
-        if response.is_a?(String) && names[response]
-          @response_from_names = true
-          names[response]
-        else
-          Schema.real_parse(response, names)
+        if response.is_a?(String) && names
+          fullname = Name.make_fullname(response, default_namespace)
+          return names[fullname] if names.include?(fullname)
         end
+
+        Schema.real_parse(response, names, default_namespace)
       end
 
       def parse_errors(errors, names)
         unless errors.is_a?(Array)
           raise ProtocolParseError, "Errors property not an Array: #{errors}"
         end
-        Schema.real_parse(errors, names)
+        Schema.real_parse(errors, names, default_namespace)
       end
     end
   end

Modified: avro/trunk/lang/ruby/lib/avro/schema.rb
URL: http://svn.apache.org/viewvc/avro/trunk/lang/ruby/lib/avro/schema.rb?rev=1458349&r1=1458348&r2=1458349&view=diff
==============================================================================
--- avro/trunk/lang/ruby/lib/avro/schema.rb (original)
+++ avro/trunk/lang/ruby/lib/avro/schema.rb Tue Mar 19 15:44:37 2013
@@ -37,7 +37,7 @@ module Avro
     end
 
     # Build Avro Schema from data parsed out of JSON string.
-    def self.real_parse(json_obj, names=nil)
+    def self.real_parse(json_obj, names=nil, default_namespace=nil)
       if json_obj.is_a? Hash
         type = json_obj['type']
         raise SchemaParseError, %Q(No "type" property: #{json_obj}) if type.nil?
@@ -54,7 +54,7 @@ module Avro
 
         elsif NAMED_TYPES_SYM.include? type_sym
           name = json_obj['name']
-          namespace = json_obj['namespace']
+          namespace = json_obj.include?('namespace') ? json_obj['namespace'] : default_namespace
           case type_sym
           when :fixed
             size = json_obj['size']
@@ -72,9 +72,9 @@ module Avro
         else
           case type_sym
           when :array
-            return ArraySchema.new(json_obj['items'], names)
+            return ArraySchema.new(json_obj['items'], names, default_namespace)
           when :map
-            return MapSchema.new(json_obj['values'], names)
+            return MapSchema.new(json_obj['values'], names, default_namespace)
           else
             raise SchemaParseError.new("Unknown Valid Type: #{type}")
           end
@@ -82,7 +82,7 @@ module Avro
 
       elsif json_obj.is_a? Array
         # JSON array (union)
-        return UnionSchema.new(json_obj, names)
+        return UnionSchema.new(json_obj, names, default_namespace)
       elsif PRIMITIVE_TYPES.include? json_obj
         return PrimitiveSchema.new(json_obj)
       else
@@ -146,16 +146,21 @@ module Avro
       type_sym.hash
     end
 
-    def subparse(json_obj, names=nil)
+    def subparse(json_obj, names=nil, namespace=nil)
+      if json_obj.is_a?(String) && names
+        fullname = Name.make_fullname(json_obj, namespace)
+        return names[fullname] if names.include?(fullname)
+      end
+
       begin
-        Schema.real_parse(json_obj, names)
+        Schema.real_parse(json_obj, names, namespace)
       rescue => e
         raise e if e.is_a? SchemaParseError
         raise SchemaParseError, "Sub-schema for #{self.class.name} not a valid Avro schema.
Bad schema: #{json_obj}"
       end
     end
 
-    def to_avro
+    def to_avro(names=nil)
       {'type' => type}
     end
 
@@ -171,7 +176,11 @@ module Avro
         names = Name.add_name(names, self)
       end
 
-      def to_avro
+      def to_avro(names=Set.new)
+        if @name
+          return fullname if names.include?(fullname)
+          names << fullname
+        end
         props = {'name' => @name}
         props.merge!('namespace' => @namespace) if @namespace
         super.merge props
@@ -185,7 +194,7 @@ module Avro
     class RecordSchema < NamedSchema
       attr_reader :fields
 
-      def self.make_field_objects(field_data, names)
+      def self.make_field_objects(field_data, names, namespace=nil)
         field_objects, field_names = [], Set.new
         field_data.each_with_index do |field, i|
           if field.respond_to?(:[]) # TODO(jmhodges) wtffffff
@@ -193,7 +202,7 @@ module Avro
             name = field['name']
             default = field['default']
             order = field['order']
-            new_field = Field.new(type, name, default, order, names)
+            new_field = Field.new(type, name, default, order, names, namespace)
             # make sure field name has not been used yet
             if field_names.include?(new_field.name)
               raise SchemaParseError, "Field name #{new_field.name.inspect} is already in
use"
@@ -210,18 +219,21 @@ module Avro
       def initialize(name, namespace, fields, names=nil, schema_type=:record)
         if schema_type == :request || schema_type == 'request'
           @type_sym = schema_type.to_sym
+          @namespace = namespace
         else
           super(schema_type, name, namespace, names)
         end
-        @fields = RecordSchema.make_field_objects(fields, names)
+        @fields = RecordSchema.make_field_objects(fields, names, self.namespace)
       end
 
       def fields_hash
         @fields_hash ||= fields.inject({}){|hsh, field| hsh[field.name] = field; hsh }
       end
 
-      def to_avro
-        hsh = super.merge('fields' => @fields.map {|f| f.to_avro } )
+      def to_avro(names=Set.new)
+        hsh = super
+        return hsh unless hsh.is_a?(Hash)
+        hsh['fields'] = @fields.map {|f| f.to_avro(names) }
         if type_sym == :request
           hsh['fields']
         else
@@ -231,73 +243,42 @@ module Avro
     end
 
     class ArraySchema < Schema
-      attr_reader :items, :items_schema_from_names
-      def initialize(items, names=nil)
-        @items_schema_from_names = false
+      attr_reader :items
 
+      def initialize(items, names=nil, default_namespace=nil)
         super(:array)
-
-        if items.is_a?(String) && names.has_key?(items)
-          @items = names[items]
-          @items_schema_from_names = true
-        else
-          @items = subparse(items, names)
-        end
+        @items = subparse(items, names, default_namespace)
       end
 
-      def to_avro
-        name_or_json = if items_schema_from_names
-                         items.fullname
-                       else
-                         items.to_avro
-                       end
-        super.merge('items' => name_or_json)
+      def to_avro(names=Set.new)
+        super.merge('items' => items.to_avro(names))
       end
     end
 
     class MapSchema < Schema
-      attr_reader :values, :values_schema_from_names
+      attr_reader :values
 
-      def initialize(values, names=nil)
-        @values_schema_from_names = false
+      def initialize(values, names=nil, default_namespace=nil)
         super(:map)
-        if values.is_a?(String) && names.has_key?(values)
-          values_schema = names[values]
-          @values_schema_from_names = true
-        else
-          values_schema = subparse(values, names)
-        end
-        @values = values_schema
+        @values = subparse(values, names, default_namespace)
       end
 
-      def to_avro
-        to_dump = super
-        if values_schema_from_names
-          to_dump['values'] = values
-        else
-          to_dump['values'] = values.to_avro
-        end
-        to_dump
+      def to_avro(names=Set.new)
+        super.merge('values' => values.to_avro(names))
       end
     end
 
     class UnionSchema < Schema
-      attr_reader :schemas, :schema_from_names_indices
-      def initialize(schemas, names=nil)
+      attr_reader :schemas
+
+      def initialize(schemas, names=nil, default_namespace=nil)
         super(:union)
 
         schema_objects = []
-        @schema_from_names_indices = []
         schemas.each_with_index do |schema, i|
-          from_names = false
-          if schema.is_a?(String) && names.has_key?(schema)
-            new_schema = names[schema]
-            from_names = true
-          else
-            new_schema = subparse(schema, names)
-          end
-
+          new_schema = subparse(schema, names, default_namespace)
           ns_type = new_schema.type_sym
+
           if VALID_TYPES_SYM.include?(ns_type) &&
               !NAMED_TYPES_SYM.include?(ns_type) &&
               schema_objects.any?{|o| o.type_sym == ns_type }
@@ -306,24 +287,13 @@ module Avro
             raise SchemaParseError, "Unions cannot contain other unions"
           else
             schema_objects << new_schema
-            @schema_from_names_indices << i if from_names
           end
           @schemas = schema_objects
         end
       end
 
-      def to_avro
-        # FIXME(jmhodges) this from_name pattern is really weird and
-        # seems code-smelly.
-        to_dump = []
-        schemas.each_with_index do |schema, i|
-          if schema_from_names_indices.include?(i)
-            to_dump << schema.fullname
-          else
-            to_dump << schema.to_avro
-          end
-        end
-        to_dump
+      def to_avro(names=Set.new)
+        schemas.map {|schema| schema.to_avro(names) }
       end
     end
 
@@ -338,8 +308,9 @@ module Avro
         @symbols = symbols
       end
 
-      def to_avro
-        super.merge('symbols' => symbols)
+      def to_avro(names=Set.new)
+        avro = super
+        avro.is_a?(Hash) ? avro.merge('symbols' => symbols) : avro
       end
     end
 
@@ -355,7 +326,7 @@ module Avro
         end
       end
 
-      def to_avro
+      def to_avro(names=nil)
         hsh = super
         hsh.size == 1 ? type : hsh
       end
@@ -372,36 +343,27 @@ module Avro
         @size = size
       end
 
-      def to_avro
-        super.merge('size' => @size)
+      def to_avro(names=Set.new)
+        avro = super
+        avro.is_a?(Hash) ? avro.merge('size' => size) : avro
       end
     end
 
     class Field < Schema
-      attr_reader :type, :name, :default, :order, :type_from_names
-      def initialize(type, name, default=nil, order=nil, names=nil)
-        @type_from_names = false
-        if type.is_a?(String) && names && names.has_key?(type)
-          type_schema = names[type]
-          @type_from_names = true
-        else
-          type_schema = subparse(type, names)
-        end
-        @type = type_schema
+      attr_reader :type, :name, :default, :order
+
+      def initialize(type, name, default=nil, order=nil, names=nil, namespace=nil)
+        @type = subparse(type, names, namespace)
         @name = name
         @default = default
         @order = order
       end
 
-      def to_avro
-        sigh_type = type_from_names ? type.fullname : type.to_avro
-        hsh = {
-          'name' => name,
-          'type' => sigh_type
-        }
-        hsh['default'] = default if default
-        hsh['order'] = order if order
-        hsh
+      def to_avro(names=Set.new)
+        {'name' => name, 'type' => type.to_avro(names)}.tap do |avro|
+          avro['default'] = default if default
+          avro['order'] = order if order
+        end
       end
     end
   end

Modified: avro/trunk/lang/ruby/test/test_protocol.rb
URL: http://svn.apache.org/viewvc/avro/trunk/lang/ruby/test/test_protocol.rb?rev=1458349&r1=1458348&r2=1458349&view=diff
==============================================================================
--- avro/trunk/lang/ruby/test/test_protocol.rb (original)
+++ avro/trunk/lang/ruby/test/test_protocol.rb Tue Mar 19 15:44:37 2013
@@ -189,4 +189,11 @@ EOS
       assert_equal original, round_trip
     end
   end
+
+  def test_namespaces
+    protocol = Protocol.parse(EXAMPLES.first.protocol_string)
+    protocol.types.each do |type|
+      assert_equal type.namespace, 'com.acme'
+    end
+  end
 end

Added: avro/trunk/lang/ruby/test/test_schema.rb
URL: http://svn.apache.org/viewvc/avro/trunk/lang/ruby/test/test_schema.rb?rev=1458349&view=auto
==============================================================================
--- avro/trunk/lang/ruby/test/test_schema.rb (added)
+++ avro/trunk/lang/ruby/test/test_schema.rb Tue Mar 19 15:44:37 2013
@@ -0,0 +1,134 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'test_help'
+
+class TestSchema < Test::Unit::TestCase
+  def test_default_namespace
+    schema = Avro::Schema.parse <<-SCHEMA
+      {"type": "record", "name": "OuterRecord", "fields": [
+        {"name": "field1", "type": {
+          "type": "record", "name": "InnerRecord", "fields": []
+        }},
+        {"name": "field2", "type": "InnerRecord"}
+      ]}
+    SCHEMA
+
+    assert_equal schema.name, 'OuterRecord'
+    assert_equal schema.fullname, 'OuterRecord'
+    assert_nil schema.namespace
+
+    schema.fields.each do |field|
+      assert_equal field.type.name, 'InnerRecord'
+      assert_equal field.type.fullname, 'InnerRecord'
+      assert_nil field.type.namespace
+    end
+  end
+
+  def test_inherited_namespace
+    schema = Avro::Schema.parse <<-SCHEMA
+      {"type": "record", "name": "OuterRecord", "namespace": "my.name.space",
+       "fields": [
+          {"name": "definition", "type": {
+            "type": "record", "name": "InnerRecord", "fields": []
+          }},
+          {"name": "relativeReference", "type": "InnerRecord"},
+          {"name": "absoluteReference", "type": "my.name.space.InnerRecord"}
+      ]}
+    SCHEMA
+
+    assert_equal schema.name, 'OuterRecord'
+    assert_equal schema.fullname, 'my.name.space.OuterRecord'
+    assert_equal schema.namespace, 'my.name.space'
+    schema.fields.each do |field|
+      assert_equal field.type.name, 'InnerRecord'
+      assert_equal field.type.fullname, 'my.name.space.InnerRecord'
+      assert_equal field.type.namespace, 'my.name.space'
+    end
+  end
+
+  def test_inherited_namespace_from_dotted_name
+    schema = Avro::Schema.parse <<-SCHEMA
+      {"type": "record", "name": "my.name.space.OuterRecord", "fields": [
+        {"name": "definition", "type": {
+          "type": "enum", "name": "InnerEnum", "symbols": ["HELLO", "WORLD"]
+        }},
+        {"name": "relativeReference", "type": "InnerEnum"},
+        {"name": "absoluteReference", "type": "my.name.space.InnerEnum"}
+      ]}
+    SCHEMA
+
+    assert_equal schema.name, 'OuterRecord'
+    assert_equal schema.fullname, 'my.name.space.OuterRecord'
+    assert_equal schema.namespace, 'my.name.space'
+    schema.fields.each do |field|
+      assert_equal field.type.name, 'InnerEnum'
+      assert_equal field.type.fullname, 'my.name.space.InnerEnum'
+      assert_equal field.type.namespace, 'my.name.space'
+    end
+  end
+
+  def test_nested_namespaces
+    schema = Avro::Schema.parse <<-SCHEMA
+      {"type": "record", "name": "outer.OuterRecord", "fields": [
+        {"name": "middle", "type": {
+          "type": "record", "name": "middle.MiddleRecord", "fields": [
+            {"name": "inner", "type": {
+              "type": "record", "name": "InnerRecord", "fields": [
+                {"name": "recursive", "type": "MiddleRecord"}
+              ]
+            }}
+          ]
+        }}
+      ]}
+    SCHEMA
+
+    assert_equal schema.name, 'OuterRecord'
+    assert_equal schema.fullname, 'outer.OuterRecord'
+    assert_equal schema.namespace, 'outer'
+    middle = schema.fields.first.type
+    assert_equal middle.name, 'MiddleRecord'
+    assert_equal middle.fullname, 'middle.MiddleRecord'
+    assert_equal middle.namespace, 'middle'
+    inner = middle.fields.first.type
+    assert_equal inner.name, 'InnerRecord'
+    assert_equal inner.fullname, 'middle.InnerRecord'
+    assert_equal inner.namespace, 'middle'
+    assert_equal inner.fields.first.type, middle
+  end
+
+  def test_to_avro_includes_namespaces
+    schema = Avro::Schema.parse <<-SCHEMA
+      {"type": "record", "name": "my.name.space.OuterRecord", "fields": [
+        {"name": "definition", "type": {
+          "type": "fixed", "name": "InnerFixed", "size": 16
+        }},
+        {"name": "reference", "type": "InnerFixed"}
+      ]}
+    SCHEMA
+
+    assert_equal schema.to_avro, {
+      'type' => 'record', 'name' => 'OuterRecord', 'namespace' => 'my.name.space',
+      'fields' => [
+        {'name' => 'definition', 'type' => {
+          'type' => 'fixed', 'name' => 'InnerFixed', 'namespace' => 'my.name.space',
+          'size' => 16
+        }},
+        {'name' => 'reference', 'type' => 'my.name.space.InnerFixed'}
+      ]
+    }
+  end
+end

Propchange: avro/trunk/lang/ruby/test/test_schema.rb
------------------------------------------------------------------------------
    svn:eol-style = native



Mime
View raw message