avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jmhod...@apache.org
Subject svn commit: r951270 - in /avro/branches/branch-1.3: ./ lang/ruby/lib/avro/data_file.rb lang/ruby/lib/avro/io.rb lang/ruby/test/test_datafile.rb
Date Fri, 04 Jun 2010 05:28:11 GMT
Author: jmhodges
Date: Fri Jun  4 05:28:11 2010
New Revision: 951270

URL: http://svn.apache.org/viewvc?rev=951270&view=rev
Log:
AVRO-489. Skipping complex objects in the ruby impl. merge from trunk

Modified:
    avro/branches/branch-1.3/   (props changed)
    avro/branches/branch-1.3/lang/ruby/lib/avro/data_file.rb
    avro/branches/branch-1.3/lang/ruby/lib/avro/io.rb
    avro/branches/branch-1.3/lang/ruby/test/test_datafile.rb

Propchange: avro/branches/branch-1.3/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jun  4 05:28:11 2010
@@ -1,2 +1,2 @@
 /avro/trunk:944035,944049
-/hadoop/avro/trunk:930458-930459,935526,938347
+/hadoop/avro/trunk:930458-930459,930461,935526,938347

Modified: avro/branches/branch-1.3/lang/ruby/lib/avro/data_file.rb
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.3/lang/ruby/lib/avro/data_file.rb?rev=951270&r1=951269&r2=951270&view=diff
==============================================================================
--- avro/branches/branch-1.3/lang/ruby/lib/avro/data_file.rb (original)
+++ avro/branches/branch-1.3/lang/ruby/lib/avro/data_file.rb Fri Jun  4 05:28:11 2010
@@ -46,7 +46,7 @@ module Avro
       yield io if block_given?
       io
     ensure
-      io.close if block_given?
+      io.close if block_given? && io
     end
 
     class << self

Modified: avro/branches/branch-1.3/lang/ruby/lib/avro/io.rb
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.3/lang/ruby/lib/avro/io.rb?rev=951270&r1=951269&r2=951270&view=diff
==============================================================================
--- avro/branches/branch-1.3/lang/ruby/lib/avro/io.rb (original)
+++ avro/branches/branch-1.3/lang/ruby/lib/avro/io.rb Fri Jun  4 05:28:11 2010
@@ -516,6 +516,42 @@ module Avro
           raise AvroError, "Unknown schema type: #{schm.type}"
         end
       end
+
+      def skip_fixed(writers_schema, decoder)
+        decoder.skip(writers_schema.size)
+      end
+
+      def skip_enum(writers_schema, decoder)
+        decoder.skip_int
+      end
+
+      def skip_array(writers_schema, decoder)
+        skip_blocks(decoder) { skip_data(writers_schema.items, decoder) }
+      end
+
+      def skip_map(writers_schema, decoder)
+        skip_blocks(decoder) {
+          decoder.skip_string
+          skip_data(writers_schema.values, decoder)
+        }
+      end
+
+      def skip_record(writers_schema, decoder)
+        writers_schema.fields.each{|f| skip_data(f.type, decoder) }
+      end
+
+      private
+      def skip_blocks(decoder, &blk)
+        block_count = decoder.read_long
+        while block_count != 0
+          if block_count < 0
+            decoder.skip(decoder.read_long)
+          else
+            block_count.times &blk
+          end
+          block_count = decoder.read_long
+        end
+      end
     end # DatumReader
 
     # DatumWriter for generic ruby objects

Modified: avro/branches/branch-1.3/lang/ruby/test/test_datafile.rb
URL: http://svn.apache.org/viewvc/avro/branches/branch-1.3/lang/ruby/test/test_datafile.rb?rev=951270&r1=951269&r2=951270&view=diff
==============================================================================
--- avro/branches/branch-1.3/lang/ruby/test/test_datafile.rb (original)
+++ avro/branches/branch-1.3/lang/ruby/test/test_datafile.rb Fri Jun  4 05:28:11 2010
@@ -48,4 +48,58 @@ JSON
     end
   end
 
+  def test_differing_schemas_with_complex_objects
+    writer_schema = <<-JSON
+{ "type": "record",
+  "name": "something",
+  "fields": [
+    {"name": "something_fixed", "type": {"name": "inner_fixed",
+                                         "type": "fixed", "size": 3}},
+    {"name": "something_enum", "type": {"name": "inner_enum",
+                                        "type": "enum",
+                                        "symbols": ["hello", "goodbye"]}},
+    {"name": "something_array", "type": {"type": "array", "items": "int"}},
+    {"name": "something_map", "type": {"type": "map", "values": "int"}},
+    {"name": "something_record", "type": {"name": "inner_record",
+                                          "type": "record",
+                                          "fields": [
+                                            {"name": "inner", "type": "int"}
+                                          ]}},
+    {"name": "username", "type": "string"}
+]}
+JSON
+
+    data = [{"username" => "john",
+              "something_fixed" => "foo",
+              "something_enum" => "hello",
+              "something_array" => [1,2,3],
+              "something_map" => {"a" => 1, "b" => 2},
+              "something_record" => {"inner" => 2},
+              "something_error" => {"code" => 403}
+            },
+            {"username" => "ryan",
+              "something_fixed" => "bar",
+              "something_enum" => "goodbye",
+              "something_array" => [1,2,3],
+              "something_map" => {"a" => 2, "b" => 6},
+              "something_record" => {"inner" => 1},
+              "something_error" => {"code" => 401}
+            }]
+
+    Avro::DataFile.open('data.avr', 'w', writer_schema) do |dw|
+      data.each{|d| dw << d }
+    end
+
+    %w[fixed enum record error array map union].each do |s|
+      reader = Yajl.load(writer_schema)
+      reader['fields'] = reader['fields'].reject{|f| f['type']['type'] == s}
+      Avro::DataFile.open('data.avr', 'r', Yajl.dump(reader)) do |dr|
+        dr.each_with_index do |obj, i|
+          reader['fields'].each do |field|
+            assert_equal data[i][field['name']], obj[field['name']]
+          end
+        end
+      end
+    end
+  end
 end



Mime
View raw message