Author: jmhodges
Date: Sat Apr 3 02:21:58 2010
New Revision: 930461
URL: http://svn.apache.org/viewvc?rev=930461&view=rev
Log:
AVRO-489. Skipping complex objects in the ruby impl.
Modified:
hadoop/avro/trunk/lang/ruby/lib/avro/data_file.rb
hadoop/avro/trunk/lang/ruby/lib/avro/io.rb
hadoop/avro/trunk/lang/ruby/test/test_datafile.rb
Modified: hadoop/avro/trunk/lang/ruby/lib/avro/data_file.rb
URL:
http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/ruby/lib/avro/data_file.rb?rev=930461&r1=930460&r2=930461&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/ruby/lib/avro/data_file.rb (original)
+++ hadoop/avro/trunk/lang/ruby/lib/avro/data_file.rb Sat Apr 3 02:21:58 2010
@@ -46,7 +46,7 @@ module Avro
yield io if block_given?
io
ensure
- io.close if block_given?
+ io.close if block_given? && io
end
class << self
Modified: hadoop/avro/trunk/lang/ruby/lib/avro/io.rb
URL:
http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/ruby/lib/avro/io.rb?rev=930461&r1=930460&r2=930461&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/ruby/lib/avro/io.rb (original)
+++ hadoop/avro/trunk/lang/ruby/lib/avro/io.rb Sat Apr 3 02:21:58 2010
@@ -516,6 +516,42 @@ module Avro
raise AvroError, "Unknown schema type: #{schm.type}"
end
end
+
+ def skip_fixed(writers_schema, decoder)
+ decoder.skip(writers_schema.size)
+ end
+
+ def skip_enum(writers_schema, decoder)
+ decoder.skip_int
+ end
+
+ def skip_array(writers_schema, decoder)
+ skip_blocks(decoder) { skip_data(writers_schema.items, decoder) }
+ end
+
+ def skip_map(writers_schema, decoder)
+ skip_blocks(decoder) {
+ decoder.skip_string
+ skip_data(writers_schema.values, decoder)
+ }
+ end
+
+ def skip_record(writers_schema, decoder)
+ writers_schema.fields.each{|f| skip_data(f.type, decoder) }
+ end
+
+ private
+ def skip_blocks(decoder, &blk)
+ block_count = decoder.read_long
+ while block_count != 0
+ if block_count < 0
+ decoder.skip(decoder.read_long)
+ else
+ block_count.times &blk
+ end
+ block_count = decoder.read_long
+ end
+ end
end # DatumReader
# DatumWriter for generic ruby objects
Modified: hadoop/avro/trunk/lang/ruby/test/test_datafile.rb
URL:
http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/ruby/test/test_datafile.rb?rev=930461&r1=930460&r2=930461&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/ruby/test/test_datafile.rb (original)
+++ hadoop/avro/trunk/lang/ruby/test/test_datafile.rb Sat Apr 3 02:21:58 2010
@@ -48,4 +48,58 @@ JSON
end
end
+ def test_differing_schemas_with_complex_objects
+ writer_schema = <<-JSON
+{ "type": "record",
+ "name": "something",
+ "fields": [
+ {"name": "something_fixed", "type": {"name": "inner_fixed",
+ "type": "fixed", "size": 3}},
+ {"name": "something_enum", "type": {"name": "inner_enum",
+ "type": "enum",
+ "symbols": ["hello", "goodbye"]}},
+ {"name": "something_array", "type": {"type": "array", "items": "int"}},
+ {"name": "something_map", "type": {"type": "map", "values": "int"}},
+ {"name": "something_record", "type": {"name": "inner_record",
+ "type": "record",
+ "fields": [
+ {"name": "inner", "type": "int"}
+ ]}},
+ {"name": "username", "type": "string"}
+]}
+JSON
+
+ data = [{"username" => "john",
+ "something_fixed" => "foo",
+ "something_enum" => "hello",
+ "something_array" => [1,2,3],
+ "something_map" => {"a" => 1, "b" => 2},
+ "something_record" => {"inner" => 2},
+ "something_error" => {"code" => 403}
+ },
+ {"username" => "ryan",
+ "something_fixed" => "bar",
+ "something_enum" => "goodbye",
+ "something_array" => [1,2,3],
+ "something_map" => {"a" => 2, "b" => 6},
+ "something_record" => {"inner" => 1},
+ "something_error" => {"code" => 401}
+ }]
+
+ Avro::DataFile.open('data.avr', 'w', writer_schema) do |dw|
+ data.each{|d| dw << d }
+ end
+
+ %w[fixed enum record error array map union].each do |s|
+ reader = Yajl.load(writer_schema)
+ reader['fields'] = reader['fields'].reject{|f| f['type']['type'] == s}
+ Avro::DataFile.open('data.avr', 'r', Yajl.dump(reader)) do |dr|
+ dr.each_with_index do |obj, i|
+ reader['fields'].each do |field|
+ assert_equal data[i][field['name']], obj[field['name']]
+ end
+ end
+ end
+ end
+ end
end
