Author: jmhodges
Date: Sat Apr 3 02:18:12 2010
New Revision: 930459
URL: http://svn.apache.org/viewvc?rev=930459&view=rev
Log:
AVRO-486. DataFile.open for the ruby side
Modified:
hadoop/avro/trunk/lang/ruby/lib/avro/data_file.rb
hadoop/avro/trunk/lang/ruby/test/test_datafile.rb
Modified: hadoop/avro/trunk/lang/ruby/lib/avro/data_file.rb
URL:
http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/ruby/lib/avro/data_file.rb?rev=930459&r1=930458&r2=930459&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/ruby/lib/avro/data_file.rb (original)
+++ hadoop/avro/trunk/lang/ruby/lib/avro/data_file.rb Sat Apr 3 02:18:12 2010
@@ -29,6 +29,39 @@ module Avro
class DataFileError < AvroError; end
+ def self.open(file_path, mode='r', schema=nil)
+ schema = Avro::Schema.parse(schema) if schema
+ case mode
+ when 'w'
+ unless schema
+ raise DataFileError, "Writing an Avro file requires a schema."
+ end
+ io = open_writer(File.open(file_path, 'wb'), schema)
+ when 'r'
+ io = open_reader(File.open(file_path, 'rb'), schema)
+ else
+ raise DataFileError, "Only modes 'r' and 'w' allowed. You gave
#{mode.inspect}."
+ end
+
+ yield io if block_given?
+ io
+ ensure
+ io.close if block_given?
+ end
+
+ class << self
+ private
+ def open_writer(file, schema)
+ writer = Avro::IO::DatumWriter.new(schema)
+ Avro::DataFile::Writer.new(file, writer, schema)
+ end
+
+ def open_reader(file, schema)
+ reader = Avro::IO::DatumReader.new(nil, schema)
+ Avro::DataFile::Reader.new(file, reader)
+ end
+ end
+
class Writer
def self.generate_sync_marker
OpenSSL::Random.random_bytes(16)
Modified: hadoop/avro/trunk/lang/ruby/test/test_datafile.rb
URL:
http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/ruby/test/test_datafile.rb?rev=930459&r1=930458&r2=930459&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/ruby/test/test_datafile.rb (original)
+++ hadoop/avro/trunk/lang/ruby/test/test_datafile.rb Sat Apr 3 02:18:12 2010
@@ -14,7 +14,7 @@ class TestDataFile < Test::Unit::TestCas
end
end
- def test_differing_schemas
+ def test_differing_schemas_with_primitives
writer_schema = <<-JSON
{ "type": "record",
"name": "User",
@@ -28,15 +28,9 @@ JSON
data = [{"username" => "john", "age" => 25, "verified" => true},
{"username" => "ryan", "age" => 23, "verified" => false}]
- file = File.open('data.avr', 'wb')
- schema = Avro::Schema.parse(writer_schema)
- writer = Avro::IO::DatumWriter.new(schema)
- dw = Avro::DataFile::Writer.new(file, writer, schema)
- data.each{|h| dw << h }
- dw.close
-
- file = File.open('data.avr', 'r+')
- dr = Avro::DataFile::Reader.new(file, Avro::IO::DatumReader.new)
+ Avro::DataFile.open('data.avr', 'w', writer_schema) do |dw|
+ data.each{|h| dw << h }
+ end
# extract the username only from the avro serialized file
reader_schema = <<-JSON
@@ -47,13 +41,11 @@ JSON
]}
JSON
- reader = Avro::IO::DatumReader.new(nil, Avro::Schema.parse(reader_schema))
- dr = Avro::DataFile::Reader.new(file, reader)
- run = false
- dr.each_with_index do |record, i|
- run = true
- assert_equal data[i]['username'], record['username']
+ Avro::DataFile.open('data.avr', 'r', reader_schema) do |dr|
+ dr.each_with_index do |record, i|
+ assert_equal data[i]['username'], record['username']
+ end
end
- assert run, "enumerable is run through at least once"
end
+
end
