Author: shv Date: Wed Sep 28 03:11:40 2011 New Revision: 1176694 URL: http://svn.apache.org/viewvc?rev=1176694&view=rev Log: MAPREDUCE-3039. Upgrade to Avro 1.5.3. Contributed by Joep Rottinghuis.
Modified: hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt hadoop/common/branches/branch-0.22/mapreduce/build.xml hadoop/common/branches/branch-0.22/mapreduce/ivy.xml hadoop/common/branches/branch-0.22/mapreduce/ivy/libraries.properties hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/ivy.xml hadoop/common/branches/branch-0.22/mapreduce/src/contrib/fairscheduler/ivy.xml hadoop/common/branches/branch-0.22/mapreduce/src/contrib/gridmix/ivy.xml hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mrunit/ivy.xml hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mumak/ivy.xml hadoop/common/branches/branch-0.22/mapreduce/src/contrib/raid/ivy.xml hadoop/common/branches/branch-0.22/mapreduce/src/contrib/streaming/ivy.xml hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java Modified: hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt?rev=1176694&r1=1176693&r2=1176694&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt Wed Sep 28 03:11:40 2011 @@ -223,6 +223,8 @@ Release 0.22.0 - Unreleased HADOOP-7106. Reorganize project SVN layout to "unsplit" the projects. (todd, nigel) + MAPREDUCE-3039. Upgrade to Avro 1.5.3. (Joep Rottinghuis via shv) + OPTIMIZATIONS MAPREDUCE-1354. Enhancements to JobTracker for better performance and Modified: hadoop/common/branches/branch-0.22/mapreduce/build.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/build.xml?rev=1176694&r1=1176693&r2=1176694&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/mapreduce/build.xml (original) +++ hadoop/common/branches/branch-0.22/mapreduce/build.xml Wed Sep 28 03:11:40 2011 @@ -339,7 +339,7 @@ <target name="avro-generate" depends="init"> <mkdir dir="${build.src}/org/apache/hadoop/mapreduce/jobhistory"/> - <taskdef name="protocol" classname="org.apache.avro.specific.ProtocolTask"> + <taskdef name="protocol" classname="org.apache.avro.compiler.specific.ProtocolTask"> <classpath refid="classpath" /> </taskdef> <protocol destdir="${build.src}"> Modified: hadoop/common/branches/branch-0.22/mapreduce/ivy.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/mapreduce/ivy.xml (original) +++ hadoop/common/branches/branch-0.22/mapreduce/ivy.xml Wed Sep 28 03:11:40 2011 @@ -97,12 +97,18 @@ <dependency org="org.apache.lucene" name="lucene-core" rev="${lucene-core.version}" conf="javadoc->default"/> - <dependency org="org.apache.hadoop" name="avro" rev="${avro.version}" - conf="common->default"> + <dependency org="org.apache.avro" name="avro" + rev="${avro.version}" conf="common->default"> <exclude module="ant"/> <exclude module="jetty"/> - <exclude module="slf4j-simple"/> - </dependency> + <exclude module="slf4j-api"/> + </dependency> + <dependency org="org.apache.avro" name="avro-compiler" + rev="${avro.version}" conf="common->default"> + <exclude module="ant"/> + <exclude module="jetty"/> + <exclude module="slf4j-api"/> + </dependency> <dependency org="org.mockito" name="mockito-all" rev="${mockito-all.version}" conf="test->default"/> <dependency org="org.mortbay.jetty" name="jetty-servlet-tester" rev="${jetty.version}" Modified: hadoop/common/branches/branch-0.22/mapreduce/ivy/libraries.properties URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/ivy/libraries.properties?rev=1176694&r1=1176693&r2=1176694&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/mapreduce/ivy/libraries.properties (original) +++ hadoop/common/branches/branch-0.22/mapreduce/ivy/libraries.properties Wed Sep 28 03:11:40 2011 @@ -20,7 +20,7 @@ ant-task.version=2.0.10 #This property has to be updated synchronously with aop.xml aspectj.version=1.6.5 -avro.version=1.3.2 +avro.version=1.5.3 checkstyle.version=4.2 Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/ivy.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/ivy.xml (original) +++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/ivy.xml Wed Sep 28 03:11:40 2011 @@ -69,11 +69,5 @@ rev="${servlet-api-2.5.version}" conf="common->master"/> <dependency org="commons-httpclient" name="commons-httpclient" rev="${commons-httpclient.version}" conf="common->master"/> - <dependency org="org.apache.hadoop" name="avro" - rev="${avro.version}" conf="common->default"> - <exclude module="ant"/> - <exclude module="jetty"/> - <exclude module="slf4j-simple"/> - </dependency> </dependencies> </ivy-module> Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/fairscheduler/ivy.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/fairscheduler/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/fairscheduler/ivy.xml (original) +++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/fairscheduler/ivy.xml Wed Sep 28 03:11:40 2011 @@ -64,26 +64,10 @@ name="junit" rev="${junit.version}" conf="common->default"/> - <dependency org="org.apache.hadoop" - name="avro" - rev="${avro.version}" - conf="common->default"> - <exclude module="ant"/> - <exclude module="jetty"/> - <exclude module="slf4j-simple"/> - </dependency> <dependency org="org.codehaus.jackson" name="jackson-mapper-asl" rev="${jackson.version}" conf="common->default"/> - <dependency org="com.thoughtworks.paranamer" - name="paranamer" - rev="${paranamer.version}" - conf="common->default"/> - <dependency org="com.thoughtworks.paranamer" - name="paranamer-ant" - rev="${paranamer.version}" - conf="common->default"/> <dependency org="org.mortbay.jetty" name="jetty-util" rev="${jetty-util.version}" Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/gridmix/ivy.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/gridmix/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/gridmix/ivy.xml (original) +++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/gridmix/ivy.xml Wed Sep 28 03:11:40 2011 @@ -102,14 +102,6 @@ name="commons-cli" rev="${commons-cli.version}" conf="common->default"/> - <dependency org="org.apache.hadoop" - name="avro" - rev="${avro.version}" - conf="common->default"> - <exclude module="ant"/> - <exclude module="jetty"/> - <exclude module="slf4j-simple"/> - </dependency> <dependency org="org.codehaus.jackson" name="jackson-mapper-asl" rev="${jackson.version}" @@ -118,9 +110,5 @@ name="jackson-core-asl" rev="${jackson.version}" conf="common->default"/> - <dependency org="com.thoughtworks.paranamer" - name="paranamer" - rev="${paranamer.version}" - conf="common->default"/> </dependencies> </ivy-module> Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mrunit/ivy.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mrunit/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mrunit/ivy.xml (original) +++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mrunit/ivy.xml Wed Sep 28 03:11:40 2011 @@ -45,11 +45,6 @@ <dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}" conf="common->default"/> <dependency org="log4j" name="log4j" rev="${log4j.version}" conf="common->master"/> <dependency org="junit" name="junit" rev="${junit.version}" conf="common->master"/> - <dependency org="org.apache.hadoop" name="avro" rev="${avro.version}" conf="common->default"> - <exclude module="ant"/> - <exclude module="jetty"/> - <exclude module="slf4j-simple"/> - </dependency> </dependencies> </ivy-module> Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mumak/ivy.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mumak/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mumak/ivy.xml (original) +++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mumak/ivy.xml Wed Sep 28 03:11:40 2011 @@ -114,13 +114,5 @@ name="commons-cli" rev="${commons-cli.version}" conf="common->default"/> - <dependency org="org.apache.hadoop" - name="avro" - rev="${avro.version}" - conf="common->default"> - <exclude module="ant"/> - <exclude module="jetty"/> - <exclude module="slf4j-simple"/> - </dependency> </dependencies> </ivy-module> Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/raid/ivy.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/raid/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/raid/ivy.xml (original) +++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/raid/ivy.xml Wed Sep 28 03:11:40 2011 @@ -102,14 +102,6 @@ name="commons-cli" rev="${commons-cli.version}" conf="common->default"/> - <dependency org="org.apache.hadoop" - name="avro" - rev="${avro.version}" - conf="common->default"> - <exclude module="ant"/> - <exclude module="jetty"/> - <exclude module="slf4j-simple"/> - </dependency> <dependency org="org.codehaus.jackson" name="jackson-mapper-asl" rev="${jackson.version}" Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/streaming/ivy.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/streaming/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/streaming/ivy.xml (original) +++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/streaming/ivy.xml Wed Sep 28 03:11:40 2011 @@ -68,12 +68,6 @@ rev="${commons-httpclient.version}" conf="common->default"/> <dependency org="log4j" name="log4j" rev="${log4j.version}" conf="common->master"/> - <dependency org="org.apache.hadoop" name="avro" - rev="${avro.version}" conf="common->default"> - <exclude module="ant"/> - <exclude module="jetty"/> - <exclude module="slf4j-simple"/> - </dependency> <dependency org="org.slf4j" name="slf4j-api" rev="${slf4j-api.version}" conf="common->master"/> </dependencies> Modified: hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java?rev=1176694&r1=1176693&r2=1176694&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java (original) +++ hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java Wed Sep 28 03:11:40 2011 @@ -20,10 +20,14 @@ package org.apache.hadoop.mapreduce.jobh import java.io.Closeable; import java.io.DataInputStream; -import java.io.IOException; import java.io.EOFException; -import java.io.StringBufferInputStream; +import java.io.IOException; +import org.apache.avro.Schema; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.Decoder; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.specific.SpecificDatumReader; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FileSystem; @@ -32,13 +36,6 @@ import org.apache.hadoop.mapreduce.Count import org.apache.hadoop.mapreduce.CounterGroup; import org.apache.hadoop.mapreduce.Counters; -import org.apache.avro.Schema; -import org.apache.avro.AvroRuntimeException; -import org.apache.avro.io.Decoder; -import org.apache.avro.io.JsonDecoder; -import org.apache.avro.io.DatumReader; -import org.apache.avro.specific.SpecificDatumReader; - @InterfaceAudience.Private @InterfaceStability.Unstable public class EventReader implements Closeable { @@ -46,7 +43,7 @@ public class EventReader implements Clos private Schema schema; private DataInputStream in; private Decoder decoder; - private DatumReader reader; + private DatumReader<Event> reader; /** * Create a new Event Reader @@ -73,8 +70,8 @@ public class EventReader implements Clos } this.schema = Schema.parse(in.readLine()); - this.reader = new SpecificDatumReader(schema); - this.decoder = new JsonDecoder(schema, in); + this.reader = new SpecificDatumReader<Event>(schema); + this.decoder = DecoderFactory.get().jsonDecoder(schema, in); } /** @@ -82,11 +79,10 @@ public class EventReader implements Clos * @return the next event * @throws IOException */ - @SuppressWarnings("unchecked") public HistoryEvent getNextEvent() throws IOException { Event wrapper; try { - wrapper = (Event)reader.read(null, decoder); + wrapper = reader.read(null, decoder); } catch (EOFException e) { // at EOF return null; } Modified: hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java?rev=1176694&r1=1176693&r2=1176694&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java (original) +++ hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java Wed Sep 28 03:11:40 2011 @@ -19,20 +19,18 @@ package org.apache.hadoop.mapreduce.jobhistory; import java.io.IOException; -import java.util.Iterator; - -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.mapreduce.Counter; -import org.apache.hadoop.mapreduce.CounterGroup; -import org.apache.hadoop.mapreduce.Counters; import org.apache.avro.Schema; -import org.apache.avro.io.Encoder; -import org.apache.avro.io.JsonEncoder; +import org.apache.avro.generic.GenericData; import org.apache.avro.io.DatumWriter; +import org.apache.avro.io.Encoder; +import org.apache.avro.io.EncoderFactory; import org.apache.avro.specific.SpecificDatumWriter; -import org.apache.avro.generic.GenericData; import org.apache.avro.util.Utf8; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.mapreduce.Counter; +import org.apache.hadoop.mapreduce.CounterGroup; +import org.apache.hadoop.mapreduce.Counters; /** * Event Writer is an utility class used to write events to the underlying @@ -54,7 +52,7 @@ class EventWriter { out.writeBytes("\n"); out.writeBytes(Event.SCHEMA$.toString()); out.writeBytes("\n"); - this.encoder = new JsonEncoder(Event.SCHEMA$, out); + this.encoder = EncoderFactory.get().jsonEncoder(Event.SCHEMA$, out); } synchronized void write(HistoryEvent event) throws IOException { Modified: hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java?rev=1176694&r1=1176693&r2=1176694&view=diff ============================================================================== --- hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java (original) +++ hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java Wed Sep 28 03:11:40 2011 @@ -57,7 +57,7 @@ public class JobSubmittedEvent implement datum.userName = new Utf8(userName); datum.submitTime = submitTime; datum.jobConfPath = new Utf8(jobConfPath); - Map<Utf8, Utf8> jobAcls = new HashMap<Utf8, Utf8>(); + Map<CharSequence, CharSequence> jobAcls = new HashMap<CharSequence, CharSequence>(); for (Entry<JobACL, AccessControlList> entry : jobACLs.entrySet()) { jobAcls.put(new Utf8(entry.getKey().getAclName()), new Utf8( entry.getValue().getAclString()));