This is an automated email from the ASF dual-hosted git repository.
steveloughran pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9318632ebfb HADOOP-19928. Move to Avro 1.12.1 (#8567)
9318632ebfb is described below
commit 9318632ebfbe22cc1bd9b9ed78c89fd225f62a67
Author: Steve Loughran <[email protected]>
AuthorDate: Tue Jun 30 19:21:23 2026 +0100
HADOOP-19928. Move to Avro 1.12.1 (#8567)
Contains code contributed by Anthropic Claude
Contributed by Steve Loughran
---
LICENSE-binary | 2 +-
.../io/serializer/avro/TestAvroSerialization.java | 91 +++++++++++++----
.../hadoop/mapreduce/jobhistory/TestEvents.java | 111 ++++++++++++---------
hadoop-project/pom.xml | 2 +-
4 files changed, 136 insertions(+), 70 deletions(-)
diff --git a/LICENSE-binary b/LICENSE-binary
index 780902ae05c..c5d6d6f1763 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -363,7 +363,7 @@ io.swagger:swagger-annotations:1.5.4
javax.inject:javax.inject:1
net.java.dev.jna:jna:5.2.0
net.minidev:accessors-smart:1.21
-org.apache.avro:avro:1.11.5
+org.apache.avro:avro:1.12.1
org.apache.commons:commons-compress:1.26.1
org.apache.commons:commons-configuration2:2.10.1
org.apache.commons:commons-csv:1.9.0
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java
index dc040d16e7e..3f658754137 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java
@@ -18,8 +18,9 @@
package org.apache.hadoop.io.serializer.avro;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNull;
+import java.util.Objects;
+
+import static org.assertj.core.api.Assertions.assertThat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.serializer.SerializationFactory;
@@ -35,7 +36,7 @@ public void testSpecific() throws Exception {
AvroRecord before = new AvroRecord();
before.setIntField(5);
AvroRecord after = SerializationTestUtil.testSerialization(conf, before);
- assertEquals(before, after);
+ assertThat(after).isEqualTo(before);
}
@Test
@@ -45,14 +46,14 @@ public void testReflectPkg() throws Exception {
conf.set(AvroReflectSerialization.AVRO_REFLECT_PACKAGES,
before.getClass().getPackage().getName());
Record after = SerializationTestUtil.testSerialization(conf, before);
- assertEquals(before, after);
+ assertThat(after).isEqualTo(before);
}
@Test
public void testAcceptHandlingPrimitivesAndArrays() throws Exception {
SerializationFactory factory = new SerializationFactory(conf);
- assertNull(factory.getSerializer(byte[].class));
- assertNull(factory.getSerializer(byte.class));
+ assertThat(factory.getSerializer(byte[].class)).isNull();
+ assertThat(factory.getSerializer(byte.class)).isNull();
}
@Test
@@ -62,18 +63,34 @@ public void testReflectInnerClass() throws Exception {
conf.set(AvroReflectSerialization.AVRO_REFLECT_PACKAGES,
before.getClass().getPackage().getName());
InnerRecord after = SerializationTestUtil.testSerialization(conf, before);
- assertEquals(before, after);
+ assertThat(after).isEqualTo(before);
}
@Test
public void testReflect() throws Exception {
RefSerializable before = new RefSerializable();
before.x = 10;
- RefSerializable after =
+ RefSerializable after =
SerializationTestUtil.testSerialization(conf, before);
- assertEquals(before, after);
+ assertThat(after).isEqualTo(before);
+ }
+
+ /**
+ * Round-trip a reflect-serializable record that carries a String field.
+ * String handling is the part of the Avro API most affected by changes to
+ * {@code org.apache.avro.util.Utf8}, so this guards the reflect
serialization
+ * path across Avro upgrades.
+ */
+ @Test
+ public void testReflectStringField() throws Exception {
+ StringRecord before = new StringRecord();
+ before.x = 10;
+ before.s = "value";
+ StringRecord after =
+ SerializationTestUtil.testSerialization(conf, before);
+ assertThat(after).isEqualTo(before);
}
-
+
public static class InnerRecord {
public int x = 7;
@@ -84,20 +101,21 @@ public int hashCode() {
@Override
public boolean equals(Object obj) {
- if (this == obj)
+ if (this == obj) {
return true;
- if (obj == null)
+ }
+ if (obj == null) {
return false;
- if (getClass() != obj.getClass())
+ }
+ if (getClass() != obj.getClass()) {
return false;
+ }
final InnerRecord other = (InnerRecord) obj;
- if (x != other.x)
- return false;
- return true;
+ return x == other.x;
}
}
- public static class RefSerializable implements AvroReflectSerializable {
+ public static final class RefSerializable implements AvroReflectSerializable
{
public int x = 7;
@Override
@@ -107,16 +125,45 @@ public int hashCode() {
@Override
public boolean equals(Object obj) {
- if (this == obj)
+ if (this == obj) {
return true;
- if (obj == null)
+ }
+ if (obj == null) {
return false;
- if (getClass() != obj.getClass())
+ }
+ if (getClass() != obj.getClass()) {
return false;
+ }
final RefSerializable other = (RefSerializable) obj;
- if (x != other.x)
+ return x == other.x;
+ }
+ }
+
+ public static final class StringRecord implements AvroReflectSerializable {
+ public int x = 7;
+ public String s = "";
+
+ @Override
+ public int hashCode() {
+ return x * 31 + (s == null ? 0 : s.hashCode());
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ final StringRecord other = (StringRecord) obj;
+ if (x != other.x) {
return false;
- return true;
+ }
+ return Objects.equals(s, other.s);
}
}
}
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
index 1819a641999..80882e5aca6 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.mapreduce.jobhistory;
import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -72,7 +70,7 @@ public void testTaskAttemptFinishedEvent() throws Exception {
assertThat(test.getTaskId()).isEqualTo(tid);
assertThat(test.getTaskStatus()).isEqualTo("TEST");
assertThat(test.getTaskType()).isEqualTo(TaskType.REDUCE);
- assertEquals(234, test.getStartTime());
+ assertThat(test.getStartTime()).isEqualTo(234);
}
/**
@@ -121,88 +119,109 @@ public void testTaskUpdated() throws Exception {
/*
* test EventReader EventReader should read the list of events and return
* instance of HistoryEvent Different HistoryEvent should have a different
- * datum.
+ * datum. Uses the Avro-Json encoding.
*/
@Test
@Timeout(value = 10)
public void testEvents() throws Exception {
+ verifyEvents(getEvents(EventWriter.WriteMode.JSON));
+ }
+
+ /*
+ * As {@link #testEvents()} but exercising the Avro-Binary encoding, which is
+ * the format job history files are written in. This guards the
+ * SpecificDatumWriter/SpecificDatumReader binary round-trip across Avro
+ * upgrades.
+ */
+ @Test
+ @Timeout(value = 10)
+ public void testEventsBinary() throws Exception {
+ verifyEvents(getEvents(EventWriter.WriteMode.BINARY));
+ }
+
+ private void verifyEvents(byte[] events) throws Exception {
EventReader reader = new EventReader(new DataInputStream(
- new ByteArrayInputStream(getEvents())));
+ new ByteArrayInputStream(events)));
HistoryEvent e = reader.getNextEvent();
- assertTrue(e.getEventType().equals(EventType.JOB_PRIORITY_CHANGED));
- assertEquals("ID", ((JobPriorityChange)
e.getDatum()).getJobid().toString());
+ assertThat(e.getEventType()).isEqualTo(EventType.JOB_PRIORITY_CHANGED);
+ assertThat(((JobPriorityChange) e.getDatum()).getJobid().toString())
+ .isEqualTo("ID");
e = reader.getNextEvent();
- assertTrue(e.getEventType().equals(EventType.JOB_STATUS_CHANGED));
- assertEquals("ID", ((JobStatusChanged)
e.getDatum()).getJobid().toString());
+ assertThat(e.getEventType()).isEqualTo(EventType.JOB_STATUS_CHANGED);
+ assertThat(((JobStatusChanged) e.getDatum()).getJobid().toString())
+ .isEqualTo("ID");
e = reader.getNextEvent();
- assertTrue(e.getEventType().equals(EventType.TASK_UPDATED));
- assertEquals("ID", ((TaskUpdated) e.getDatum()).getTaskid().toString());
+ assertThat(e.getEventType()).isEqualTo(EventType.TASK_UPDATED);
+ assertThat(((TaskUpdated) e.getDatum()).getTaskid().toString())
+ .isEqualTo("ID");
e = reader.getNextEvent();
- assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_KILLED));
- assertEquals(taskId,
- ((TaskAttemptUnsuccessfulCompletion)
e.getDatum()).getTaskid().toString());
+ assertThat(e.getEventType()).isEqualTo(EventType.REDUCE_ATTEMPT_KILLED);
+ assertThat(((TaskAttemptUnsuccessfulCompletion)
e.getDatum()).getTaskid().toString())
+ .isEqualTo(taskId);
e = reader.getNextEvent();
- assertTrue(e.getEventType().equals(EventType.JOB_KILLED));
- assertEquals("ID",
- ((JobUnsuccessfulCompletion) e.getDatum()).getJobid().toString());
+ assertThat(e.getEventType()).isEqualTo(EventType.JOB_KILLED);
+ assertThat(((JobUnsuccessfulCompletion)
e.getDatum()).getJobid().toString())
+ .isEqualTo("ID");
e = reader.getNextEvent();
- assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_STARTED));
- assertEquals(taskId,
- ((TaskAttemptStarted) e.getDatum()).getTaskid().toString());
+ assertThat(e.getEventType()).isEqualTo(EventType.REDUCE_ATTEMPT_STARTED);
+ assertThat(((TaskAttemptStarted) e.getDatum()).getTaskid().toString())
+ .isEqualTo(taskId);
e = reader.getNextEvent();
- assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_FINISHED));
- assertEquals(taskId,
- ((TaskAttemptFinished) e.getDatum()).getTaskid().toString());
+ assertThat(e.getEventType()).isEqualTo(EventType.REDUCE_ATTEMPT_FINISHED);
+ assertThat(((TaskAttemptFinished) e.getDatum()).getTaskid().toString())
+ .isEqualTo(taskId);
e = reader.getNextEvent();
- assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_KILLED));
- assertEquals(taskId,
- ((TaskAttemptUnsuccessfulCompletion)
e.getDatum()).getTaskid().toString());
+ assertThat(e.getEventType()).isEqualTo(EventType.REDUCE_ATTEMPT_KILLED);
+ assertThat(((TaskAttemptUnsuccessfulCompletion)
e.getDatum()).getTaskid().toString())
+ .isEqualTo(taskId);
e = reader.getNextEvent();
- assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_KILLED));
- assertEquals(taskId,
- ((TaskAttemptUnsuccessfulCompletion)
e.getDatum()).getTaskid().toString());
+ assertThat(e.getEventType()).isEqualTo(EventType.REDUCE_ATTEMPT_KILLED);
+ assertThat(((TaskAttemptUnsuccessfulCompletion)
e.getDatum()).getTaskid().toString())
+ .isEqualTo(taskId);
e = reader.getNextEvent();
- assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_STARTED));
- assertEquals(taskId,
- ((TaskAttemptStarted) e.getDatum()).getTaskid().toString());
+ assertThat(e.getEventType()).isEqualTo(EventType.REDUCE_ATTEMPT_STARTED);
+ assertThat(((TaskAttemptStarted) e.getDatum()).getTaskid().toString())
+ .isEqualTo(taskId);
e = reader.getNextEvent();
- assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_FINISHED));
- assertEquals(taskId,
- ((TaskAttemptFinished) e.getDatum()).getTaskid().toString());
+ assertThat(e.getEventType()).isEqualTo(EventType.REDUCE_ATTEMPT_FINISHED);
+ assertThat(((TaskAttemptFinished) e.getDatum()).getTaskid().toString())
+ .isEqualTo(taskId);
e = reader.getNextEvent();
- assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_KILLED));
- assertEquals(taskId,
- ((TaskAttemptUnsuccessfulCompletion)
e.getDatum()).getTaskid().toString());
+ assertThat(e.getEventType()).isEqualTo(EventType.REDUCE_ATTEMPT_KILLED);
+ assertThat(((TaskAttemptUnsuccessfulCompletion)
e.getDatum()).getTaskid().toString())
+ .isEqualTo(taskId);
e = reader.getNextEvent();
- assertTrue(e.getEventType().equals(EventType.REDUCE_ATTEMPT_KILLED));
- assertEquals(taskId,
- ((TaskAttemptUnsuccessfulCompletion)
e.getDatum()).getTaskid().toString());
+ assertThat(e.getEventType()).isEqualTo(EventType.REDUCE_ATTEMPT_KILLED);
+ assertThat(((TaskAttemptUnsuccessfulCompletion)
e.getDatum()).getTaskid().toString())
+ .isEqualTo(taskId);
reader.close();
}
- /*
- * makes array of bytes with History events
+ /**
+ * makes array of bytes with History events, encoded in the given write mode.
+ * @param mode write mode.
+ * @return the marshalled output
+ * @throws Exception failure
*/
- private byte[] getEvents() throws Exception {
+ private byte[] getEvents(EventWriter.WriteMode mode) throws Exception {
ByteArrayOutputStream output = new ByteArrayOutputStream();
FSDataOutputStream fsOutput = new FSDataOutputStream(output,
new FileSystem.Statistics("scheme"));
- EventWriter writer = new EventWriter(fsOutput,
- EventWriter.WriteMode.JSON);
+ EventWriter writer = new EventWriter(fsOutput, mode);
writer.write(getJobPriorityChangedEvent());
writer.write(getJobStatusChangedEvent());
writer.write(getTaskUpdatedEvent());
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index a92998ce262..b6df5330cd2 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -62,7 +62,7 @@
<java.security.egd>file:///dev/urandom</java.security.egd>
<!-- avro version -->
- <avro.version>1.11.5</avro.version>
+ <avro.version>1.12.1</avro.version>
<!-- jersey version -->
<jersey2.version>2.46</jersey2.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
