This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new c3622a87c4 Make BeamAvroFileIOTest independent of Beam's write order,
fixes #9279 (#9280)
c3622a87c4 is described below
commit c3622a87c441b162b79c58cf6ab9ace5fc0ee281
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Thu Nov 5 10:50:28 2026 +0100
Make BeamAvroFileIOTest independent of Beam's write order, fixes #9279
(#9280)
---
.../beam/pipeline/handler/BeamAvroFileIOTest.java | 38 ++++++++++++++--------
1 file changed, 24 insertions(+), 14 deletions(-)
diff --git
a/plugins/engines/beam/src/test/java/org/apache/hop/beam/pipeline/handler/BeamAvroFileIOTest.java
b/plugins/engines/beam/src/test/java/org/apache/hop/beam/pipeline/handler/BeamAvroFileIOTest.java
index 98bbc945ab..1829b2720c 100644
---
a/plugins/engines/beam/src/test/java/org/apache/hop/beam/pipeline/handler/BeamAvroFileIOTest.java
+++
b/plugins/engines/beam/src/test/java/org/apache/hop/beam/pipeline/handler/BeamAvroFileIOTest.java
@@ -18,7 +18,6 @@
package org.apache.hop.beam.pipeline.handler;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -109,25 +108,36 @@ class BeamAvroFileIOTest {
Path file = tempDir.resolve("rows.avro");
writeTwoRows(file);
- try (DataFileStream<GenericRecord> in =
- new DataFileStream<>(Files.newInputStream(file), new
GenericDatumReader<>())) {
- GenericRecord first = in.next();
- assertEquals("alice", first.get("name").toString());
- assertEquals(1L, first.get("id"));
- GenericRecord second = in.next();
- assertEquals("bob", second.get("name").toString());
- assertEquals(2L, second.get("id"));
- assertFalse(in.hasNext());
- }
-
- assertEquals(List.of("keep-me|alice|1", "keep-me|bob|2"), readBack(file,
null));
+ // Beam doesn't keep element order through a write: compare without order.
+ assertEquals(List.of("alice|1", "bob|2"), sorted(recordsInFile(file)));
+ assertEquals(List.of("keep-me|alice|1", "keep-me|bob|2"),
sorted(readBack(file, null)));
}
@Test
void rowLimitCapsWhatOneWorkerEmits() throws Exception {
Path file = tempDir.resolve("limited.avro");
writeTwoRows(file);
- assertEquals(List.of("keep-me|alice|1"), readBack(file, "1"));
+ // The limit keeps the first record of the file, whichever row Beam wrote
first.
+ String first = recordsInFile(file).get(0);
+ assertEquals(List.of("keep-me|" + first), readBack(file, "1"));
+ }
+
+ private static List<String> recordsInFile(Path file) throws Exception {
+ List<String> records = new ArrayList<>();
+ try (DataFileStream<GenericRecord> in =
+ new DataFileStream<>(Files.newInputStream(file), new
GenericDatumReader<>())) {
+ while (in.hasNext()) {
+ GenericRecord record = in.next();
+ records.add(record.get("name") + "|" + record.get("id"));
+ }
+ }
+ return records;
+ }
+
+ private static List<String> sorted(List<String> values) {
+ List<String> copy = new ArrayList<>(values);
+ Collections.sort(copy);
+ return copy;
}
@Test