This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new effb726a4f An Avro record converts from text, with its schema or the
field's, fi… (#9561)
effb726a4f is described below
commit effb726a4ffe9cf4cf25f7f91941cc2c07b3e746
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Thu Nov 19 11:01:03 2026 +0100
An Avro record converts from text, with its schema or the field's, fi…
(#9561)
* An Avro record converts from text, with its schema or the field's, fixes
#9560
* fix remarks
---
core/src/main/java/org/apache/hop/core/Const.java | 13 +
.../hop/core/row/value/ValueMetaAvroRecord.java | 285 +++++++++++++++++++--
.../core/row/value/ValueMetaAvroRecordTest.java | 198 ++++++++++++++
.../modules/ROOT/pages/variables.adoc | 1 +
.../transforms/0131-avro-record-convert.hpl | 198 ++++++++++++++
.../datasets/golden-0131-avro-record-convert.csv | 4 +
.../transforms/main-0131-avro-record-convert.hwf | 80 ++++++
.../dataset/golden-0131-avro-record-convert.json | 24 ++
.../unit-test/0131-avro-record-convert UNIT.json | 32 +++
9 files changed, 817 insertions(+), 18 deletions(-)
diff --git a/core/src/main/java/org/apache/hop/core/Const.java
b/core/src/main/java/org/apache/hop/core/Const.java
index a467f14645..72aeee47d2 100644
--- a/core/src/main/java/org/apache/hop/core/Const.java
+++ b/core/src/main/java/org/apache/hop/core/Const.java
@@ -639,6 +639,19 @@ public class Const {
public static final String HOP_EMPTY_STRING_DIFFERS_FROM_NULL =
"HOP_EMPTY_STRING_DIFFERS_FROM_NULL";
+ /**
+ * Write an Avro record as text with its schema, so the text can be
converted back to a record
+ * without one.
+ */
+ @Variable(
+ value = "N",
+ description =
+ "Set this variable to Y to write an Avro record as text with its
schema and its data in"
+ + " Avro's JSON encoding, so the text converts back to the
record without a schema."
+ + " With N only the data is written")
+ public static final String HOP_AVRO_RECORD_STRING_WITH_SCHEMA =
+ "HOP_AVRO_RECORD_STRING_WITH_SCHEMA";
+
/**
* System wide flag to allow non-strict string to number conversion for
backward compatibility. If
* this setting is set to "Y", an string starting with digits will be
converted successfully into
diff --git
a/core/src/main/java/org/apache/hop/core/row/value/ValueMetaAvroRecord.java
b/core/src/main/java/org/apache/hop/core/row/value/ValueMetaAvroRecord.java
index 6ee5deac50..7ee522cb5f 100644
--- a/core/src/main/java/org/apache/hop/core/row/value/ValueMetaAvroRecord.java
+++ b/core/src/main/java/org/apache/hop/core/row/value/ValueMetaAvroRecord.java
@@ -17,6 +17,8 @@
package org.apache.hop.core.row.value;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
@@ -25,7 +27,10 @@ import java.io.EOFException;
import java.io.IOException;
import java.io.OutputStream;
import java.net.SocketTimeoutException;
+import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
+import java.util.LinkedHashMap;
+import java.util.Map;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumReader;
@@ -35,8 +40,10 @@ import org.apache.avro.io.BinaryDecoder;
import org.apache.avro.io.BinaryEncoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.EncoderFactory;
-import org.apache.avro.io.JsonDecoder;
+import org.apache.avro.io.JsonEncoder;
+import org.apache.avro.util.Utf8;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.Const;
import org.apache.hop.core.exception.HopEofException;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopFileException;
@@ -57,6 +64,8 @@ public class ValueMetaAvroRecord extends ValueMetaBase {
private Schema schema;
private static final String CONST_SCHEMA = "schema";
+ private static final String CONST_DATA = "data";
+ private static final ObjectMapper JSON_MAPPER = new ObjectMapper();
private static final String CONST_SPECIFIED = " specified.";
private static final String CONST_UNKNOWN_STORAGE_TYPE = " : Unknown storage
type ";
private static final String CONST_SCHEMA_NEEDED =
@@ -65,6 +74,13 @@ public class ValueMetaAvroRecord extends ValueMetaBase {
/** Reject a corrupt length instead of allocating it. Sample rows are far
smaller than this. */
private static final int MAX_AVRO_BYTES = 32 * 1024 * 1024;
+ /**
+ * Write a record as text with its schema (see {@link
#convertGenericRecordToString}), so it can
+ * be read back without one. Off by default: the text is then only the data,
as before.
+ */
+ private final boolean stringWithSchema =
+
Const.toBoolean(System.getProperty(Const.HOP_AVRO_RECORD_STRING_WITH_SCHEMA,
"N"));
+
public ValueMetaAvroRecord() {
super(null, IValueMeta.TYPE_AVRO);
}
@@ -119,7 +135,7 @@ public class ValueMetaAvroRecord extends ValueMetaBase {
case STORAGE_TYPE_NORMAL:
try {
String jsonString = (String) object;
- return convertStringToGenericRecord(jsonString);
+ return convertStringToGenericRecord(jsonString, schema);
} catch (Exception e) {
throw new HopValueException(
"Error converting a JSON representation of an Avro
GenericRecord to a native representation",
@@ -136,7 +152,9 @@ public class ValueMetaAvroRecord extends ValueMetaBase {
}
/**
- * Convert the record to both schema and data in a single JSON block...
+ * Convert the record to both schema and data in a single JSON block, the
data in Avro's JSON
+ * encoding: <code>{ "schema" : ..., "data" : ... }</code>. {@link
+ * #convertStringToGenericRecord(String)} reads it back.
*
* @param genericRecord The record to convert to JSON
* @return The JSON representation of a generic Avro record
@@ -146,33 +164,237 @@ public class ValueMetaAvroRecord extends ValueMetaBase {
throws HopValueException {
try {
Schema schema = genericRecord.getSchema();
- String schemaJson = schema.toString();
- String dataJson = genericRecord.toString();
+ ByteArrayOutputStream data = new ByteArrayOutputStream();
+ JsonEncoder encoder = EncoderFactory.get().jsonEncoder(schema, data);
+ new GenericDatumWriter<GenericRecord>(schema).write(genericRecord,
encoder);
+ encoder.flush();
- return "{ \"schema\" : " + schemaJson + ", \"data\" : " + dataJson + "
}";
+ return "{ \"schema\" : "
+ + schema
+ + ", \"data\" : "
+ + data.toString(StandardCharsets.UTF_8)
+ + " }";
} catch (Exception e) {
throw new HopValueException(
"Unable to convert an Avro record to a JSON String using the
provided schema", e);
}
}
+ /** Read a record from the schema and data JSON {@link
#convertGenericRecordToString} writes. */
public static GenericRecord convertStringToGenericRecord(String jsonString)
throws HopValueException {
+ return convertStringToGenericRecord(jsonString, null);
+ }
+
+ /**
+ * Read a record from text: either the schema and data JSON {@link
#convertGenericRecordToString}
+ * writes, or only the data (as a record's toString() writes it) when a
schema is given. The data
+ * can be in Avro's JSON encoding or plain JSON values.
+ *
+ * @param jsonString The text to read
+ * @param schema The schema of data without one, or null
+ * @return The record
+ */
+ public static GenericRecord convertStringToGenericRecord(String jsonString,
Schema schema)
+ throws HopValueException {
+ JsonNode json;
try {
- // Convert schema AND data to JSON...
- //
- JSONObject json = (JSONObject) new JSONParser().parse(jsonString);
- JSONObject schemaObject = (JSONObject) json.get(CONST_SCHEMA);
- JSONObject dataObject = (JSONObject) json.get("data");
+ json = JSON_MAPPER.readTree(jsonString);
+ } catch (Exception e) {
+ throw new HopValueException("Unable to read an Avro record from text:
not valid JSON", e);
+ }
+ if (json != null && json.has(CONST_SCHEMA) && json.has(CONST_DATA)) {
+ Schema textSchema;
+ try {
+ textSchema = new
Schema.Parser().parse(json.get(CONST_SCHEMA).toString());
+ } catch (Exception e) {
+ throw new HopValueException("Unable to read the Avro schema in the
text", e);
+ }
+ return (GenericRecord) jsonToAvro(json.get(CONST_DATA), textSchema);
+ }
+ if (schema == null) {
+ throw new HopValueException(
+ "Unable to read an Avro record from text without a schema: the text
holds no"
+ + " \"schema\" and the field has none");
+ }
+ return (GenericRecord) jsonToAvro(json, schema);
+ }
+
+ /**
+ * One JSON value as the Avro value of the schema. A union value can be
wrapped in its type, as
+ * Avro's JSON encoding writes it, or plain, as a record's toString() writes
it: then the first
+ * type it fits is used.
+ */
+ private static Object jsonToAvro(JsonNode node, Schema schema) throws
HopValueException {
+ boolean isNull = node == null || node.isNull();
+ switch (schema.getType()) {
+ case NULL:
+ if (isNull) {
+ return null;
+ }
+ break;
+ case BOOLEAN:
+ if (!isNull && node.isBoolean()) {
+ return node.booleanValue();
+ }
+ break;
+ case INT:
+ if (!isNull && node.isIntegralNumber() && node.canConvertToInt()) {
+ return node.intValue();
+ }
+ break;
+ case LONG:
+ if (!isNull && node.isIntegralNumber() && node.canConvertToLong()) {
+ return node.longValue();
+ }
+ break;
+ case FLOAT:
+ if (!isNull && node.isNumber()) {
+ return node.floatValue();
+ }
+ if (nonFiniteNumber(node) instanceof Double nonFinite) {
+ return nonFinite.floatValue();
+ }
+ break;
+ case DOUBLE:
+ if (!isNull && node.isNumber()) {
+ return node.doubleValue();
+ }
+ if (nonFiniteNumber(node) instanceof Double nonFinite) {
+ return nonFinite;
+ }
+ break;
+ case STRING:
+ if (!isNull && node.isTextual()) {
+ return new Utf8(node.textValue());
+ }
+ break;
+ case BYTES:
+ if (!isNull && node.isTextual()) {
+ return
ByteBuffer.wrap(node.textValue().getBytes(StandardCharsets.ISO_8859_1));
+ }
+ break;
+ case FIXED:
+ if (!isNull && node.isTextual()) {
+ return new GenericData.Fixed(
+ schema, node.textValue().getBytes(StandardCharsets.ISO_8859_1));
+ }
+ // A record's toString() writes the bytes as an array of numbers: [1,
2, 3, -6]
+ if (fixedBytes(node, schema.getFixedSize()) instanceof byte[] bytes) {
+ return new GenericData.Fixed(schema, bytes);
+ }
+ break;
+ case ENUM:
+ if (!isNull && node.isTextual() &&
schema.hasEnumSymbol(node.textValue())) {
+ return new GenericData.EnumSymbol(schema, node.textValue());
+ }
+ break;
+ case ARRAY:
+ if (!isNull && node.isArray()) {
+ GenericData.Array<Object> array = new
GenericData.Array<>(node.size(), schema);
+ for (JsonNode element : node) {
+ array.add(jsonToAvro(element, schema.getElementType()));
+ }
+ return array;
+ }
+ break;
+ case MAP:
+ if (!isNull && node.isObject()) {
+ Map<Utf8, Object> map = new LinkedHashMap<>();
+ for (Map.Entry<String, JsonNode> entry : node.properties()) {
+ map.put(new Utf8(entry.getKey()), jsonToAvro(entry.getValue(),
schema.getValueType()));
+ }
+ return map;
+ }
+ break;
+ case RECORD:
+ // Only an object with no keys the record doesn't have: in a union of
records written by
+ // toString(), a record whose fields are all optional would otherwise
match any object.
+ if (!isNull && node.isObject() && onlyFieldsOf(node, schema)) {
+ GenericData.Record avroRecord = new GenericData.Record(schema);
+ for (Schema.Field field : schema.getFields()) {
+ JsonNode value = node.get(field.name());
+ if (value == null && field.hasDefaultValue()) {
+ avroRecord.put(field.pos(),
GenericData.get().getDefaultValue(field));
+ } else {
+ avroRecord.put(field.pos(), jsonToAvro(value, field.schema()));
+ }
+ }
+ return avroRecord;
+ }
+ break;
+ case UNION:
+ return unionToAvro(node, schema);
+ default:
+ break;
+ }
+ throw new HopValueException(
+ "Unable to read " + (isNull ? "null" : node.toString()) + " as Avro
type " + schema);
+ }
- Schema schema = new Schema.Parser().parse(schemaObject.toJSONString());
+ /** NaN and the infinities, which JSON writers (toString(), Avro's encoder)
write as text. */
+ private static Double nonFiniteNumber(JsonNode node) {
+ if (node == null || !node.isTextual()) {
+ return null;
+ }
+ return switch (node.textValue()) {
+ case "NaN" -> Double.NaN;
+ case "Infinity" -> Double.POSITIVE_INFINITY;
+ case "-Infinity" -> Double.NEGATIVE_INFINITY;
+ default -> null;
+ };
+ }
- JsonDecoder jsonDecoder = DecoderFactory.get().jsonDecoder(schema,
dataObject.toJSONString());
- GenericDatumReader<GenericRecord> genericDatumReader = new
GenericDatumReader<>(schema);
- return genericDatumReader.read(null, jsonDecoder);
- } catch (Exception e) {
- throw new HopValueException("Unable to convert a String to an Avro
record", e);
+ private static byte[] fixedBytes(JsonNode node, int size) {
+ if (node == null || !node.isArray() || node.size() != size) {
+ return null;
+ }
+ byte[] bytes = new byte[size];
+ for (int i = 0; i < size; i++) {
+ JsonNode element = node.get(i);
+ if (!element.isIntegralNumber()
+ || !element.canConvertToInt()
+ || element.intValue() < Byte.MIN_VALUE
+ || element.intValue() > Byte.MAX_VALUE) {
+ return null;
+ }
+ bytes[i] = (byte) element.intValue();
+ }
+ return bytes;
+ }
+
+ private static boolean onlyFieldsOf(JsonNode node, Schema recordSchema) {
+ for (Map.Entry<String, JsonNode> entry : node.properties()) {
+ if (recordSchema.getField(entry.getKey()) == null) {
+ return false;
+ }
}
+ return true;
+ }
+
+ private static Object unionToAvro(JsonNode node, Schema union) throws
HopValueException {
+ // Avro's JSON encoding: {"type name": value}
+ if (node != null && node.isObject() && node.size() == 1) {
+ String typeName = node.fieldNames().next();
+ for (Schema type : union.getTypes()) {
+ if (typeName.equals(type.getFullName()) ||
typeName.equals(type.getName())) {
+ return jsonToAvro(node.get(typeName), type);
+ }
+ }
+ }
+ // Plain JSON: the first type the value fits
+ for (Schema type : union.getTypes()) {
+ try {
+ return jsonToAvro(node, type);
+ } catch (HopValueException e) {
+ // Try the next type of the union
+ }
+ }
+ throw new HopValueException(
+ "Unable to read "
+ + (node == null ? "null" : node.toString())
+ + " as any type of Avro union "
+ + union);
}
/**
@@ -260,7 +482,12 @@ public class ValueMetaAvroRecord extends ValueMetaBase {
case IValueMeta.TYPE_AVRO:
string =
switch (storageType) {
- case STORAGE_TYPE_NORMAL -> object == null ? null :
object.toString();
+ case STORAGE_TYPE_NORMAL ->
+ object == null
+ ? null
+ : stringWithSchema
+ ? convertGenericRecordToString((GenericRecord)
object)
+ : object.toString();
default ->
throw new HopValueException(
this
@@ -293,6 +520,28 @@ public class ValueMetaAvroRecord extends ValueMetaBase {
}
}
+ /**
+ * An Avro record from another value: a record is copied. Text is read as
the schema and data JSON
+ * {@link #convertGenericRecordToString} writes, or as only the data when
this field has a schema.
+ */
+ @Override
+ public Object convertData(IValueMeta meta2, Object data2) throws
HopValueException {
+ if (data2 == null) {
+ return null;
+ }
+ if (meta2 != null && meta2.getType() == TYPE_AVRO) {
+ return cloneValueData(data2);
+ }
+ if (meta2 != null && meta2.isString()) {
+ String json = meta2.getString(data2);
+ if (StringUtils.isEmpty(json)) {
+ return null;
+ }
+ return convertStringToGenericRecord(json, schema);
+ }
+ throw new HopValueException(this + " : unable to convert " + meta2 + "
data to an Avro record");
+ }
+
@Override
public Object cloneValueData(Object object) throws HopValueException {
if (object == null) {
diff --git
a/core/src/test/java/org/apache/hop/core/row/value/ValueMetaAvroRecordTest.java
b/core/src/test/java/org/apache/hop/core/row/value/ValueMetaAvroRecordTest.java
index cc5618fca2..db6ebffb7f 100644
---
a/core/src/test/java/org/apache/hop/core/row/value/ValueMetaAvroRecordTest.java
+++
b/core/src/test/java/org/apache/hop/core/row/value/ValueMetaAvroRecordTest.java
@@ -21,6 +21,7 @@ package org.apache.hop.core.row.value;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import com.fasterxml.jackson.databind.JsonNode;
@@ -33,8 +34,11 @@ import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.util.Utf8;
+import org.apache.hop.core.Const;
import org.apache.hop.core.HopClientEnvironment;
+import org.apache.hop.core.exception.HopValueException;
import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.RowMeta;
import org.apache.hop.core.util.TestUtil;
import org.json.simple.JSONObject;
@@ -262,6 +266,200 @@ class ValueMetaAvroRecordTest {
assertEquals(5L, loaded[2]);
}
+ /** By default the text is only the data: a field with a schema reads it
back. */
+ @Test
+ void aRecordComesBackFromItsTextWithTheFieldsSchema() throws Exception {
+ GenericRecord genericRecord = generateGenericRecord();
+ ValueMetaAvroRecord valueMeta = new ValueMetaAvroRecord("test",
genericRecord.getSchema());
+ String text = valueMeta.getString(genericRecord);
+ assertEquals(genericRecord.toString(), text);
+
+ Object back = valueMeta.convertData(new ValueMetaString("text"), text);
+
+ assertEquals(genericRecord.getSchema(), ((GenericRecord)
back).getSchema());
+ verifyGenericRecords(genericRecord, (GenericRecord) back);
+ }
+
+ /** Only the data and no schema on the field: nothing to read it with. */
+ @Test
+ void textWithoutASchemaNeedsTheFieldsSchema() throws Exception {
+ GenericRecord genericRecord = generateGenericRecord();
+ String text = genericRecord.toString();
+
+ ValueMetaAvroRecord withoutSchema = new ValueMetaAvroRecord("test");
+ HopValueException thrown =
+ assertThrows(
+ HopValueException.class,
+ () -> withoutSchema.convertData(new ValueMetaString("text"),
text));
+ assertTrue(thrown.getMessage().contains("schema"), thrown.getMessage());
+ }
+
+ /** With HOP_AVRO_RECORD_STRING_WITH_SCHEMA=Y the text carries the schema:
no schema needed. */
+ @Test
+ void textWithItsSchemaComesBackWithoutOne() throws Exception {
+ GenericRecord genericRecord = generateGenericRecord();
+ String text;
+ System.setProperty(Const.HOP_AVRO_RECORD_STRING_WITH_SCHEMA, "Y");
+ try {
+ text = new ValueMetaAvroRecord("test",
genericRecord.getSchema()).getString(genericRecord);
+ } finally {
+ System.clearProperty(Const.HOP_AVRO_RECORD_STRING_WITH_SCHEMA);
+ }
+ assertTrue(text.startsWith("{ \"schema\" : "), text);
+
+ Object back = new ValueMetaAvroRecord("test").convertData(new
ValueMetaString("text"), text);
+
+ assertEquals(genericRecord.getSchema(), ((GenericRecord)
back).getSchema());
+ verifyGenericRecords(genericRecord, (GenericRecord) back);
+ }
+
+ /** The schema and data helpers round trip, also for a nullable (union)
field. */
+ @Test
+ void schemaAndDataTextRoundTrips() throws Exception {
+ GenericRecord genericRecord = generateGenericRecord();
+
+ GenericRecord back =
+ ValueMetaAvroRecord.convertStringToGenericRecord(
+ ValueMetaAvroRecord.convertGenericRecordToString(genericRecord));
+
+ verifyGenericRecords(genericRecord, back);
+ }
+
+ /** Schema and data text as earlier versions wrote it, the data as
toString() gives it. */
+ @Test
+ void schemaAndDataTextWithPlainDataIsRead() throws Exception {
+ GenericRecord genericRecord = generateGenericRecord();
+ String text =
+ "{ \"schema\" : " + genericRecord.getSchema() + ", \"data\" : " +
genericRecord + " }";
+
+ GenericRecord back =
ValueMetaAvroRecord.convertStringToGenericRecord(text);
+
+ verifyGenericRecords(genericRecord, back);
+ }
+
+ /**
+ * toString() doesn't say which record of a union a value is: the reader
must not take a record
+ * whose fields are all optional for an object with other keys.
+ */
+ @Test
+ void aUnionOfRecordsReadsBackAsTheRightRecord() throws Exception {
+ Schema schema =
+ new Schema.Parser()
+ .parse(
+
"{\"type\":\"record\",\"name\":\"Outer\",\"fields\":[{\"name\":\"two\",\"type\":["
+ +
"{\"type\":\"record\",\"name\":\"A\",\"fields\":[{\"name\":\"a\","
+ + "\"type\":[\"null\",\"string\"],\"default\":null}]},"
+ +
"{\"type\":\"record\",\"name\":\"B\",\"fields\":[{\"name\":\"b\","
+ + "\"type\":\"string\"}]}]}]}");
+ Schema recordB = schema.getField("two").schema().getTypes().get(1);
+ GenericRecord b = new GenericData.Record(recordB);
+ b.put("b", "bee");
+ GenericRecord outer = new GenericData.Record(schema);
+ outer.put("two", b);
+
+ GenericRecord back =
ValueMetaAvroRecord.convertStringToGenericRecord(outer.toString(), schema);
+
+ GenericRecord two = (GenericRecord) back.get("two");
+ assertEquals("B", two.getSchema().getName());
+ assertEquals("bee", two.get("b").toString());
+ }
+
+ /** A fixed value, as toString() writes it ([1, 2, 3, -6]) and as Avro's
JSON encoding does. */
+ @Test
+ void aFixedValueReadsBackFromBothTexts() throws Exception {
+ Schema schema =
+ new Schema.Parser()
+ .parse(
+
"{\"type\":\"record\",\"name\":\"R\",\"fields\":[{\"name\":\"hash\",\"type\":"
+ + "{\"type\":\"fixed\",\"name\":\"Hash\",\"size\":4}}]}");
+ GenericRecord original = new GenericData.Record(schema);
+ original.put(
+ "hash", new GenericData.Fixed(schema.getField("hash").schema(), new
byte[] {1, 2, 3, -6}));
+
+ GenericRecord fromPlain =
+ ValueMetaAvroRecord.convertStringToGenericRecord(original.toString(),
schema);
+ GenericRecord fromEncoded =
+ ValueMetaAvroRecord.convertStringToGenericRecord(
+ ValueMetaAvroRecord.convertGenericRecordToString(original));
+
+ assertEquals(original, fromPlain);
+ assertEquals(original, fromEncoded);
+ }
+
+ /** NaN and the infinities are written as text by toString() and by Avro's
JSON encoding. */
+ @Test
+ void nanAndInfinityReadBackFromBothTexts() throws Exception {
+ Schema schema =
+ new Schema.Parser()
+ .parse(
+ "{\"type\":\"record\",\"name\":\"R\",\"fields\":["
+ + "{\"name\":\"d\",\"type\":\"double\"},"
+ + "{\"name\":\"f\",\"type\":\"float\"},"
+ + "{\"name\":\"n\",\"type\":[\"null\",\"double\"]}]}");
+ GenericRecord original = new GenericData.Record(schema);
+ original.put("d", Double.NaN);
+ original.put("f", Float.NEGATIVE_INFINITY);
+ original.put("n", Double.POSITIVE_INFINITY);
+
+ GenericRecord fromPlain =
+ ValueMetaAvroRecord.convertStringToGenericRecord(original.toString(),
schema);
+ GenericRecord fromEncoded =
+ ValueMetaAvroRecord.convertStringToGenericRecord(
+ ValueMetaAvroRecord.convertGenericRecordToString(original));
+
+ for (GenericRecord back : new GenericRecord[] {fromPlain, fromEncoded}) {
+ assertTrue(Double.isNaN((Double) back.get("d")));
+ assertEquals(Float.NEGATIVE_INFINITY, back.get("f"));
+ assertEquals(Double.POSITIVE_INFINITY, back.get("n"));
+ }
+ }
+
+ /** Text file input, CSV input and the like read an Avro record field this
way. */
+ @Test
+ void aRecordIsReadFromText() throws Exception {
+ GenericRecord genericRecord = generateGenericRecord();
+ ValueMetaAvroRecord valueMeta = new ValueMetaAvroRecord("test",
genericRecord.getSchema());
+ String text = valueMeta.getString(genericRecord);
+
+ Object read =
+ valueMeta.convertDataFromString(
+ text, new ValueMetaString("text"), null, null,
IValueMeta.TRIM_TYPE_NONE);
+
+ verifyGenericRecords(genericRecord, (GenericRecord) read);
+ }
+
+ @Test
+ void aRecordIsCopied() throws Exception {
+ GenericRecord genericRecord = generateGenericRecord();
+ ValueMetaAvroRecord valueMeta = new ValueMetaAvroRecord("test",
genericRecord.getSchema());
+
+ Object copy =
+ valueMeta.convertData(
+ new ValueMetaAvroRecord("other", genericRecord.getSchema()),
genericRecord);
+
+ assertNotSame(genericRecord, copy);
+ verifyGenericRecords(genericRecord, (GenericRecord) copy);
+ }
+
+ @Test
+ void emptyTextIsNoRecord() throws Exception {
+ ValueMetaAvroRecord valueMeta = new ValueMetaAvroRecord("test");
+
+ assertNull(valueMeta.convertData(new ValueMetaString("text"), ""));
+ assertNull(valueMeta.convertData(new ValueMetaString("text"), null));
+ }
+
+ @Test
+ void textThatIsNoRecordOrAnotherTypeIsAnError() {
+ ValueMetaAvroRecord valueMeta = new ValueMetaAvroRecord("test");
+
+ assertThrows(
+ HopValueException.class,
+ () -> valueMeta.convertData(new ValueMetaString("text"), "not json"));
+ assertThrows(
+ HopValueException.class, () -> valueMeta.convertData(new
ValueMetaInteger("n"), 1L));
+ }
+
private GenericRecord generateGenericRecord() {
Schema schema = new Schema.Parser().parse(SCHEMA_JSON);
GenericRecord genericRecord = new GenericData.Record(schema);
diff --git a/docs/hop-user-manual/modules/ROOT/pages/variables.adoc
b/docs/hop-user-manual/modules/ROOT/pages/variables.adoc
index 15850ea4a1..88d3349409 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/variables.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/variables.adoc
@@ -245,6 +245,7 @@ Otherwise by default NULL is ignored by the MIN aggregate
and MIN is set to the
See also the variable HOP_AGGREGATION_ALL_NULLS_ARE_ZERO.
|HOP_ALLOW_EMPTY_FIELD_NAMES_AND_TYPES|N|Set this variable to Y to allow your
pipeline to pass 'null' fields and/or empty types.
|HOP_ALLOW_UNCONSUMED_MAIN_INPUT|N|Set this variable to Y to start a pipeline
that has main hops into transforms that do not consume input (for example
existing files whose upstream produces no rows). Default N fails init and stops
on leftover input. Verify still reports an error. See
xref:pipeline/pipeline-sources.adoc[Pipeline sources].
+|HOP_AVRO_RECORD_STRING_WITH_SCHEMA|N|Set this variable to Y to write an Avro
record as text with its schema and its data in Avro's JSON encoding: `{
"schema" : ..., "data" : ... }`. That text converts back to the record without
a schema, for example in xref:pipeline/transforms/selectvalues.adoc[Select
values]. With N only the data is written, as a record's `toString()` gives it;
that converts back only to a field that has a schema.
|HOP_BINARY_FIELDS_AVOID_HEX_PREVIEW|false|Set this variable to true to show
the content of binary fields as text, for example for CLOB database fields, in
preview windows, in the results of the
xref:hop-gui/perspective-database.adoc[Database perspective] and in the preview
of a Parquet file.
By default a binary value is shown as its hexadecimal representation.
Hop only reads this variable from the configuration perspective.
diff --git a/integration-tests/transforms/0131-avro-record-convert.hpl
b/integration-tests/transforms/0131-avro-record-convert.hpl
new file mode 100644
index 0000000000..8ff698c74b
--- /dev/null
+++ b/integration-tests/transforms/0131-avro-record-convert.hpl
@@ -0,0 +1,198 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0131-avro-record-convert</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Select values converts text holding an Avro record with its
schema to an Avro Record field and back to text (#9560)</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/18 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/18 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>text</from>
+ <to>text to record</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>text to record</from>
+ <to>record to text</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>record to text</from>
+ <to>validate</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>text</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>id</name>
+ <precision>-1</precision>
+ <type>Integer</type>
+ </field>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>text</name>
+ <precision>-1</precision>
+ <type>String</type>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>1</item>
+ <item>{ "schema" :
{"type":"record","name":"person","namespace":"hop.it","fields":[{"name":"id","type":"long"},{"name":"name","type":["null","string"]},{"name":"score","type":["null","double"]}]},
"data" : {"id":1,"name":{"string":"Ada"},"score":{"double":9.5}} }</item>
+ </line>
+ <line>
+ <item>2</item>
+ <item>{ "schema" :
{"type":"record","name":"person","namespace":"hop.it","fields":[{"name":"id","type":"long"},{"name":"name","type":["null","string"]},{"name":"score","type":["null","double"]}]},
"data" : {"id": 2, "name": "Grace", "score": null} }</item>
+ </line>
+ <line>
+ <item>3</item>
+ <item/>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>text to record</name>
+ <type>SelectValues</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <select_unspecified>N</select_unspecified>
+ <meta>
+ <name>text</name>
+ <rename>record</rename>
+ <type>Avro Record</type>
+ <length>-2</length>
+ <precision>-2</precision>
+ <conversion_mask/>
+ <date_format_lenient>false</date_format_lenient>
+ <date_format_locale/>
+ <date_format_timezone/>
+ <lenient_string_to_number>false</lenient_string_to_number>
+ <encoding/>
+ <decimal_symbol/>
+ <grouping_symbol/>
+ <currency_symbol/>
+ <storage_type/>
+ </meta>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>record to text</name>
+ <type>SelectValues</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <select_unspecified>N</select_unspecified>
+ <meta>
+ <name>record</name>
+ <rename>text_out</rename>
+ <type>String</type>
+ <length>-2</length>
+ <precision>-2</precision>
+ <conversion_mask/>
+ <date_format_lenient>false</date_format_lenient>
+ <date_format_locale/>
+ <date_format_timezone/>
+ <lenient_string_to_number>false</lenient_string_to_number>
+ <encoding/>
+ <decimal_symbol/>
+ <grouping_symbol/>
+ <currency_symbol/>
+ <storage_type/>
+ </meta>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>480</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>672</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/transforms/datasets/golden-0131-avro-record-convert.csv
b/integration-tests/transforms/datasets/golden-0131-avro-record-convert.csv
new file mode 100644
index 0000000000..7eca3f585e
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-0131-avro-record-convert.csv
@@ -0,0 +1,4 @@
+id,text_out
+1,"{""id"": 1, ""name"": ""Ada"", ""score"": 9.5}"
+2,"{""id"": 2, ""name"": ""Grace"", ""score"": null}"
+3,
diff --git a/integration-tests/transforms/main-0131-avro-record-convert.hwf
b/integration-tests/transforms/main-0131-avro-record-convert.hwf
new file mode 100644
index 0000000000..d26df2acfd
--- /dev/null
+++ b/integration-tests/transforms/main-0131-avro-record-convert.hwf
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0131-avro-record-convert</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/11/18 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/18 12:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>96</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0131-avro-record-convert UNIT</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0131-avro-record-convert UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>288</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>0131-avro-record-convert UNIT</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-0131-avro-record-convert.json
b/integration-tests/transforms/metadata/dataset/golden-0131-avro-record-convert.json
new file mode 100644
index 0000000000..3bdcb05bc5
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-0131-avro-record-convert.json
@@ -0,0 +1,24 @@
+{
+ "base_filename": "golden-0131-avro-record-convert.csv",
+ "name": "golden-0131-avro-record-convert",
+ "description": "Avro records read from schema and data text, written back as
their data; empty text stays empty",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": -1,
+ "field_name": "id",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "text_out",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
diff --git
a/integration-tests/transforms/metadata/unit-test/0131-avro-record-convert
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0131-avro-record-convert
UNIT.json
new file mode 100644
index 0000000000..2c019cd4c8
--- /dev/null
+++ b/integration-tests/transforms/metadata/unit-test/0131-avro-record-convert
UNIT.json
@@ -0,0 +1,32 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "${HOP_UNIT_TESTS_FOLDER}",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ },
+ {
+ "transform_field": "text_out",
+ "data_set_field": "text_out"
+ }
+ ],
+ "field_order": [
+ "id"
+ ],
+ "data_set_name": "golden-0131-avro-record-convert",
+ "transform_name": "validate"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0131-avro-record-convert UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0131-avro-record-convert.hpl"
+}