This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 39e5457189 Issue #2358 : Run Avro file input and output on Beam (#9210)
39e5457189 is described below
commit 39e5457189778d1960b48041da8f7c65c362dd06
Author: Matt Casters <[email protected]>
AuthorDate: Mon Nov 2 16:15:45 2026 +0100
Issue #2358 : Run Avro file input and output on Beam (#9210)
Write one Avro file with AvroIO and read each named file back onto the
incoming row. Copy number, partition number, nested paths, and enums
are refused while the graph is built.
---
.../pages/pipeline/transforms/avro-file-input.adoc | 4 +
.../pipeline/transforms/avro-file-output.adoc | 4 +
plugins/engines/beam/pom.xml | 6 +
.../apache/hop/beam/core/fn/AvroFieldMapping.java | 45 +++
.../apache/hop/beam/core/fn/HopRowToAvroFn.java | 113 ++++++++
.../apache/hop/beam/core/fn/ReadAvroFileFn.java | 113 ++++++++
.../HopPipelineMetaToBeamPipelineConverter.java | 10 +-
.../handler/BeamAvroInputTransformHandler.java | 125 +++++++++
.../handler/BeamAvroOutputTransformHandler.java | 290 +++++++++++++++++++
.../java/org/apache/hop/beam/util/BeamConst.java | 2 +
.../beam/src/main/resources/dependencies.xml | 1 +
.../beam/pipeline/handler/BeamAvroFileIOTest.java | 307 +++++++++++++++++++++
.../hop/avro/transforms/avrooutput/AvroOutput.java | 22 +-
13 files changed, 1036 insertions(+), 6 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-file-input.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-file-input.adoc
index 9b5dd36859..582fcdc8e5 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-file-input.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-file-input.adoc
@@ -58,3 +58,7 @@ Note: This name has to be unique in a single pipeline.
|===
+== Beam
+
+On a Beam runner each file is read by one worker, and the records are appended
to the row that named the file. A positive row limit is the most records that
worker emits across the files it reads. A null or empty filename is skipped.
The native Spark engine stays excluded.
+
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-file-output.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-file-output.adoc
index 898cc6ff95..3e4d3504bd 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-file-output.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/avro-file-output.adoc
@@ -126,3 +126,7 @@ AVRO_TYPE_INT = 4 +
AVRO_TYPE_LONG = 5 +
AVRO_TYPE_STRING = 6 +
AVRO_TYPE_ENUM = 7
+
+== Beam
+
+Binary file output writes one Avro file with `AvroIO`. The filename, including
the date and time, is resolved when the pipeline is submitted. A transform copy
number or a partition number in the filename is refused. Nested Avro paths and
enum fields are refused. Binary-field and JSON-field output still run as an
ordinary Beam transform and add a column to the row.
diff --git a/plugins/engines/beam/pom.xml b/plugins/engines/beam/pom.xml
index a33b418fe3..8151ec8b5a 100644
--- a/plugins/engines/beam/pom.xml
+++ b/plugins/engines/beam/pom.xml
@@ -1686,6 +1686,12 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-tech-avro</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.apache.hop</groupId>
<artifactId>hop-transform-addsequence</artifactId>
diff --git
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/core/fn/AvroFieldMapping.java
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/core/fn/AvroFieldMapping.java
new file mode 100644
index 0000000000..c2a62fef38
--- /dev/null
+++
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/core/fn/AvroFieldMapping.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.beam.core.fn;
+
+import java.io.Serializable;
+
+/** One flat Avro output field. The type codes match {@code AvroOutputField}.
*/
+public class AvroFieldMapping implements Serializable {
+ private final String hopName;
+ private final String avroName;
+ private final int avroType;
+
+ public AvroFieldMapping(String hopName, String avroName, int avroType) {
+ this.hopName = hopName;
+ this.avroName = avroName;
+ this.avroType = avroType;
+ }
+
+ public String getHopName() {
+ return hopName;
+ }
+
+ public String getAvroName() {
+ return avroName;
+ }
+
+ public int getAvroType() {
+ return avroType;
+ }
+}
diff --git
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/core/fn/HopRowToAvroFn.java
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/core/fn/HopRowToAvroFn.java
new file mode 100644
index 0000000000..d6afed4ac3
--- /dev/null
+++
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/core/fn/HopRowToAvroFn.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.beam.core.fn;
+
+import java.util.List;
+import lombok.RequiredArgsConstructor;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.hop.beam.core.BeamHop;
+import org.apache.hop.beam.core.HopRow;
+import org.apache.hop.core.exception.HopRuntimeException;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.JsonRowMeta;
+import org.apache.hop.pipeline.Pipeline;
+
+/** Maps one Hop row onto a flat Avro record. Nested paths are rejected before
this runs. */
+@RequiredArgsConstructor
+public class HopRowToAvroFn extends DoFn<HopRow, GenericRecord> {
+ static final int TYPE_BOOLEAN = 1;
+ static final int TYPE_DOUBLE = 2;
+ static final int TYPE_FLOAT = 3;
+ static final int TYPE_INT = 4;
+ static final int TYPE_LONG = 5;
+ static final int TYPE_STRING = 6;
+
+ private final String transformName;
+ private final String rowMetaJson;
+ private final String schemaJson;
+ private final List<AvroFieldMapping> fields;
+
+ private transient Schema schema;
+ private transient IRowMeta rowMeta;
+ private transient int[] indexes;
+ private transient Counter readCounter;
+ private transient Counter outputCounter;
+ private transient Counter errorCounter;
+
+ @Setup
+ public void setup() {
+ readCounter = Metrics.counter(Pipeline.METRIC_NAME_READ, transformName);
+ outputCounter = Metrics.counter(Pipeline.METRIC_NAME_OUTPUT,
transformName);
+ errorCounter = Metrics.counter(Pipeline.METRIC_NAME_ERROR, transformName);
+ try {
+ BeamHop.init();
+ schema = new Schema.Parser().parse(schemaJson);
+ rowMeta = JsonRowMeta.fromJson(rowMetaJson);
+ indexes = new int[fields.size()];
+ for (int i = 0; i < fields.size(); i++) {
+ indexes[i] = rowMeta.indexOfValue(fields.get(i).getHopName());
+ if (indexes[i] < 0) {
+ throw new HopRuntimeException(
+ "Field [" + fields.get(i).getHopName() + "] couldn't be found in
the input stream");
+ }
+ }
+ Metrics.counter(Pipeline.METRIC_NAME_INIT, transformName).inc();
+ } catch (Exception e) {
+ errorCounter.inc();
+ throw new HopRuntimeException("Unable to initialize Avro output " +
transformName, e);
+ }
+ }
+
+ @ProcessElement
+ public void process(@Element HopRow row, OutputReceiver<GenericRecord>
output) {
+ readCounter.inc();
+ try {
+ GenericRecord record = new GenericData.Record(schema);
+ Object[] values = row.getRow();
+ for (int i = 0; i < fields.size(); i++) {
+ AvroFieldMapping field = fields.get(i);
+ record.put(field.getAvroName(), value(values, indexes[i],
field.getAvroType()));
+ }
+ output.output(record);
+ outputCounter.inc();
+ } catch (Exception e) {
+ errorCounter.inc();
+ throw new HopRuntimeException("Unable to convert a row for Avro output "
+ transformName, e);
+ }
+ }
+
+ private Object value(Object[] values, int index, int avroType) throws
Exception {
+ if (rowMeta.isNull(values, index)) {
+ return null;
+ }
+ return switch (avroType) {
+ case TYPE_INT -> rowMeta.getInteger(values, index).intValue();
+ case TYPE_LONG -> rowMeta.getInteger(values, index);
+ case TYPE_FLOAT -> rowMeta.getNumber(values, index).floatValue();
+ case TYPE_DOUBLE -> rowMeta.getNumber(values, index);
+ case TYPE_BOOLEAN -> rowMeta.getBoolean(values, index);
+ case TYPE_STRING -> rowMeta.getString(values, index);
+ default -> throw new HopRuntimeException("Unsupported Avro type " +
avroType);
+ };
+ }
+}
diff --git
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/core/fn/ReadAvroFileFn.java
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/core/fn/ReadAvroFileFn.java
new file mode 100644
index 0000000000..6aaa04d97a
--- /dev/null
+++
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/core/fn/ReadAvroFileFn.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.beam.core.fn;
+
+import java.io.InputStream;
+import java.nio.channels.Channels;
+import java.util.Arrays;
+import lombok.RequiredArgsConstructor;
+import org.apache.avro.file.DataFileStream;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.beam.sdk.io.FileSystems;
+import org.apache.beam.sdk.io.fs.MatchResult;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.hop.beam.core.BeamHop;
+import org.apache.hop.beam.core.HopRow;
+import org.apache.hop.core.exception.HopRuntimeException;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.JsonRowMeta;
+import org.apache.hop.pipeline.Pipeline;
+
+/**
+ * Reads the Avro data file named by one incoming field and appends each
record to that row.
+ *
+ * <p>{@code AvroIO.readFiles} returns records without the path, so it cannot
put the record back on
+ * the row that named the file. This reads the file with the same {@code
DataFileStream} AvroIO
+ * uses, on the worker that holds the row. A positive row limit is the most
records this worker
+ * emits, across every file it sees.
+ */
+@RequiredArgsConstructor
+public class ReadAvroFileFn extends DoFn<HopRow, HopRow> {
+ private final String transformName;
+ private final String rowMetaJson;
+ private final String filenameField;
+ private final int rowsLimit;
+
+ private transient IRowMeta rowMeta;
+ private transient int filenameIndex;
+ private transient int inputSize;
+ private transient int emitted;
+ private transient Counter readCounter;
+ private transient Counter outputCounter;
+ private transient Counter errorCounter;
+
+ @Setup
+ public void setup() {
+ readCounter = Metrics.counter(Pipeline.METRIC_NAME_READ, transformName);
+ outputCounter = Metrics.counter(Pipeline.METRIC_NAME_OUTPUT,
transformName);
+ errorCounter = Metrics.counter(Pipeline.METRIC_NAME_ERROR, transformName);
+ try {
+ BeamHop.init();
+ rowMeta = JsonRowMeta.fromJson(rowMetaJson);
+ filenameIndex = rowMeta.indexOfValue(filenameField);
+ if (filenameIndex < 0) {
+ throw new HopRuntimeException("Filename field '" + filenameField + "'
was not found");
+ }
+ inputSize = rowMeta.size();
+ Metrics.counter(Pipeline.METRIC_NAME_INIT, transformName).inc();
+ } catch (Exception e) {
+ errorCounter.inc();
+ throw new HopRuntimeException("Unable to initialize Avro file input " +
transformName, e);
+ }
+ }
+
+ @ProcessElement
+ public void process(@Element HopRow row, OutputReceiver<HopRow> output) {
+ if (rowsLimit > 0 && emitted >= rowsLimit) {
+ return;
+ }
+ try {
+ String filename = rowMeta.getString(row.getRow(), filenameIndex);
+ if (filename == null || filename.isEmpty()) {
+ return;
+ }
+ MatchResult.Metadata metadata =
FileSystems.matchSingleFileSpec(filename);
+ try (InputStream in =
Channels.newInputStream(FileSystems.open(metadata.resourceId()));
+ DataFileStream<GenericRecord> stream =
+ new DataFileStream<>(in, new GenericDatumReader<>())) {
+ while (stream.hasNext() && (rowsLimit <= 0 || emitted < rowsLimit)) {
+ GenericRecord record = stream.next();
+ Object[] source = row.getRow();
+ Object[] copy = Arrays.copyOf(source, inputSize + 1);
+ copy[inputSize] = record;
+ output.output(new HopRow(copy));
+ emitted++;
+ readCounter.inc();
+ outputCounter.inc();
+ }
+ }
+ } catch (Exception e) {
+ errorCounter.inc();
+ throw new HopRuntimeException(
+ "Unable to read an Avro file for transform " + transformName, e);
+ }
+ }
+}
diff --git
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/HopPipelineMetaToBeamPipelineConverter.java
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/HopPipelineMetaToBeamPipelineConverter.java
index b4c97c46b9..6f9cd84b05 100644
---
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/HopPipelineMetaToBeamPipelineConverter.java
+++
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/HopPipelineMetaToBeamPipelineConverter.java
@@ -43,6 +43,8 @@ import
org.apache.hop.beam.engines.IBeamPipelineEngineRunConfiguration;
import
org.apache.hop.beam.engines.dataflow.BeamDataFlowPipelineRunConfiguration;
import org.apache.hop.beam.metadata.RunnerType;
import org.apache.hop.beam.pipeline.handler.BeamAddSequenceTransformHandler;
+import org.apache.hop.beam.pipeline.handler.BeamAvroInputTransformHandler;
+import org.apache.hop.beam.pipeline.handler.BeamAvroOutputTransformHandler;
import org.apache.hop.beam.pipeline.handler.BeamGenericTransformHandler;
import org.apache.hop.beam.pipeline.handler.BeamMemoryGroupByTransformHandler;
import org.apache.hop.beam.pipeline.handler.BeamMergeJoinTransformHandler;
@@ -90,7 +92,9 @@ public class HopPipelineMetaToBeamPipelineConverter {
BeamConst.STRING_MERGE_JOIN_PLUGIN_ID,
BeamConst.STRING_BEAM_ROW_GENERATOR_PLUGIN_ID,
BeamConst.STRING_MEMORY_GROUP_BY_PLUGIN_ID,
- BeamConst.STRING_ADD_SEQUENCE_PLUGIN_ID);
+ BeamConst.STRING_ADD_SEQUENCE_PLUGIN_ID,
+ BeamConst.STRING_AVRO_FILE_INPUT_PLUGIN_ID,
+ BeamConst.STRING_AVRO_OUTPUT_PLUGIN_ID);
/**
* Transform meta classes that Beam refuses to run at all, mapped to the
user-facing reason. The
@@ -267,6 +271,10 @@ public class HopPipelineMetaToBeamPipelineConverter {
BeamConst.STRING_MEMORY_GROUP_BY_PLUGIN_ID, new
BeamMemoryGroupByTransformHandler());
transformHandlers.put(
BeamConst.STRING_ADD_SEQUENCE_PLUGIN_ID, new
BeamAddSequenceTransformHandler());
+ transformHandlers.put(
+ BeamConst.STRING_AVRO_FILE_INPUT_PLUGIN_ID, new
BeamAvroInputTransformHandler());
+ transformHandlers.put(
+ BeamConst.STRING_AVRO_OUTPUT_PLUGIN_ID, new
BeamAvroOutputTransformHandler());
genericTransformHandler = new BeamGenericTransformHandler();
}
diff --git
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/handler/BeamAvroInputTransformHandler.java
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/handler/BeamAvroInputTransformHandler.java
new file mode 100644
index 0000000000..eba72cafe5
--- /dev/null
+++
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/handler/BeamAvroInputTransformHandler.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.beam.pipeline.handler;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.hop.avro.transforms.avroinput.AvroFileInputMeta;
+import org.apache.hop.beam.core.HopRow;
+import org.apache.hop.beam.core.coder.HopRowCoder;
+import org.apache.hop.beam.core.fn.ReadAvroFileFn;
+import org.apache.hop.beam.engines.IBeamPipelineEngineRunConfiguration;
+import org.apache.hop.beam.pipeline.IBeamPipelineTransformHandler;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.JsonRowMeta;
+import org.apache.hop.core.util.Utils;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+
+/**
+ * Issue #2358: Avro file input on Beam.
+ *
+ * <p>Each incoming row names a file. The file is read on one worker and each
record is appended to
+ * that row as an Avro record field. The file keeps the schema stored in its
header.
+ */
+public class BeamAvroInputTransformHandler extends BeamBaseTransformHandler
+ implements IBeamPipelineTransformHandler {
+
+ @Override
+ public boolean isInput() {
+ return false;
+ }
+
+ @Override
+ public boolean isOutput() {
+ return false;
+ }
+
+ @Override
+ public void handleTransform(
+ ILogChannel log,
+ IVariables variables,
+ String runConfigurationName,
+ IBeamPipelineEngineRunConfiguration runConfiguration,
+ String dataSamplersJson,
+ IHopMetadataProvider metadataProvider,
+ PipelineMeta pipelineMeta,
+ TransformMeta transformMeta,
+ Map<String, PCollection<HopRow>> transformCollectionMap,
+ Pipeline pipeline,
+ IRowMeta rowMeta,
+ List<TransformMeta> previousTransforms,
+ PCollection<HopRow> input,
+ String parentLogChannelId)
+ throws HopException {
+
+ if (input == null || previousTransforms == null ||
previousTransforms.isEmpty()) {
+ throw new HopException(
+ "Avro file input '" + transformMeta.getName() + "' needs one
incoming transform");
+ }
+
+ AvroFileInputMeta meta = new AvroFileInputMeta();
+ loadTransformMetadata(meta, transformMeta, metadataProvider, pipelineMeta);
+
+ String filenameField = variables.resolve(meta.getDataFilenameField());
+ if (Utils.isEmpty(filenameField)) {
+ throw new HopException(
+ "Avro file input '" + transformMeta.getName() + "' has no filename
field");
+ }
+ if (rowMeta == null || rowMeta.indexOfValue(filenameField) < 0) {
+ throw new HopException(
+ "Avro file input '"
+ + transformMeta.getName()
+ + "' cannot find filename field '"
+ + filenameField
+ + "'");
+ }
+
+ int rowsLimit = -1;
+ if (!Utils.isEmpty(meta.getRowsLimit())) {
+ rowsLimit = Const.toIntExpanded(variables.resolve(meta.getRowsLimit()),
-1);
+ }
+
+ PCollection<HopRow> read =
+ input
+ .apply(
+ transformMeta.getName() + " Read Avro",
+ ParDo.of(
+ new ReadAvroFileFn(
+ transformMeta.getName(),
+ JsonRowMeta.toJson(rowMeta),
+ filenameField,
+ rowsLimit)))
+ .setCoder(new HopRowCoder());
+
+ transformCollectionMap.put(transformMeta.getName(), read);
+ log.logBasic(
+ "Handled transform (AVRO FILE INPUT) : "
+ + transformMeta.getName()
+ + ", filename field "
+ + filenameField);
+ }
+}
diff --git
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/handler/BeamAvroOutputTransformHandler.java
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/handler/BeamAvroOutputTransformHandler.java
new file mode 100644
index 0000000000..1d686eed4c
--- /dev/null
+++
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/handler/BeamAvroOutputTransformHandler.java
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.beam.pipeline.handler;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.avro.Schema;
+import org.apache.avro.file.CodecFactory;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.extensions.avro.coders.AvroCoder;
+import org.apache.beam.sdk.extensions.avro.io.AvroIO;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.hop.avro.transforms.avrooutput.AvroOutput;
+import org.apache.hop.avro.transforms.avrooutput.AvroOutputField;
+import org.apache.hop.avro.transforms.avrooutput.AvroOutputMeta;
+import org.apache.hop.beam.core.HopRow;
+import org.apache.hop.beam.core.fn.AvroFieldMapping;
+import org.apache.hop.beam.core.fn.HopRowToAvroFn;
+import org.apache.hop.beam.engines.IBeamPipelineEngineRunConfiguration;
+import org.apache.hop.beam.pipeline.IBeamPipelineTransformHandler;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.JsonRowMeta;
+import org.apache.hop.core.util.Utils;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+
+/**
+ * Issue #2358: binary-file Avro output on Beam.
+ *
+ * <p>One file, written with {@code AvroIO.writeGenericRecords} and {@code
withoutSharding()}. Date
+ * and time are resolved on the machine that submits the pipeline.
Binary-field and JSON-field
+ * output keep the generic handler, because those modes add a column to the
row instead of writing a
+ * file.
+ */
+public class BeamAvroOutputTransformHandler extends BeamBaseTransformHandler
+ implements IBeamPipelineTransformHandler {
+
+ @Override
+ public boolean isInput() {
+ return false;
+ }
+
+ @Override
+ public boolean isOutput() {
+ return false;
+ }
+
+ @Override
+ public void handleTransform(
+ ILogChannel log,
+ IVariables variables,
+ String runConfigurationName,
+ IBeamPipelineEngineRunConfiguration runConfiguration,
+ String dataSamplersJson,
+ IHopMetadataProvider metadataProvider,
+ PipelineMeta pipelineMeta,
+ TransformMeta transformMeta,
+ Map<String, PCollection<HopRow>> transformCollectionMap,
+ Pipeline pipeline,
+ IRowMeta rowMeta,
+ List<TransformMeta> previousTransforms,
+ PCollection<HopRow> input,
+ String parentLogChannelId)
+ throws HopException {
+
+ if (input == null || previousTransforms == null ||
previousTransforms.isEmpty()) {
+ throw new HopException(
+ "Avro output '" + transformMeta.getName() + "' needs one incoming
transform");
+ }
+
+ AvroOutputMeta meta = new AvroOutputMeta();
+ loadTransformMetadata(meta, transformMeta, metadataProvider, pipelineMeta);
+
+ if
(!AvroOutputMeta.OUTPUT_TYPES[AvroOutputMeta.OUTPUT_TYPE_BINARY_FILE].equals(
+ meta.getOutputType())) {
+ new BeamGenericTransformHandler()
+ .handleTransform(
+ log,
+ variables,
+ runConfigurationName,
+ runConfiguration,
+ dataSamplersJson,
+ metadataProvider,
+ pipelineMeta,
+ transformMeta,
+ transformCollectionMap,
+ pipeline,
+ rowMeta,
+ previousTransforms,
+ input,
+ parentLogChannelId);
+ return;
+ }
+
+ if (meta.isTransformNrInFilename() || meta.isPartNrInFilename()) {
+ throw new HopException(
+ "Avro output '"
+ + transformMeta.getName()
+ + "' cannot put a transform copy or a partition number in the
filename on Beam."
+ + " The Beam write is one file for the whole collection.");
+ }
+ if (rowMeta == null) {
+ throw new HopException(
+ "Avro output '" + transformMeta.getName() + "' has no incoming row
layout");
+ }
+
+ String filename = meta.buildFilename(meta.getFileName(), variables, 0,
null, false, null, 0);
+ if (Utils.isEmpty(filename)) {
+ throw new HopException("Avro output '" + transformMeta.getName() + "'
has no filename");
+ }
+
+ List<AvroOutputField> fields = copyFields(meta);
+ if (fields.isEmpty()) {
+ throw new HopException("Avro output '" + transformMeta.getName() + "'
has no output fields");
+ }
+ List<AvroFieldMapping> mappings = mappings(transformMeta.getName(),
fields, rowMeta);
+
+ Schema schema;
+ if (meta.isCreateSchemaFile()) {
+ schema =
+ AvroOutput.createAvroSchema(
+ new ArrayList<>(fields),
+ "",
+ meta.getDoc(),
+ meta.getRecordName(),
+ meta.getNamespace());
+ if (meta.isWriteSchemaFile()) {
+ writeSchemaFile(meta, variables, schema);
+ }
+ } else {
+ schema = readSchemaFile(meta, variables);
+ }
+
+ createParentIfRequested(meta, filename);
+
+ PCollection<GenericRecord> records =
+ input
+ .apply(
+ transformMeta.getName() + " To Avro",
+ ParDo.of(
+ new HopRowToAvroFn(
+ transformMeta.getName(),
+ JsonRowMeta.toJson(rowMeta),
+ schema.toString(),
+ mappings)))
+ .setCoder(AvroCoder.of(schema));
+
+ AvroIO.Write<GenericRecord> write =
+ AvroIO.writeGenericRecords(schema).to(filename).withoutSharding();
+ String compression = meta.getCompressionType();
+ if (!Utils.isEmpty(compression) && !"none".equalsIgnoreCase(compression)) {
+ write = write.withCodec(CodecFactory.fromString(compression));
+ }
+ records.apply(transformMeta.getName() + " Write", write);
+
+ // Binary-file output passes the incoming row on, the same way the local
transform does.
+ transformCollectionMap.put(transformMeta.getName(), input);
+ log.logBasic(
+ "Handled transform (AVRO OUTPUT) : " + transformMeta.getName() + ",
file " + filename);
+ }
+
+ private static List<AvroOutputField> copyFields(AvroOutputMeta meta) {
+ List<AvroOutputField> fields = new ArrayList<>();
+ if (meta.getOutputFields() == null) {
+ return fields;
+ }
+ for (AvroOutputField field : meta.getOutputFields()) {
+ if (field != null) {
+ fields.add(new AvroOutputField(field));
+ }
+ }
+ return fields;
+ }
+
+ private static List<AvroFieldMapping> mappings(
+ String transformName, List<AvroOutputField> fields, IRowMeta rowMeta)
throws HopException {
+ List<AvroFieldMapping> mappings = new ArrayList<>();
+ for (AvroOutputField field : fields) {
+ String avroName = field.getAvroName();
+ if (avroName.startsWith("$.")) {
+ avroName = avroName.substring(2);
+ }
+ if (avroName.contains(".")) {
+ throw new HopException(
+ "Avro output '"
+ + transformName
+ + "' cannot write nested path '"
+ + avroName
+ + "' on Beam");
+ }
+ if (field.getAvroType() == AvroOutputField.AVRO_TYPE_ENUM) {
+ throw new HopException(
+ "Avro output '"
+ + transformName
+ + "' cannot write enum field '"
+ + avroName
+ + "' on Beam");
+ }
+ if (rowMeta.indexOfValue(field.getName()) < 0) {
+ throw new HopException(
+ "Field [" + field.getName() + "] couldn't be found in the input
stream!");
+ }
+ mappings.add(new AvroFieldMapping(field.getName(), avroName,
field.getAvroType()));
+ }
+ return mappings;
+ }
+
+ private static void writeSchemaFile(AvroOutputMeta meta, IVariables
variables, Schema schema)
+ throws HopException {
+ if (Utils.isEmpty(meta.getSchemaFileName())) {
+ throw new HopException(
+ "Avro output is set to write a schema file, but no schema filename
is set");
+ }
+ String schemaFilename =
+ meta.buildFilename(meta.getSchemaFileName(), variables, 0, null,
false, null, 0);
+ createParentIfRequested(meta, schemaFilename);
+ try (OutputStream out = HopVfs.getOutputStream(schemaFilename, false)) {
+ out.write(schema.toString(true).getBytes(StandardCharsets.UTF_8));
+ } catch (Exception e) {
+ throw new HopException("Unable to write Avro schema file " +
schemaFilename, e);
+ }
+ }
+
+ private static Schema readSchemaFile(AvroOutputMeta meta, IVariables
variables)
+ throws HopException {
+ if (Utils.isEmpty(meta.getSchemaFileName())) {
+ throw new HopException("Avro output has no schema file and is not
creating one");
+ }
+ String schemaFilename = variables.resolve(meta.getSchemaFileName());
+ try (InputStream in = HopVfs.getInputStream(schemaFilename, variables)) {
+ return new Schema.Parser().parse(in);
+ } catch (Exception e) {
+ throw new HopException("Unable to read Avro schema file " +
schemaFilename, e);
+ }
+ }
+
+ private static void createParentIfRequested(AvroOutputMeta meta, String
filename)
+ throws HopException {
+ FileObject parent = null;
+ try {
+ parent = HopVfs.getFileObject(filename).getParent();
+ if (parent != null && !parent.exists()) {
+ if (meta.isCreateParentFolder()) {
+ parent.createFolder();
+ } else {
+ throw new HopException("Parent folder does not exist for " +
filename);
+ }
+ }
+ } catch (HopException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new HopException("Unable to prepare the parent folder for " +
filename, e);
+ } finally {
+ if (parent != null) {
+ try {
+ parent.close();
+ } catch (Exception ignored) {
+ // closing the parent is best-effort
+ }
+ }
+ }
+ }
+}
diff --git
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/util/BeamConst.java
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/util/BeamConst.java
index 00bde8bfe3..dc33957a9d 100644
--- a/plugins/engines/beam/src/main/java/org/apache/hop/beam/util/BeamConst.java
+++ b/plugins/engines/beam/src/main/java/org/apache/hop/beam/util/BeamConst.java
@@ -45,6 +45,8 @@ public class BeamConst {
public static final String STRING_BEAM_KAFKA_PRODUCE_PLUGIN_ID =
"BeamKafkaProduce";
public static final String STRING_BEAM_ROW_GENERATOR_PLUGIN_ID =
"RowGenerator";
public static final String STRING_ADD_SEQUENCE_PLUGIN_ID = "Sequence";
+ public static final String STRING_AVRO_FILE_INPUT_PLUGIN_ID =
"AvroFileInput";
+ public static final String STRING_AVRO_OUTPUT_PLUGIN_ID = "AvroOutput";
/** Mask fields. Banned by plugin id so the Beam engine does not have to
load the transform. */
public static final String STRING_MASK_FIELDS_PLUGIN_ID = "MaskFields";
diff --git a/plugins/engines/beam/src/main/resources/dependencies.xml
b/plugins/engines/beam/src/main/resources/dependencies.xml
index 75b046b4f4..5a382399bc 100644
--- a/plugins/engines/beam/src/main/resources/dependencies.xml
+++ b/plugins/engines/beam/src/main/resources/dependencies.xml
@@ -18,6 +18,7 @@
<dependencies>
<folder>../../transforms/addsequence</folder>
+ <folder>../../tech/avro</folder>
<folder>../../transforms/constant</folder>
<folder>../../transforms/filterrows</folder>
<folder>../../transforms/groupby</folder>
diff --git
a/plugins/engines/beam/src/test/java/org/apache/hop/beam/pipeline/handler/BeamAvroFileIOTest.java
b/plugins/engines/beam/src/test/java/org/apache/hop/beam/pipeline/handler/BeamAvroFileIOTest.java
new file mode 100644
index 0000000000..98bbc945ab
--- /dev/null
+++
b/plugins/engines/beam/src/test/java/org/apache/hop/beam/pipeline/handler/BeamAvroFileIOTest.java
@@ -0,0 +1,307 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.beam.pipeline.handler;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import org.apache.avro.Schema;
+import org.apache.avro.file.DataFileStream;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.hop.avro.transforms.avroinput.AvroFileInputMeta;
+import org.apache.hop.avro.transforms.avrooutput.AvroOutput;
+import org.apache.hop.avro.transforms.avrooutput.AvroOutputField;
+import org.apache.hop.avro.transforms.avrooutput.AvroOutputMeta;
+import org.apache.hop.beam.core.HopRow;
+import org.apache.hop.beam.core.coder.HopRowCoder;
+import org.apache.hop.beam.pipeline.HopPipelineMetaToBeamPipelineConverter;
+import org.apache.hop.beam.util.BeamConst;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/** Issue #2358: Avro file input and output on the Beam Direct runner. */
+class BeamAvroFileIOTest {
+
+ private static final List<String> SEEN = Collections.synchronizedList(new
ArrayList<>());
+
+ @TempDir Path tempDir;
+
+ @BeforeAll
+ static void initHopEnvironment() throws Exception {
+ HopEnvironment.init();
+ PluginRegistry.init();
+ }
+
+ @Test
+ void staticSchemaUsesTheRecordNameAndNullableFields() throws Exception {
+ AvroOutputMeta meta = fileMeta(tempDir.resolve("schema.avro"));
+ Schema schema =
+ AvroOutput.createAvroSchema(
+ new ArrayList<>(meta.getOutputFields()),
+ "",
+ meta.getDoc(),
+ meta.getRecordName(),
+ meta.getNamespace());
+
+ assertEquals("row", schema.getName());
+ assertEquals("org.apache.hop.test", schema.getNamespace());
+ assertNotNull(schema.getField("name"));
+ assertNotNull(schema.getField("id"));
+ assertEquals(Schema.Type.UNION,
schema.getField("name").schema().getType());
+ assertEquals(Schema.Type.STRING,
schema.getField("name").schema().getTypes().get(1).getType());
+ }
+
+ @Test
+ void avroPluginIdsAreExplicitBeamHandlers() {
+ assertTrue(
+
HopPipelineMetaToBeamPipelineConverter.EXPLICIT_HANDLER_PLUGIN_IDS.contains(
+ BeamConst.STRING_AVRO_OUTPUT_PLUGIN_ID));
+ assertTrue(
+
HopPipelineMetaToBeamPipelineConverter.EXPLICIT_HANDLER_PLUGIN_IDS.contains(
+ BeamConst.STRING_AVRO_FILE_INPUT_PLUGIN_ID));
+ }
+
+ @Test
+ void writeThenReadKeepsTheIncomingRow() throws Exception {
+ Path file = tempDir.resolve("rows.avro");
+ writeTwoRows(file);
+
+ try (DataFileStream<GenericRecord> in =
+ new DataFileStream<>(Files.newInputStream(file), new
GenericDatumReader<>())) {
+ GenericRecord first = in.next();
+ assertEquals("alice", first.get("name").toString());
+ assertEquals(1L, first.get("id"));
+ GenericRecord second = in.next();
+ assertEquals("bob", second.get("name").toString());
+ assertEquals(2L, second.get("id"));
+ assertFalse(in.hasNext());
+ }
+
+ assertEquals(List.of("keep-me|alice|1", "keep-me|bob|2"), readBack(file,
null));
+ }
+
+ @Test
+ void rowLimitCapsWhatOneWorkerEmits() throws Exception {
+ Path file = tempDir.resolve("limited.avro");
+ writeTwoRows(file);
+ assertEquals(List.of("keep-me|alice|1"), readBack(file, "1"));
+ }
+
+ @Test
+ void missingFilenameFieldFailsWhileTheGraphIsBuilt() {
+ AvroFileInputMeta meta = new AvroFileInputMeta();
+ meta.setDataFilenameField("missing");
+ meta.setOutputFieldName("avro");
+ HopException missing =
+ assertThrows(HopException.class, () -> readGraph(meta, stringRow(),
"x"));
+ assertTrue(missing.getMessage().contains("missing"));
+ }
+
+ @Test
+ void copyNumberInTheFilenameIsRefused() throws Exception {
+ AvroOutputMeta meta = fileMeta(tempDir.resolve("refused.avro"));
+ meta.setTransformNrInFilename(true);
+ HopException refused =
+ assertThrows(
+ HopException.class,
+ () ->
+ new BeamAvroOutputTransformHandler()
+ .handleTransform(
+ LogChannel.GENERAL,
+ new Variables(),
+ "direct",
+ null,
+ null,
+ new MemoryMetadataProvider(),
+ new PipelineMeta(),
+ new TransformMeta("AvroOutput", "write", meta),
+ new HashMap<>(),
+ Pipeline.create(),
+ valueRow(),
+ List.of(upstream()),
+ twoRows(Pipeline.create()),
+ null));
+ assertTrue(refused.getMessage().contains("partition"));
+ }
+
+ private void writeTwoRows(Path file) throws Exception {
+ AvroOutputMeta meta = fileMeta(file);
+ Pipeline pipeline = Pipeline.create();
+ new BeamAvroOutputTransformHandler()
+ .handleTransform(
+ LogChannel.GENERAL,
+ new Variables(),
+ "direct",
+ null,
+ null,
+ new MemoryMetadataProvider(),
+ new PipelineMeta(),
+ new TransformMeta("AvroOutput", "write", meta),
+ new HashMap<>(),
+ pipeline,
+ valueRow(),
+ List.of(upstream()),
+ twoRows(pipeline),
+ null);
+ pipeline.run().waitUntilFinish();
+ }
+
+ private static List<String> readBack(Path file, String rowsLimit) throws
Exception {
+ SEEN.clear();
+ AvroFileInputMeta meta = new AvroFileInputMeta();
+ meta.setDataFilenameField("file");
+ meta.setOutputFieldName("avro");
+ meta.setRowsLimit(rowsLimit);
+ Pipeline pipeline = Pipeline.create();
+ PCollection<HopRow> rows =
+ pipeline.apply(
+ Create.of(new HopRow(new Object[]
{file.toAbsolutePath().toString(), "keep-me"}))
+ .withCoder(new HopRowCoder()));
+ HashMap<String, PCollection<HopRow>> map = new HashMap<>();
+ new BeamAvroInputTransformHandler()
+ .handleTransform(
+ LogChannel.GENERAL,
+ new Variables(),
+ "direct",
+ null,
+ null,
+ new MemoryMetadataProvider(),
+ new PipelineMeta(),
+ new TransformMeta("AvroFileInput", "read", meta),
+ map,
+ pipeline,
+ fileRow(),
+ List.of(upstream()),
+ rows,
+ null);
+ map.get("read").apply(ParDo.of(new CollectRecord()));
+ pipeline.run().waitUntilFinish();
+ return new ArrayList<>(SEEN);
+ }
+
+ private static void readGraph(AvroFileInputMeta meta, IRowMeta rowMeta,
String value)
+ throws Exception {
+ Pipeline pipeline = Pipeline.create();
+ PCollection<HopRow> rows =
+ pipeline.apply(Create.of(new HopRow(new Object[]
{value})).withCoder(new HopRowCoder()));
+ new BeamAvroInputTransformHandler()
+ .handleTransform(
+ LogChannel.GENERAL,
+ new Variables(),
+ "direct",
+ null,
+ null,
+ new MemoryMetadataProvider(),
+ new PipelineMeta(),
+ new TransformMeta("AvroFileInput", "read", meta),
+ new HashMap<>(),
+ pipeline,
+ rowMeta,
+ List.of(upstream()),
+ rows,
+ null);
+ }
+
+ /** DirectRunner serializes this function. Results go to {@link #SEEN}. */
+ static class CollectRecord extends DoFn<HopRow, Void> {
+ @ProcessElement
+ public void process(@Element HopRow row) {
+ Object[] values = row.getRow();
+ GenericRecord record = (GenericRecord) values[2];
+ SEEN.add(values[1] + "|" + record.get("name") + "|" + record.get("id"));
+ }
+ }
+
+ private static AvroOutputMeta fileMeta(Path file) {
+ AvroOutputMeta meta = new AvroOutputMeta();
+ meta.setDefault();
+ meta.setFileName(file.toAbsolutePath().toString());
+
meta.setOutputType(AvroOutputMeta.OUTPUT_TYPES[AvroOutputMeta.OUTPUT_TYPE_BINARY_FILE]);
+ meta.setCreateSchemaFile(true);
+ meta.setWriteSchemaFile(false);
+ meta.setRecordName("row");
+ meta.setNamespace("org.apache.hop.test");
+ meta.setDoc("test");
+ meta.setCompressionType("none");
+ meta.setOutputFields(
+ List.of(
+ new AvroOutputField("name", "name",
AvroOutputField.AVRO_TYPE_STRING, true),
+ new AvroOutputField("id", "id", AvroOutputField.AVRO_TYPE_LONG,
true)));
+ return meta;
+ }
+
+ private static TransformMeta upstream() {
+ TransformMeta upstream = new TransformMeta();
+ upstream.setName("upstream");
+ return upstream;
+ }
+
+ private static PCollection<HopRow> twoRows(Pipeline pipeline) {
+ return pipeline.apply(
+ Create.of(new HopRow(new Object[] {"alice", 1L}), new HopRow(new
Object[] {"bob", 2L}))
+ .withCoder(new HopRowCoder()));
+ }
+
+ private static IRowMeta valueRow() {
+ RowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("name"));
+ rowMeta.addValueMeta(new ValueMetaInteger("id"));
+ return rowMeta;
+ }
+
+ private static IRowMeta fileRow() {
+ RowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("file"));
+ rowMeta.addValueMeta(new ValueMetaString("tag"));
+ return rowMeta;
+ }
+
+ private static IRowMeta stringRow() {
+ RowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("file"));
+ return rowMeta;
+ }
+}
diff --git
a/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avrooutput/AvroOutput.java
b/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avrooutput/AvroOutput.java
index 2d8ffcdf20..2420d6ff6c 100644
---
a/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avrooutput/AvroOutput.java
+++
b/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avrooutput/AvroOutput.java
@@ -134,10 +134,22 @@ public class AvroOutput extends
BaseTransform<AvroOutputMeta, AvroOutputData> {
public Schema createAvroSchema(List<AvroOutputField> avroFields, String
parentPath)
throws HopException {
- // Get standard schema stuff
- String doc = meta.getDoc();
- String recordName = meta.getRecordName();
- String namespace = meta.getNamespace();
+ return createAvroSchema(
+ avroFields, parentPath, meta.getDoc(), meta.getRecordName(),
meta.getNamespace());
+ }
+
+ /**
+ * Builds the Avro record for these fields. Nested paths are expressed with
dots in the Avro name.
+ * The field list is mutated: nested siblings are removed while their parent
record is built, so
+ * callers pass a copy.
+ */
+ public static Schema createAvroSchema(
+ List<AvroOutputField> avroFields,
+ String parentPath,
+ String doc,
+ String recordName,
+ String namespace)
+ throws HopException {
// do not want to have to deal with $. paths.
if (parentPath.startsWith("$.")) {
@@ -211,7 +223,7 @@ public class AvroOutput extends
BaseTransform<AvroOutputMeta, AvroOutputData> {
}
subPath = subPath.substring(0, subPath.length() - 1);
- Schema subSchema = createAvroSchema(subFields, subPath);
+ Schema subSchema = createAvroSchema(subFields, subPath, doc,
recordName, namespace);
Schema outSchema = subSchema;
if (nullable) {
Schema nullSchema = Schema.create(Schema.Type.NULL);