This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 0545dae22e Data set input performance improvements, fixes #9435 (#9436)
0545dae22e is described below
commit 0545dae22e7f08fd1e4f6ae00985b89ea04d3cd8
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Thu Nov 12 19:46:22 2026 +0100
Data set input performance improvements, fixes #9435 (#9436)
---
.../main/java/org/apache/hop/testing/DataSet.java | 10 ++
.../org/apache/hop/testing/DataSetCsvUtil.java | 53 ++++--
.../transforms/datasetinput/DataSetInput.java | 12 +-
.../hop/testing/DataSetCsvUtilReadRowsTest.java | 171 +++++++++++++++++++
.../transforms/datasetinput/DataSetInputTest.java | 183 +++++++++++++++++++++
5 files changed, 413 insertions(+), 16 deletions(-)
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSet.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSet.java
index 1feb07da99..bab3ea016e 100644
--- a/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSet.java
+++ b/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSet.java
@@ -199,6 +199,16 @@ public class DataSet extends HopMetadataBase implements
Cloneable, IHopMetadata
return DataSetCsvUtil.getAllRows(variables, this);
}
+ /**
+ * Reads the rows of this data set one at a time, without loading them all.
+ *
+ * @param handler receives each row in file order; return false to stop
reading
+ */
+ public void readRows(IVariables variables, DataSetCsvUtil.IRowHandler
handler)
+ throws HopException {
+ DataSetCsvUtil.readRows(variables, this, handler);
+ }
+
/**
* Calculate the row metadata for the data set fields needed for the given
location.
*
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvUtil.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvUtil.java
index c0e5e1c05a..4a14b98e56 100644
---
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvUtil.java
+++
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvUtil.java
@@ -81,20 +81,53 @@ public class DataSetCsvUtil {
}
}
+ /** Receives the rows of a data set one at a time, in file order. */
+ @FunctionalInterface
+ public interface IRowHandler {
+ /**
+ * @param row the next row, in the layout of {@link
DataSet#getSetRowMeta()}
+ * @return true to go on reading, false to stop
+ */
+ boolean handleRow(Object[] row) throws HopException;
+ }
+
public static final List<Object[]> getAllRows(IVariables variables, DataSet
dataSet)
throws HopException {
+ List<Object[]> rows = new ArrayList<>();
+ readRows(variables, dataSet, rows::add);
+ return rows;
+ }
+
+ /**
+ * Reads the rows of a data set and hands each one to the handler as soon as
it is parsed, so a
+ * large data set never has to fit in memory. A data set without a file has
no rows yet: the
+ * handler is not called.
+ *
+ * @param handler receives each row; return false to stop reading
+ */
+ public static final void readRows(IVariables variables, DataSet dataSet,
IRowHandler handler)
+ throws HopException {
IRowMeta setRowMeta = dataSet.getSetRowMeta();
setValueFormats(setRowMeta);
String dataSetFilename = dataSet.getActualDataSetFilename(variables);
- List<Object[]> rows = new ArrayList<>();
- final ValueMetaString constantValueMeta = new ValueMetaString("constant");
+
+ // One String meta per field, prepared once: it carries the field's mask
and symbols for the
+ // conversion. The field metadata keeps its number and date formats from
row to row.
+ int nrFields = setRowMeta.size();
+ IValueMeta[] valueMetas = new IValueMeta[nrFields];
+ IValueMeta[] stringMetas = new IValueMeta[nrFields];
+ for (int i = 0; i < nrFields; i++) {
+ valueMetas[i] = setRowMeta.getValueMeta(i);
+ stringMetas[i] = new ValueMetaString("constant");
+ stringMetas[i].setConversionMetadata(valueMetas[i]);
+ }
try {
FileObject file = HopVfs.getFileObject(dataSetFilename);
if (!file.exists()) {
// This is fine. We haven't put rows in yet.
//
- return rows;
+ return;
}
try (Reader reader =
@@ -102,20 +135,18 @@ public class DataSetCsvUtil {
CSVParser csvParser = new CSVParser(reader,
getCsvFormat(setRowMeta)); ) {
for (CSVRecord csvRecord : csvParser) {
if (csvRecord.getRecordNumber() > 1) {
- Object[] row = RowDataUtil.allocateRowData(setRowMeta.size());
- for (int i = 0; i < setRowMeta.size(); i++) {
- IValueMeta valueMeta = setRowMeta.getValueMeta(i).clone();
- constantValueMeta.setConversionMetadata(valueMeta);
+ Object[] row = RowDataUtil.allocateRowData(nrFields);
+ for (int i = 0; i < nrFields; i++) {
if (i < csvRecord.size()) {
- String value = csvRecord.get(i);
- row[i] = csvValueToField(valueMeta, constantValueMeta, value);
+ row[i] = csvValueToField(valueMetas[i], stringMetas[i],
csvRecord.get(i));
}
}
- rows.add(row);
+ if (!handler.handleRow(row)) {
+ return;
+ }
}
}
}
- return rows;
} catch (Exception e) {
throw new HopException(
"Unable to get all rows for CSV data set '" + dataSet.getName() +
"'", e);
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/transforms/datasetinput/DataSetInput.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/transforms/datasetinput/DataSetInput.java
index d033b9cc5f..1956194349 100644
---
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/transforms/datasetinput/DataSetInput.java
+++
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/transforms/datasetinput/DataSetInput.java
@@ -17,7 +17,6 @@
package org.apache.hop.testing.transforms.datasetinput;
-import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.row.RowMeta;
@@ -70,10 +69,13 @@ public class DataSetInput extends
BaseTransform<DataSetInputMeta, DataSetInputDa
data.outputRowMeta.addRowMeta(data.dataSet.getSetRowMeta());
}
- List<Object[]> rows = data.dataSet.getAllRows(this, getLogChannel());
- for (Object[] row : rows) {
- putRow(data.outputRowMeta, row);
- }
+ // Write every row as soon as it is read: the data set is never held in
memory as a whole.
+ data.dataSet.readRows(
+ this,
+ row -> {
+ putRow(data.outputRowMeta, row);
+ return !isStopped();
+ });
setOutputDone();
return false;
diff --git
a/plugins/misc/testing/src/test/java/org/apache/hop/testing/DataSetCsvUtilReadRowsTest.java
b/plugins/misc/testing/src/test/java/org/apache/hop/testing/DataSetCsvUtilReadRowsTest.java
new file mode 100644
index 0000000000..0d356e4ce1
--- /dev/null
+++
b/plugins/misc/testing/src/test/java/org/apache/hop/testing/DataSetCsvUtilReadRowsTest.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.testing;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.math.BigDecimal;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.variables.Variables;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/** Reading a data set one row at a time gives the rows that reading it whole
gives. */
+class DataSetCsvUtilReadRowsTest {
+
+ @TempDir Path tempDir;
+
+ private Variables variables;
+
+ @BeforeAll
+ static void initHop() throws HopException {
+ HopEnvironment.init();
+ }
+
+ @BeforeEach
+ void setUp() {
+ variables = new Variables();
+ variables.setVariable(DataSet.VARIABLE_HOP_DATASETS_FOLDER,
tempDir.toString());
+ }
+
+ @Test
+ void everyFieldTypeIsReadBackOverManyRows() throws Exception {
+ DataSet dataSet = dataSet("types");
+ Date first = new Date(1_700_000_000_000L);
+ Date second = new Date(1_700_000_123_456L);
+ Object[][] written = {
+ {1L, "Alice", 3.14, new BigDecimal("12345.678"), first, true, new byte[]
{1, 2, (byte) 255}},
+ {2L, "Bob", -0.5, new BigDecimal("-1"), second, false, new byte[] {0}},
+ {3L, null, null, null, null, null, null},
+ {4L, "Carol, \"quoted\"", 1e-20, new BigDecimal("0.000001"), first,
true, new byte[] {9}}
+ };
+ try (DataSetCsvWriter writer =
+ new DataSetCsvWriter(variables, dataSet, dataSet.getSetRowMeta())) {
+ for (Object[] row : written) {
+ writer.writeRow(row);
+ }
+ }
+
+ List<Object[]> streamed = new ArrayList<>();
+ DataSetCsvUtil.readRows(variables, dataSet, streamed::add);
+ List<Object[]> loaded = DataSetCsvUtil.getAllRows(variables, dataSet);
+
+ assertEquals(written.length, streamed.size());
+ assertEquals(written.length, loaded.size());
+ for (int r = 0; r < written.length; r++) {
+ for (int f = 0; f < written[r].length; f++) {
+ Object expected = written[r][f];
+ Object actual = streamed.get(r)[f];
+ if (expected instanceof byte[] bytes) {
+ assertArrayEquals(bytes, (byte[]) actual);
+ assertArrayEquals(bytes, (byte[]) loaded.get(r)[f]);
+ } else if (expected instanceof BigDecimal number) {
+ assertEquals(0, number.compareTo((BigDecimal) actual), "row " + r +
" field " + f);
+ assertEquals(actual, loaded.get(r)[f]);
+ } else {
+ if (expected == null && f == 1) {
+ // CSV has no null for text: an empty String field reads back as
an empty string
+ expected = "";
+ }
+ assertEquals(expected, actual, "row " + r + " field " + f);
+ assertEquals(actual, loaded.get(r)[f]);
+ }
+ }
+ }
+ }
+
+ @Test
+ void readingStopsWhenTheHandlerSaysSo() throws Exception {
+ DataSet dataSet = dataSet("stop");
+ try (DataSetCsvWriter writer =
+ new DataSetCsvWriter(variables, dataSet, dataSet.getSetRowMeta())) {
+ for (long i = 1; i <= 10; i++) {
+ writer.writeRow(new Object[] {i, "row " + i, null, null, null, null,
null});
+ }
+ }
+
+ List<Object[]> rows = new ArrayList<>();
+ DataSetCsvUtil.readRows(
+ variables,
+ dataSet,
+ row -> {
+ rows.add(row);
+ return rows.size() < 3;
+ });
+
+ assertEquals(3, rows.size());
+ assertEquals(3L, rows.get(2)[0]);
+ }
+
+ @Test
+ void aDataSetWithoutAFileHasNoRows() throws Exception {
+ DataSet dataSet = dataSet("missing");
+
+ List<Object[]> rows = new ArrayList<>();
+ DataSetCsvUtil.readRows(variables, dataSet, rows::add);
+
+ assertTrue(rows.isEmpty());
+ assertTrue(DataSetCsvUtil.getAllRows(variables, dataSet).isEmpty());
+ }
+
+ @Test
+ void aShortRecordLeavesTheMissingFieldsEmpty() throws Exception {
+ DataSet dataSet = dataSet("short");
+ Files.writeString(
+ tempDir.resolve("short.csv"),
+ "id,name,amount,total,created,active,payload\n7,Dave\n",
+ StandardCharsets.UTF_8);
+
+ List<Object[]> rows = DataSetCsvUtil.getAllRows(variables, dataSet);
+
+ assertEquals(1, rows.size());
+ assertEquals(7L, rows.get(0)[0]);
+ assertEquals("Dave", rows.get(0)[1]);
+ for (int f = 2; f < 7; f++) {
+ assertNull(rows.get(0)[f]);
+ }
+ }
+
+ private static DataSet dataSet(String name) {
+ DataSet dataSet = new DataSet();
+ dataSet.setName(name);
+ dataSet.setBaseFilename(name + ".csv");
+ dataSet.setFields(
+ List.of(
+ new DataSetField("id", IValueMeta.TYPE_INTEGER, -1, 0, "", "0"),
+ new DataSetField("name", IValueMeta.TYPE_STRING, -1, -1, "", ""),
+ new DataSetField("amount", IValueMeta.TYPE_NUMBER, -1, -1, "", ""),
+ new DataSetField("total", IValueMeta.TYPE_BIGNUMBER, -1, -1, "",
""),
+ new DataSetField("created", IValueMeta.TYPE_DATE, -1, -1, "", ""),
+ new DataSetField("active", IValueMeta.TYPE_BOOLEAN, -1, -1, "",
""),
+ new DataSetField("payload", IValueMeta.TYPE_BINARY, -1, -1, "",
"")));
+ return dataSet;
+ }
+}
diff --git
a/plugins/misc/testing/src/test/java/org/apache/hop/testing/transforms/datasetinput/DataSetInputTest.java
b/plugins/misc/testing/src/test/java/org/apache/hop/testing/transforms/datasetinput/DataSetInputTest.java
new file mode 100644
index 0000000000..1ca3d31f3b
--- /dev/null
+++
b/plugins/misc/testing/src/test/java/org/apache/hop/testing/transforms/datasetinput/DataSetInputTest.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.testing.transforms.datasetinput;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+import java.nio.file.Path;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.hop.core.BlockingRowSet;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.apache.hop.testing.DataSet;
+import org.apache.hop.testing.DataSetCsvWriter;
+import org.apache.hop.testing.DataSetField;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+class DataSetInputTest {
+
+ @TempDir Path tempDir;
+
+ private TransformMockHelper<DataSetInputMeta, DataSetInputData> mockHelper;
+ private MemoryMetadataProvider provider;
+
+ @BeforeAll
+ static void initHop() throws HopException {
+ HopEnvironment.init();
+ }
+
+ @BeforeEach
+ void setUp() {
+ mockHelper =
+ new TransformMockHelper<>("Data set input", DataSetInputMeta.class,
DataSetInputData.class);
+ when(mockHelper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+ .thenReturn(mockHelper.iLogChannel);
+ when(mockHelper.pipeline.isRunning()).thenReturn(true);
+ when(mockHelper.transformMeta.getCopies(any())).thenReturn(1);
+ when(mockHelper.transformMeta.isPartitioned()).thenReturn(false);
+ provider = new MemoryMetadataProvider();
+ }
+
+ @AfterEach
+ void tearDown() {
+ mockHelper.cleanUp();
+ }
+
+ @Test
+ void everyRowIsWrittenInFileOrder() throws Exception {
+ DataSet dataSet = save("orders");
+ write(dataSet, 5);
+
+ DataSetInput transform = createTransform("orders", false);
+ BlockingRowSet output = output(transform);
+ assertTrue(transform.init());
+ assertFalse(transform.processRow());
+
+ for (long i = 1; i <= 5; i++) {
+ Object[] row = output.getRowWait(1, TimeUnit.SECONDS);
+ assertEquals(i, row[0]);
+ assertEquals("row " + i, row[1]);
+ }
+ assertNull(output.getRowWait(100, TimeUnit.MILLISECONDS));
+ assertTrue(output.isDone());
+ }
+
+ @Test
+ void aDataSetWithoutAFileGivesNoRows() throws Exception {
+ save("empty");
+
+ DataSetInput transform = createTransform("empty", false);
+ BlockingRowSet output = output(transform);
+ assertTrue(transform.init());
+ assertFalse(transform.processRow());
+
+ assertNull(output.getRowWait(100, TimeUnit.MILLISECONDS));
+ assertTrue(output.isDone());
+ }
+
+ /** The rows are written while the file is read, so a stop ends the reading
too. */
+ @Test
+ void readingStopsWhenThePipelineIsStopped() throws Exception {
+ DataSet dataSet = save("stopped");
+ write(dataSet, 50);
+
+ DataSetInput transform = createTransform("stopped", true);
+ BlockingRowSet output = output(transform);
+ assertTrue(transform.init());
+ assertFalse(transform.processRow());
+
+ int rows = 0;
+ while (output.getRowWait(100, TimeUnit.MILLISECONDS) != null) {
+ rows++;
+ }
+ assertEquals(3, rows);
+ }
+
+ private DataSet save(String name) throws HopException {
+ DataSet dataSet = new DataSet();
+ dataSet.setName(name);
+ dataSet.setFolderName(tempDir.toString());
+ dataSet.setBaseFilename(name + ".csv");
+ dataSet.setFields(
+ List.of(
+ new DataSetField("id", IValueMeta.TYPE_INTEGER, -1, 0, "", "0"),
+ new DataSetField("name", IValueMeta.TYPE_STRING, -1, -1, "", "")));
+ provider.getSerializer(DataSet.class).save(dataSet);
+ return dataSet;
+ }
+
+ private void write(DataSet dataSet, int rows) throws Exception {
+ try (DataSetCsvWriter writer =
+ new DataSetCsvWriter(mockHelper.pipeline, dataSet,
dataSet.getSetRowMeta())) {
+ for (long i = 1; i <= rows; i++) {
+ writer.writeRow(new Object[] {i, "row " + i});
+ }
+ }
+ }
+
+ private static BlockingRowSet output(DataSetInput transform) {
+ BlockingRowSet rowSet = new BlockingRowSet(100);
+ rowSet.setThreadNameFromToCopy("Data set input", 0, "next", 0);
+ transform.addRowSetToOutputRowSets(rowSet);
+ return rowSet;
+ }
+
+ /**
+ * @param stopAfterThreeRows stop the transform once it has written its
third row
+ */
+ private DataSetInput createTransform(String dataSetName, boolean
stopAfterThreeRows) {
+ DataSetInputMeta meta = new DataSetInputMeta();
+ meta.setDataSetName(dataSetName);
+ when(mockHelper.transformMeta.getTransform()).thenReturn(meta);
+ DataSetInput transform =
+ new DataSetInput(
+ mockHelper.transformMeta,
+ meta,
+ new DataSetInputData(),
+ 0,
+ mockHelper.pipelineMeta,
+ mockHelper.pipeline) {
+ private int written;
+
+ @Override
+ public void putRow(IRowMeta rowMeta, Object[] row) throws
HopTransformException {
+ super.putRow(rowMeta, row);
+ if (stopAfterThreeRows && ++written == 3) {
+ setStopped(true);
+ }
+ }
+ };
+ transform.setMetadataProvider(provider);
+ return transform;
+ }
+}