This is an automated email from the ASF dual-hosted git repository.

bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git


The following commit(s) were added to refs/heads/main by this push:
     new 1ad0ab702f Join rows: do not clone every cached row for every output 
row, fixes … (#9480)
1ad0ab702f is described below

commit 1ad0ab702f82619e3a8da7134406958685307f76
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Tue Nov 17 09:30:49 2026 +0100

    Join rows: do not clone every cached row for every output row, fixes … 
(#9480)
    
    * Join rows: do not clone every cached row for every output row, fixes #9478
    
    * fix remarks
---
 .../ROOT/pages/pipeline/transforms/joinrows.adoc   |   3 +
 .../hop/pipeline/transforms/joinrows/JoinRows.java |  29 ++-
 .../pipeline/transforms/joinrows/JoinRowsData.java |   6 +
 .../joinrows/JoinRowsCachedValuesTest.java         | 276 +++++++++++++++++++++
 4 files changed, 309 insertions(+), 5 deletions(-)

diff --git 
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/joinrows.adoc 
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/joinrows.adoc
index 0effd1ef39..9ee998b25b 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/joinrows.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/joinrows.adoc
@@ -47,6 +47,9 @@ To build a cartesian product on these engines, add the same 
constant field to bo
 See 
xref:pipeline/beam/getting-started-with-beam.adoc#_unsupported_transforms[Getting
 started with Beam] and 
xref:pipeline/pipeline-run-configurations/native-spark-pipeline-engine.adoc#transforms-not-supported-on-native-spark[Transforms
 not supported on Native Spark].
 ====
 
+NOTE: With several copies of this transform, set the transforms of the other 
streams to Copy rows.
+Otherwise each copy only receives a part of those streams and combinations go 
missing.
+
 NOTE: The transform gives the same result on the 
xref:pipeline/pipeline-run-configurations/single-threaded-pipeline-engine.adoc[single
 threaded engine] as on the Hop engine.
 That engine runs the previous transforms first, so all the rows of the other 
streams are there when Join Rows starts.
 
diff --git 
a/plugins/transforms/joinrows/src/main/java/org/apache/hop/pipeline/transforms/joinrows/JoinRows.java
 
b/plugins/transforms/joinrows/src/main/java/org/apache/hop/pipeline/transforms/joinrows/JoinRows.java
index 8db4e80edd..4416462010 100644
--- 
a/plugins/transforms/joinrows/src/main/java/org/apache/hop/pipeline/transforms/joinrows/JoinRows.java
+++ 
b/plugins/transforms/joinrows/src/main/java/org/apache/hop/pipeline/transforms/joinrows/JoinRows.java
@@ -35,6 +35,7 @@ import org.apache.hop.core.exception.HopFileException;
 import org.apache.hop.core.io.CountingInputStream;
 import org.apache.hop.core.io.CountingOutputStream;
 import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
 import org.apache.hop.core.row.RowMeta;
 import org.apache.hop.i18n.BaseMessages;
 import org.apache.hop.pipeline.Pipeline;
@@ -87,6 +88,7 @@ public class JoinRows extends BaseTransform<JoinRowsMeta, 
JoinRowsData> {
       data.joinrow = new Object[rowSetsSize][];
       data.rs = new IRowSet[rowSetsSize];
       data.cache = new List[rowSetsSize];
+      data.cloneCachedRow = new boolean[rowSetsSize];
       data.position = new int[rowSetsSize];
       data.fileOutputStream = new OutputStream[rowSetsSize];
       data.dataOutputStream = new DataOutputStream[rowSetsSize];
@@ -237,8 +239,12 @@ public class JoinRows extends BaseTransform<JoinRowsMeta, 
JoinRowsData> {
         rowData = data.cache[filenr].get(data.position[data.filenr]);
 
         // Don't forget to clone the data to protect it against data 
alteration downstream.
+        // Only a value that can be changed in place needs that: outputRow() 
copies the values
+        // into a new row, so the cached row itself never leaves this 
transform.
         //
-        rowData = data.fileRowMeta[filenr].cloneRow(rowData);
+        if (data.cloneCachedRow[filenr]) {
+          rowData = data.fileRowMeta[filenr].cloneRow(rowData);
+        }
 
         data.position[filenr]++;
 
@@ -315,10 +321,9 @@ public class JoinRows extends BaseTransform<JoinRowsMeta, 
JoinRowsData> {
       Object[] sum = new Object[data.outputRowMeta.size()];
       int sumIndex = 0;
       for (int f = 0; f <= data.filenr; f++) {
-        for (int c = 0; c < data.fileRowMeta[f].size(); c++) {
-          sum[sumIndex] = data.joinrow[f][c];
-          sumIndex++;
-        }
+        int size = data.fileRowMeta[f].size();
+        System.arraycopy(data.joinrow[f], 0, sum, sumIndex, size);
+        sumIndex += size;
       }
 
       if (meta.getCondition() != null && !meta.getCondition().isEmpty()) {
@@ -345,6 +350,19 @@ public class JoinRows extends BaseTransform<JoinRowsMeta, 
JoinRowsData> {
     return true;
   }
 
+  /**
+   * @return true when the row has a field whose values can be changed in 
place, so that a copy of
+   *     the row needs a copy of those values as well
+   */
+  private static boolean requiresRealClone(IRowMeta rowMeta) {
+    for (IValueMeta valueMeta : rowMeta.getValueMetaList()) {
+      if (valueMeta.requiresRealClone()) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   private boolean cacheInputRow() throws HopException {
     // /////////////////////////////
     // Read from input channels //
@@ -391,6 +409,7 @@ public class JoinRows extends BaseTransform<JoinRowsMeta, 
JoinRowsData> {
       if (data.fileRowMeta[data.filenr] == null) {
         // The first row is used as meta-data, clone it for safety
         data.fileRowMeta[data.filenr] = rowSet.getRowMeta().clone();
+        data.cloneCachedRow[data.filenr] = 
requiresRealClone(data.fileRowMeta[data.filenr]);
       }
 
       
data.fileRowMeta[data.filenr].writeData(data.dataOutputStream[data.filenr], 
rowData);
diff --git 
a/plugins/transforms/joinrows/src/main/java/org/apache/hop/pipeline/transforms/joinrows/JoinRowsData.java
 
b/plugins/transforms/joinrows/src/main/java/org/apache/hop/pipeline/transforms/joinrows/JoinRowsData.java
index ca374456e1..aaba09dc20 100644
--- 
a/plugins/transforms/joinrows/src/main/java/org/apache/hop/pipeline/transforms/joinrows/JoinRowsData.java
+++ 
b/plugins/transforms/joinrows/src/main/java/org/apache/hop/pipeline/transforms/joinrows/JoinRowsData.java
@@ -41,6 +41,12 @@ public class JoinRowsData extends BaseTransformData 
implements ITransformData {
   public IRowSet[] rs;
   public List<Object[]>[] cache;
 
+  /**
+   * Per stream: does it have fields whose values can be changed in place 
(Binary, Serializable)? A
+   * row from the cache of such a stream is cloned before it goes into an 
output row.
+   */
+  public boolean[] cloneCachedRow;
+
   public boolean caching;
 
   public OutputStream[] fileOutputStream;
diff --git 
a/plugins/transforms/joinrows/src/test/java/org/apache/hop/pipeline/transforms/joinrows/JoinRowsCachedValuesTest.java
 
b/plugins/transforms/joinrows/src/test/java/org/apache/hop/pipeline/transforms/joinrows/JoinRowsCachedValuesTest.java
new file mode 100644
index 0000000000..7c712311d1
--- /dev/null
+++ 
b/plugins/transforms/joinrows/src/test/java/org/apache/hop/pipeline/transforms/joinrows/JoinRowsCachedValuesTest.java
@@ -0,0 +1,276 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.joinrows;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.annotations.Transform;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.plugins.TransformPluginType;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaBinary;
+import org.apache.hop.core.row.value.ValueMetaDate;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.pipeline.Pipeline;
+import org.apache.hop.pipeline.PipelineHopMeta;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.RowProducer;
+import org.apache.hop.pipeline.SingleThreadedPipelineExecutor;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.pipeline.transform.ITransform;
+import org.apache.hop.pipeline.transform.ITransformMeta;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.dummy.DummyMeta;
+import org.apache.hop.pipeline.transforms.injector.InjectorMeta;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+/**
+ * The rows Join Rows keeps in memory are used for many output rows. A row of 
the last stream is
+ * read again for every output row, so whatever a transform downstream does to 
its values, the next
+ * output row has to come out with the values that went in.
+ *
+ * <p>Every input has a String, an Integer, a Date and a Binary field, with 
nulls in its second row.
+ * The transform that reads the output overwrites the bytes of the Binary 
value of the last stream,
+ * as a transform that changes a value in place would. A Serializable field is 
left out: Join Rows
+ * writes the rows of every stream but the main one to a temporary file, and a 
Serializable value
+ * can not be written there.
+ *
+ * <p>The values of the main stream, and of a stream between the main and the 
last one, are shared
+ * by the output rows they are part of. That is how it always was, and it is 
not changed here.
+ */
+class JoinRowsCachedValuesTest {
+
+  private static final Duration TIMEOUT = Duration.ofSeconds(20);
+  private static final String[] FIELDS = {"name", "nr", "day", "bytes"};
+
+  @BeforeAll
+  static void initHop() throws Exception {
+    HopEnvironment.init();
+    PluginRegistry registry = PluginRegistry.getInstance();
+    for (ITransformMeta meta : List.of(new JoinRowsMeta(), new InjectorMeta(), 
new DummyMeta())) {
+      if (registry.getPluginId(TransformPluginType.class, meta) == null) {
+        registry.registerPluginClass(
+            meta.getClass().getName(), TransformPluginType.class, 
Transform.class);
+      }
+    }
+  }
+
+  @Test
+  void rowsKeptInMemoryKeepTheirValues() {
+    List<Input> inputs = List.of(new Input("main", 4), new Input("other", 3));
+    Output output = assertTimeoutPreemptively(TIMEOUT, () -> runJoin(inputs, 
"main", 500));
+
+    assertEquals(product(inputs), output.rows);
+    assertEveryBinaryValueOfTheLastStreamIsItsOwnCopy(output);
+  }
+
+  @Test
+  void rowsKeptInMemoryKeepTheirValuesWithThreeInputs() {
+    // The main stream sits in the middle: the two other streams are the ones 
kept in memory.
+    List<Input> inputs = List.of(new Input("a", 3), new Input("main", 3), new 
Input("b", 2));
+    Output output = assertTimeoutPreemptively(TIMEOUT, () -> runJoin(inputs, 
"main", 500));
+
+    assertEquals(product(List.of(inputs.get(1), inputs.get(0), 
inputs.get(2))), output.rows);
+    assertEveryBinaryValueOfTheLastStreamIsItsOwnCopy(output);
+  }
+
+  @Test
+  void rowsReadBackFromTheTemporaryFileKeepTheirValues() {
+    // A cache size below the number of rows: the rows of the other stream are 
read from disk.
+    List<Input> inputs = List.of(new Input("main", 4), new Input("other", 5));
+    Output output = assertTimeoutPreemptively(TIMEOUT, () -> runJoin(inputs, 
"main", 2));
+
+    assertEquals(product(inputs), output.rows);
+    assertEveryBinaryValueOfTheLastStreamIsItsOwnCopy(output);
+  }
+
+  @Test
+  void oneStreamInMemoryAndOneOnDisk() {
+    // "a" fits in the cache, "b" does not.
+    List<Input> inputs = List.of(new Input("main", 2), new Input("a", 3), new 
Input("b", 4));
+    Output output = assertTimeoutPreemptively(TIMEOUT, () -> runJoin(inputs, 
"main", 3));
+
+    assertEquals(product(inputs), output.rows);
+    assertEveryBinaryValueOfTheLastStreamIsItsOwnCopy(output);
+  }
+
+  /** An input of Join Rows with the four fields, named after the input, and 
this many rows. */
+  private record Input(String name, int nrRows) {}
+
+  /** The output rows as text, and the Binary values of the last stream. */
+  private static final class Output {
+    final List<String> rows = new ArrayList<>();
+    final List<byte[]> lastStreamBinaryValues = new ArrayList<>();
+  }
+
+  private static Object[] row(Input input, int n) {
+    if (n == 2) {
+      // nulls in every field that could need a copy
+      return new Object[] {input.name() + n, (long) n, null, null};
+    }
+    return new Object[] {
+      input.name() + n,
+      (long) n,
+      new Date(1_700_000_000_000L + n * 86_400_000L),
+      (input.name() + "-bytes-" + n).getBytes()
+    };
+  }
+
+  private static String text(Object[] row, int offset) {
+    Object[] values = Arrays.copyOfRange(row, offset, offset + FIELDS.length);
+    if (values[3] != null) {
+      values[3] = new String((byte[]) values[3]);
+    }
+    if (values[2] != null) {
+      values[2] = ((Date) values[2]).getTime();
+    }
+    return Arrays.toString(values);
+  }
+
+  /** The complete cartesian product as text. The values of the last input 
vary fastest. */
+  private static List<String> product(List<Input> fieldOrder) {
+    List<String> expected = List.of("");
+    for (Input input : fieldOrder) {
+      List<String> combined = new ArrayList<>();
+      for (String prefix : expected) {
+        for (int n = 1; n <= input.nrRows(); n++) {
+          combined.add(prefix + text(row(input, n), 0));
+        }
+      }
+      expected = combined;
+    }
+    return expected;
+  }
+
+  /**
+   * No two output rows share the Binary value of the last stream: otherwise a 
change to one output
+   * row shows up in another.
+   */
+  private static void assertEveryBinaryValueOfTheLastStreamIsItsOwnCopy(Output 
output) {
+    Map<byte[], Boolean> seen = new IdentityHashMap<>();
+    for (byte[] value : output.lastStreamBinaryValues) {
+      assertNull(seen.put(value, Boolean.TRUE), "a Binary value is in two 
output rows");
+    }
+    assertTrue(seen.size() > 1, "there were Binary values to check");
+  }
+
+  private static Output runJoin(List<Input> inputs, String mainName, int 
cacheSize)
+      throws Exception {
+    PipelineMeta pipelineMeta = new PipelineMeta();
+    pipelineMeta.setName("join-rows-cached-values");
+
+    List<TransformMeta> inputTransforms = new ArrayList<>();
+    for (Input input : inputs) {
+      inputTransforms.add(addTransform(pipelineMeta, input.name(), new 
InjectorMeta()));
+    }
+
+    JoinRowsMeta joinRowsMeta = new JoinRowsMeta();
+    joinRowsMeta.setDefault();
+    joinRowsMeta.setCacheSize(cacheSize);
+    joinRowsMeta.setMainTransformName(mainName);
+    joinRowsMeta.setDirectory(System.getProperty("java.io.tmpdir"));
+    TransformMeta join = addTransform(pipelineMeta, "join", joinRowsMeta);
+    TransformMeta dummy = addTransform(pipelineMeta, "output", new 
DummyMeta());
+
+    for (TransformMeta inputTransform : inputTransforms) {
+      pipelineMeta.addPipelineHop(new PipelineHopMeta(inputTransform, join));
+    }
+    pipelineMeta.addPipelineHop(new PipelineHopMeta(join, dummy));
+    joinRowsMeta.searchInfoAndTargetTransforms(pipelineMeta.getTransforms());
+
+    Pipeline pipeline = new LocalPipelineEngine(pipelineMeta);
+    pipeline.setPipelineType(PipelineMeta.PipelineType.SingleThreaded);
+    pipeline.prepareExecution();
+
+    Map<String, RowProducer> producers = new HashMap<>();
+    for (Input input : inputs) {
+      producers.put(input.name(), pipeline.addRowProducer(input.name(), 0));
+    }
+
+    // The main stream comes first in the output row, the other streams follow 
in hop order.
+    int nrStreams = inputs.size();
+    Output output = new Output();
+    ITransform joinTransform = pipeline.getTransform("join", 0);
+    joinTransform.addRowListener(
+        new RowAdapter() {
+          @Override
+          public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) {
+            StringBuilder text = new StringBuilder();
+            for (int stream = 0; stream < nrStreams; stream++) {
+              text.append(text(row, stream * FIELDS.length));
+            }
+            output.rows.add(text.toString());
+
+            // Now do what a transform downstream may do: change a Binary 
value in place.
+            byte[] bytes = (byte[]) row[(nrStreams - 1) * FIELDS.length + 3];
+            if (bytes != null) {
+              output.lastStreamBinaryValues.add(bytes);
+              Arrays.fill(bytes, (byte) '#');
+            }
+          }
+        });
+
+    pipeline.startThreads();
+
+    for (Input input : inputs) {
+      IRowMeta rowMeta = new RowMeta();
+      rowMeta.addValueMeta(new ValueMetaString(input.name() + "_" + 
FIELDS[0]));
+      rowMeta.addValueMeta(new ValueMetaInteger(input.name() + "_" + 
FIELDS[1]));
+      rowMeta.addValueMeta(new ValueMetaDate(input.name() + "_" + FIELDS[2]));
+      rowMeta.addValueMeta(new ValueMetaBinary(input.name() + "_" + 
FIELDS[3]));
+      RowProducer producer = producers.get(input.name());
+      for (int n = 1; n <= input.nrRows(); n++) {
+        producer.putRow(rowMeta, row(input, n));
+      }
+    }
+
+    SingleThreadedPipelineExecutor executor = new 
SingleThreadedPipelineExecutor(pipeline);
+    assertTrue(executor.init());
+    try {
+      executor.oneIteration();
+    } finally {
+      executor.dispose();
+    }
+    assertEquals(0, joinTransform.getErrors());
+    return output;
+  }
+
+  private static TransformMeta addTransform(
+      PipelineMeta pipelineMeta, String name, ITransformMeta meta) {
+    String pluginId = 
PluginRegistry.getInstance().getPluginId(TransformPluginType.class, meta);
+    TransformMeta transformMeta = new TransformMeta(pluginId, name, meta);
+    pipelineMeta.addTransform(transformMeta);
+    return transformMeta;
+  }
+}

Reply via email to