This is an automated email from the ASF dual-hosted git repository.

bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git


The following commit(s) were added to refs/heads/main by this push:
     new 859a6274de Make the Hop GUI row samplers cheaper per row, fixes #9307 
(#9308)
859a6274de is described below

commit 859a6274de819f3d6dd847cf7f428a99c24c165f
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Fri Nov 6 07:23:14 2026 +0100

    Make the Hop GUI row samplers cheaper per row, fixes #9307 (#9308)
---
 .../hopgui/file/pipeline/HopGuiPipelineGraph.java  |   9 +-
 .../file/shared/PipelineRowSamplerHelper.java      | 248 +++++++++++++++------
 .../file/shared/PipelineRowSamplerHelperTest.java  | 134 +++++++++++
 3 files changed, 320 insertions(+), 71 deletions(-)

diff --git 
a/ui/src/main/java/org/apache/hop/ui/hopgui/file/pipeline/HopGuiPipelineGraph.java
 
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/pipeline/HopGuiPipelineGraph.java
index adeab15e1f..6080e9943f 100644
--- 
a/ui/src/main/java/org/apache/hop/ui/hopgui/file/pipeline/HopGuiPipelineGraph.java
+++ 
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/pipeline/HopGuiPipelineGraph.java
@@ -35,6 +35,7 @@ import java.util.Objects;
 import java.util.Set;
 import java.util.Timer;
 import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
 import lombok.Getter;
 import lombok.Setter;
 import org.apache.commons.lang3.StringUtils;
@@ -6575,8 +6576,8 @@ public class HopGuiPipelineGraph extends 
HopGuiAbstractGraph
 
   private void addRowsSamplerToPipeline(IPipelineEngine<PipelineMeta> 
pipeline) {
     try {
-      outputRowsMap = new HashMap<>();
-      outputHopRowsMap = new HashMap<>();
+      outputRowsMap = new ConcurrentHashMap<>();
+      outputHopRowsMap = new ConcurrentHashMap<>();
       PipelineRowSamplerHelper.addRowSamplersToPipeline(pipeline, 
outputRowsMap, outputHopRowsMap);
     } catch (Exception e) {
       // Ignore: run config not local or sample type not set
@@ -6592,8 +6593,8 @@ public class HopGuiPipelineGraph extends 
HopGuiAbstractGraph
     if (pipeline == null) {
       return;
     }
-    outputRowsMap = new HashMap<>();
-    outputHopRowsMap = new HashMap<>();
+    outputRowsMap = new ConcurrentHashMap<>();
+    outputHopRowsMap = new ConcurrentHashMap<>();
     PipelineRowSamplerHelper.addRowSamplersToPipeline(pipeline, outputRowsMap, 
outputHopRowsMap);
   }
 
diff --git 
a/ui/src/main/java/org/apache/hop/ui/hopgui/file/shared/PipelineRowSamplerHelper.java
 
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/shared/PipelineRowSamplerHelper.java
index f4412eedd7..ba9cb007a4 100644
--- 
a/ui/src/main/java/org/apache/hop/ui/hopgui/file/shared/PipelineRowSamplerHelper.java
+++ 
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/shared/PipelineRowSamplerHelper.java
@@ -17,13 +17,12 @@
 
 package org.apache.hop.ui.hopgui.file.shared;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.LinkedList;
+import java.util.AbstractList;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Map;
-import java.util.Random;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.ThreadLocalRandom;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hop.core.Const;
 import org.apache.hop.core.IRowSet;
@@ -48,6 +47,11 @@ import org.apache.hop.pipeline.transform.RowToAdapter;
  * <p>Supports both transform-level samples ({@code IRowListener} / {@code 
putRow}) and hop-level
  * samples ({@code IRowToListener} / {@code putRowTo}) for target hops such as 
Filter and
  * Switch/Case.
+ *
+ * <p>The listeners run for every row the sampled transform writes, so they do 
as little as possible
+ * per row: each listener resolves its buffer once, and a row is only cloned 
when it is kept. First
+ * stops cloning once the buffer is full, Random clones only the rows that 
land in the reservoir,
+ * and Last keeps a fixed ring of rows instead of a linked list.
  */
 public final class PipelineRowSamplerHelper {
 
@@ -135,11 +139,6 @@ public final class PipelineRowSamplerHelper {
     if (meta == null) {
       return;
     }
-    // Created once per pipeline start and shared by all the row listeners 
below
-    @SuppressWarnings("java:S2119")
-    final Random random = new Random();
-    // Per-key counters for Random sampling (transform name or hop key)
-    final Map<String, AtomicInteger> rowCounters = new ConcurrentHashMap<>();
     for (final String transformName : meta.getTransformNames()) {
       IEngineComponent component = pipeline.findComponent(transformName, 0);
       if (component == null) {
@@ -149,18 +148,18 @@ public final class PipelineRowSamplerHelper {
       // Transform-level samples (main putRow path)
       component.addRowListener(
           new RowAdapter() {
+            private RowSampler sampler;
+
             @Override
             public void rowWrittenEvent(IRowMeta rowMeta, Object[] row)
                 throws HopTransformException {
-              applySample(
-                  targetMap,
-                  transformName,
-                  rowMeta,
-                  row,
-                  sampleType,
-                  sampleSize,
-                  random,
-                  rowCounters);
+              RowSampler s = sampler;
+              if (s == null) {
+                s = createSampler(sampleType, sampleSize, rowMeta);
+                targetMap.put(transformName, s.getRowBuffer());
+                sampler = s;
+              }
+              s.sample(rowMeta, row);
             }
           });
 
@@ -168,72 +167,187 @@ public final class PipelineRowSamplerHelper {
       if (hopTargetMap != null) {
         component.addRowToListener(
             new RowToAdapter() {
+              // One sampler per target row set, so the hop key is built once 
per hop, not per row
+              private final Map<IRowSet, RowSampler> samplers = new 
ConcurrentHashMap<>();
+
               @Override
               public void rowWrittenTo(IRowMeta rowMeta, Object[] row, IRowSet 
rowSet)
                   throws HopTransformException {
-                String key = hopKey(rowSet);
-                if (StringUtils.isEmpty(key) || key.equals(HOP_KEY_SEPARATOR)) 
{
+                if (rowSet == null) {
                   return;
                 }
-                applySample(
-                    hopTargetMap, key, rowMeta, row, sampleType, sampleSize, 
random, rowCounters);
+                RowSampler s = samplers.get(rowSet);
+                if (s == null) {
+                  String key = hopKey(rowSet);
+                  if (StringUtils.isEmpty(key) || 
key.equals(HOP_KEY_SEPARATOR)) {
+                    return;
+                  }
+                  s = createSampler(sampleType, sampleSize, rowMeta);
+                  hopTargetMap.put(key, s.getRowBuffer());
+                  samplers.put(rowSet, s);
+                }
+                s.sample(rowMeta, row);
               }
             });
       }
     }
   }
 
-  /** Apply First/Last/Random sampling into the map. */
-  private static void applySample(
-      Map<String, RowBuffer> targetMap,
-      String key,
-      IRowMeta rowMeta,
-      Object[] row,
-      SampleType sampleType,
-      int sampleSize,
-      Random random,
-      Map<String, AtomicInteger> rowCounters)
-      throws HopTransformException {
-    RowBuffer rowBuffer = targetMap.get(key);
-    if (rowBuffer == null) {
-      rowBuffer = new RowBuffer(rowMeta);
-      if (sampleType == SampleType.Last) {
-        rowBuffer.setBuffer(Collections.synchronizedList(new LinkedList<>()));
-      } else {
-        rowBuffer.setBuffer(Collections.synchronizedList(new ArrayList<>()));
-      }
-      targetMap.put(key, rowBuffer);
-    }
+  /**
+   * Create the sampler for one transform or hop. Package-private for tests.
+   *
+   * @param sampleType First, Last or Random
+   * @param sampleSize the number of rows to keep
+   * @param rowMeta the layout of the sampled rows
+   * @return a new sampler with an empty buffer
+   */
+  static RowSampler createSampler(SampleType sampleType, int sampleSize, 
IRowMeta rowMeta) {
+    return switch (sampleType) {
+      case First -> new FirstRowsSampler(rowMeta, sampleSize);
+      case Last -> new LastRowsSampler(rowMeta, sampleSize);
+      case Random -> new RandomRowsSampler(rowMeta, sampleSize);
+      default -> throw new IllegalArgumentException("Unsupported sample type " 
+ sampleType);
+    };
+  }
+
+  private static Object[] copyRow(IRowMeta rowMeta, Object[] row) throws 
HopTransformException {
     try {
-      row = rowMeta.cloneRow(row);
+      return rowMeta.cloneRow(row);
     } catch (HopValueException e) {
       throw new HopTransformException("Error copying row for preview", e);
     }
-    switch (sampleType) {
-      case First:
-        if (rowBuffer.size() < sampleSize) {
-          rowBuffer.addRow(row);
+  }
+
+  /**
+   * Keeps a sample of the rows of one transform or hop in a {@link 
RowBuffer}. Readers lock the
+   * buffer's list ({@code synchronized (rowBuffer.getBuffer())}) to get a 
consistent view, so every
+   * change to the list happens under that same lock.
+   */
+  abstract static class RowSampler {
+    protected final RowBuffer rowBuffer;
+    protected final int sampleSize;
+
+    RowSampler(RowBuffer rowBuffer, int sampleSize) {
+      this.rowBuffer = rowBuffer;
+      this.sampleSize = sampleSize;
+    }
+
+    RowBuffer getRowBuffer() {
+      return rowBuffer;
+    }
+
+    abstract void sample(IRowMeta rowMeta, Object[] row) throws 
HopTransformException;
+  }
+
+  /** Keeps the first rows, then ignores the rest without cloning them. */
+  static final class FirstRowsSampler extends RowSampler {
+    private volatile boolean full;
+
+    FirstRowsSampler(IRowMeta rowMeta, int sampleSize) {
+      super(new RowBuffer(rowMeta), sampleSize);
+    }
+
+    @Override
+    void sample(IRowMeta rowMeta, Object[] row) throws HopTransformException {
+      if (full) {
+        return;
+      }
+      List<Object[]> rows = rowBuffer.getBuffer();
+      synchronized (rows) {
+        if (rows.size() < sampleSize) {
+          rows.add(copyRow(rowMeta, row));
         }
-        break;
-      case Last:
-        rowBuffer.addRow(0, row);
-        if (rowBuffer.size() > sampleSize) {
-          rowBuffer.removeRow(rowBuffer.size() - 1);
+        full = rows.size() >= sampleSize;
+      }
+    }
+  }
+
+  /** Keeps the last rows in a fixed ring, newest first. */
+  static final class LastRowsSampler extends RowSampler {
+    private final LastRowsList rows;
+
+    LastRowsSampler(IRowMeta rowMeta, int sampleSize) {
+      super(new RowBuffer(rowMeta), sampleSize);
+      rows = new LastRowsList(sampleSize);
+      rowBuffer.setBuffer(rows);
+    }
+
+    @Override
+    void sample(IRowMeta rowMeta, Object[] row) throws HopTransformException {
+      rows.addNewest(copyRow(rowMeta, row));
+    }
+  }
+
+  /**
+   * Keeps a uniform random sample (reservoir sampling). Only the rows that 
enter the reservoir are
+   * cloned.
+   */
+  static final class RandomRowsSampler extends RowSampler {
+    private long rowsSeen;
+
+    RandomRowsSampler(IRowMeta rowMeta, int sampleSize) {
+      super(new RowBuffer(rowMeta), sampleSize);
+    }
+
+    @Override
+    void sample(IRowMeta rowMeta, Object[] row) throws HopTransformException {
+      List<Object[]> rows = rowBuffer.getBuffer();
+      synchronized (rows) {
+        long seen = ++rowsSeen;
+        if (seen <= sampleSize) {
+          rows.add(copyRow(rowMeta, row));
+          return;
         }
-        break;
-      case Random:
-        int nrRows = rowCounters.computeIfAbsent(key, k -> new 
AtomicInteger(0)).incrementAndGet();
-        if (rowBuffer.size() < sampleSize) {
-          rowBuffer.addRow(row);
-        } else {
-          int randomIndex = random.nextInt(nrRows);
-          if (randomIndex < sampleSize) {
-            rowBuffer.setRow(randomIndex, row);
-          }
+        long index = ThreadLocalRandom.current().nextLong(seen);
+        if (index < sampleSize) {
+          rows.set((int) index, copyRow(rowMeta, row));
         }
-        break;
-      default:
-        break;
+      }
+    }
+  }
+
+  /**
+   * A fixed-size ring of rows exposed as a list, newest row at index 0. 
Adding a row once the ring
+   * is full replaces the oldest one, without allocating a node or shifting 
elements. All access
+   * locks the list itself, the same lock {@code Collections.synchronizedList} 
uses, so readers can
+   * keep doing {@code synchronized (rowBuffer.getBuffer())}.
+   */
+  static final class LastRowsList extends AbstractList<Object[]> {
+    private final Object[][] ring;
+    private int next;
+    private int size;
+
+    LastRowsList(int capacity) {
+      ring = new Object[capacity][];
+    }
+
+    synchronized void addNewest(Object[] row) {
+      ring[next] = row;
+      next = (next + 1) % ring.length;
+      if (size < ring.length) {
+        size++;
+      }
+    }
+
+    @Override
+    public synchronized Object[] get(int index) {
+      if (index < 0 || index >= size) {
+        throw new IndexOutOfBoundsException("Index " + index + ", size " + 
size);
+      }
+      return ring[Math.floorMod(next - 1 - index, ring.length)];
+    }
+
+    @Override
+    public synchronized int size() {
+      return size;
+    }
+
+    @Override
+    public synchronized void clear() {
+      Arrays.fill(ring, null);
+      next = 0;
+      size = 0;
+      modCount++;
     }
   }
 }
diff --git 
a/ui/src/test/java/org/apache/hop/ui/hopgui/file/shared/PipelineRowSamplerHelperTest.java
 
b/ui/src/test/java/org/apache/hop/ui/hopgui/file/shared/PipelineRowSamplerHelperTest.java
new file mode 100644
index 0000000000..47da41ca0e
--- /dev/null
+++ 
b/ui/src/test/java/org/apache/hop/ui/hopgui/file/shared/PipelineRowSamplerHelperTest.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.ui.hopgui.file.shared;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaDate;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import 
org.apache.hop.pipeline.engines.local.LocalPipelineRunConfiguration.SampleType;
+import 
org.apache.hop.ui.hopgui.file.shared.PipelineRowSamplerHelper.RowSampler;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class PipelineRowSamplerHelperTest {
+
+  private static final int SAMPLE_SIZE = 10;
+
+  private IRowMeta rowMeta;
+
+  @BeforeEach
+  void setUp() {
+    rowMeta = new RowMeta();
+    rowMeta.addValueMeta(new ValueMetaInteger("nr"));
+    rowMeta.addValueMeta(new ValueMetaDate("date"));
+  }
+
+  private static Object[] row(long nr) {
+    return new Object[] {nr, new Date(nr)};
+  }
+
+  private static List<Long> numbers(RowSampler sampler) {
+    List<Object[]> rows = sampler.getRowBuffer().getBuffer();
+    List<Long> numbers = new ArrayList<>();
+    synchronized (rows) {
+      for (Object[] r : rows) {
+        numbers.add((Long) r[0]);
+      }
+    }
+    return numbers;
+  }
+
+  private RowSampler feed(SampleType sampleType, int rowCount) throws 
Exception {
+    RowSampler sampler = PipelineRowSamplerHelper.createSampler(sampleType, 
SAMPLE_SIZE, rowMeta);
+    for (long nr = 1; nr <= rowCount; nr++) {
+      sampler.sample(rowMeta, row(nr));
+    }
+    return sampler;
+  }
+
+  @Test
+  void firstKeepsTheFirstRows() throws Exception {
+    assertEquals(List.of(1L, 2L, 3L), numbers(feed(SampleType.First, 3)));
+    assertEquals(
+        List.of(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L), 
numbers(feed(SampleType.First, 1000)));
+  }
+
+  @Test
+  void lastKeepsTheLastRowsNewestFirst() throws Exception {
+    assertEquals(List.of(3L, 2L, 1L), numbers(feed(SampleType.Last, 3)));
+    assertEquals(
+        List.of(1000L, 999L, 998L, 997L, 996L, 995L, 994L, 993L, 992L, 991L),
+        numbers(feed(SampleType.Last, 1000)));
+  }
+
+  @Test
+  void lastAtExactlyTheSampleSize() throws Exception {
+    assertEquals(
+        List.of(10L, 9L, 8L, 7L, 6L, 5L, 4L, 3L, 2L, 1L),
+        numbers(feed(SampleType.Last, SAMPLE_SIZE)));
+    assertEquals(
+        List.of(11L, 10L, 9L, 8L, 7L, 6L, 5L, 4L, 3L, 2L),
+        numbers(feed(SampleType.Last, SAMPLE_SIZE + 1)));
+  }
+
+  @Test
+  void randomKeepsAllRowsWhileThereAreFewerThanTheSampleSize() throws 
Exception {
+    assertEquals(List.of(1L, 2L, 3L), numbers(feed(SampleType.Random, 3)));
+  }
+
+  @Test
+  void randomKeepsSampleSizeDistinctInputRows() throws Exception {
+    List<Long> numbers = numbers(feed(SampleType.Random, 10_000));
+    assertEquals(SAMPLE_SIZE, numbers.size());
+    Set<Long> distinct = new HashSet<>(numbers);
+    assertEquals(SAMPLE_SIZE, distinct.size());
+    for (long nr : numbers) {
+      assertTrue(nr >= 1 && nr <= 10_000, "row " + nr + " was never written");
+    }
+    // With 10,000 rows, a sample made only of the first rows means the 
reservoir never replaced
+    assertTrue(numbers.stream().anyMatch(nr -> nr > SAMPLE_SIZE));
+  }
+
+  @Test
+  void keptRowsAreCopies() throws Exception {
+    for (SampleType sampleType : List.of(SampleType.First, SampleType.Last, 
SampleType.Random)) {
+      RowSampler sampler = PipelineRowSamplerHelper.createSampler(sampleType, 
SAMPLE_SIZE, rowMeta);
+      Object[] written = row(42);
+      sampler.sample(rowMeta, written);
+
+      // A later transform may replace values in the array it received, as 
Value Mapper does
+      written[0] = 0L;
+      written[1] = new Date(0L);
+
+      Object[] kept = sampler.getRowBuffer().getBuffer().get(0);
+      assertNotSame(written, kept, sampleType.name());
+      assertEquals(42L, kept[0], sampleType.name());
+      assertEquals(42L, ((Date) kept[1]).getTime(), sampleType.name());
+    }
+  }
+}

Reply via email to