This is an automated email from the ASF dual-hosted git repository.

bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git


The following commit(s) were added to refs/heads/main by this push:
     new 28fe41ac30 Token replacement is slow because it rereads the template 
for every row, fixes #9350 (#9352)
28fe41ac30 is described below

commit 28fe41ac30a12ce1409f272ef798afcaa4748872
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Fri Nov 6 21:10:05 2026 +0100

    Token replacement is slow because it rereads the template for every row, 
fixes #9350 (#9352)
---
 .../tokenreplacement/TokenReplacement.java         | 243 +++++++++++++-------
 .../tokenreplacement/TokenReplacementData.java     |  31 +++
 .../transforms/tokenreplacement/TokenTemplate.java | 122 ++++++++++
 .../tokenreplacement/TokenReplacementTest.java     | 251 +++++++++++++++++++++
 .../tokenreplacement/TokenTemplateTest.java        | 138 +++++++++++
 5 files changed, 701 insertions(+), 84 deletions(-)

diff --git 
a/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacement.java
 
b/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacement.java
index 74cb1a4512..26bdc49135 100644
--- 
a/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacement.java
+++ 
b/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacement.java
@@ -22,14 +22,14 @@ import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.io.Reader;
-import java.io.StringReader;
-import java.io.StringWriter;
-import java.io.Writer;
 import java.nio.ByteBuffer;
 import java.nio.CharBuffer;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
 import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.vfs2.FileObject;
 import org.apache.hop.core.Const;
@@ -39,6 +39,7 @@ import org.apache.hop.core.exception.HopValueException;
 import org.apache.hop.core.io.CountingInputStream;
 import org.apache.hop.core.io.CountingOutputStream;
 import org.apache.hop.core.row.RowDataUtil;
+import org.apache.hop.core.util.StringUtil;
 import org.apache.hop.core.util.Utils;
 import org.apache.hop.core.vfs.HopVfs;
 import org.apache.hop.i18n.BaseMessages;
@@ -127,48 +128,47 @@ public class TokenReplacement extends 
BaseTransform<TokenReplacementMeta, TokenR
       }
     }
 
-    // Create token resolver
-    TokenResolver resolver = new TokenResolver();
-
-    for (TokenReplacementField field : meta.getTokenReplacementFields()) {
-      if (data.inputRowMeta.indexOfValue(field.getName()) >= 0) {
-        String fieldValue = resolve(data.inputRowMeta.getString(r, 
field.getName(), null));
-        if (fieldValue == null
-            && !BooleanUtils.toBoolean(
-                
Const.getEnvironmentVariable("KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL", "N"))) {
-          fieldValue = Const.nullToEmpty(fieldValue);
-        }
-        resolver.addToken(field.getTokenName(), fieldValue);
-      } else {
-        throw new HopValueException("Field " + field.getName() + 
CONST_NOT_FOUND_ON_INPUT_STREAM);
-      }
+    // The value of every token field in this row, by its position in the 
transform's field list
+    if (data.tokenValuePositions == null) {
+      prepareTokens();
     }
+    String[] tokenValues = getTokenValues(r);
 
-    Reader reader;
+    // Text and field input: the template, split into fixed text and tokens.
+    // File input: streamed through a reader, so large files are never read 
into memory at once.
+    TokenTemplate template = null;
+    Reader reader = null;
     String inputFilename = "";
 
     if (meta.getInputType().equalsIgnoreCase("text")) {
-      reader =
-          new TokenReplacingReader(
-              resolver,
-              new StringReader(meta.getInputText()),
-              resolve(meta.getTokenStartString()),
-              resolve(meta.getTokenEndString()));
+      if (data.textTemplate == null) {
+        data.textTemplate =
+            TokenTemplate.parse(
+                Const.NVL(meta.getInputText(), ""),
+                data.tokenStartMarker,
+                data.tokenEndMarker,
+                data.tokenValuePositions);
+      }
+      template = data.textTemplate;
 
     } else if (meta.getInputType().equalsIgnoreCase(CONST_FIELD)) {
-      if (data.inputRowMeta.indexOfValue(meta.getInputFieldName()) >= 0) {
-        String inputString = data.inputRowMeta.getString(r, 
meta.getInputFieldName(), "");
-        reader =
-            new TokenReplacingReader(
-                resolver,
-                new StringReader(inputString),
-                resolve(meta.getTokenStartString()),
-                resolve(meta.getTokenEndString()));
-
-      } else {
-        throw new HopValueException(
-            "Input field " + meta.getInputFieldName() + 
CONST_NOT_FOUND_ON_INPUT_STREAM);
+      if (data.inputFieldIndex < 0) {
+        data.inputFieldIndex = 
data.inputRowMeta.indexOfValue(meta.getInputFieldName());
+        if (data.inputFieldIndex < 0) {
+          throw new HopValueException(
+              "Input field " + meta.getInputFieldName() + 
CONST_NOT_FOUND_ON_INPUT_STREAM);
+        }
+      }
+      String inputString = Const.NVL(data.inputRowMeta.getString(r, 
data.inputFieldIndex), "");
+      // Rows often share the same template: split it again only when it 
changes
+      if (data.fieldTemplate == null || 
!inputString.equals(data.fieldTemplateText)) {
+        data.fieldTemplate =
+            TokenTemplate.parse(
+                inputString, data.tokenStartMarker, data.tokenEndMarker, 
data.tokenValuePositions);
+        data.fieldTemplateText = inputString;
       }
+      template = data.fieldTemplate;
+
     } else if (meta.getInputType().equalsIgnoreCase("file")) {
       if (meta.isInputFileNameInField()) {
         if 
(data.inputRowMeta.indexOfValue(resolve(meta.getInputFileNameField())) >= 0) {
@@ -194,11 +194,11 @@ public class TokenReplacement extends 
BaseTransform<TokenReplacementMeta, TokenR
       String inputEncoding = Const.NVL(resolve(meta.getOutputFileEncoding()), 
Const.UTF_8);
       reader =
           new TokenReplacingReader(
-              resolver,
+              getTokenResolver(tokenValues),
               new InputStreamReader(
                   data.currentCountingInputStream, 
Charset.forName(inputEncoding)),
-              resolve(meta.getTokenStartString()),
-              resolve(meta.getTokenEndString()));
+              data.tokenStartMarker,
+              data.tokenEndMarker);
 
       if (meta.isAddInputFileNameToResult()) {
         ResultFile resultFile =
@@ -214,12 +214,9 @@ public class TokenReplacement extends 
BaseTransform<TokenReplacementMeta, TokenR
       throw new HopException("Unsupported input type " + meta.getInputType());
     }
 
-    Writer stringWriter = null;
     OutputStream bufferedWriter = null;
 
-    if (meta.getOutputType().equalsIgnoreCase(CONST_FIELD)) {
-      stringWriter = new StringWriter(5000);
-    } else {
+    if (!meta.getOutputType().equalsIgnoreCase(CONST_FIELD)) {
       if (meta.getOutputType().equalsIgnoreCase("file")) {
 
         if (inputFilename.equals(outputFilename)) {
@@ -241,58 +238,55 @@ public class TokenReplacement extends 
BaseTransform<TokenReplacementMeta, TokenR
     }
 
     String output = "";
+    boolean toField = meta.getOutputType().equalsIgnoreCase(CONST_FIELD);
+    StringBuilder text = data.outputText;
+    text.setLength(0);
 
     try {
-      char[] cbuf = new char[5000];
-      int length = 0;
-      while ((length = reader.read(cbuf)) > 0) {
-        if (meta.getOutputType().equalsIgnoreCase(CONST_FIELD)) {
-          stringWriter.write(cbuf, 0, length);
-        } else if (meta.getOutputType().equalsIgnoreCase("file")) {
-          CharBuffer cBuffer = CharBuffer.wrap(cbuf, 0, length);
-          ByteBuffer bBuffer = 
Charset.forName(meta.getOutputFileEncoding()).encode(cBuffer);
-          byte[] bytes = new byte[bBuffer.limit()];
-          bBuffer.get(bytes);
-          bufferedWriter.write(bytes);
-        } // No else.  Anything else will be thrown to a Hop exception prior 
to getting here.
-        cbuf = new char[5000];
+      if (template != null) {
+        template.appendTo(tokenValues, text);
+        if (!toField) {
+          writeText(bufferedWriter, text);
+        }
+      } else {
+        int length;
+        while ((length = reader.read(data.readBuffer)) > 0) {
+          if (toField) {
+            text.append(data.readBuffer, 0, length);
+          } else {
+            writeText(bufferedWriter, CharBuffer.wrap(data.readBuffer, 0, 
length));
+          }
+        }
       }
 
-      if (meta.getOutputType().equalsIgnoreCase(CONST_FIELD)) {
-        output += stringWriter.toString();
-      } else if (meta.getOutputType().equalsIgnoreCase("file")) {
+      if (toField) {
+        output = text.toString();
+      } else {
         bufferedWriter.write(meta.getOutputFileFormatString().getBytes());
       }
     } catch (IOException ex) {
       throw new HopException(ex.getMessage(), ex);
     } finally {
-      try {
-        if (data.currentCountingInputStream != null) {
-          long bytesIn = data.currentCountingInputStream.getCount();
-          dataVolumeIn = (dataVolumeIn != null ? dataVolumeIn : 0L) + bytesIn;
-          if (meta.getInputType().equalsIgnoreCase("file")
-              && !Utils.isEmpty(inputFilename)
-              && bytesIn > 0) {
-            try {
-              FileObject inFile = HopVfs.getFileObject(inputFilename, 
variables);
-              LineageFileIoEmitter.emitTransformFileIo(
-                  this, FileIoOperation.READ, inFile, null, bytesIn, true, 
null);
-            } catch (Exception ignored) {
-              // optional lineage
+      if (reader != null) {
+        try {
+          if (data.currentCountingInputStream != null) {
+            long bytesIn = data.currentCountingInputStream.getCount();
+            dataVolumeIn = (dataVolumeIn != null ? dataVolumeIn : 0L) + 
bytesIn;
+            if (!Utils.isEmpty(inputFilename) && bytesIn > 0) {
+              try {
+                FileObject inFile = HopVfs.getFileObject(inputFilename, 
variables);
+                LineageFileIoEmitter.emitTransformFileIo(
+                    this, FileIoOperation.READ, inFile, null, bytesIn, true, 
null);
+              } catch (Exception ignored) {
+                // optional lineage
+              }
             }
           }
+          data.currentCountingInputStream = null;
+          reader.close();
+        } catch (IOException ex) {
+          throw new HopException(ex.getMessage(), ex);
         }
-        data.currentCountingInputStream = null;
-        reader.close();
-        if (stringWriter != null) {
-          stringWriter.close();
-        }
-
-        reader = null;
-        stringWriter = null;
-
-      } catch (IOException ex) {
-        throw new HopException(ex.getMessage(), ex);
       }
     }
 
@@ -311,6 +305,87 @@ public class TokenReplacement extends 
BaseTransform<TokenReplacementMeta, TokenR
     return result;
   }
 
+  /**
+   * Looks up, once, what every row needs: the position of each token field, 
the token markers and
+   * whether an empty string differs from null.
+   */
+  private void prepareTokens() throws HopValueException {
+    List<TokenReplacementField> fields = meta.getTokenReplacementFields();
+    int[] indexes = new int[fields.size()];
+    Map<String, Integer> positions = new HashMap<>();
+    for (int i = 0; i < indexes.length; i++) {
+      TokenReplacementField field = fields.get(i);
+      indexes[i] = data.inputRowMeta.indexOfValue(field.getName());
+      if (indexes[i] < 0) {
+        throw new HopValueException("Field " + field.getName() + 
CONST_NOT_FOUND_ON_INPUT_STREAM);
+      }
+      // When two fields share a token name, the last one wins, as before
+      positions.put(field.getTokenName(), i);
+    }
+    data.tokenStartMarker = resolve(meta.getTokenStartString());
+    data.tokenEndMarker = resolve(meta.getTokenEndString());
+    data.emptyStringDiffersFromNull =
+        BooleanUtils.toBoolean(
+            
Const.getEnvironmentVariable("KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL", "N"));
+    data.tokenFieldIndexes = indexes;
+    data.tokenValuePositions = positions;
+  }
+
+  /**
+   * The value of every token field in a row. Variables in a value are 
resolved; a null value
+   * becomes an empty string unless KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL is 
set, and then keeps its
+   * token as written.
+   */
+  private String[] getTokenValues(Object[] row) throws HopValueException {
+    String[] values = new String[data.tokenFieldIndexes.length];
+    for (int i = 0; i < values.length; i++) {
+      String value = resolveValue(data.inputRowMeta.getString(row, 
data.tokenFieldIndexes[i]));
+      if (value == null && !data.emptyStringDiffersFromNull) {
+        value = "";
+      }
+      values[i] = value;
+    }
+    return values;
+  }
+
+  /**
+   * Resolves the variables in a field value. A value without a variable 
expression comes out of
+   * {@link #resolve(String)} unchanged, so that (synchronized) call is 
skipped for it.
+   */
+  private String resolveValue(String value) {
+    if (Utils.isEmpty(value) || !mayContainVariables(value)) {
+      return value;
+    }
+    return resolve(value);
+  }
+
+  /** True when a text holds one of the expressions that {@link 
#resolve(String)} replaces. */
+  static boolean mayContainVariables(String text) {
+    return text.contains(StringUtil.UNIX_OPEN)
+        || text.contains(StringUtil.WINDOWS_OPEN)
+        || text.contains(StringUtil.HEX_OPEN)
+        || text.contains(StringUtil.RESOLVER_OPEN);
+  }
+
+  /** The token values of a row by token name, for the reader of an input 
file. */
+  private TokenResolver getTokenResolver(String[] tokenValues) {
+    TokenResolver resolver = new TokenResolver();
+    List<TokenReplacementField> fields = meta.getTokenReplacementFields();
+    for (int i = 0; i < tokenValues.length; i++) {
+      resolver.addToken(fields.get(i).getTokenName(), tokenValues[i]);
+    }
+    return resolver;
+  }
+
+  /** Writes text to an output file in the output file encoding. */
+  private void writeText(OutputStream outputStream, CharSequence text) throws 
IOException {
+    if (data.outputCharset == null) {
+      data.outputCharset = Charset.forName(meta.getOutputFileEncoding());
+    }
+    ByteBuffer bytes = data.outputCharset.encode(CharBuffer.wrap(text));
+    outputStream.write(bytes.array(), bytes.arrayOffset() + bytes.position(), 
bytes.remaining());
+  }
+
   @Override
   public boolean init() {
 
diff --git 
a/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacementData.java
 
b/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacementData.java
index e2fedf9a78..4cd73738cd 100644
--- 
a/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacementData.java
+++ 
b/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacementData.java
@@ -18,10 +18,12 @@
 package org.apache.hop.pipeline.transforms.tokenreplacement;
 
 import java.io.OutputStream;
+import java.nio.charset.Charset;
 import java.text.DateFormatSymbols;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import org.apache.hop.core.io.CountingInputStream;
 import org.apache.hop.core.row.IRowMeta;
 import org.apache.hop.pipeline.transform.BaseTransformData;
@@ -51,6 +53,35 @@ public class TokenReplacementData extends BaseTransformData 
implements ITransfor
 
   public int rowNumber;
 
+  /** Position of each token field in the input row, looked up on the first 
row. */
+  public int[] tokenFieldIndexes;
+
+  /** Token name to the position of its value in the token values of a row. */
+  public Map<String, Integer> tokenValuePositions;
+
+  public String tokenStartMarker;
+  public String tokenEndMarker;
+  public boolean emptyStringDiffersFromNull;
+
+  /** The template typed in the transform, split once. */
+  public TokenTemplate textTemplate;
+
+  /** Position of the template field in the input row, -1 until it is looked 
up. */
+  public int inputFieldIndex = -1;
+
+  /** The last template read from the template field, and how it was split. */
+  public String fieldTemplateText;
+
+  public TokenTemplate fieldTemplate;
+
+  /** Reused for every row: the output text, and the buffer an input file is 
read through. */
+  public final StringBuilder outputText = new StringBuilder();
+
+  public final char[] readBuffer = new char[5000];
+
+  /** The output file encoding, looked up on the first write. */
+  public Charset outputCharset;
+
   public TokenReplacementData() {
     super();
 
diff --git 
a/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenTemplate.java
 
b/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenTemplate.java
new file mode 100644
index 0000000000..878bcea10a
--- /dev/null
+++ 
b/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenTemplate.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.tokenreplacement;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.hop.core.util.Utils;
+
+/**
+ * A template split once into fixed text and tokens, so a row only fills in 
the values. It follows
+ * the same rules as {@link TokenReplacingReader}: a token runs from the start 
marker to the first
+ * end marker after it; a token without a value is kept as it is written, 
markers included; a start
+ * marker without an end marker is kept with the rest of the text; values are 
inserted as they are,
+ * never searched for tokens again.
+ */
+public class TokenTemplate {
+
+  /** The fixed text before each token, and after the last one: one more than 
there are tokens. */
+  private final String[] texts;
+
+  /** For each token: the position of its value in the values of a row, or -1 
when there is none. */
+  private final int[] valuePositions;
+
+  /** For each token: the token as written in the template, markers included. 
*/
+  private final String[] writtenTokens;
+
+  private TokenTemplate(String[] texts, int[] valuePositions, String[] 
writtenTokens) {
+    this.texts = texts;
+    this.valuePositions = valuePositions;
+    this.writtenTokens = writtenTokens;
+  }
+
+  /**
+   * Splits a template into fixed text and tokens.
+   *
+   * @param template the template
+   * @param startMarker the text a token starts with, for example "${"
+   * @param endMarker the text a token ends with, for example "}"
+   * @param valuePositions the position of the value of each token name in the 
values of a row
+   * @return the template, ready to fill in
+   */
+  public static TokenTemplate parse(
+      String template, String startMarker, String endMarker, Map<String, 
Integer> valuePositions) {
+    if (Utils.isEmpty(startMarker) || Utils.isEmpty(endMarker)) {
+      throw new IllegalArgumentException("Token start / end marker may not be 
null or empty");
+    }
+    List<String> texts = new ArrayList<>();
+    List<Integer> positions = new ArrayList<>();
+    List<String> written = new ArrayList<>();
+
+    StringBuilder text = new StringBuilder();
+    int position = 0;
+    while (position < template.length()) {
+      int start = template.indexOf(startMarker, position);
+      if (start < 0) {
+        break;
+      }
+      int nameStart = start + startMarker.length();
+      int end = template.indexOf(endMarker, nameStart);
+      if (end < 0) {
+        // No end marker: the start marker and the rest are plain text
+        break;
+      }
+      text.append(template, position, start);
+      texts.add(text.toString());
+      text.setLength(0);
+
+      String name = template.substring(nameStart, end);
+      Integer valuePosition = valuePositions.get(name);
+      positions.add(valuePosition == null ? -1 : valuePosition);
+      written.add(template.substring(start, end + endMarker.length()));
+
+      position = end + endMarker.length();
+    }
+    text.append(template, position, template.length());
+    texts.add(text.toString());
+
+    int[] valuePositionArray = new int[positions.size()];
+    for (int i = 0; i < valuePositionArray.length; i++) {
+      valuePositionArray[i] = positions.get(i);
+    }
+    return new TokenTemplate(
+        texts.toArray(new String[0]), valuePositionArray, written.toArray(new 
String[0]));
+  }
+
+  /**
+   * Fills in the values of one row.
+   *
+   * @param values the values of the row, by position; a null value keeps the 
token as written
+   * @param output where the result is appended
+   */
+  public void appendTo(String[] values, StringBuilder output) {
+    for (int i = 0; i < valuePositions.length; i++) {
+      output.append(texts[i]);
+      int valuePosition = valuePositions[i];
+      String value = valuePosition < 0 ? null : values[valuePosition];
+      output.append(value == null ? writtenTokens[i] : value);
+    }
+    output.append(texts[texts.length - 1]);
+  }
+
+  /** The number of tokens in the template. */
+  public int getTokenCount() {
+    return valuePositions.length;
+  }
+}
diff --git 
a/plugins/transforms/tokenreplacement/src/test/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacementTest.java
 
b/plugins/transforms/tokenreplacement/src/test/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacementTest.java
new file mode 100644
index 0000000000..808bdbbda2
--- /dev/null
+++ 
b/plugins/transforms/tokenreplacement/src/test/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacementTest.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.tokenreplacement;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Consumer;
+import org.apache.hop.core.IRowSet;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+class TokenReplacementTest {
+
+  private static final String LETTER = "Dear ${name}, you live in ${city}. 
${unknown} ${";
+
+  // More than 500 rows: BaseTransform switches the way it reads rows after 500
+  private static final int ROWS = 1_200;
+
+  @TempDir Path tempDir;
+
+  private TransformMockHelper<TokenReplacementMeta, TokenReplacementData> 
helper;
+
+  @BeforeEach
+  void setUp() {
+    helper =
+        new TransformMockHelper<>(
+            "Token replacement", TokenReplacementMeta.class, 
TokenReplacementData.class);
+    when(helper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+        .thenReturn(helper.iLogChannel);
+    when(helper.pipeline.isRunning()).thenReturn(true);
+  }
+
+  @AfterEach
+  void tearDown() {
+    helper.cleanUp();
+  }
+
+  @Test
+  void fillsATemplateTypedInTheTransform() throws Exception {
+    List<Object[]> input = new ArrayList<>();
+    for (int i = 0; i < ROWS; i++) {
+      input.add(new Object[] {"Name " + i, "City " + (i % 7), null});
+    }
+
+    List<Object[]> output = run(textToField(LETTER), input, null);
+
+    assertEquals(ROWS, output.size());
+    for (int i = 0; i < ROWS; i++) {
+      assertEquals(
+          "Dear Name " + i + ", you live in City " + (i % 7) + ". ${unknown} 
${",
+          output.get(i)[3],
+          "row " + i);
+    }
+  }
+
+  @Test
+  void resolvesVariablesInFieldValues() throws Exception {
+    List<Object[]> input = new ArrayList<>();
+    input.add(new Object[] {"${GREETING} there", "100% sure", null});
+    input.add(new Object[] {"%%GREETING%%", "$[41,42]", null});
+    input.add(new Object[] {"plain text", "${NOT_SET}", null});
+
+    List<Object[]> output =
+        run(
+            textToField("${name}|${city}"),
+            input,
+            transform -> transform.setVariable("GREETING", "Hello"));
+
+    assertEquals("Hello there|100% sure", output.get(0)[3]);
+    assertEquals("Hello|AB", output.get(1)[3]);
+    assertEquals("plain text|${NOT_SET}", output.get(2)[3]);
+  }
+
+  @Test
+  void anEmptyValueReplacesTheTokenWithNothing() throws Exception {
+    List<Object[]> input = new ArrayList<>();
+    input.add(new Object[] {null, "", null});
+
+    List<Object[]> output = run(textToField("[${name}][${city}]"), input, 
null);
+
+    assertEquals("[][]", output.get(0)[3]);
+  }
+
+  @Test
+  void anEmptyValueAtTheStartKeepsTheRestOfTheText() throws Exception {
+    // The old reader gave an empty result for the whole row here
+    List<Object[]> input = new ArrayList<>();
+    input.add(new Object[] {"", "Ghent", null});
+
+    List<Object[]> output = run(textToField("${name} lives in ${city}"), 
input, null);
+
+    assertEquals(" lives in Ghent", output.get(0)[3]);
+  }
+
+  @Test
+  void fillsATemplateFromAField() throws Exception {
+    List<Object[]> input = new ArrayList<>();
+    for (int i = 0; i < ROWS; i++) {
+      // The template changes every few rows
+      String template = (i / 3) % 2 == 0 ? "A ${name} in ${city}" : "${city}: 
B ${name}";
+      input.add(new Object[] {"n" + i, "c" + i, template});
+    }
+    TokenReplacementMeta meta = textToField("not used");
+    meta.setInputType("Field");
+    meta.setInputFieldName("template");
+
+    List<Object[]> output = run(meta, input, null);
+
+    for (int i = 0; i < ROWS; i++) {
+      String expected = (i / 3) % 2 == 0 ? "A n" + i + " in c" + i : "c" + i + 
": B n" + i;
+      assertEquals(expected, output.get(i)[3], "row " + i);
+    }
+  }
+
+  @Test
+  void writesToAFile() throws Exception {
+    List<Object[]> input = new ArrayList<>();
+    input.add(new Object[] {"Ann", "Ghent", null});
+    input.add(new Object[] {"Bob", "Liège", null});
+    TokenReplacementMeta meta = textToField("${name} - ${city}");
+    meta.setOutputType("File");
+    meta.setOutputFileName(tempDir.resolve("letters.txt").toString());
+    meta.setOutputFileFormat("UNIX");
+    meta.setOutputFileEncoding("UTF-8");
+
+    List<Object[]> output = run(meta, input, null);
+
+    assertEquals(2, output.size());
+    Path written = tempDir.resolve("letters.txt");
+    assertTrue(Files.exists(written));
+    assertEquals("Ann - Ghent\nBob - Liège\n", Files.readString(written, 
StandardCharsets.UTF_8));
+  }
+
+  @Test
+  void readsATemplateFile() throws Exception {
+    Path templateFile = tempDir.resolve("template.txt");
+    Files.writeString(templateFile, "Hello ${name} from ${city}", 
StandardCharsets.UTF_8);
+    List<Object[]> input = new ArrayList<>();
+    input.add(new Object[] {"Ann", "Ghent", null});
+    TokenReplacementMeta meta = textToField("not used");
+    meta.setInputType("File");
+    meta.setInputFileName(templateFile.toString());
+
+    List<Object[]> output = run(meta, input, null);
+
+    assertEquals("Hello Ann from Ghent", output.get(0)[3]);
+  }
+
+  @Test
+  void onlyExpressionsAreSentToTheVariableResolver() {
+    assertTrue(TokenReplacement.mayContainVariables("a ${b}"));
+    assertTrue(TokenReplacement.mayContainVariables("%%b%%"));
+    assertTrue(TokenReplacement.mayContainVariables("$[41]"));
+    assertTrue(TokenReplacement.mayContainVariables("#{a:b}"));
+    assertFalse(TokenReplacement.mayContainVariables("100% $ # { } plain"));
+  }
+
+  private static TokenReplacementMeta textToField(String template) {
+    TokenReplacementMeta meta = new TokenReplacementMeta();
+    meta.setInputType("Text");
+    meta.setInputText(template);
+    meta.setOutputType("Field");
+    meta.setOutputFieldName("result");
+    List<TokenReplacementField> fields = new ArrayList<>();
+    fields.add(field("name", "name"));
+    fields.add(field("city", "city"));
+    meta.setTokenReplacementFields(fields);
+    return meta;
+  }
+
+  private static TokenReplacementField field(String name, String tokenName) {
+    TokenReplacementField field = new TokenReplacementField();
+    field.setName(name);
+    field.setTokenName(tokenName);
+    return field;
+  }
+
+  private List<Object[]> run(
+      TokenReplacementMeta meta, List<Object[]> input, 
Consumer<TokenReplacement> beforeInit)
+      throws Exception {
+    TokenReplacement transform =
+        new TokenReplacement(
+            helper.transformMeta,
+            meta,
+            new TokenReplacementData(),
+            0,
+            helper.pipelineMeta,
+            helper.pipeline);
+    if (beforeInit != null) {
+      beforeInit.accept(transform);
+    }
+    transform.init();
+
+    IRowSet rowSet = helper.getMockInputRowSet(input);
+    // BaseTransform also reads with getRowImmediate() (every 500 rows): same 
rows
+    when(rowSet.getRowImmediate()).thenAnswer(invocation -> rowSet.getRow());
+    IRowMeta rowMeta = new RowMeta();
+    rowMeta.addValueMeta(new ValueMetaString("name"));
+    rowMeta.addValueMeta(new ValueMetaString("city"));
+    rowMeta.addValueMeta(new ValueMetaString("template"));
+    when(rowSet.getRowMeta()).thenReturn(rowMeta);
+    transform.setInputRowSets(new ArrayList<>(List.of(rowSet)));
+
+    List<Object[]> output = new ArrayList<>();
+    transform.addRowListener(
+        new RowAdapter() {
+          @Override
+          public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) {
+            output.add(row);
+          }
+        });
+
+    while (transform.processRow()) {
+      // keep going until the transform is done
+    }
+    transform.dispose();
+    return output;
+  }
+}
diff --git 
a/plugins/transforms/tokenreplacement/src/test/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenTemplateTest.java
 
b/plugins/transforms/tokenreplacement/src/test/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenTemplateTest.java
new file mode 100644
index 0000000000..420dc4b95b
--- /dev/null
+++ 
b/plugins/transforms/tokenreplacement/src/test/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenTemplateTest.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.tokenreplacement;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+/** The template must give exactly what {@link TokenReplacingReader} gives for 
the same input. */
+class TokenTemplateTest {
+
+  private static final String[] NAMES = {"a", "b", "name", ""};
+
+  @Test
+  void fillsInTokens() {
+    assertEquals(
+        "Dear Ann, you live in Ghent.",
+        fill("Dear ${name}, you live in ${city}.", "${", "}", "name", "Ann", 
"city", "Ghent"));
+  }
+
+  @Test
+  void keepsTokensWithoutAValueAsWritten() {
+    assertEquals("Hi ${nobody} and ${}", fill("Hi ${nobody} and ${}", "${", 
"}", "a", "x"));
+  }
+
+  @Test
+  void keepsAnUnfinishedTokenWithTheRestOfTheText() {
+    assertEquals("Hi x ${a and more", fill("Hi ${a} ${a and more", "${", "}", 
"a", "x"));
+  }
+
+  @Test
+  void emptyValueAtTheStartKeepsTheRestOfTheText() {
+    // The old reader returned nothing at all for this row
+    assertEquals(" lives here", fill("${name} lives here", "${", "}", "name", 
""));
+  }
+
+  @Test
+  void doesNotLookForTokensInValues() {
+    assertEquals("<${b}>", fill("<${a}>", "${", "}", "a", "${b}", "b", 
"nested"));
+  }
+
+  @Test
+  void refusesEmptyMarkers() {
+    Map<String, Integer> positions = Map.of();
+    assertThrows(
+        IllegalArgumentException.class, () -> TokenTemplate.parse("x", "", 
"}", positions));
+    assertThrows(
+        IllegalArgumentException.class, () -> TokenTemplate.parse("x", "${", 
"", positions));
+  }
+
+  @ParameterizedTest
+  @CsvSource({"${,}", "{{,}}", "[,]", "<%,%>", "$,$", "##,#"})
+  void givesTheSameAsTheReaderForRandomTemplates(String start, String end) 
throws IOException {
+    Random random = new Random(start.hashCode() * 31L + end.hashCode());
+    String alphabet = "ab xyz" + start + end + "name";
+    for (int run = 0; run < 5_000; run++) {
+      String template = randomText(random, alphabet, random.nextInt(40));
+
+      // Values may hold markers and token names too; some are missing (null). 
No empty values:
+      // the old reader ends its output at an empty value at the start of a 
read, see
+      // emptyValueAtTheStartKeepsTheRestOfTheText
+      Map<String, Integer> positions = new HashMap<>();
+      String[] values = new String[NAMES.length];
+      TokenResolver resolver = new TokenResolver();
+      for (int i = 0; i < NAMES.length; i++) {
+        positions.put(NAMES[i], i);
+        values[i] =
+            random.nextInt(5) == 0 ? null : randomText(random, alphabet, 1 + 
random.nextInt(7));
+        resolver.addToken(NAMES[i], values[i]);
+      }
+
+      StringBuilder actual = new StringBuilder();
+      TokenTemplate.parse(template, start, end, positions).appendTo(values, 
actual);
+
+      assertEquals(
+          readThroughReader(template, start, end, resolver),
+          actual.toString(),
+          "template [" + template + "] with markers " + start + " and " + end);
+    }
+  }
+
+  private static String fill(String template, String start, String end, 
String... nameValues) {
+    Map<String, Integer> positions = new HashMap<>();
+    String[] values = new String[nameValues.length / 2];
+    for (int i = 0; i < values.length; i++) {
+      positions.put(nameValues[i * 2], i);
+      values[i] = nameValues[i * 2 + 1];
+    }
+    StringBuilder output = new StringBuilder();
+    TokenTemplate.parse(template, start, end, positions).appendTo(values, 
output);
+    return output.toString();
+  }
+
+  private static String readThroughReader(
+      String template, String start, String end, TokenResolver resolver) 
throws IOException {
+    StringBuilder output = new StringBuilder();
+    try (TokenReplacingReader reader =
+        new TokenReplacingReader(resolver, new StringReader(template), start, 
end)) {
+      char[] buffer = new char[7];
+      int length;
+      while ((length = reader.read(buffer)) > 0) {
+        output.append(buffer, 0, length);
+      }
+    }
+    return output.toString();
+  }
+
+  private static String randomText(Random random, String alphabet, int length) 
{
+    StringBuilder text = new StringBuilder();
+    for (int i = 0; i < length; i++) {
+      text.append(alphabet.charAt(random.nextInt(alphabet.length())));
+    }
+    return text.toString();
+  }
+}

Reply via email to