This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 28fe41ac30 Token replacement is slow because it rereads the template
for every row, fixes #9350 (#9352)
28fe41ac30 is described below
commit 28fe41ac30a12ce1409f272ef798afcaa4748872
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Fri Nov 6 21:10:05 2026 +0100
Token replacement is slow because it rereads the template for every row,
fixes #9350 (#9352)
---
.../tokenreplacement/TokenReplacement.java | 243 +++++++++++++-------
.../tokenreplacement/TokenReplacementData.java | 31 +++
.../transforms/tokenreplacement/TokenTemplate.java | 122 ++++++++++
.../tokenreplacement/TokenReplacementTest.java | 251 +++++++++++++++++++++
.../tokenreplacement/TokenTemplateTest.java | 138 +++++++++++
5 files changed, 701 insertions(+), 84 deletions(-)
diff --git
a/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacement.java
b/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacement.java
index 74cb1a4512..26bdc49135 100644
---
a/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacement.java
+++
b/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacement.java
@@ -22,14 +22,14 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.Reader;
-import java.io.StringReader;
-import java.io.StringWriter;
-import java.io.Writer;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.vfs2.FileObject;
import org.apache.hop.core.Const;
@@ -39,6 +39,7 @@ import org.apache.hop.core.exception.HopValueException;
import org.apache.hop.core.io.CountingInputStream;
import org.apache.hop.core.io.CountingOutputStream;
import org.apache.hop.core.row.RowDataUtil;
+import org.apache.hop.core.util.StringUtil;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.vfs.HopVfs;
import org.apache.hop.i18n.BaseMessages;
@@ -127,48 +128,47 @@ public class TokenReplacement extends
BaseTransform<TokenReplacementMeta, TokenR
}
}
- // Create token resolver
- TokenResolver resolver = new TokenResolver();
-
- for (TokenReplacementField field : meta.getTokenReplacementFields()) {
- if (data.inputRowMeta.indexOfValue(field.getName()) >= 0) {
- String fieldValue = resolve(data.inputRowMeta.getString(r,
field.getName(), null));
- if (fieldValue == null
- && !BooleanUtils.toBoolean(
-
Const.getEnvironmentVariable("KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL", "N"))) {
- fieldValue = Const.nullToEmpty(fieldValue);
- }
- resolver.addToken(field.getTokenName(), fieldValue);
- } else {
- throw new HopValueException("Field " + field.getName() +
CONST_NOT_FOUND_ON_INPUT_STREAM);
- }
+ // The value of every token field in this row, by its position in the
transform's field list
+ if (data.tokenValuePositions == null) {
+ prepareTokens();
}
+ String[] tokenValues = getTokenValues(r);
- Reader reader;
+ // Text and field input: the template, split into fixed text and tokens.
+ // File input: streamed through a reader, so large files are never read
into memory at once.
+ TokenTemplate template = null;
+ Reader reader = null;
String inputFilename = "";
if (meta.getInputType().equalsIgnoreCase("text")) {
- reader =
- new TokenReplacingReader(
- resolver,
- new StringReader(meta.getInputText()),
- resolve(meta.getTokenStartString()),
- resolve(meta.getTokenEndString()));
+ if (data.textTemplate == null) {
+ data.textTemplate =
+ TokenTemplate.parse(
+ Const.NVL(meta.getInputText(), ""),
+ data.tokenStartMarker,
+ data.tokenEndMarker,
+ data.tokenValuePositions);
+ }
+ template = data.textTemplate;
} else if (meta.getInputType().equalsIgnoreCase(CONST_FIELD)) {
- if (data.inputRowMeta.indexOfValue(meta.getInputFieldName()) >= 0) {
- String inputString = data.inputRowMeta.getString(r,
meta.getInputFieldName(), "");
- reader =
- new TokenReplacingReader(
- resolver,
- new StringReader(inputString),
- resolve(meta.getTokenStartString()),
- resolve(meta.getTokenEndString()));
-
- } else {
- throw new HopValueException(
- "Input field " + meta.getInputFieldName() +
CONST_NOT_FOUND_ON_INPUT_STREAM);
+ if (data.inputFieldIndex < 0) {
+ data.inputFieldIndex =
data.inputRowMeta.indexOfValue(meta.getInputFieldName());
+ if (data.inputFieldIndex < 0) {
+ throw new HopValueException(
+ "Input field " + meta.getInputFieldName() +
CONST_NOT_FOUND_ON_INPUT_STREAM);
+ }
+ }
+ String inputString = Const.NVL(data.inputRowMeta.getString(r,
data.inputFieldIndex), "");
+ // Rows often share the same template: split it again only when it
changes
+ if (data.fieldTemplate == null ||
!inputString.equals(data.fieldTemplateText)) {
+ data.fieldTemplate =
+ TokenTemplate.parse(
+ inputString, data.tokenStartMarker, data.tokenEndMarker,
data.tokenValuePositions);
+ data.fieldTemplateText = inputString;
}
+ template = data.fieldTemplate;
+
} else if (meta.getInputType().equalsIgnoreCase("file")) {
if (meta.isInputFileNameInField()) {
if
(data.inputRowMeta.indexOfValue(resolve(meta.getInputFileNameField())) >= 0) {
@@ -194,11 +194,11 @@ public class TokenReplacement extends
BaseTransform<TokenReplacementMeta, TokenR
String inputEncoding = Const.NVL(resolve(meta.getOutputFileEncoding()),
Const.UTF_8);
reader =
new TokenReplacingReader(
- resolver,
+ getTokenResolver(tokenValues),
new InputStreamReader(
data.currentCountingInputStream,
Charset.forName(inputEncoding)),
- resolve(meta.getTokenStartString()),
- resolve(meta.getTokenEndString()));
+ data.tokenStartMarker,
+ data.tokenEndMarker);
if (meta.isAddInputFileNameToResult()) {
ResultFile resultFile =
@@ -214,12 +214,9 @@ public class TokenReplacement extends
BaseTransform<TokenReplacementMeta, TokenR
throw new HopException("Unsupported input type " + meta.getInputType());
}
- Writer stringWriter = null;
OutputStream bufferedWriter = null;
- if (meta.getOutputType().equalsIgnoreCase(CONST_FIELD)) {
- stringWriter = new StringWriter(5000);
- } else {
+ if (!meta.getOutputType().equalsIgnoreCase(CONST_FIELD)) {
if (meta.getOutputType().equalsIgnoreCase("file")) {
if (inputFilename.equals(outputFilename)) {
@@ -241,58 +238,55 @@ public class TokenReplacement extends
BaseTransform<TokenReplacementMeta, TokenR
}
String output = "";
+ boolean toField = meta.getOutputType().equalsIgnoreCase(CONST_FIELD);
+ StringBuilder text = data.outputText;
+ text.setLength(0);
try {
- char[] cbuf = new char[5000];
- int length = 0;
- while ((length = reader.read(cbuf)) > 0) {
- if (meta.getOutputType().equalsIgnoreCase(CONST_FIELD)) {
- stringWriter.write(cbuf, 0, length);
- } else if (meta.getOutputType().equalsIgnoreCase("file")) {
- CharBuffer cBuffer = CharBuffer.wrap(cbuf, 0, length);
- ByteBuffer bBuffer =
Charset.forName(meta.getOutputFileEncoding()).encode(cBuffer);
- byte[] bytes = new byte[bBuffer.limit()];
- bBuffer.get(bytes);
- bufferedWriter.write(bytes);
- } // No else. Anything else will be thrown to a Hop exception prior
to getting here.
- cbuf = new char[5000];
+ if (template != null) {
+ template.appendTo(tokenValues, text);
+ if (!toField) {
+ writeText(bufferedWriter, text);
+ }
+ } else {
+ int length;
+ while ((length = reader.read(data.readBuffer)) > 0) {
+ if (toField) {
+ text.append(data.readBuffer, 0, length);
+ } else {
+ writeText(bufferedWriter, CharBuffer.wrap(data.readBuffer, 0,
length));
+ }
+ }
}
- if (meta.getOutputType().equalsIgnoreCase(CONST_FIELD)) {
- output += stringWriter.toString();
- } else if (meta.getOutputType().equalsIgnoreCase("file")) {
+ if (toField) {
+ output = text.toString();
+ } else {
bufferedWriter.write(meta.getOutputFileFormatString().getBytes());
}
} catch (IOException ex) {
throw new HopException(ex.getMessage(), ex);
} finally {
- try {
- if (data.currentCountingInputStream != null) {
- long bytesIn = data.currentCountingInputStream.getCount();
- dataVolumeIn = (dataVolumeIn != null ? dataVolumeIn : 0L) + bytesIn;
- if (meta.getInputType().equalsIgnoreCase("file")
- && !Utils.isEmpty(inputFilename)
- && bytesIn > 0) {
- try {
- FileObject inFile = HopVfs.getFileObject(inputFilename,
variables);
- LineageFileIoEmitter.emitTransformFileIo(
- this, FileIoOperation.READ, inFile, null, bytesIn, true,
null);
- } catch (Exception ignored) {
- // optional lineage
+ if (reader != null) {
+ try {
+ if (data.currentCountingInputStream != null) {
+ long bytesIn = data.currentCountingInputStream.getCount();
+ dataVolumeIn = (dataVolumeIn != null ? dataVolumeIn : 0L) +
bytesIn;
+ if (!Utils.isEmpty(inputFilename) && bytesIn > 0) {
+ try {
+ FileObject inFile = HopVfs.getFileObject(inputFilename,
variables);
+ LineageFileIoEmitter.emitTransformFileIo(
+ this, FileIoOperation.READ, inFile, null, bytesIn, true,
null);
+ } catch (Exception ignored) {
+ // optional lineage
+ }
}
}
+ data.currentCountingInputStream = null;
+ reader.close();
+ } catch (IOException ex) {
+ throw new HopException(ex.getMessage(), ex);
}
- data.currentCountingInputStream = null;
- reader.close();
- if (stringWriter != null) {
- stringWriter.close();
- }
-
- reader = null;
- stringWriter = null;
-
- } catch (IOException ex) {
- throw new HopException(ex.getMessage(), ex);
}
}
@@ -311,6 +305,87 @@ public class TokenReplacement extends
BaseTransform<TokenReplacementMeta, TokenR
return result;
}
+ /**
+ * Looks up, once, what every row needs: the position of each token field,
the token markers and
+ * whether an empty string differs from null.
+ */
+ private void prepareTokens() throws HopValueException {
+ List<TokenReplacementField> fields = meta.getTokenReplacementFields();
+ int[] indexes = new int[fields.size()];
+ Map<String, Integer> positions = new HashMap<>();
+ for (int i = 0; i < indexes.length; i++) {
+ TokenReplacementField field = fields.get(i);
+ indexes[i] = data.inputRowMeta.indexOfValue(field.getName());
+ if (indexes[i] < 0) {
+ throw new HopValueException("Field " + field.getName() +
CONST_NOT_FOUND_ON_INPUT_STREAM);
+ }
+ // When two fields share a token name, the last one wins, as before
+ positions.put(field.getTokenName(), i);
+ }
+ data.tokenStartMarker = resolve(meta.getTokenStartString());
+ data.tokenEndMarker = resolve(meta.getTokenEndString());
+ data.emptyStringDiffersFromNull =
+ BooleanUtils.toBoolean(
+
Const.getEnvironmentVariable("KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL", "N"));
+ data.tokenFieldIndexes = indexes;
+ data.tokenValuePositions = positions;
+ }
+
+ /**
+ * The value of every token field in a row. Variables in a value are
resolved; a null value
+ * becomes an empty string unless KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL is
set, and then keeps its
+ * token as written.
+ */
+ private String[] getTokenValues(Object[] row) throws HopValueException {
+ String[] values = new String[data.tokenFieldIndexes.length];
+ for (int i = 0; i < values.length; i++) {
+ String value = resolveValue(data.inputRowMeta.getString(row,
data.tokenFieldIndexes[i]));
+ if (value == null && !data.emptyStringDiffersFromNull) {
+ value = "";
+ }
+ values[i] = value;
+ }
+ return values;
+ }
+
+ /**
+ * Resolves the variables in a field value. A value without a variable
expression comes out of
+ * {@link #resolve(String)} unchanged, so that (synchronized) call is
skipped for it.
+ */
+ private String resolveValue(String value) {
+ if (Utils.isEmpty(value) || !mayContainVariables(value)) {
+ return value;
+ }
+ return resolve(value);
+ }
+
+ /** True when a text holds one of the expressions that {@link
#resolve(String)} replaces. */
+ static boolean mayContainVariables(String text) {
+ return text.contains(StringUtil.UNIX_OPEN)
+ || text.contains(StringUtil.WINDOWS_OPEN)
+ || text.contains(StringUtil.HEX_OPEN)
+ || text.contains(StringUtil.RESOLVER_OPEN);
+ }
+
+ /** The token values of a row by token name, for the reader of an input
file. */
+ private TokenResolver getTokenResolver(String[] tokenValues) {
+ TokenResolver resolver = new TokenResolver();
+ List<TokenReplacementField> fields = meta.getTokenReplacementFields();
+ for (int i = 0; i < tokenValues.length; i++) {
+ resolver.addToken(fields.get(i).getTokenName(), tokenValues[i]);
+ }
+ return resolver;
+ }
+
+ /** Writes text to an output file in the output file encoding. */
+ private void writeText(OutputStream outputStream, CharSequence text) throws
IOException {
+ if (data.outputCharset == null) {
+ data.outputCharset = Charset.forName(meta.getOutputFileEncoding());
+ }
+ ByteBuffer bytes = data.outputCharset.encode(CharBuffer.wrap(text));
+ outputStream.write(bytes.array(), bytes.arrayOffset() + bytes.position(),
bytes.remaining());
+ }
+
@Override
public boolean init() {
diff --git
a/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacementData.java
b/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacementData.java
index e2fedf9a78..4cd73738cd 100644
---
a/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacementData.java
+++
b/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacementData.java
@@ -18,10 +18,12 @@
package org.apache.hop.pipeline.transforms.tokenreplacement;
import java.io.OutputStream;
+import java.nio.charset.Charset;
import java.text.DateFormatSymbols;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import org.apache.hop.core.io.CountingInputStream;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.pipeline.transform.BaseTransformData;
@@ -51,6 +53,35 @@ public class TokenReplacementData extends BaseTransformData
implements ITransfor
public int rowNumber;
+ /** Position of each token field in the input row, looked up on the first
row. */
+ public int[] tokenFieldIndexes;
+
+ /** Token name to the position of its value in the token values of a row. */
+ public Map<String, Integer> tokenValuePositions;
+
+ public String tokenStartMarker;
+ public String tokenEndMarker;
+ public boolean emptyStringDiffersFromNull;
+
+ /** The template typed in the transform, split once. */
+ public TokenTemplate textTemplate;
+
+ /** Position of the template field in the input row, -1 until it is looked
up. */
+ public int inputFieldIndex = -1;
+
+ /** The last template read from the template field, and how it was split. */
+ public String fieldTemplateText;
+
+ public TokenTemplate fieldTemplate;
+
+ /** Reused for every row: the output text, and the buffer an input file is
read through. */
+ public final StringBuilder outputText = new StringBuilder();
+
+ public final char[] readBuffer = new char[5000];
+
+ /** The output file encoding, looked up on the first write. */
+ public Charset outputCharset;
+
public TokenReplacementData() {
super();
diff --git
a/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenTemplate.java
b/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenTemplate.java
new file mode 100644
index 0000000000..878bcea10a
--- /dev/null
+++
b/plugins/transforms/tokenreplacement/src/main/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenTemplate.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.tokenreplacement;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.hop.core.util.Utils;
+
+/**
+ * A template split once into fixed text and tokens, so a row only fills in
the values. It follows
+ * the same rules as {@link TokenReplacingReader}: a token runs from the start
marker to the first
+ * end marker after it; a token without a value is kept as it is written,
markers included; a start
+ * marker without an end marker is kept with the rest of the text; values are
inserted as they are,
+ * never searched for tokens again.
+ */
+public class TokenTemplate {
+
+ /** The fixed text before each token, and after the last one: one more than
there are tokens. */
+ private final String[] texts;
+
+ /** For each token: the position of its value in the values of a row, or -1
when there is none. */
+ private final int[] valuePositions;
+
+ /** For each token: the token as written in the template, markers included.
*/
+ private final String[] writtenTokens;
+
+ private TokenTemplate(String[] texts, int[] valuePositions, String[]
writtenTokens) {
+ this.texts = texts;
+ this.valuePositions = valuePositions;
+ this.writtenTokens = writtenTokens;
+ }
+
+ /**
+ * Splits a template into fixed text and tokens.
+ *
+ * @param template the template
+ * @param startMarker the text a token starts with, for example "${"
+ * @param endMarker the text a token ends with, for example "}"
+ * @param valuePositions the position of the value of each token name in the
values of a row
+ * @return the template, ready to fill in
+ */
+ public static TokenTemplate parse(
+ String template, String startMarker, String endMarker, Map<String,
Integer> valuePositions) {
+ if (Utils.isEmpty(startMarker) || Utils.isEmpty(endMarker)) {
+ throw new IllegalArgumentException("Token start / end marker may not be
null or empty");
+ }
+ List<String> texts = new ArrayList<>();
+ List<Integer> positions = new ArrayList<>();
+ List<String> written = new ArrayList<>();
+
+ StringBuilder text = new StringBuilder();
+ int position = 0;
+ while (position < template.length()) {
+ int start = template.indexOf(startMarker, position);
+ if (start < 0) {
+ break;
+ }
+ int nameStart = start + startMarker.length();
+ int end = template.indexOf(endMarker, nameStart);
+ if (end < 0) {
+ // No end marker: the start marker and the rest are plain text
+ break;
+ }
+ text.append(template, position, start);
+ texts.add(text.toString());
+ text.setLength(0);
+
+ String name = template.substring(nameStart, end);
+ Integer valuePosition = valuePositions.get(name);
+ positions.add(valuePosition == null ? -1 : valuePosition);
+ written.add(template.substring(start, end + endMarker.length()));
+
+ position = end + endMarker.length();
+ }
+ text.append(template, position, template.length());
+ texts.add(text.toString());
+
+ int[] valuePositionArray = new int[positions.size()];
+ for (int i = 0; i < valuePositionArray.length; i++) {
+ valuePositionArray[i] = positions.get(i);
+ }
+ return new TokenTemplate(
+ texts.toArray(new String[0]), valuePositionArray, written.toArray(new
String[0]));
+ }
+
+ /**
+ * Fills in the values of one row.
+ *
+ * @param values the values of the row, by position; a null value keeps the
token as written
+ * @param output where the result is appended
+ */
+ public void appendTo(String[] values, StringBuilder output) {
+ for (int i = 0; i < valuePositions.length; i++) {
+ output.append(texts[i]);
+ int valuePosition = valuePositions[i];
+ String value = valuePosition < 0 ? null : values[valuePosition];
+ output.append(value == null ? writtenTokens[i] : value);
+ }
+ output.append(texts[texts.length - 1]);
+ }
+
+ /** The number of tokens in the template. */
+ public int getTokenCount() {
+ return valuePositions.length;
+ }
+}
diff --git
a/plugins/transforms/tokenreplacement/src/test/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacementTest.java
b/plugins/transforms/tokenreplacement/src/test/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacementTest.java
new file mode 100644
index 0000000000..808bdbbda2
--- /dev/null
+++
b/plugins/transforms/tokenreplacement/src/test/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenReplacementTest.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.tokenreplacement;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Consumer;
+import org.apache.hop.core.IRowSet;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+class TokenReplacementTest {
+
+ private static final String LETTER = "Dear ${name}, you live in ${city}.
${unknown} ${";
+
+ // More than 500 rows: BaseTransform switches the way it reads rows after 500
+ private static final int ROWS = 1_200;
+
+ @TempDir Path tempDir;
+
+ private TransformMockHelper<TokenReplacementMeta, TokenReplacementData>
helper;
+
+ @BeforeEach
+ void setUp() {
+ helper =
+ new TransformMockHelper<>(
+ "Token replacement", TokenReplacementMeta.class,
TokenReplacementData.class);
+ when(helper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+ .thenReturn(helper.iLogChannel);
+ when(helper.pipeline.isRunning()).thenReturn(true);
+ }
+
+ @AfterEach
+ void tearDown() {
+ helper.cleanUp();
+ }
+
+ @Test
+ void fillsATemplateTypedInTheTransform() throws Exception {
+ List<Object[]> input = new ArrayList<>();
+ for (int i = 0; i < ROWS; i++) {
+ input.add(new Object[] {"Name " + i, "City " + (i % 7), null});
+ }
+
+ List<Object[]> output = run(textToField(LETTER), input, null);
+
+ assertEquals(ROWS, output.size());
+ for (int i = 0; i < ROWS; i++) {
+ assertEquals(
+ "Dear Name " + i + ", you live in City " + (i % 7) + ". ${unknown}
${",
+ output.get(i)[3],
+ "row " + i);
+ }
+ }
+
+ @Test
+ void resolvesVariablesInFieldValues() throws Exception {
+ List<Object[]> input = new ArrayList<>();
+ input.add(new Object[] {"${GREETING} there", "100% sure", null});
+ input.add(new Object[] {"%%GREETING%%", "$[41,42]", null});
+ input.add(new Object[] {"plain text", "${NOT_SET}", null});
+
+ List<Object[]> output =
+ run(
+ textToField("${name}|${city}"),
+ input,
+ transform -> transform.setVariable("GREETING", "Hello"));
+
+ assertEquals("Hello there|100% sure", output.get(0)[3]);
+ assertEquals("Hello|AB", output.get(1)[3]);
+ assertEquals("plain text|${NOT_SET}", output.get(2)[3]);
+ }
+
+ @Test
+ void anEmptyValueReplacesTheTokenWithNothing() throws Exception {
+ List<Object[]> input = new ArrayList<>();
+ input.add(new Object[] {null, "", null});
+
+ List<Object[]> output = run(textToField("[${name}][${city}]"), input,
null);
+
+ assertEquals("[][]", output.get(0)[3]);
+ }
+
+ @Test
+ void anEmptyValueAtTheStartKeepsTheRestOfTheText() throws Exception {
+ // The old reader gave an empty result for the whole row here
+ List<Object[]> input = new ArrayList<>();
+ input.add(new Object[] {"", "Ghent", null});
+
+ List<Object[]> output = run(textToField("${name} lives in ${city}"),
input, null);
+
+ assertEquals(" lives in Ghent", output.get(0)[3]);
+ }
+
+ @Test
+ void fillsATemplateFromAField() throws Exception {
+ List<Object[]> input = new ArrayList<>();
+ for (int i = 0; i < ROWS; i++) {
+ // The template changes every few rows
+ String template = (i / 3) % 2 == 0 ? "A ${name} in ${city}" : "${city}:
B ${name}";
+ input.add(new Object[] {"n" + i, "c" + i, template});
+ }
+ TokenReplacementMeta meta = textToField("not used");
+ meta.setInputType("Field");
+ meta.setInputFieldName("template");
+
+ List<Object[]> output = run(meta, input, null);
+
+ for (int i = 0; i < ROWS; i++) {
+ String expected = (i / 3) % 2 == 0 ? "A n" + i + " in c" + i : "c" + i +
": B n" + i;
+ assertEquals(expected, output.get(i)[3], "row " + i);
+ }
+ }
+
+ @Test
+ void writesToAFile() throws Exception {
+ List<Object[]> input = new ArrayList<>();
+ input.add(new Object[] {"Ann", "Ghent", null});
+ input.add(new Object[] {"Bob", "Liège", null});
+ TokenReplacementMeta meta = textToField("${name} - ${city}");
+ meta.setOutputType("File");
+ meta.setOutputFileName(tempDir.resolve("letters.txt").toString());
+ meta.setOutputFileFormat("UNIX");
+ meta.setOutputFileEncoding("UTF-8");
+
+ List<Object[]> output = run(meta, input, null);
+
+ assertEquals(2, output.size());
+ Path written = tempDir.resolve("letters.txt");
+ assertTrue(Files.exists(written));
+ assertEquals("Ann - Ghent\nBob - Liège\n", Files.readString(written,
StandardCharsets.UTF_8));
+ }
+
+ @Test
+ void readsATemplateFile() throws Exception {
+ Path templateFile = tempDir.resolve("template.txt");
+ Files.writeString(templateFile, "Hello ${name} from ${city}",
StandardCharsets.UTF_8);
+ List<Object[]> input = new ArrayList<>();
+ input.add(new Object[] {"Ann", "Ghent", null});
+ TokenReplacementMeta meta = textToField("not used");
+ meta.setInputType("File");
+ meta.setInputFileName(templateFile.toString());
+
+ List<Object[]> output = run(meta, input, null);
+
+ assertEquals("Hello Ann from Ghent", output.get(0)[3]);
+ }
+
+ @Test
+ void onlyExpressionsAreSentToTheVariableResolver() {
+ assertTrue(TokenReplacement.mayContainVariables("a ${b}"));
+ assertTrue(TokenReplacement.mayContainVariables("%%b%%"));
+ assertTrue(TokenReplacement.mayContainVariables("$[41]"));
+ assertTrue(TokenReplacement.mayContainVariables("#{a:b}"));
+ assertFalse(TokenReplacement.mayContainVariables("100% $ # { } plain"));
+ }
+
+ private static TokenReplacementMeta textToField(String template) {
+ TokenReplacementMeta meta = new TokenReplacementMeta();
+ meta.setInputType("Text");
+ meta.setInputText(template);
+ meta.setOutputType("Field");
+ meta.setOutputFieldName("result");
+ List<TokenReplacementField> fields = new ArrayList<>();
+ fields.add(field("name", "name"));
+ fields.add(field("city", "city"));
+ meta.setTokenReplacementFields(fields);
+ return meta;
+ }
+
+ private static TokenReplacementField field(String name, String tokenName) {
+ TokenReplacementField field = new TokenReplacementField();
+ field.setName(name);
+ field.setTokenName(tokenName);
+ return field;
+ }
+
+ private List<Object[]> run(
+ TokenReplacementMeta meta, List<Object[]> input,
Consumer<TokenReplacement> beforeInit)
+ throws Exception {
+ TokenReplacement transform =
+ new TokenReplacement(
+ helper.transformMeta,
+ meta,
+ new TokenReplacementData(),
+ 0,
+ helper.pipelineMeta,
+ helper.pipeline);
+ if (beforeInit != null) {
+ beforeInit.accept(transform);
+ }
+ transform.init();
+
+ IRowSet rowSet = helper.getMockInputRowSet(input);
+ // BaseTransform also reads with getRowImmediate() (every 500 rows): same
rows
+ when(rowSet.getRowImmediate()).thenAnswer(invocation -> rowSet.getRow());
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("name"));
+ rowMeta.addValueMeta(new ValueMetaString("city"));
+ rowMeta.addValueMeta(new ValueMetaString("template"));
+ when(rowSet.getRowMeta()).thenReturn(rowMeta);
+ transform.setInputRowSets(new ArrayList<>(List.of(rowSet)));
+
+ List<Object[]> output = new ArrayList<>();
+ transform.addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) {
+ output.add(row);
+ }
+ });
+
+ while (transform.processRow()) {
+ // keep going until the transform is done
+ }
+ transform.dispose();
+ return output;
+ }
+}
diff --git
a/plugins/transforms/tokenreplacement/src/test/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenTemplateTest.java
b/plugins/transforms/tokenreplacement/src/test/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenTemplateTest.java
new file mode 100644
index 0000000000..420dc4b95b
--- /dev/null
+++
b/plugins/transforms/tokenreplacement/src/test/java/org/apache/hop/pipeline/transforms/tokenreplacement/TokenTemplateTest.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.tokenreplacement;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+/** The template must give exactly what {@link TokenReplacingReader} gives for
the same input. */
+class TokenTemplateTest {
+
+ private static final String[] NAMES = {"a", "b", "name", ""};
+
+ @Test
+ void fillsInTokens() {
+ assertEquals(
+ "Dear Ann, you live in Ghent.",
+ fill("Dear ${name}, you live in ${city}.", "${", "}", "name", "Ann",
"city", "Ghent"));
+ }
+
+ @Test
+ void keepsTokensWithoutAValueAsWritten() {
+ assertEquals("Hi ${nobody} and ${}", fill("Hi ${nobody} and ${}", "${",
"}", "a", "x"));
+ }
+
+ @Test
+ void keepsAnUnfinishedTokenWithTheRestOfTheText() {
+ assertEquals("Hi x ${a and more", fill("Hi ${a} ${a and more", "${", "}",
"a", "x"));
+ }
+
+ @Test
+ void emptyValueAtTheStartKeepsTheRestOfTheText() {
+ // The old reader returned nothing at all for this row
+ assertEquals(" lives here", fill("${name} lives here", "${", "}", "name",
""));
+ }
+
+ @Test
+ void doesNotLookForTokensInValues() {
+ assertEquals("<${b}>", fill("<${a}>", "${", "}", "a", "${b}", "b",
"nested"));
+ }
+
+ @Test
+ void refusesEmptyMarkers() {
+ Map<String, Integer> positions = Map.of();
+ assertThrows(
+ IllegalArgumentException.class, () -> TokenTemplate.parse("x", "",
"}", positions));
+ assertThrows(
+ IllegalArgumentException.class, () -> TokenTemplate.parse("x", "${",
"", positions));
+ }
+
+ @ParameterizedTest
+ @CsvSource({"${,}", "{{,}}", "[,]", "<%,%>", "$,$", "##,#"})
+ void givesTheSameAsTheReaderForRandomTemplates(String start, String end)
throws IOException {
+ Random random = new Random(start.hashCode() * 31L + end.hashCode());
+ String alphabet = "ab xyz" + start + end + "name";
+ for (int run = 0; run < 5_000; run++) {
+ String template = randomText(random, alphabet, random.nextInt(40));
+
+ // Values may hold markers and token names too; some are missing (null).
No empty values:
+ // the old reader ends its output at an empty value at the start of a
read, see
+ // emptyValueAtTheStartKeepsTheRestOfTheText
+ Map<String, Integer> positions = new HashMap<>();
+ String[] values = new String[NAMES.length];
+ TokenResolver resolver = new TokenResolver();
+ for (int i = 0; i < NAMES.length; i++) {
+ positions.put(NAMES[i], i);
+ values[i] =
+ random.nextInt(5) == 0 ? null : randomText(random, alphabet, 1 +
random.nextInt(7));
+ resolver.addToken(NAMES[i], values[i]);
+ }
+
+ StringBuilder actual = new StringBuilder();
+ TokenTemplate.parse(template, start, end, positions).appendTo(values,
actual);
+
+ assertEquals(
+ readThroughReader(template, start, end, resolver),
+ actual.toString(),
+ "template [" + template + "] with markers " + start + " and " + end);
+ }
+ }
+
+ private static String fill(String template, String start, String end,
String... nameValues) {
+ Map<String, Integer> positions = new HashMap<>();
+ String[] values = new String[nameValues.length / 2];
+ for (int i = 0; i < values.length; i++) {
+ positions.put(nameValues[i * 2], i);
+ values[i] = nameValues[i * 2 + 1];
+ }
+ StringBuilder output = new StringBuilder();
+ TokenTemplate.parse(template, start, end, positions).appendTo(values,
output);
+ return output.toString();
+ }
+
+ private static String readThroughReader(
+ String template, String start, String end, TokenResolver resolver)
throws IOException {
+ StringBuilder output = new StringBuilder();
+ try (TokenReplacingReader reader =
+ new TokenReplacingReader(resolver, new StringReader(template), start,
end)) {
+ char[] buffer = new char[7];
+ int length;
+ while ((length = reader.read(buffer)) > 0) {
+ output.append(buffer, 0, length);
+ }
+ }
+ return output.toString();
+ }
+
+ private static String randomText(Random random, String alphabet, int length)
{
+ StringBuilder text = new StringBuilder();
+ for (int i = 0; i < length; i++) {
+ text.append(alphabet.charAt(random.nextInt(alphabet.length())));
+ }
+ return text.toString();
+ }
+}