This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 237d8d8bdb Calculator gets lazily converted operands as values, fixes
#9507 (#9516)
237d8d8bdb is described below
commit 237d8d8bdbcc7f3e598b239487553253cbf34d72
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Tue Nov 17 09:36:15 2026 +0100
Calculator gets lazily converted operands as values, fixes #9507 (#9516)
* Calculator gets lazily converted operands as values, fixes #9507
* fix remarks
---
.../transforms/0011-calculator-lazy-conversion.hpl | 216 +++++++++++++++++++++
.../datasets/golden-calculator-lazy-conversion.csv | 3 +
.../files/calculator-lazy-conversion.csv | 3 +
.../transforms/main-0011-calculator.hwf | 3 +
.../dataset/golden-calculator-lazy-conversion.json | 24 +++
.../0011-calculator-lazy-conversion UNIT.json | 33 ++++
.../pipeline/transforms/calculator/Calculator.java | 47 ++++-
.../transforms/calculator/CalculatorUnitTest.java | 101 ++++++++++
8 files changed, 424 insertions(+), 6 deletions(-)
diff --git a/integration-tests/transforms/0011-calculator-lazy-conversion.hpl
b/integration-tests/transforms/0011-calculator-lazy-conversion.hpl
new file mode 100644
index 0000000000..2448d3bc46
--- /dev/null
+++ b/integration-tests/transforms/0011-calculator-lazy-conversion.hpl
@@ -0,0 +1,216 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <pipeline_version/>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <pipeline_status>0</pipeline_status>
+ <parameters/>
+ <name>0011-calculator-lazy-conversion</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>CSV file input with lazy conversion keeps the bytes read from
the file. The
+ calculations get the values: lower case of a String field, and the sum
of two Integer
+ fields, one of them empty.</description>
+ <extended_description/>
+ <created_user>-</created_user>
+ <modified_user>-</modified_user>
+ <created_date>2026/11/17 09:00:00.000</created_date>
+ <modified_date>2026/11/17 09:00:00.000</modified_date>
+ <created_hop_version/>
+ <modified_hop_version>2.20.0-SNAPSHOT</modified_hop_version>
+ </info>
+ <transform>
+ <type>CSVInput</type>
+ <name>Lazy CSV</name>
+ <filename>${PROJECT_HOME}/files/calculator-lazy-conversion.csv</filename>
+ <filename_field/>
+ <include_filename>N</include_filename>
+ <rownum_field/>
+ <header>Y</header>
+ <separator>,</separator>
+ <enclosure>"</enclosure>
+ <buffer_size>50000</buffer_size>
+ <lazy_conversion>Y</lazy_conversion>
+ <add_filename_result>N</add_filename_result>
+ <parallel>N</parallel>
+ <encoding>UTF-8</encoding>
+ <newline_possible>N</newline_possible>
+ <schemaDefinition/>
+ <ignoreFields>N</ignoreFields>
+ <namingScheme/>
+ <fields>
+ <field>
+ <name>text</name>
+ <length>-1</length>
+ <type>String</type>
+ <format/>
+ <trim_type>none</trim_type>
+ <precision>-1</precision>
+ <currency/>
+ <decimal/>
+ <group/>
+ </field>
+ <field>
+ <name>a</name>
+ <length>-1</length>
+ <type>Integer</type>
+ <format/>
+ <trim_type>none</trim_type>
+ <precision>-1</precision>
+ <currency/>
+ <decimal/>
+ <group/>
+ </field>
+ <field>
+ <name>b</name>
+ <length>-1</length>
+ <type>Integer</type>
+ <format/>
+ <trim_type>none</trim_type>
+ <precision>-1</precision>
+ <currency/>
+ <decimal/>
+ <group/>
+ </field>
+ </fields>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>128</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>Calculator</type>
+ <name>calculate</name>
+ <failIfNoFile>Y</failIfNoFile>
+ <calculation>
+ <calc_type>LOWER_CASE</calc_type>
+ <conversion_mask/>
+ <currency_symbol/>
+ <decimal_symbol/>
+ <field_a>text</field_a>
+ <field_b/>
+ <field_c/>
+ <field_name>lower</field_name>
+ <grouping_symbol/>
+ <remove>N</remove>
+ <value_length>-1</value_length>
+ <value_precision>-1</value_precision>
+ <value_type>String</value_type>
+ </calculation>
+ <calculation>
+ <calc_type>ADD</calc_type>
+ <conversion_mask/>
+ <currency_symbol/>
+ <decimal_symbol/>
+ <field_a>a</field_a>
+ <field_b>b</field_b>
+ <field_c/>
+ <field_name>sum</field_name>
+ <grouping_symbol/>
+ <remove>N</remove>
+ <value_length>-1</value_length>
+ <value_precision>-1</value_precision>
+ <value_type>Integer</value_type>
+ </calculation>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>320</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>SelectValues</type>
+ <name>clean</name>
+ <fields>
+ <field>
+ <name>lower</name>
+ </field>
+ <field>
+ <name>sum</name>
+ </field>
+ <select_unspecified>N</select_unspecified>
+ </fields>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>512</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>Dummy</type>
+ <name>verify</name>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>704</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <order>
+ <hop>
+ <from>Lazy CSV</from>
+ <to>calculate</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>calculate</from>
+ <to>clean</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>clean</from>
+ <to>verify</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <notepads/>
+ <attributes/>
+ <transform_error_handling/>
+</pipeline>
diff --git
a/integration-tests/transforms/datasets/golden-calculator-lazy-conversion.csv
b/integration-tests/transforms/datasets/golden-calculator-lazy-conversion.csv
new file mode 100644
index 0000000000..5c74415058
--- /dev/null
+++
b/integration-tests/transforms/datasets/golden-calculator-lazy-conversion.csv
@@ -0,0 +1,3 @@
+lower,sum
+mixed case,7
+,
diff --git a/integration-tests/transforms/files/calculator-lazy-conversion.csv
b/integration-tests/transforms/files/calculator-lazy-conversion.csv
new file mode 100644
index 0000000000..43e7ea8cdd
--- /dev/null
+++ b/integration-tests/transforms/files/calculator-lazy-conversion.csv
@@ -0,0 +1,3 @@
+text,a,b
+MiXed Case,3,4
+,,5
diff --git a/integration-tests/transforms/main-0011-calculator.hwf
b/integration-tests/transforms/main-0011-calculator.hwf
index f48a67f054..b77f057d8e 100644
--- a/integration-tests/transforms/main-0011-calculator.hwf
+++ b/integration-tests/transforms/main-0011-calculator.hwf
@@ -65,6 +65,9 @@ limitations under the License.
<test_name>
<name>0011-calculator-dates UNIT</name>
</test_name>
+ <test_name>
+ <name>0011-calculator-lazy-conversion UNIT</name>
+ </test_name>
</test_names>
<parallel>N</parallel>
<xloc>320</xloc>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-calculator-lazy-conversion.json
b/integration-tests/transforms/metadata/dataset/golden-calculator-lazy-conversion.json
new file mode 100644
index 0000000000..7b77749662
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-calculator-lazy-conversion.json
@@ -0,0 +1,24 @@
+{
+ "base_filename": "golden-calculator-lazy-conversion.csv",
+ "name": "golden-calculator-lazy-conversion",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "lower"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_format": "####0;-####0",
+ "field_name": "sum"
+ }
+ ],
+ "folder_name": ""
+}
diff --git
a/integration-tests/transforms/metadata/unit-test/0011-calculator-lazy-conversion
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0011-calculator-lazy-conversion
UNIT.json
new file mode 100644
index 0000000000..bd24d09932
--- /dev/null
+++
b/integration-tests/transforms/metadata/unit-test/0011-calculator-lazy-conversion
UNIT.json
@@ -0,0 +1,33 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": true,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "lower",
+ "data_set_field": "lower"
+ },
+ {
+ "transform_field": "sum",
+ "data_set_field": "sum"
+ }
+ ],
+ "field_order": [
+ "lower",
+ "sum"
+ ],
+ "transform_name": "verify",
+ "data_set_name": "golden-calculator-lazy-conversion"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0011-calculator-lazy-conversion UNIT",
+ "description": "",
+ "trans_test_tweaks": [],
+ "persist_filename": "",
+ "pipeline_filename": "./0011-calculator-lazy-conversion.hpl",
+ "test_type": "UNIT_TEST"
+}
diff --git
a/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/Calculator.java
b/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/Calculator.java
index e3658443a1..a058980316 100644
---
a/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/Calculator.java
+++
b/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/Calculator.java
@@ -37,6 +37,9 @@ import org.apache.hop.pipeline.transform.TransformMeta;
@SuppressWarnings("java:S1104")
public class Calculator extends BaseTransform<CalculatorMeta, CalculatorData> {
+ /** Per field: its metadata in normal storage, for a lazily converted field.
*/
+ private IValueMeta[] normalStorageMetas;
+
private static final Class<?> PKG = CalculatorMeta.class;
public static final String CONST_FOR_CALCULATION = " for calculation #";
@@ -196,6 +199,35 @@ public class Calculator extends
BaseTransform<CalculatorMeta, CalculatorData> {
return true;
}
+ /**
+ * The metadata of a field as a calculation sees it: in normal storage. A
lazily converted field
+ * (CSV input with lazy conversion) keeps its raw bytes in the row that goes
on, but a calculation
+ * works on the value, so it gets the value and metadata in normal storage.
+ */
+ private IValueMeta normalStorageMeta(int index) {
+ IValueMeta valueMeta = data.getCalcRowMeta().getValueMeta(index);
+ if (valueMeta.getStorageType() == IValueMeta.STORAGE_TYPE_NORMAL) {
+ return valueMeta;
+ }
+ if (normalStorageMetas == null) {
+ normalStorageMetas = new IValueMeta[data.getCalcRowMeta().size()];
+ }
+ if (normalStorageMetas[index] == null) {
+ IValueMeta normal = valueMeta.clone();
+ normal.setStorageType(IValueMeta.STORAGE_TYPE_NORMAL);
+ normalStorageMetas[index] = normal;
+ }
+ return normalStorageMetas[index];
+ }
+
+ private static Object toNormalStorage(IValueMeta valueMeta, Object value)
+ throws HopValueException {
+ if (value == null || valueMeta.getStorageType() ==
IValueMeta.STORAGE_TYPE_NORMAL) {
+ return value;
+ }
+ return valueMeta.convertToNormalStorageType(value);
+ }
+
/**
* @param inputRowMeta the input row metadata
* @param r the input row (data)
@@ -216,24 +248,27 @@ public class Calculator extends
BaseTransform<CalculatorMeta, CalculatorData> {
Object dataA = null;
if (data.getFieldIndexes()[i].indexA >= 0) {
- metaA =
data.getCalcRowMeta().getValueMeta(data.getFieldIndexes()[i].indexA);
- dataA = calcData[data.getFieldIndexes()[i].indexA];
+ int indexA = data.getFieldIndexes()[i].indexA;
+ metaA = normalStorageMeta(indexA);
+ dataA = toNormalStorage(data.getCalcRowMeta().getValueMeta(indexA),
calcData[indexA]);
}
IValueMeta metaB = null;
Object dataB = null;
if (data.getFieldIndexes()[i].indexB >= 0) {
- metaB =
data.getCalcRowMeta().getValueMeta(data.getFieldIndexes()[i].indexB);
- dataB = calcData[data.getFieldIndexes()[i].indexB];
+ int indexB = data.getFieldIndexes()[i].indexB;
+ metaB = normalStorageMeta(indexB);
+ dataB = toNormalStorage(data.getCalcRowMeta().getValueMeta(indexB),
calcData[indexB]);
}
IValueMeta metaC = null;
Object dataC = null;
if (data.getFieldIndexes()[i].indexC >= 0) {
- metaC =
data.getCalcRowMeta().getValueMeta(data.getFieldIndexes()[i].indexC);
- dataC = calcData[data.getFieldIndexes()[i].indexC];
+ int indexC = data.getFieldIndexes()[i].indexC;
+ metaC = normalStorageMeta(indexC);
+ dataC = toNormalStorage(data.getCalcRowMeta().getValueMeta(indexC),
calcData[indexC]);
}
CalculationType calcType = fn.getCalcType();
diff --git
a/plugins/transforms/calculator/src/test/java/org/apache/hop/pipeline/transforms/calculator/CalculatorUnitTest.java
b/plugins/transforms/calculator/src/test/java/org/apache/hop/pipeline/transforms/calculator/CalculatorUnitTest.java
index 727ccf767c..ba20cc09be 100644
---
a/plugins/transforms/calculator/src/test/java/org/apache/hop/pipeline/transforms/calculator/CalculatorUnitTest.java
+++
b/plugins/transforms/calculator/src/test/java/org/apache/hop/pipeline/transforms/calculator/CalculatorUnitTest.java
@@ -30,9 +30,11 @@ import static org.mockito.Mockito.when;
import java.math.BigDecimal;
import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import org.apache.hop.core.Const;
import org.apache.hop.core.HopEnvironment;
@@ -1015,4 +1017,103 @@ class CalculatorUnitTest {
fail();
}
}
+
+ /**
+ * After a CSV file input with lazy conversion the fields hold the bytes
read from the file. The
+ * calculations get the values, as they do without lazy conversion.
+ */
+ @Test
+ void lazilyConvertedOperandsGiveTheResultsOfConvertedOnes() throws
HopException {
+ String[][] text = {{"MiXed Case", "3", "4"}, {"", "", "5"}};
+
+ List<List<Object>> converted = calculate(text, false);
+ List<List<Object>> lazy = calculate(text, true);
+
+ assertEquals("mixed case", converted.get(0).get(0));
+ assertEquals(7L, converted.get(0).get(1));
+ assertEquals(converted, lazy);
+ }
+
+ /** Lower case of the text field and the sum of the two numbers, for every
row. */
+ private List<List<Object>> calculate(String[][] text, boolean lazy) throws
HopException {
+ RowMeta inputRowMeta = new RowMeta();
+ inputRowMeta.addValueMeta(new ValueMetaString("text"));
+ inputRowMeta.addValueMeta(new ValueMetaInteger("a"));
+ inputRowMeta.addValueMeta(new ValueMetaInteger("b"));
+ for (IValueMeta valueMeta : inputRowMeta.getValueMetaList()) {
+ // As CSV file input reads a field with lazy conversion
+ valueMeta.setStorageType(IValueMeta.STORAGE_TYPE_BINARY_STRING);
+
valueMeta.setStorageMetadata(ValueMetaFactory.createParseMeta(valueMeta));
+ }
+ Object[][] rows = new Object[text.length][];
+ for (int r = 0; r < text.length; r++) {
+ rows[r] = new Object[text[r].length];
+ for (int f = 0; f < text[r].length; f++) {
+ byte[] bytes = text[r][f].getBytes(StandardCharsets.UTF_8);
+ rows[r][f] = lazy ? bytes :
inputRowMeta.getValueMeta(f).convertToNormalStorageType(bytes);
+ }
+ }
+ if (!lazy) {
+ for (IValueMeta valueMeta : inputRowMeta.getValueMetaList()) {
+ valueMeta.setStorageType(IValueMeta.STORAGE_TYPE_NORMAL);
+ }
+ }
+ IRowSet inputRowSet = smh.getMockInputRowSet(rows);
+ inputRowSet.setRowMeta(inputRowMeta);
+
+ CalculatorMeta meta = new CalculatorMeta();
+ meta.getFunctions()
+ .add(
+ new CalculatorMetaFunction(
+ "lower",
+ CalculationType.LOWER_CASE,
+ "text",
+ null,
+ null,
+ "String",
+ 0,
+ 0,
+ "",
+ "",
+ "",
+ "",
+ false));
+ meta.getFunctions()
+ .add(
+ new CalculatorMetaFunction(
+ "sum",
+ CalculationType.ADD,
+ "a",
+ "b",
+ null,
+ "Integer",
+ 0,
+ 0,
+ "",
+ "",
+ "",
+ "",
+ false));
+
+ Calculator calculator =
+ new Calculator(
+ smh.transformMeta, meta, new CalculatorData(), 0,
smh.pipelineMeta, smh.pipeline);
+ calculator.addRowSetToInputRowSets(inputRowSet);
+ calculator.setInputRowMeta(inputRowMeta);
+ calculator.init();
+
+ List<List<Object>> results = new ArrayList<>();
+ calculator.addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) {
+ results.add(Arrays.asList(row[3], row[4]));
+ }
+ });
+ while (calculator.processRow()) {
+ // one row per call
+ }
+ assertEquals(text.length, results.size());
+ return results;
+ }
}