This is an automated email from the ASF dual-hosted git repository.

bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git


The following commit(s) were added to refs/heads/main by this push:
     new 237d8d8bdb Calculator gets lazily converted operands as values, fixes 
#9507 (#9516)
237d8d8bdb is described below

commit 237d8d8bdbcc7f3e598b239487553253cbf34d72
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Tue Nov 17 09:36:15 2026 +0100

    Calculator gets lazily converted operands as values, fixes #9507 (#9516)
    
    * Calculator gets lazily converted operands as values, fixes #9507
    
    * fix remarks
---
 .../transforms/0011-calculator-lazy-conversion.hpl | 216 +++++++++++++++++++++
 .../datasets/golden-calculator-lazy-conversion.csv |   3 +
 .../files/calculator-lazy-conversion.csv           |   3 +
 .../transforms/main-0011-calculator.hwf            |   3 +
 .../dataset/golden-calculator-lazy-conversion.json |  24 +++
 .../0011-calculator-lazy-conversion UNIT.json      |  33 ++++
 .../pipeline/transforms/calculator/Calculator.java |  47 ++++-
 .../transforms/calculator/CalculatorUnitTest.java  | 101 ++++++++++
 8 files changed, 424 insertions(+), 6 deletions(-)

diff --git a/integration-tests/transforms/0011-calculator-lazy-conversion.hpl 
b/integration-tests/transforms/0011-calculator-lazy-conversion.hpl
new file mode 100644
index 0000000000..2448d3bc46
--- /dev/null
+++ b/integration-tests/transforms/0011-calculator-lazy-conversion.hpl
@@ -0,0 +1,216 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+  <info>
+    <pipeline_version/>
+    <capture_transform_performance>N</capture_transform_performance>
+    
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+    
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+    <pipeline_status>0</pipeline_status>
+    <parameters/>
+    <name>0011-calculator-lazy-conversion</name>
+    <name_sync_with_filename>Y</name_sync_with_filename>
+    <description>CSV file input with lazy conversion keeps the bytes read from 
the file. The
+      calculations get the values: lower case of a String field, and the sum 
of two Integer
+      fields, one of them empty.</description>
+    <extended_description/>
+    <created_user>-</created_user>
+    <modified_user>-</modified_user>
+    <created_date>2026/11/17 09:00:00.000</created_date>
+    <modified_date>2026/11/17 09:00:00.000</modified_date>
+    <created_hop_version/>
+    <modified_hop_version>2.20.0-SNAPSHOT</modified_hop_version>
+  </info>
+  <transform>
+    <type>CSVInput</type>
+    <name>Lazy CSV</name>
+    <filename>${PROJECT_HOME}/files/calculator-lazy-conversion.csv</filename>
+    <filename_field/>
+    <include_filename>N</include_filename>
+    <rownum_field/>
+    <header>Y</header>
+    <separator>,</separator>
+    <enclosure>"</enclosure>
+    <buffer_size>50000</buffer_size>
+    <lazy_conversion>Y</lazy_conversion>
+    <add_filename_result>N</add_filename_result>
+    <parallel>N</parallel>
+    <encoding>UTF-8</encoding>
+    <newline_possible>N</newline_possible>
+    <schemaDefinition/>
+    <ignoreFields>N</ignoreFields>
+    <namingScheme/>
+    <fields>
+      <field>
+        <name>text</name>
+        <length>-1</length>
+        <type>String</type>
+        <format/>
+        <trim_type>none</trim_type>
+        <precision>-1</precision>
+        <currency/>
+        <decimal/>
+        <group/>
+      </field>
+      <field>
+        <name>a</name>
+        <length>-1</length>
+        <type>Integer</type>
+        <format/>
+        <trim_type>none</trim_type>
+        <precision>-1</precision>
+        <currency/>
+        <decimal/>
+        <group/>
+      </field>
+      <field>
+        <name>b</name>
+        <length>-1</length>
+        <type>Integer</type>
+        <format/>
+        <trim_type>none</trim_type>
+        <precision>-1</precision>
+        <currency/>
+        <decimal/>
+        <group/>
+      </field>
+    </fields>
+    <distribute>Y</distribute>
+    <copies>1</copies>
+    <GUI>
+      <xloc>128</xloc>
+      <yloc>96</yloc>
+    </GUI>
+    <description/>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <attributes/>
+  </transform>
+  <transform>
+    <type>Calculator</type>
+    <name>calculate</name>
+    <failIfNoFile>Y</failIfNoFile>
+    <calculation>
+      <calc_type>LOWER_CASE</calc_type>
+      <conversion_mask/>
+      <currency_symbol/>
+      <decimal_symbol/>
+      <field_a>text</field_a>
+      <field_b/>
+      <field_c/>
+      <field_name>lower</field_name>
+      <grouping_symbol/>
+      <remove>N</remove>
+      <value_length>-1</value_length>
+      <value_precision>-1</value_precision>
+      <value_type>String</value_type>
+    </calculation>
+    <calculation>
+      <calc_type>ADD</calc_type>
+      <conversion_mask/>
+      <currency_symbol/>
+      <decimal_symbol/>
+      <field_a>a</field_a>
+      <field_b>b</field_b>
+      <field_c/>
+      <field_name>sum</field_name>
+      <grouping_symbol/>
+      <remove>N</remove>
+      <value_length>-1</value_length>
+      <value_precision>-1</value_precision>
+      <value_type>Integer</value_type>
+    </calculation>
+    <distribute>Y</distribute>
+    <copies>1</copies>
+    <GUI>
+      <xloc>320</xloc>
+      <yloc>96</yloc>
+    </GUI>
+    <description/>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <attributes/>
+  </transform>
+  <transform>
+    <type>SelectValues</type>
+    <name>clean</name>
+    <fields>
+      <field>
+        <name>lower</name>
+      </field>
+      <field>
+        <name>sum</name>
+      </field>
+      <select_unspecified>N</select_unspecified>
+    </fields>
+    <distribute>Y</distribute>
+    <copies>1</copies>
+    <GUI>
+      <xloc>512</xloc>
+      <yloc>96</yloc>
+    </GUI>
+    <description/>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <attributes/>
+  </transform>
+  <transform>
+    <type>Dummy</type>
+    <name>verify</name>
+    <distribute>Y</distribute>
+    <copies>1</copies>
+    <GUI>
+      <xloc>704</xloc>
+      <yloc>96</yloc>
+    </GUI>
+    <description/>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <attributes/>
+  </transform>
+  <order>
+    <hop>
+      <from>Lazy CSV</from>
+      <to>calculate</to>
+      <enabled>Y</enabled>
+    </hop>
+    <hop>
+      <from>calculate</from>
+      <to>clean</to>
+      <enabled>Y</enabled>
+    </hop>
+    <hop>
+      <from>clean</from>
+      <to>verify</to>
+      <enabled>Y</enabled>
+    </hop>
+  </order>
+  <notepads/>
+  <attributes/>
+  <transform_error_handling/>
+</pipeline>
diff --git 
a/integration-tests/transforms/datasets/golden-calculator-lazy-conversion.csv 
b/integration-tests/transforms/datasets/golden-calculator-lazy-conversion.csv
new file mode 100644
index 0000000000..5c74415058
--- /dev/null
+++ 
b/integration-tests/transforms/datasets/golden-calculator-lazy-conversion.csv
@@ -0,0 +1,3 @@
+lower,sum
+mixed case,7
+,
diff --git a/integration-tests/transforms/files/calculator-lazy-conversion.csv 
b/integration-tests/transforms/files/calculator-lazy-conversion.csv
new file mode 100644
index 0000000000..43e7ea8cdd
--- /dev/null
+++ b/integration-tests/transforms/files/calculator-lazy-conversion.csv
@@ -0,0 +1,3 @@
+text,a,b
+MiXed Case,3,4
+,,5
diff --git a/integration-tests/transforms/main-0011-calculator.hwf 
b/integration-tests/transforms/main-0011-calculator.hwf
index f48a67f054..b77f057d8e 100644
--- a/integration-tests/transforms/main-0011-calculator.hwf
+++ b/integration-tests/transforms/main-0011-calculator.hwf
@@ -65,6 +65,9 @@ limitations under the License.
         <test_name>
           <name>0011-calculator-dates UNIT</name>
         </test_name>
+        <test_name>
+          <name>0011-calculator-lazy-conversion UNIT</name>
+        </test_name>
       </test_names>
       <parallel>N</parallel>
       <xloc>320</xloc>
diff --git 
a/integration-tests/transforms/metadata/dataset/golden-calculator-lazy-conversion.json
 
b/integration-tests/transforms/metadata/dataset/golden-calculator-lazy-conversion.json
new file mode 100644
index 0000000000..7b77749662
--- /dev/null
+++ 
b/integration-tests/transforms/metadata/dataset/golden-calculator-lazy-conversion.json
@@ -0,0 +1,24 @@
+{
+  "base_filename": "golden-calculator-lazy-conversion.csv",
+  "name": "golden-calculator-lazy-conversion",
+  "description": "",
+  "dataset_fields": [
+    {
+      "field_comment": "",
+      "field_length": -1,
+      "field_type": 2,
+      "field_precision": -1,
+      "field_format": "",
+      "field_name": "lower"
+    },
+    {
+      "field_comment": "",
+      "field_length": -1,
+      "field_type": 5,
+      "field_precision": 0,
+      "field_format": "####0;-####0",
+      "field_name": "sum"
+    }
+  ],
+  "folder_name": ""
+}
diff --git 
a/integration-tests/transforms/metadata/unit-test/0011-calculator-lazy-conversion
 UNIT.json 
b/integration-tests/transforms/metadata/unit-test/0011-calculator-lazy-conversion
 UNIT.json
new file mode 100644
index 0000000000..bd24d09932
--- /dev/null
+++ 
b/integration-tests/transforms/metadata/unit-test/0011-calculator-lazy-conversion
 UNIT.json 
@@ -0,0 +1,33 @@
+{
+  "variableValues": [],
+  "database_replacements": [],
+  "autoOpening": true,
+  "basePath": "",
+  "golden_data_sets": [
+    {
+      "field_mappings": [
+        {
+          "transform_field": "lower",
+          "data_set_field": "lower"
+        },
+        {
+          "transform_field": "sum",
+          "data_set_field": "sum"
+        }
+      ],
+      "field_order": [
+        "lower",
+        "sum"
+      ],
+      "transform_name": "verify",
+      "data_set_name": "golden-calculator-lazy-conversion"
+    }
+  ],
+  "input_data_sets": [],
+  "name": "0011-calculator-lazy-conversion UNIT",
+  "description": "",
+  "trans_test_tweaks": [],
+  "persist_filename": "",
+  "pipeline_filename": "./0011-calculator-lazy-conversion.hpl",
+  "test_type": "UNIT_TEST"
+}
diff --git 
a/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/Calculator.java
 
b/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/Calculator.java
index e3658443a1..a058980316 100644
--- 
a/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/Calculator.java
+++ 
b/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/Calculator.java
@@ -37,6 +37,9 @@ import org.apache.hop.pipeline.transform.TransformMeta;
 @SuppressWarnings("java:S1104")
 public class Calculator extends BaseTransform<CalculatorMeta, CalculatorData> {
 
+  /** Per field: its metadata in normal storage, for a lazily converted field. 
*/
+  private IValueMeta[] normalStorageMetas;
+
   private static final Class<?> PKG = CalculatorMeta.class;
   public static final String CONST_FOR_CALCULATION = " for calculation #";
 
@@ -196,6 +199,35 @@ public class Calculator extends 
BaseTransform<CalculatorMeta, CalculatorData> {
     return true;
   }
 
+  /**
+   * The metadata of a field as a calculation sees it: in normal storage. A 
lazily converted field
+   * (CSV input with lazy conversion) keeps its raw bytes in the row that goes 
on, but a calculation
+   * works on the value, so it gets the value and metadata in normal storage.
+   */
+  private IValueMeta normalStorageMeta(int index) {
+    IValueMeta valueMeta = data.getCalcRowMeta().getValueMeta(index);
+    if (valueMeta.getStorageType() == IValueMeta.STORAGE_TYPE_NORMAL) {
+      return valueMeta;
+    }
+    if (normalStorageMetas == null) {
+      normalStorageMetas = new IValueMeta[data.getCalcRowMeta().size()];
+    }
+    if (normalStorageMetas[index] == null) {
+      IValueMeta normal = valueMeta.clone();
+      normal.setStorageType(IValueMeta.STORAGE_TYPE_NORMAL);
+      normalStorageMetas[index] = normal;
+    }
+    return normalStorageMetas[index];
+  }
+
+  private static Object toNormalStorage(IValueMeta valueMeta, Object value)
+      throws HopValueException {
+    if (value == null || valueMeta.getStorageType() == 
IValueMeta.STORAGE_TYPE_NORMAL) {
+      return value;
+    }
+    return valueMeta.convertToNormalStorageType(value);
+  }
+
   /**
    * @param inputRowMeta the input row metadata
    * @param r the input row (data)
@@ -216,24 +248,27 @@ public class Calculator extends 
BaseTransform<CalculatorMeta, CalculatorData> {
         Object dataA = null;
 
         if (data.getFieldIndexes()[i].indexA >= 0) {
-          metaA = 
data.getCalcRowMeta().getValueMeta(data.getFieldIndexes()[i].indexA);
-          dataA = calcData[data.getFieldIndexes()[i].indexA];
+          int indexA = data.getFieldIndexes()[i].indexA;
+          metaA = normalStorageMeta(indexA);
+          dataA = toNormalStorage(data.getCalcRowMeta().getValueMeta(indexA), 
calcData[indexA]);
         }
 
         IValueMeta metaB = null;
         Object dataB = null;
 
         if (data.getFieldIndexes()[i].indexB >= 0) {
-          metaB = 
data.getCalcRowMeta().getValueMeta(data.getFieldIndexes()[i].indexB);
-          dataB = calcData[data.getFieldIndexes()[i].indexB];
+          int indexB = data.getFieldIndexes()[i].indexB;
+          metaB = normalStorageMeta(indexB);
+          dataB = toNormalStorage(data.getCalcRowMeta().getValueMeta(indexB), 
calcData[indexB]);
         }
 
         IValueMeta metaC = null;
         Object dataC = null;
 
         if (data.getFieldIndexes()[i].indexC >= 0) {
-          metaC = 
data.getCalcRowMeta().getValueMeta(data.getFieldIndexes()[i].indexC);
-          dataC = calcData[data.getFieldIndexes()[i].indexC];
+          int indexC = data.getFieldIndexes()[i].indexC;
+          metaC = normalStorageMeta(indexC);
+          dataC = toNormalStorage(data.getCalcRowMeta().getValueMeta(indexC), 
calcData[indexC]);
         }
 
         CalculationType calcType = fn.getCalcType();
diff --git 
a/plugins/transforms/calculator/src/test/java/org/apache/hop/pipeline/transforms/calculator/CalculatorUnitTest.java
 
b/plugins/transforms/calculator/src/test/java/org/apache/hop/pipeline/transforms/calculator/CalculatorUnitTest.java
index 727ccf767c..ba20cc09be 100644
--- 
a/plugins/transforms/calculator/src/test/java/org/apache/hop/pipeline/transforms/calculator/CalculatorUnitTest.java
+++ 
b/plugins/transforms/calculator/src/test/java/org/apache/hop/pipeline/transforms/calculator/CalculatorUnitTest.java
@@ -30,9 +30,11 @@ import static org.mockito.Mockito.when;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import org.apache.hop.core.Const;
 import org.apache.hop.core.HopEnvironment;
@@ -1015,4 +1017,103 @@ class CalculatorUnitTest {
       fail();
     }
   }
+
+  /**
+   * After a CSV file input with lazy conversion the fields hold the bytes 
read from the file. The
+   * calculations get the values, as they do without lazy conversion.
+   */
+  @Test
+  void lazilyConvertedOperandsGiveTheResultsOfConvertedOnes() throws 
HopException {
+    String[][] text = {{"MiXed Case", "3", "4"}, {"", "", "5"}};
+
+    List<List<Object>> converted = calculate(text, false);
+    List<List<Object>> lazy = calculate(text, true);
+
+    assertEquals("mixed case", converted.get(0).get(0));
+    assertEquals(7L, converted.get(0).get(1));
+    assertEquals(converted, lazy);
+  }
+
+  /** Lower case of the text field and the sum of the two numbers, for every 
row. */
+  private List<List<Object>> calculate(String[][] text, boolean lazy) throws 
HopException {
+    RowMeta inputRowMeta = new RowMeta();
+    inputRowMeta.addValueMeta(new ValueMetaString("text"));
+    inputRowMeta.addValueMeta(new ValueMetaInteger("a"));
+    inputRowMeta.addValueMeta(new ValueMetaInteger("b"));
+    for (IValueMeta valueMeta : inputRowMeta.getValueMetaList()) {
+      // As CSV file input reads a field with lazy conversion
+      valueMeta.setStorageType(IValueMeta.STORAGE_TYPE_BINARY_STRING);
+      
valueMeta.setStorageMetadata(ValueMetaFactory.createParseMeta(valueMeta));
+    }
+    Object[][] rows = new Object[text.length][];
+    for (int r = 0; r < text.length; r++) {
+      rows[r] = new Object[text[r].length];
+      for (int f = 0; f < text[r].length; f++) {
+        byte[] bytes = text[r][f].getBytes(StandardCharsets.UTF_8);
+        rows[r][f] = lazy ? bytes : 
inputRowMeta.getValueMeta(f).convertToNormalStorageType(bytes);
+      }
+    }
+    if (!lazy) {
+      for (IValueMeta valueMeta : inputRowMeta.getValueMetaList()) {
+        valueMeta.setStorageType(IValueMeta.STORAGE_TYPE_NORMAL);
+      }
+    }
+    IRowSet inputRowSet = smh.getMockInputRowSet(rows);
+    inputRowSet.setRowMeta(inputRowMeta);
+
+    CalculatorMeta meta = new CalculatorMeta();
+    meta.getFunctions()
+        .add(
+            new CalculatorMetaFunction(
+                "lower",
+                CalculationType.LOWER_CASE,
+                "text",
+                null,
+                null,
+                "String",
+                0,
+                0,
+                "",
+                "",
+                "",
+                "",
+                false));
+    meta.getFunctions()
+        .add(
+            new CalculatorMetaFunction(
+                "sum",
+                CalculationType.ADD,
+                "a",
+                "b",
+                null,
+                "Integer",
+                0,
+                0,
+                "",
+                "",
+                "",
+                "",
+                false));
+
+    Calculator calculator =
+        new Calculator(
+            smh.transformMeta, meta, new CalculatorData(), 0, 
smh.pipelineMeta, smh.pipeline);
+    calculator.addRowSetToInputRowSets(inputRowSet);
+    calculator.setInputRowMeta(inputRowMeta);
+    calculator.init();
+
+    List<List<Object>> results = new ArrayList<>();
+    calculator.addRowListener(
+        new RowAdapter() {
+          @Override
+          public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) {
+            results.add(Arrays.asList(row[3], row[4]));
+          }
+        });
+    while (calculator.processRow()) {
+      // one row per call
+    }
+    assertEquals(text.length, results.size());
+    return results;
+  }
 }

Reply via email to