This is an automated email from the ASF dual-hosted git repository.

bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git


The following commit(s) were added to refs/heads/main by this push:
     new 2975077e61 Row normaliser: normalise every input field when no fields 
are listed… (#9446)
2975077e61 is described below

commit 2975077e61626c8f223cb7ac62e63dbadd921e3a
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Sat Nov 14 12:49:06 2026 +0100

    Row normaliser: normalise every input field when no fields are listed… 
(#9446)
    
    * Row normaliser: normalise every input field when no fields are listed, 
fixes #9397
    
    * fix remarks
---
 .../pages/pipeline/transforms/rownormaliser.adoc   |  13 +-
 .../transforms/0020-row-normaliser-all-fields.hpl  | 159 +++++++++++++++
 .../datasets/golden-row-normaliser-all-fields.csv  |   7 +
 .../transforms/main-0020-row-normaliser.hwf        |   3 +
 .../dataset/golden-row-normaliser-all-fields.json  |  24 +++
 .../0020-row-normaliser-all-fields UNIT.json       |  30 +++
 .../pipeline/transforms/normaliser/Normaliser.java | 187 ++++++++----------
 .../transforms/normaliser/NormaliserData.java      |  20 +-
 .../transforms/normaliser/NormaliserDialog.java    |  49 ++++-
 .../transforms/normaliser/NormaliserMeta.java      | 203 +++++++++++++++----
 .../normaliser/messages/messages_en_US.properties  |   5 +
 .../normaliser/NormaliserMetaInjectionTest.java    |   1 +
 .../transforms/normaliser/NormaliserMetaTest.java  | 167 +++++++++++++++-
 .../transforms/normaliser/NormaliserTest.java      | 219 +++++++++++++++++++++
 .../src/test/resources/normaliser-transform.xml    |   1 +
 15 files changed, 919 insertions(+), 169 deletions(-)

diff --git 
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/rownormaliser.adoc
 
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/rownormaliser.adoc
index 8c92709504..3fe51a73b3 100644
--- 
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/rownormaliser.adoc
+++ 
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/rownormaliser.adoc
@@ -31,6 +31,12 @@ For every row it reads, the transform writes one row per 
type. Each input field
 
 A type can fill each new field from only one input field: listing two fields 
with the same type and the same new field is an error.
 
+== Normalising every input field
+
+When the **Fields table** is empty, the transform normalises every field it 
receives. Each input field is a type of its own, named after the field, and its 
value goes into the **Value field**. This works when the input fields are only 
known when the pipeline runs, for example in a pipeline that profiles any file 
or table it is given.
+
+Every input field is normalised this way, so no field is passed on unchanged: 
the output has only the type field and the value field. When the input fields 
have different data types the value field is a String, as described below. The 
value field keeps a length or precision only when all input fields have the 
same one. While you design a pipeline whose input fields are not known yet, the 
transform shows the value field to the transforms after it as a String.
+
 == Data types of the new fields
 
 A new field is filled from a different input field on every row, so its data 
type depends on all of them:
@@ -49,8 +55,9 @@ NOTE: Before Hop 2.20, a new field took the data type of the 
first input field f
 |Option|Description
 |Transform name|Name of the transform, this name has to be unique in a single 
pipeline.
 |Typefield|The name of the type field (**key** in the example below).
+|Value field|The name of the field that holds the values when the **Fields 
table** is empty and every input field is normalised. The default is **value**. 
Give it a name other than the type field's: otherwise it is renamed in the 
output, to **value_1** for example.
 |Fields table|
-A list of the fields you want to normalize; you must set the following 
properties for each selected field:
+A list of the fields you want to normalize. Leave it empty to normalise every 
input field into the **Value field**. You must set the following properties for 
each listed field:
 
 * Fieldname: Name of the fields to normalize, as you get them from the input 
transform.
 * Type: Give a string to classify the field (you can use the same field names, 
or input custom strings).
@@ -71,7 +78,7 @@ A list of the fields you want to normalize; you must set the 
following propertie
 |===
 
 === Normalized data (example 1)
-Set **Typefield** = "key" and use the **Get Fields** button to load all the 
fields for normalization, set also **New field** = "value" in all rows. The 
result is:
+Set **Typefield** = "key" and **Value field** = "value", and leave the 
**Fields table** empty. The result is the same as using the **Get Fields** 
button to list all the fields and setting **New field** = "value" in all rows, 
but it follows the fields of the input, whatever they are:
 
 [options="header"]
 |===
@@ -91,7 +98,7 @@ Set **Typefield** = "key" and use the **Get Fields** button 
to load all the fiel
 |===
 
 === Normalized data (example 2)
-Similar to example 1, but remove the **RecordID** field from the **Fields 
table**. The result is:
+Set **Typefield** = "key", use the **Get Fields** button to list all the 
fields, remove the **RecordID** row from the **Fields table** and set **New 
field** = "value" in the other rows. **RecordID** is not listed, so it is 
passed on unchanged. The result is:
 
 [options="header"]
 |===
diff --git a/integration-tests/transforms/0020-row-normaliser-all-fields.hpl 
b/integration-tests/transforms/0020-row-normaliser-all-fields.hpl
new file mode 100644
index 0000000000..9b0cf8a041
--- /dev/null
+++ b/integration-tests/transforms/0020-row-normaliser-all-fields.hpl
@@ -0,0 +1,159 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+  <info>
+    <name>0020-row-normaliser-all-fields</name>
+    <name_sync_with_filename>Y</name_sync_with_filename>
+    <description/>
+    <extended_description/>
+    <pipeline_version/>
+    <pipeline_type>Normal</pipeline_type>
+    <parameters>
+    </parameters>
+    <capture_transform_performance>N</capture_transform_performance>
+    
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+    
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+    <created_user>-</created_user>
+    <created_date>2021/05/31 11:24:13.758</created_date>
+    <modified_user>-</modified_user>
+    <modified_date>2021/05/31 11:24:13.758</modified_date>
+    <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
+    <is_key_private>N</is_key_private>
+  </info>
+  <notepads>
+  </notepads>
+  <order>
+    <hop>
+      <from>Sample data</from>
+      <to>Normalise all fields</to>
+      <enabled>Y</enabled>
+    </hop>
+    <hop>
+      <from>Normalise all fields</from>
+      <to>Verify</to>
+      <enabled>Y</enabled>
+    </hop>
+  </order>
+  <transform>
+    <name>Normalise all fields</name>
+    <type>Normaliser</type>
+    <description/>
+    <distribute>Y</distribute>
+    <custom_distribution/>
+    <copies>1</copies>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <typefield>column</typefield>
+    <valuefield>content</valuefield>
+    <fields>
+    </fields>
+    <attributes/>
+    <GUI>
+      <xloc>272</xloc>
+      <yloc>96</yloc>
+    </GUI>
+  </transform>
+  <transform>
+    <name>Sample data</name>
+    <type>DataGrid</type>
+    <description/>
+    <distribute>Y</distribute>
+    <custom_distribution/>
+    <copies>1</copies>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <fields>
+      <field>
+        <name>name</name>
+        <type>String</type>
+        <format/>
+        <currency/>
+        <decimal/>
+        <group/>
+        <length>-1</length>
+        <precision>-1</precision>
+        <set_empty_string>N</set_empty_string>
+      </field>
+      <field>
+        <name>age</name>
+        <type>Integer</type>
+        <format/>
+        <currency/>
+        <decimal/>
+        <group/>
+        <length>-1</length>
+        <precision>-1</precision>
+        <set_empty_string>N</set_empty_string>
+      </field>
+      <field>
+        <name>city</name>
+        <type>String</type>
+        <format/>
+        <currency/>
+        <decimal/>
+        <group/>
+        <length>-1</length>
+        <precision>-1</precision>
+        <set_empty_string>N</set_empty_string>
+      </field>
+    </fields>
+    <data>
+      <line>
+        <item>Ann</item>
+        <item>31</item>
+        <item>Paris</item>
+      </line>
+      <line>
+        <item>Bob</item>
+        <item/>
+        <item>Rome</item>
+      </line>
+    </data>
+    <attributes/>
+    <GUI>
+      <xloc>128</xloc>
+      <yloc>96</yloc>
+    </GUI>
+  </transform>
+  <transform>
+    <name>Verify</name>
+    <type>Dummy</type>
+    <description/>
+    <distribute>Y</distribute>
+    <custom_distribution/>
+    <copies>1</copies>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <attributes/>
+    <GUI>
+      <xloc>416</xloc>
+      <yloc>96</yloc>
+    </GUI>
+  </transform>
+  <transform_error_handling>
+  </transform_error_handling>
+  <attributes/>
+</pipeline>
diff --git 
a/integration-tests/transforms/datasets/golden-row-normaliser-all-fields.csv 
b/integration-tests/transforms/datasets/golden-row-normaliser-all-fields.csv
new file mode 100644
index 0000000000..951775c0f2
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-row-normaliser-all-fields.csv
@@ -0,0 +1,7 @@
+column,content
+name,Ann
+age,31
+city,Paris
+name,Bob
+age,
+city,Rome
diff --git a/integration-tests/transforms/main-0020-row-normaliser.hwf 
b/integration-tests/transforms/main-0020-row-normaliser.hwf
index bc6d6a3766..f0a0c7a785 100644
--- a/integration-tests/transforms/main-0020-row-normaliser.hwf
+++ b/integration-tests/transforms/main-0020-row-normaliser.hwf
@@ -63,6 +63,9 @@ limitations under the License.
         <test_name>
           <name>0020-row-normaliser-mixed-types UNIT</name>
         </test_name>
+        <test_name>
+          <name>0020-row-normaliser-all-fields UNIT</name>
+        </test_name>
       </test_names>
       <parallel>N</parallel>
       <xloc>272</xloc>
diff --git 
a/integration-tests/transforms/metadata/dataset/golden-row-normaliser-all-fields.json
 
b/integration-tests/transforms/metadata/dataset/golden-row-normaliser-all-fields.json
new file mode 100644
index 0000000000..0e62d690a6
--- /dev/null
+++ 
b/integration-tests/transforms/metadata/dataset/golden-row-normaliser-all-fields.json
@@ -0,0 +1,24 @@
+{
+  "base_filename": "golden-row-normaliser-all-fields.csv",
+  "name": "golden-row-normaliser-all-fields",
+  "description": "",
+  "dataset_fields": [
+    {
+      "field_comment": "",
+      "field_length": -1,
+      "field_type": 2,
+      "field_precision": -1,
+      "field_format": "",
+      "field_name": "column"
+    },
+    {
+      "field_comment": "",
+      "field_length": -1,
+      "field_type": 2,
+      "field_precision": -1,
+      "field_format": "",
+      "field_name": "content"
+    }
+  ],
+  "folder_name": ""
+}
\ No newline at end of file
diff --git 
a/integration-tests/transforms/metadata/unit-test/0020-row-normaliser-all-fields
 UNIT.json 
b/integration-tests/transforms/metadata/unit-test/0020-row-normaliser-all-fields
 UNIT.json
new file mode 100644
index 0000000000..9dda44bb9c
--- /dev/null
+++ 
b/integration-tests/transforms/metadata/unit-test/0020-row-normaliser-all-fields
 UNIT.json  
@@ -0,0 +1,30 @@
+{
+  "variableValues": [],
+  "database_replacements": [],
+  "autoOpening": true,
+  "basePath": "",
+  "golden_data_sets": [
+    {
+      "field_mappings": [
+        {
+          "transform_field": "column",
+          "data_set_field": "column"
+        },
+        {
+          "transform_field": "content",
+          "data_set_field": "content"
+        }
+      ],
+      "field_order": [],
+      "transform_name": "Verify",
+      "data_set_name": "golden-row-normaliser-all-fields"
+    }
+  ],
+  "input_data_sets": [],
+  "name": "0020-row-normaliser-all-fields UNIT",
+  "description": "",
+  "trans_test_tweaks": [],
+  "persist_filename": "",
+  "pipeline_filename": "./0020-row-normaliser-all-fields.hpl",
+  "test_type": "UNIT_TEST"
+}
\ No newline at end of file
diff --git 
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/Normaliser.java
 
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/Normaliser.java
index dfda2c2768..4ff54446a5 100644
--- 
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/Normaliser.java
+++ 
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/Normaliser.java
@@ -18,9 +18,10 @@
 package org.apache.hop.pipeline.transforms.normaliser;
 
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Set;
+import java.util.Map;
+import java.util.stream.IntStream;
 import org.apache.hop.core.exception.HopException;
 import org.apache.hop.core.row.IValueMeta;
 import org.apache.hop.i18n.BaseMessages;
@@ -28,6 +29,7 @@ import org.apache.hop.pipeline.Pipeline;
 import org.apache.hop.pipeline.PipelineMeta;
 import org.apache.hop.pipeline.transform.BaseTransform;
 import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.normaliser.NormaliserData.Placement;
 
 /** Normalise de-normalised input data. */
 public class Normaliser extends BaseTransform<NormaliserMeta, NormaliserData> {
@@ -53,121 +55,30 @@ public class Normaliser extends 
BaseTransform<NormaliserMeta, NormaliserData> {
       return false;
     }
 
-    if (first) { // INITIALISE
-
+    if (first) {
       first = false;
-
-      data.inputRowMeta = getInputRowMeta();
-      data.outputRowMeta = data.inputRowMeta.clone();
-      meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, 
metadataProvider);
-      data.typeToPlacements = new HashMap<>();
-      String typeValue;
-      int dataFieldNr;
-
-      // The normalised fields are the last ones of the output row, in this 
order.
-      //
-      List<String> normNames = meta.getNormalisedFieldNames();
-      int firstNormIndex = data.outputRowMeta.size() - normNames.size();
-
-      String duplicate = meta.getDuplicateMapping();
-      if (duplicate != null) {
-        throw new HopException(duplicate);
-      }
-
-      // Get a unique list of occurrences...
-      //
-      data.type_occ = new ArrayList<>();
-      data.maxlen = 0;
-
-      for (NormaliserField field : meta.getNormaliserFields()) {
-        typeValue = field.getValue();
-        if (!data.type_occ.contains(typeValue)) {
-          data.type_occ.add(typeValue);
-        }
-        if (typeValue.length() > data.maxlen) {
-          data.maxlen = typeValue.length();
-        }
-
-        // For every type, the fields that get normalised under it and where 
each one goes: into
-        // the normalised field it names, whatever order the fields are listed 
in.
-        //
-        dataFieldNr = data.inputRowMeta.indexOfValue(field.getName());
-        if (dataFieldNr < 0) {
-          logError(
-              BaseMessages.getString(
-                  PKG, "Normaliser.Log.CouldNotFindFieldInRow", 
field.getName()));
-          setErrors(1);
-          stopAll();
-          return false;
-        }
-        int outputIndex = firstNormIndex + normNames.indexOf(field.getNorm());
-        IValueMeta source = data.inputRowMeta.getValueMeta(dataFieldNr);
-        IValueMeta target = data.outputRowMeta.getValueMeta(outputIndex);
-        boolean copy =
-            source.getType() == target.getType()
-                && source.getStorageType() == target.getStorageType();
-        data.typeToPlacements
-            .computeIfAbsent(typeValue, k -> new ArrayList<>())
-            .add(
-                new NormaliserData.Placement(
-                    dataFieldNr, outputIndex, source, copy ? null : target));
-      }
-
-      // Which fields are not impacted? We can just copy these, leave them 
alone.
-      //
-      data.copyFieldnrs = new ArrayList<>();
-
-      Set<String> normaliserFields = meta.getFieldNames();
-      int irmSize = data.inputRowMeta.size();
-
-      for (int i = 0; i < irmSize; i++) {
-        IValueMeta v = data.inputRowMeta.getValueMeta(i);
-        // Backwards compatibility - old loop called Const.indexofstring which 
uses equalsIgnoreCase
-        if (!normaliserFields.contains(v.getName().toLowerCase())) {
-          data.copyFieldnrs.add(i);
-        }
+      if (!initialise()) {
+        return false;
       }
     }
 
-    // Modest performance improvement over millions of rows - don't 
recalculate on each loop
-    // iteration something that doesn't change
-    int typeOccSize = data.type_occ.size();
-    int copyFldNrsSz = data.copyFieldnrs.size();
+    String[] types = data.types;
+    Placement[][] placements = data.placements;
+    int[] copyFieldNrs = data.copyFieldNrs;
     int rowMetaSz = data.outputRowMeta.size();
 
-    // Modest performance improvement (large memory improvement) - re-use 
temporary objects instead
-    // of re-creating them - better for GC over time
-    String typeValue;
-    Object[] outputRowData;
-    int outputIndex, nr;
-    Object value;
-
-    // Now do the normalization
-    // Loop over the unique occurrences of the different types.
+    // One output row per type: the fields that are kept, the type, then the 
normalised fields.
     //
-    for (int e = 0; e < typeOccSize; e++) {
-      typeValue = data.type_occ.get(e);
-
-      // Create an output row per type
-      //
-      outputRowData = new Object[rowMetaSz];
-      outputIndex = 0;
-
-      // Copy the input row data, excluding the fields that are normalized...
-      //
-      for (int i = 0; i < copyFldNrsSz; i++) {
-        nr = data.copyFieldnrs.get(i);
+    for (int t = 0; t < types.length; t++) {
+      Object[] outputRowData = new Object[rowMetaSz];
+      int outputIndex = 0;
+      for (int nr : copyFieldNrs) {
         outputRowData[outputIndex++] = r[nr];
       }
+      outputRowData[outputIndex] = types[t];
 
-      // Add the typefield_value
-      //
-      outputRowData[outputIndex++] = typeValue;
-
-      // Then add the normalized fields...
-      //
-      for (NormaliserData.Placement placement : 
data.typeToPlacements.get(typeValue)) {
-        value = r[placement.inputIndex()];
+      for (Placement placement : placements[t]) {
+        Object value = r[placement.inputIndex()];
         IValueMeta target = placement.target();
         if (target != null) {
           // Either the same type in another storage, or text for a field of 
mixed types.
@@ -179,8 +90,6 @@ public class Normaliser extends 
BaseTransform<NormaliserMeta, NormaliserData> {
         outputRowData[placement.outputIndex()] = value;
       }
 
-      // The row is constructed, now give it to the next transform(s)...
-      //
       putRow(data.outputRowMeta, outputRowData);
     }
 
@@ -190,4 +99,64 @@ public class Normaliser extends 
BaseTransform<NormaliserMeta, NormaliserData> {
 
     return true;
   }
+
+  /**
+   * Works out, once, which fields are kept and where every normalised value 
goes.
+   *
+   * @return false when a field to normalise is missing from the input; the 
transform has stopped
+   */
+  private boolean initialise() throws HopException {
+    data.inputRowMeta = getInputRowMeta();
+    data.outputRowMeta = data.inputRowMeta.clone();
+    meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, 
metadataProvider);
+
+    List<NormaliserField> fields = 
meta.getFieldsToNormalise(data.inputRowMeta);
+
+    String duplicate = NormaliserMeta.getDuplicateMapping(fields);
+    if (duplicate != null) {
+      throw new HopException(duplicate);
+    }
+
+    // The normalised fields are the last ones of the output row, in this 
order.
+    //
+    List<String> normNames = NormaliserMeta.getNormalisedFieldNames(fields);
+    int firstNormIndex = data.outputRowMeta.size() - normNames.size();
+
+    // For every type, in the order it first appears, the fields that get 
normalised under it and
+    // where each one goes: into the normalised field it names, whatever order 
they are listed in.
+    //
+    Map<String, List<Placement>> typeToPlacements = new LinkedHashMap<>();
+    boolean[] normalised = new boolean[data.inputRowMeta.size()];
+    for (NormaliserField field : fields) {
+      int dataFieldNr = data.inputRowMeta.indexOfValue(field.getName());
+      if (dataFieldNr < 0) {
+        logError(
+            BaseMessages.getString(PKG, 
"Normaliser.Log.CouldNotFindFieldInRow", field.getName()));
+        setErrors(1);
+        stopAll();
+        return false;
+      }
+      normalised[dataFieldNr] = true;
+      int outputIndex = firstNormIndex + normNames.indexOf(field.getNorm());
+      IValueMeta source = data.inputRowMeta.getValueMeta(dataFieldNr);
+      IValueMeta target = data.outputRowMeta.getValueMeta(outputIndex);
+      boolean copy =
+          source.getType() == target.getType()
+              && source.getStorageType() == target.getStorageType();
+      typeToPlacements
+          .computeIfAbsent(field.getValue(), k -> new ArrayList<>())
+          .add(new Placement(dataFieldNr, outputIndex, source, copy ? null : 
target));
+    }
+    data.types = typeToPlacements.keySet().toArray(new String[0]);
+    data.placements =
+        typeToPlacements.values().stream()
+            .map(list -> list.toArray(new Placement[0]))
+            .toArray(Placement[][]::new);
+
+    // The fields that are not normalised are copied as they are. Which ones 
those are follows from
+    // the lookup above, the one getFields() uses to take the normalised 
fields out of the layout.
+    //
+    data.copyFieldNrs = IntStream.range(0, normalised.length).filter(i -> 
!normalised[i]).toArray();
+    return true;
+  }
 }
diff --git 
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserData.java
 
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserData.java
index 6bb2c777d8..1eee808880 100644
--- 
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserData.java
+++ 
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserData.java
@@ -17,8 +17,6 @@
 
 package org.apache.hop.pipeline.transforms.normaliser;
 
-import java.util.List;
-import java.util.Map;
 import org.apache.hop.core.row.IRowMeta;
 import org.apache.hop.core.row.IValueMeta;
 import org.apache.hop.pipeline.transform.BaseTransformData;
@@ -26,19 +24,17 @@ import org.apache.hop.pipeline.transform.ITransformData;
 
 @SuppressWarnings("java:S1104")
 public class NormaliserData extends BaseTransformData implements 
ITransformData {
-  public List<String> type_occ;
-  public int maxlen;
-  public List<Integer> copyFieldnrs;
-  Map<String, List<Placement>> typeToPlacements;
+  /** The types, in the order their rows are written. */
+  String[] types;
 
-  public IRowMeta inputRowMeta;
-  public IRowMeta outputRowMeta;
+  /** Per type, in the order of {@link #types}: where its input fields go. */
+  Placement[][] placements;
 
-  public NormaliserData() {
-    super();
+  /** The input fields that are copied to every output row, in input order. */
+  int[] copyFieldNrs;
 
-    type_occ = null;
-  }
+  public IRowMeta inputRowMeta;
+  public IRowMeta outputRowMeta;
 
   /**
    * Where one input field goes on the output row of its type.
diff --git 
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserDialog.java
 
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserDialog.java
index 65ef335092..16ee7ec63b 100644
--- 
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserDialog.java
+++ 
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserDialog.java
@@ -55,6 +55,10 @@ public class NormaliserDialog extends BaseTransformDialog {
 
   private Text wTypefield;
 
+  private Label wlValuefield;
+
+  private Text wValuefield;
+
   private TableView wFields;
 
   private final NormaliserMeta input;
@@ -97,12 +101,32 @@ public class NormaliserDialog extends BaseTransformDialog {
     fdTypefield.right = new FormAttachment(100, 0);
     wTypefield.setLayoutData(fdTypefield);
 
+    // Value field line: used when no fields are listed
+    wlValuefield = new Label(shell, SWT.RIGHT);
+    wlValuefield.setText(BaseMessages.getString(PKG, 
"NormaliserDialog.ValueField.Label"));
+    wlValuefield.setToolTipText(BaseMessages.getString(PKG, 
"NormaliserDialog.ValueField.Tooltip"));
+    PropsUi.setLook(wlValuefield);
+    FormData fdlValuefield = new FormData();
+    fdlValuefield.left = new FormAttachment(0, 0);
+    fdlValuefield.right = new FormAttachment(middle, -margin);
+    fdlValuefield.top = new FormAttachment(wTypefield, margin);
+    wlValuefield.setLayoutData(fdlValuefield);
+    wValuefield = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
+    wValuefield.setToolTipText(BaseMessages.getString(PKG, 
"NormaliserDialog.ValueField.Tooltip"));
+    PropsUi.setLook(wValuefield);
+    wValuefield.addModifyListener(lsMod);
+    FormData fdValuefield = new FormData();
+    fdValuefield.left = new FormAttachment(middle, 0);
+    fdValuefield.top = new FormAttachment(wTypefield, margin);
+    fdValuefield.right = new FormAttachment(100, 0);
+    wValuefield.setLayoutData(fdValuefield);
+
     Label wlFields = new Label(shell, SWT.NONE);
     wlFields.setText(BaseMessages.getString(PKG, 
"NormaliserDialog.Fields.Label"));
     PropsUi.setLook(wlFields);
     FormData fdlFields = new FormData();
     fdlFields.left = new FormAttachment(0, 0);
-    fdlFields.top = new FormAttachment(wTypefield, margin);
+    fdlFields.top = new FormAttachment(wValuefield, margin);
     wlFields.setLayoutData(fdlFields);
 
     final int fieldsCols = 3;
@@ -126,6 +150,12 @@ public class NormaliserDialog extends BaseTransformDialog {
             false);
     colinf[2].setNamingSchemeType(NamingSchemeTypes.HOP_FIELD);
 
+    // The value field is only used while no fields are listed
+    ModifyListener lsFieldsMod =
+        e -> {
+          input.setChanged();
+          enableValueField();
+        };
     wFields =
         new TableView(
             variables,
@@ -133,8 +163,9 @@ public class NormaliserDialog extends BaseTransformDialog {
             SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI,
             colinf,
             fieldsRows,
-            lsMod,
+            lsFieldsMod,
             props);
+    wFields.setContentListener(e -> enableValueField());
 
     FormData fdFields = new FormData();
     fdFields.left = new FormAttachment(0, 0);
@@ -183,6 +214,7 @@ public class NormaliserDialog extends BaseTransformDialog {
     if (input.getTypeField() != null) {
       wTypefield.setText(input.getTypeField());
     }
+    wValuefield.setText(input.getValueFieldName());
 
     for (int i = 0; i < input.getNormaliserFields().size(); i++) {
       TableItem item = wFields.table.getItem(i);
@@ -200,6 +232,14 @@ public class NormaliserDialog extends BaseTransformDialog {
 
     wFields.setRowNums();
     wFields.optWidth(true);
+    enableValueField();
+  }
+
+  /** The value field is only used when the fields table is empty. */
+  private void enableValueField() {
+    boolean noFieldsListed = wFields.nrNonEmpty() == 0;
+    wlValuefield.setEnabled(noFieldsListed);
+    wValuefield.setEnabled(noFieldsListed);
   }
 
   private void cancel() {
@@ -220,6 +260,10 @@ public class NormaliserDialog extends BaseTransformDialog {
     int i;
 
     int nrFields = wFields.nrNonEmpty();
+    if (nrFields == 0) {
+      // Listed fields leave the value field unused: keep what it was
+      input.setValueField(wValuefield.getText());
+    }
     input.getNormaliserFields().clear();
 
     for (i = 0; i < nrFields; i++) {
@@ -240,6 +284,7 @@ public class NormaliserDialog extends BaseTransformDialog {
       if (r != null && !r.isEmpty()) {
         BaseTransformDialog.getFieldsFromPrevious(
             r, wFields, 1, new int[] {1, 2}, new int[] {}, -1, -1, null);
+        enableValueField();
       }
     } catch (HopException ke) {
       new ErrorDialog(
diff --git 
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMeta.java
 
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMeta.java
index 42469332e6..5b21328231 100644
--- 
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMeta.java
+++ 
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMeta.java
@@ -18,10 +18,8 @@
 package org.apache.hop.pipeline.transforms.normaliser;
 
 import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
-import java.util.Set;
 import lombok.Getter;
 import lombok.Setter;
 import org.apache.hop.core.CheckResult;
@@ -32,6 +30,7 @@ import org.apache.hop.core.exception.HopTransformException;
 import org.apache.hop.core.row.IRowMeta;
 import org.apache.hop.core.row.IValueMeta;
 import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.util.Utils;
 import org.apache.hop.core.variables.IVariables;
 import org.apache.hop.i18n.BaseMessages;
 import org.apache.hop.metadata.api.HopMetadataProperty;
@@ -77,6 +76,8 @@ DATE      PRODUCT    Sales   Number
 public class NormaliserMeta extends BaseTransformMeta<Normaliser, 
NormaliserData> {
   private static final Class<?> PKG = NormaliserMeta.class;
 
+  public static final String DEFAULT_VALUE_FIELD = "value";
+
   /** Name of the new type-field */
   @HopMetadataProperty(
       key = "typefield",
@@ -91,32 +92,74 @@ public class NormaliserMeta extends 
BaseTransformMeta<Normaliser, NormaliserData
       injectionGroupDescription = "NormaliserMeta.Injection.FIELDS")
   private List<NormaliserField> normaliserFields;
 
+  /** Name of the field that holds the values when every input field is 
normalised. */
+  @HopMetadataProperty(
+      key = "valuefield",
+      injectionKey = "VALUEFIELD",
+      injectionKeyDescription = "NormaliserMeta.Injection.VALUEFIELD")
+  private String valueField;
+
   public NormaliserMeta() {
     super();
     this.normaliserFields = new ArrayList<>();
   }
 
-  public Set<String> getFieldNames() {
-    Set<String> fieldNames = new HashSet<>();
-
-    for (NormaliserField field : normaliserFields) {
-      if (field.getName() != null) {
-        fieldNames.add(field.getName().toLowerCase());
-      }
-    }
-    return fieldNames;
-  }
-
   @Override
   public void setDefault() {
     this.typeField = "typefield";
+    this.valueField = DEFAULT_VALUE_FIELD;
     this.normaliserFields = new ArrayList<>();
   }
 
-  /** The names of the normalised fields, in output order: in the order they 
first appear. */
-  public List<String> getNormalisedFieldNames() {
+  /**
+   * @param normaliserFields the fields to normalise; null, from metadata 
injection for example, is
+   *     an empty list: every input field is normalised then
+   */
+  public void setNormaliserFields(List<NormaliserField> normaliserFields) {
+    this.normaliserFields = normaliserFields == null ? new ArrayList<>() : 
normaliserFields;
+  }
+
+  /** True when no fields are listed: every input field is normalised then. */
+  public boolean isNormalisingAllFields() {
+    return normaliserFields == null || normaliserFields.isEmpty();
+  }
+
+  /** The name of the value field, or its default when none is set. */
+  public String getValueFieldName() {
+    return Utils.isEmpty(valueField) ? DEFAULT_VALUE_FIELD : valueField;
+  }
+
+  /**
+   * The fields to normalise: the listed fields, or with an empty list every 
input field. An input
+   * field normalised that way gets its own name as type and puts its value in 
the value field.
+   *
+   * @param inputRowMeta the fields entering the transform
+   * @return the fields to normalise
+   */
+  public List<NormaliserField> getFieldsToNormalise(IRowMeta inputRowMeta) {
+    if (!isNormalisingAllFields()) {
+      return normaliserFields;
+    }
+    String norm = getValueFieldName();
+    List<NormaliserField> fields = new ArrayList<>(inputRowMeta.size());
+    for (IValueMeta valueMeta : inputRowMeta.getValueMetaList()) {
+      NormaliserField field = new NormaliserField();
+      field.setName(valueMeta.getName());
+      field.setValue(valueMeta.getName());
+      field.setNorm(norm);
+      fields.add(field);
+    }
+    return fields;
+  }
+
+  /**
+   * The names of the normalised fields, in output order: in the order they 
first appear.
+   *
+   * @param fields the fields to normalise, see {@link 
#getFieldsToNormalise(IRowMeta)}
+   */
+  public static List<String> getNormalisedFieldNames(List<NormaliserField> 
fields) {
     List<String> names = new ArrayList<>();
-    for (NormaliserField field : normaliserFields) {
+    for (NormaliserField field : fields) {
       if (!names.contains(field.getNorm())) {
         names.add(field.getNorm());
       }
@@ -134,14 +177,28 @@ public class NormaliserMeta extends 
BaseTransformMeta<Normaliser, NormaliserData
    * transform that serializes or renders it. See issue #3636.
    *
    * @param inputRowMeta the fields entering the transform
-   * @return one value metadata per name of {@link 
#getNormalisedFieldNames()}, in the same order
+   * @return one value metadata per name of {@link 
#getNormalisedFieldNames(List)}, in the same
+   *     order
    * @throws HopTransformException when the first input field of a normalised 
field is missing
    */
   public List<IValueMeta> getNormalisedValueMetas(IRowMeta inputRowMeta)
       throws HopTransformException {
+    return getNormalisedValueMetas(inputRowMeta, 
getFieldsToNormalise(inputRowMeta));
+  }
+
+  private List<IValueMeta> getNormalisedValueMetas(
+      IRowMeta inputRowMeta, List<NormaliserField> fieldsToNormalise) throws 
HopTransformException {
+    boolean allFields = isNormalisingAllFields();
     List<IValueMeta> valueMetas = new ArrayList<>();
-    for (String normName : getNormalisedFieldNames()) {
-      List<NormaliserField> fields = getFieldsOf(normName);
+    if (allFields && fieldsToNormalise.isEmpty()) {
+      // No input fields are known, while designing a pipeline whose input is 
only known when it
+      // runs for example. The value field is there all the same, and it can 
hold any value as text.
+      //
+      valueMetas.add(new ValueMetaString(getValueFieldName()));
+      return valueMetas;
+    }
+    for (String normName : getNormalisedFieldNames(fieldsToNormalise)) {
+      List<NormaliserField> fields = getFieldsOf(fieldsToNormalise, normName);
       IValueMeta first = inputRowMeta.searchValueMeta(fields.get(0).getName());
       if (first == null) {
         throw new HopTransformException(
@@ -150,11 +207,15 @@ public class NormaliserMeta extends 
BaseTransformMeta<Normaliser, NormaliserData
       }
       boolean sameType = true;
       boolean sameStorage = true;
+      boolean sameLength = true;
+      boolean samePrecision = true;
       for (NormaliserField field : fields) {
         IValueMeta source = inputRowMeta.searchValueMeta(field.getName());
         if (source != null) {
           sameType &= source.getType() == first.getType();
           sameStorage &= source.getStorageType() == first.getStorageType();
+          sameLength &= source.getLength() == first.getLength();
+          samePrecision &= source.getPrecision() == first.getPrecision();
         }
       }
 
@@ -167,6 +228,16 @@ public class NormaliserMeta extends 
BaseTransformMeta<Normaliser, NormaliserData
           v.setStorageType(IValueMeta.STORAGE_TYPE_NORMAL);
           v.setStorageMetadata(null);
         }
+        // With every input field normalised, which one comes first is up to 
the input. The value
+        // field keeps only a length or precision they all share: the first 
field's could be too
+        // short for the values of the others, in a database table or a fixed 
width file.
+        //
+        if (allFields && !sameLength) {
+          v.setLength(-1);
+        }
+        if (allFields && !samePrecision) {
+          v.setPrecision(-1);
+        }
       }
       v.setName(normName);
       valueMetas.add(v);
@@ -184,22 +255,34 @@ public class NormaliserMeta extends 
BaseTransformMeta<Normaliser, NormaliserData
       IHopMetadataProvider metadataProvider)
       throws HopTransformException {
 
+    boolean allFields = isNormalisingAllFields();
+    List<NormaliserField> fields = getFieldsToNormalise(row);
+
     int maxlen = 0;
-    for (NormaliserField field : normaliserFields) {
+    for (NormaliserField field : fields) {
       if (field.getValue().length() > maxlen) {
         maxlen = field.getValue().length();
       }
     }
 
-    // Take the normalised fields from the input before adding anything to it.
+    // Take the normalised fields from the input before changing it.
     //
-    List<IValueMeta> normalisedValueMetas = getNormalisedValueMetas(row);
+    List<IValueMeta> normalisedValueMetas = getNormalisedValueMetas(row, 
fields);
+
+    // With every input field normalised, remove them before adding the new 
fields: the value field
+    // may have the name of an input field, value for example, and keeps it.
+    //
+    if (allFields) {
+      removeFields(row, fields);
+    }
 
     // Then add the type field!
     //
     IValueMeta typefieldValue = new ValueMetaString(typeField);
     typefieldValue.setOrigin(name);
-    typefieldValue.setLength(maxlen);
+    if (!fields.isEmpty()) {
+      typefieldValue.setLength(maxlen);
+    }
     row.addValueMeta(typefieldValue);
 
     // Add the new fields that need to be created.
@@ -209,9 +292,16 @@ public class NormaliserMeta extends 
BaseTransformMeta<Normaliser, NormaliserData
       row.addValueMeta(v);
     }
 
-    // Now remove all the normalized fields...
+    // Listed fields are removed after the new fields are added, as they 
always were: a new field
+    // with the name of a listed field is called value_1 then, and pipelines 
rely on that name.
     //
-    for (NormaliserField field : normaliserFields) {
+    if (!allFields) {
+      removeFields(row, fields);
+    }
+  }
+
+  private static void removeFields(IRowMeta row, List<NormaliserField> fields) 
{
+    for (NormaliserField field : fields) {
       int idx = row.indexOfValue(field.getName());
       if (idx >= 0) {
         row.removeValueMeta(idx);
@@ -244,11 +334,24 @@ public class NormaliserMeta extends 
BaseTransformMeta<Normaliser, NormaliserData
               transformMeta);
       remarks.add(cr);
 
+      List<NormaliserField> fields = getFieldsToNormalise(prev);
+      if (isNormalisingAllFields()) {
+        remarks.add(
+            new CheckResult(
+                ICheckResult.TYPE_RESULT_OK,
+                BaseMessages.getString(
+                    PKG,
+                    "NormaliserMeta.CheckResult.AllInputFieldsNormalised",
+                    prev.size() + "",
+                    getValueFieldName()),
+                transformMeta));
+      }
+
       boolean first = true;
       errorMessage = "";
       boolean errorFound = false;
 
-      for (NormaliserField field : normaliserFields) {
+      for (NormaliserField field : fields) {
         IValueMeta valueMeta = prev.searchValueMeta(field.getName());
         if (valueMeta == null) {
           if (first) {
@@ -271,8 +374,8 @@ public class NormaliserMeta extends 
BaseTransformMeta<Normaliser, NormaliserData
       }
       remarks.add(cr);
 
-      for (String normName : getNormalisedFieldNames()) {
-        if (hasMixedTypes(prev, normName)) {
+      for (String normName : getNormalisedFieldNames(fields)) {
+        if (hasMixedTypes(prev, fields, normName)) {
           remarks.add(
               new CheckResult(
                   ICheckResult.TYPE_RESULT_WARNING,
@@ -289,11 +392,28 @@ public class NormaliserMeta extends 
BaseTransformMeta<Normaliser, NormaliserData
       remarks.add(cr);
     }
 
-    String duplicate = getDuplicateMapping();
+    String duplicate = getDuplicateMapping(normaliserFields);
     if (duplicate != null) {
       remarks.add(new CheckResult(ICheckResult.TYPE_RESULT_ERROR, duplicate, 
transformMeta));
     }
 
+    // The type field comes first, so a new field with its name is renamed, to 
value_1 for example.
+    //
+    List<String> newFieldNames =
+        isNormalisingAllFields()
+            ? List.of(getValueFieldName())
+            : getNormalisedFieldNames(normaliserFields);
+    for (String newFieldName : newFieldNames) {
+      if (newFieldName != null && newFieldName.equalsIgnoreCase(typeField)) {
+        remarks.add(
+            new CheckResult(
+                ICheckResult.TYPE_RESULT_WARNING,
+                BaseMessages.getString(
+                    PKG, "NormaliserMeta.CheckResult.SameNameAsTypeField", 
newFieldName),
+                transformMeta));
+      }
+    }
+
     // See if we have input streams leading to this transform!
     if (input.length > 0) {
       cr =
@@ -313,15 +433,14 @@ public class NormaliserMeta extends 
BaseTransformMeta<Normaliser, NormaliserData
   }
 
   /** The fields filling a normalised field, in the order they are listed. */
-  private List<NormaliserField> getFieldsOf(String normName) {
-    return normaliserFields.stream()
-        .filter(field -> Objects.equals(normName, field.getNorm()))
-        .toList();
+  private static List<NormaliserField> getFieldsOf(List<NormaliserField> 
fields, String normName) {
+    return fields.stream().filter(field -> Objects.equals(normName, 
field.getNorm())).toList();
   }
 
   /** True when the input fields filling this normalised field are not all of 
one type. */
-  private boolean hasMixedTypes(IRowMeta inputRowMeta, String normName) {
-    return getFieldsOf(normName).stream()
+  private static boolean hasMixedTypes(
+      IRowMeta inputRowMeta, List<NormaliserField> fields, String normName) {
+    return getFieldsOf(fields, normName).stream()
             .map(field -> inputRowMeta.searchValueMeta(field.getName()))
             .filter(Objects::nonNull)
             .map(IValueMeta::getType)
@@ -334,13 +453,17 @@ public class NormaliserMeta extends 
BaseTransformMeta<Normaliser, NormaliserData
    * Two input fields filling the same normalised field on the same row leave 
no room for one of
    * them.
    *
+   * @param fields the fields to normalise, see {@link 
#getFieldsToNormalise(IRowMeta)}
    * @return a description of the first such pair, or null when there is none
    */
-  public String getDuplicateMapping() {
-    for (int i = 0; i < normaliserFields.size(); i++) {
-      NormaliserField one = normaliserFields.get(i);
-      for (int j = i + 1; j < normaliserFields.size(); j++) {
-        NormaliserField other = normaliserFields.get(j);
+  public static String getDuplicateMapping(List<NormaliserField> fields) {
+    if (fields == null) {
+      return null;
+    }
+    for (int i = 0; i < fields.size(); i++) {
+      NormaliserField one = fields.get(i);
+      for (int j = i + 1; j < fields.size(); j++) {
+        NormaliserField other = fields.get(j);
         if (Objects.equals(one.getValue(), other.getValue())
             && Objects.equals(one.getNorm(), other.getNorm())) {
           return BaseMessages.getString(
diff --git 
a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_en_US.properties
 
b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_en_US.properties
index c0f43d493d..a5a8d7dabe 100644
--- 
a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_en_US.properties
+++ 
b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_en_US.properties
@@ -29,12 +29,16 @@ NormaliserDialog.GetFields.Button=\ &Get Fields
 NormaliserDialog.Shell.Title=Row normaliser
 NormaliserDialog.TransformName.Label=Transform name 
 NormaliserDialog.TypeField.Label=Type field 
+NormaliserDialog.ValueField.Label=Value field 
+NormaliserDialog.ValueField.Tooltip=Used when the fields table is empty: every 
input field is normalised then, with its name in the type field and its value 
in this field.
 NormaliserMeta.CheckResult.AllFieldsFound=All fields to normalise are found in 
the input stream.
+NormaliserMeta.CheckResult.AllInputFieldsNormalised=No fields are listed, so 
all {0} input fields are normalised into field {1}.
 NormaliserMeta.CheckResult.CouldNotReadFieldsFromPreviousTransform=Couldn''t 
read fields from the previous transform.
 NormaliserMeta.CheckResult.DuplicateMapping=Fields {0} and {1} both fill 
normalised field {2} for type {3}. Every row has room for only one of them.
 NormaliserMeta.CheckResult.FieldsNotFound=Fields to normalise, not found in 
input stream:
 NormaliserMeta.CheckResult.MixedTypes=Normalised field {0} is filled from 
fields of different types, so it is a String holding the text of each value.
 NormaliserMeta.CheckResult.NoInputReceivedError=No input received from other 
transforms!
+NormaliserMeta.CheckResult.SameNameAsTypeField=Normalised field {0} has the 
same name as the type field, so it is renamed in the output, to {0}_1 for 
example.
 NormaliserMeta.CheckResult.TransformReceivingFieldsOK=Transform is connected 
to previous one, receiving {0} fields
 NormaliserMeta.CheckResult.TransformReceivingInfoOK=Transform is receiving 
info from other transforms.
 NormaliserMeta.Exception.UnableToFindField=Unable to find field from upstream 
transform: {0}
@@ -43,4 +47,5 @@ NormaliserMeta.Injection.NAME=Input field name
 NormaliserMeta.Injection.NORMALISED=Normalised field name
 NormaliserMeta.Injection.TYPEFIELD=The name of the type field
 NormaliserMeta.Injection.VALUE=Type field value
+NormaliserMeta.Injection.VALUEFIELD=The name of the value field, used when no 
fields are listed
 NormaliserMeta.keyword=normalize,unpivot,denormalize,columns,reshape
diff --git 
a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaInjectionTest.java
 
b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaInjectionTest.java
index bb73e9cc93..23efd9acc9 100644
--- 
a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaInjectionTest.java
+++ 
b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaInjectionTest.java
@@ -35,6 +35,7 @@ class NormaliserMetaInjectionTest extends 
BaseMetadataInjectionTestJunit5<Normal
   @Test
   void test() throws Exception {
     check("TYPEFIELD", () -> meta.getTypeField());
+    check("VALUEFIELD", () -> meta.getValueField());
     check("NAME", () -> meta.getNormaliserFields().get(0).getName());
     check("VALUE", () -> meta.getNormaliserFields().get(0).getValue());
     check("NORMALISED", () -> meta.getNormaliserFields().get(0).getNorm());
diff --git 
a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaTest.java
 
b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaTest.java
index be2131f6ea..1fe2c09fa8 100644
--- 
a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaTest.java
+++ 
b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaTest.java
@@ -18,6 +18,7 @@ package org.apache.hop.pipeline.transforms.normaliser;
 
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.mock;
@@ -60,9 +61,28 @@ class NormaliserMetaTest {
         TransformSerializationTestUtil.testSerialization(
             "/normaliser-transform.xml", NormaliserMeta.class);
 
-    // assertEquals(2, meta.getFields().size());
-    // assertEquals("fieldName", meta.getFields().get(0).getName());
-    // assertEquals("two", meta.getFields().get(0).getValue());
+    assertEquals("type", meta.getTypeField());
+    assertEquals("content", meta.getValueField());
+    assertEquals(4, meta.getNormaliserFields().size());
+    NormaliserField last = meta.getNormaliserFields().get(3);
+    assertEquals("fieldD", last.getName());
+    assertEquals("D", last.getValue());
+    assertEquals("field", last.getNorm());
+  }
+
+  /** The type field is as long as the longest type it holds. */
+  @Test
+  void theTypeFieldIsAsLongAsTheLongestType() throws Exception {
+    IRowMeta row = new RowMeta();
+    row.addValueMeta(new ValueMetaInteger("i1"));
+    row.addValueMeta(new ValueMetaInteger("i2"));
+
+    meta(field("i1", "short", "value"), field("i2", "much longer", "value"))
+        .getFields(row, "normaliser", null, null, new Variables(), null);
+
+    IValueMeta type = row.searchValueMeta("typefield");
+    assertEquals(IValueMeta.TYPE_STRING, type.getType());
+    assertEquals("much longer".length(), type.getLength());
   }
 
   @Test
@@ -199,6 +219,147 @@ class NormaliserMetaTest {
         remarks.toString());
   }
 
+  /** Issue #9397: no listed fields means every input field, into the value 
field. */
+  @Test
+  void withoutListedFieldsEveryInputFieldIsNormalisedIntoTheValueField() 
throws Exception {
+    IRowMeta row = new RowMeta();
+    row.addValueMeta(new ValueMetaInteger("i"));
+    row.addValueMeta(new ValueMetaString("a much longer name"));
+
+    NormaliserMeta meta = meta();
+    meta.setValueField("content");
+    meta.getFields(row, "normaliser", null, null, new Variables(), null);
+
+    assertArrayEquals(new String[] {"typefield", "content"}, 
row.getFieldNames());
+    assertEquals("a much longer name".length(), 
row.getValueMeta(0).getLength());
+    assertEquals(IValueMeta.TYPE_STRING, row.getValueMeta(1).getType());
+  }
+
+  /** A transform saved before the value field existed has none: it is called 
value then. */
+  @Test
+  void aMissingValueFieldIsCalledValue() throws Exception {
+    IRowMeta row = new RowMeta();
+    row.addValueMeta(new ValueMetaInteger("i"));
+
+    NormaliserMeta meta = meta();
+    meta.setValueField(null);
+    meta.getFields(row, "normaliser", null, null, new Variables(), null);
+
+    assertArrayEquals(new String[] {"typefield", "value"}, 
row.getFieldNames());
+    assertEquals(IValueMeta.TYPE_INTEGER, row.getValueMeta(1).getType());
+  }
+
+  @Test
+  void checkSaysEveryInputFieldIsNormalisedWhenNoneAreListed() {
+    IRowMeta prev = new RowMeta();
+    prev.addValueMeta(new ValueMetaInteger("i1"));
+    prev.addValueMeta(new ValueMetaInteger("i2"));
+
+    List<ICheckResult> remarks = check(meta(), prev);
+
+    assertTrue(
+        remarks.stream()
+            .anyMatch(
+                r ->
+                    r.getType() == ICheckResult.TYPE_RESULT_OK
+                        && r.getText().contains("2")
+                        && r.getText().contains("value")),
+        remarks.toString());
+    assertTrue(
+        remarks.stream().allMatch(r -> r.getType() == 
ICheckResult.TYPE_RESULT_OK),
+        remarks.toString());
+  }
+
+  /**
+   * Input fields that are not known yet, while designing a pipeline that gets 
them at run time,
+   * still leave the value field for the transforms after this one.
+   */
+  @Test
+  void withoutListedFieldsTheValueFieldIsDeclaredWhileTheInputIsUnknown() 
throws Exception {
+    IRowMeta row = new RowMeta();
+
+    meta().getFields(row, "normaliser", null, null, new Variables(), null);
+
+    assertArrayEquals(new String[] {"typefield", "value"}, 
row.getFieldNames());
+    assertEquals(IValueMeta.TYPE_STRING, row.getValueMeta(1).getType());
+  }
+
+  /**
+   * With every input field normalised, which one comes first is up to the 
input: its length would
+   * be too short for the values of the others. The value field keeps only 
what they all share.
+   */
+  @Test
+  void 
withoutListedFieldsTheValueFieldKeepsOnlyALengthAndPrecisionAllFieldsShare()
+      throws Exception {
+    IRowMeta texts = new RowMeta();
+    texts.addValueMeta(new ValueMetaString("code", 10, -1));
+    texts.addValueMeta(new ValueMetaString("description", 500, -1));
+    meta().getFields(texts, "normaliser", null, null, new Variables(), null);
+    assertEquals(-1, texts.searchValueMeta("value").getLength());
+
+    IRowMeta amounts = new RowMeta();
+    amounts.addValueMeta(new ValueMetaNumber("net", 9, 2));
+    amounts.addValueMeta(new ValueMetaNumber("rate", 9, 4));
+    meta().getFields(amounts, "normaliser", null, null, new Variables(), null);
+    IValueMeta amount = amounts.searchValueMeta("value");
+    assertEquals(9, amount.getLength());
+    assertEquals(-1, amount.getPrecision());
+
+    IRowMeta same = new RowMeta();
+    same.addValueMeta(new ValueMetaNumber("q1", 9, 2));
+    same.addValueMeta(new ValueMetaNumber("q2", 9, 2));
+    meta().getFields(same, "normaliser", null, null, new Variables(), null);
+    IValueMeta quarter = same.searchValueMeta("value");
+    assertEquals(9, quarter.getLength());
+    assertEquals(2, quarter.getPrecision());
+  }
+
+  /** The type field comes first: a new field with its name is renamed, which 
Verify points out. */
+  @Test
+  void checkWarnsAboutANewFieldWithTheNameOfTheTypeField() {
+    IRowMeta prev = new RowMeta();
+    prev.addValueMeta(new ValueMetaInteger("i1"));
+    prev.addValueMeta(new ValueMetaInteger("i2"));
+
+    NormaliserMeta allFields = meta();
+    allFields.setTypeField("Value");
+    assertTrue(
+        check(allFields, prev).stream()
+            .anyMatch(
+                r ->
+                    r.getType() == ICheckResult.TYPE_RESULT_WARNING
+                        && r.getText().contains("value")),
+        check(allFields, prev).toString());
+
+    NormaliserMeta listed = meta(field("i1", "A", "kind"), field("i2", "B", 
"kind"));
+    listed.setTypeField("kind");
+    assertTrue(
+        check(listed, prev).stream()
+            .anyMatch(
+                r ->
+                    r.getType() == ICheckResult.TYPE_RESULT_WARNING
+                        && r.getText().contains("kind")),
+        check(listed, prev).toString());
+  }
+
+  /** No field list at all, from metadata injection for example, is an empty 
one. */
+  @Test
+  void aMissingFieldListIsAnEmptyOne() {
+    IRowMeta prev = new RowMeta();
+    prev.addValueMeta(new ValueMetaInteger("i1"));
+
+    NormaliserMeta meta = meta();
+    meta.setNormaliserFields(null);
+
+    assertEquals(List.of(), meta.getNormaliserFields());
+    assertTrue(meta.isNormalisingAllFields());
+    assertNull(NormaliserMeta.getDuplicateMapping(null));
+    List<ICheckResult> remarks = check(meta, prev);
+    assertTrue(
+        remarks.stream().allMatch(r -> r.getType() == 
ICheckResult.TYPE_RESULT_OK),
+        remarks.toString());
+  }
+
   private static List<ICheckResult> check(NormaliserMeta meta, IRowMeta prev) {
     List<ICheckResult> remarks = new ArrayList<>();
     meta.check(
diff --git 
a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserTest.java
 
b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserTest.java
index ff4f2402b9..a91e3cc0cb 100644
--- 
a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserTest.java
+++ 
b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserTest.java
@@ -36,6 +36,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
+import java.util.Locale;
 import org.apache.hop.core.BlockingRowSet;
 import org.apache.hop.core.HopEnvironment;
 import org.apache.hop.core.exception.HopException;
@@ -47,6 +48,7 @@ import org.apache.hop.core.row.value.ValueMetaDate;
 import org.apache.hop.core.row.value.ValueMetaInteger;
 import org.apache.hop.core.row.value.ValueMetaNumber;
 import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.variables.Variables;
 import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
 import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
 import org.junit.jupiter.api.AfterEach;
@@ -312,6 +314,222 @@ class NormaliserTest {
     assertEquals(List.of("a", "1", "b", "2"), output.rows.stream().map(row -> 
row[1]).toList());
   }
 
+  /** The fields that are not normalised keep their order, wherever they stand 
in the input. */
+  @Test
+  void keptFieldsKeepTheirOrderAroundTheNormalisedOnes() throws Exception {
+    IRowMeta input = new RowMeta();
+    for (String name : new String[] {"id", "a", "name", "b", "city"}) {
+      input.addValueMeta(new ValueMetaString(name));
+    }
+
+    Output output =
+        normalise(
+            meta("type", field("a", "A", "value"), field("b", "B", "value")),
+            input,
+            "1",
+            "a",
+            "Ann",
+            "b",
+            "Paris");
+
+    assertArrayEquals(
+        new String[] {"id", "name", "city", "type", "value"}, 
output.rowMeta.getFieldNames());
+    assertArrayEquals(new Object[] {"1", "Ann", "Paris", "A", "a"}, 
output.rows.get(0));
+    assertArrayEquals(new Object[] {"1", "Ann", "Paris", "B", "b"}, 
output.rows.get(1));
+  }
+
+  /** A field is found in the input whatever the case it is listed in. */
+  @Test
+  void fieldNamesMatchWhateverTheirCase() throws Exception {
+    IRowMeta input = new RowMeta();
+    for (String name : new String[] {"ID", "Field_A", "FIELD_B"}) {
+      input.addValueMeta(new ValueMetaString(name));
+    }
+
+    Output output =
+        normalise(
+            meta("type", field("field_a", "A", "value"), field("field_b", "B", 
"value")),
+            input,
+            "1",
+            "a",
+            "b");
+
+    assertArrayEquals(new String[] {"ID", "type", "value"}, 
output.rowMeta.getFieldNames());
+    assertArrayEquals(new Object[] {"1", "A", "a"}, output.rows.get(0));
+    assertArrayEquals(new Object[] {"1", "B", "b"}, output.rows.get(1));
+  }
+
+  /** The rows the transform writes have the layout it declares to the 
transforms after it. */
+  @Test
+  void theRowsHaveTheLayoutTheTransformDeclares() throws Exception {
+    IRowMeta input = new RowMeta();
+    input.addValueMeta(new ValueMetaString("id"));
+    input.addValueMeta(new ValueMetaInteger("i"));
+    input.addValueMeta(new ValueMetaString("kept"));
+    input.addValueMeta(new ValueMetaNumber("n"));
+    NormaliserMeta meta = meta("type", field("i", "I", "value"), field("n", 
"N", "value"));
+
+    Output output = normalise(meta, input, "1", 2L, "k", 3.0);
+
+    IRowMeta declared = input.clone();
+    meta.getFields(declared, "Row normaliser", null, null, new Variables(), 
null);
+    assertArrayEquals(declared.getFieldNames(), 
output.rowMeta.getFieldNames());
+    for (int i = 0; i < declared.size(); i++) {
+      assertEquals(
+          declared.getValueMeta(i).getType(),
+          output.rowMeta.getValueMeta(i).getType(),
+          declared.getValueMeta(i).getName());
+    }
+  }
+
+  /**
+   * Issue #9397: without listed fields every input field is normalised, under 
its own name, into
+   * the value field. Fields of different types make that a String holding the 
text of each value.
+   */
+  @Test
+  void withoutListedFieldsEveryInputFieldIsNormalised() throws Exception {
+    IRowMeta input = new RowMeta();
+    input.addValueMeta(new ValueMetaString("name"));
+    input.addValueMeta(new ValueMetaInteger("age"));
+    input.addValueMeta(new ValueMetaString("city"));
+
+    NormaliserMeta meta = meta("column");
+    meta.setValueField("content");
+
+    Output output =
+        normaliseRows(
+            meta, input, new Object[] {"Ann", 31L, "Paris"}, new Object[] 
{"Bob", null, "Rome"});
+
+    assertArrayEquals(new String[] {"column", "content"}, 
output.rowMeta.getFieldNames());
+    assertEquals(IValueMeta.TYPE_STRING, 
output.rowMeta.getValueMeta(1).getType());
+    assertEquals(6, output.rows.size());
+    assertArrayEquals(new Object[] {"name", "Ann"}, output.rows.get(0));
+    assertArrayEquals(new Object[] {"age", "31"}, output.rows.get(1));
+    assertArrayEquals(new Object[] {"city", "Paris"}, output.rows.get(2));
+    assertArrayEquals(new Object[] {"name", "Bob"}, output.rows.get(3));
+    assertArrayEquals(new Object[] {"age", null}, output.rows.get(4));
+    assertArrayEquals(new Object[] {"city", "Rome"}, output.rows.get(5));
+  }
+
+  /** Input fields of one type keep it, and their values go through untouched. 
*/
+  @Test
+  void withoutListedFieldsFieldsOfOneTypeKeepThatType() throws Exception {
+    IRowMeta input = new RowMeta();
+    input.addValueMeta(new ValueMetaInteger("q1"));
+    input.addValueMeta(new ValueMetaInteger("q2"));
+
+    Output output = normalise(meta("quarter"), input, 10L, 20L);
+
+    assertArrayEquals(new String[] {"quarter", "value"}, 
output.rowMeta.getFieldNames());
+    assertEquals(IValueMeta.TYPE_INTEGER, 
output.rowMeta.getValueMeta(1).getType());
+    assertArrayEquals(new Object[] {"q1", 10L}, output.rows.get(0));
+    assertArrayEquals(new Object[] {"q2", 20L}, output.rows.get(1));
+  }
+
+  /**
+   * With every input field normalised, the input fields are removed before 
the new fields are
+   * added: the type and value fields keep their names when input fields had 
them.
+   */
+  @Test
+  void withoutListedFieldsTheNewFieldsKeepTheNamesOfInputFields() throws 
Exception {
+    IRowMeta input = new RowMeta();
+    input.addValueMeta(new ValueMetaString("type"));
+    input.addValueMeta(new ValueMetaString("value"));
+
+    Output output = normalise(meta("type"), input, "t", "v");
+
+    assertArrayEquals(new String[] {"type", "value"}, 
output.rowMeta.getFieldNames());
+    assertArrayEquals(new Object[] {"type", "t"}, output.rows.get(0));
+    assertArrayEquals(new Object[] {"value", "v"}, output.rows.get(1));
+  }
+
+  /**
+   * A new field with the name of a listed field is called value_1, as it 
always was: the transforms
+   * after this one read it by that name.
+   */
+  @Test
+  void aNewFieldWithTheNameOfAListedFieldKeepsItsOldName() throws Exception {
+    IRowMeta input = new RowMeta();
+    input.addValueMeta(new ValueMetaString("type"));
+    input.addValueMeta(new ValueMetaString("value"));
+
+    Output output =
+        normalise(
+            meta("kind", field("type", "T", "value"), field("value", "V", 
"value")),
+            input,
+            "t",
+            "v");
+
+    assertArrayEquals(new String[] {"kind", "value_1"}, 
output.rowMeta.getFieldNames());
+    assertArrayEquals(new Object[] {"T", "t"}, output.rows.get(0));
+    assertArrayEquals(new Object[] {"V", "v"}, output.rows.get(1));
+  }
+
+  /**
+   * The fields that are copied are the ones the declared layout keeps, also 
where lower case
+   * depends on the language: in Turkish, TITLE in lower case has a dotless i, 
so it is not title.
+   */
+  @Test
+  void theCopiedFieldsAreTheOnesTheLayoutKeepsInATurkishLocale() throws 
Exception {
+    IRowMeta input = new RowMeta();
+    input.addValueMeta(new ValueMetaString("id"));
+    input.addValueMeta(new ValueMetaString("title"));
+
+    Locale locale = Locale.getDefault();
+    Locale.setDefault(Locale.forLanguageTag("tr-TR"));
+    try {
+      Output output = normalise(meta("type", field("TITLE", "T", "value")), 
input, "1", "Dr");
+
+      assertArrayEquals(new String[] {"id", "type", "value"}, 
output.rowMeta.getFieldNames());
+      assertArrayEquals(new Object[] {"1", "T", "Dr"}, output.rows.get(0));
+    } finally {
+      Locale.setDefault(locale);
+    }
+  }
+
+  /** Every input field normalised: the declared layout matches the rows too. 
*/
+  @Test
+  void withoutListedFieldsTheRowsHaveTheLayoutTheTransformDeclares() throws 
Exception {
+    IRowMeta input = new RowMeta();
+    input.addValueMeta(new ValueMetaString("s"));
+    input.addValueMeta(new ValueMetaNumber("n"));
+    NormaliserMeta meta = meta("type");
+
+    Output output = normalise(meta, input, "a", 1.5);
+
+    IRowMeta declared = input.clone();
+    meta.getFields(declared, "Row normaliser", null, null, new Variables(), 
null);
+    assertArrayEquals(declared.getFieldNames(), 
output.rowMeta.getFieldNames());
+    assertEquals(declared.getValueMeta(1).getType(), 
output.rowMeta.getValueMeta(1).getType());
+  }
+
+  /** Without input rows the transform writes nothing and finishes without an 
error. */
+  @Test
+  void noInputRowsWriteNothing() throws Exception {
+    IRowMeta input = new RowMeta();
+    input.addValueMeta(new ValueMetaString("a"));
+    NormaliserMeta meta = meta("type", field("a", "A", "value"));
+    when(helper.transformMeta.getTransform()).thenReturn(meta);
+    Normaliser transform =
+        new Normaliser(
+            helper.transformMeta,
+            meta,
+            new NormaliserData(),
+            0,
+            helper.pipelineMeta,
+            helper.pipeline);
+    transform.init();
+    transform.setInputRowMeta(input);
+    BlockingRowSet rowSet = new BlockingRowSet(10);
+    transform.setOutputRowSets(Collections.singletonList(rowSet));
+    Normaliser spied = spy(transform);
+    doReturn(null).when(spied).getRow();
+
+    assertFalse(spied.processRow());
+    assertNull(rowSet.getRowImmediate());
+    assertEquals(0, spied.getErrors());
+  }
+
   /** A field that is not in the input stops the transform with an error. */
   @Test
   void aFieldMissingFromTheInputStopsTheTransform() throws Exception {
@@ -349,6 +567,7 @@ class NormaliserTest {
 
   private static NormaliserMeta meta(String typeField, NormaliserField... 
fields) {
     NormaliserMeta meta = new NormaliserMeta();
+    meta.setDefault();
     meta.setTypeField(typeField);
     meta.setNormaliserFields(new ArrayList<>(List.of(fields)));
     return meta;
diff --git 
a/plugins/transforms/normaliser/src/test/resources/normaliser-transform.xml 
b/plugins/transforms/normaliser/src/test/resources/normaliser-transform.xml
index 89733e17df..47598c3476 100644
--- a/plugins/transforms/normaliser/src/test/resources/normaliser-transform.xml
+++ b/plugins/transforms/normaliser/src/test/resources/normaliser-transform.xml
@@ -24,6 +24,7 @@
                <schema_name />
        </partitioning>
        <typefield>type</typefield>
+       <valuefield>content</valuefield>
        <fields>
                <field>
                        <name>fieldA</name>

Reply via email to