This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 2975077e61 Row normaliser: normalise every input field when no fields
are listed… (#9446)
2975077e61 is described below
commit 2975077e61626c8f223cb7ac62e63dbadd921e3a
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Sat Nov 14 12:49:06 2026 +0100
Row normaliser: normalise every input field when no fields are listed…
(#9446)
* Row normaliser: normalise every input field when no fields are listed,
fixes #9397
* fix remarks
---
.../pages/pipeline/transforms/rownormaliser.adoc | 13 +-
.../transforms/0020-row-normaliser-all-fields.hpl | 159 +++++++++++++++
.../datasets/golden-row-normaliser-all-fields.csv | 7 +
.../transforms/main-0020-row-normaliser.hwf | 3 +
.../dataset/golden-row-normaliser-all-fields.json | 24 +++
.../0020-row-normaliser-all-fields UNIT.json | 30 +++
.../pipeline/transforms/normaliser/Normaliser.java | 187 ++++++++----------
.../transforms/normaliser/NormaliserData.java | 20 +-
.../transforms/normaliser/NormaliserDialog.java | 49 ++++-
.../transforms/normaliser/NormaliserMeta.java | 203 +++++++++++++++----
.../normaliser/messages/messages_en_US.properties | 5 +
.../normaliser/NormaliserMetaInjectionTest.java | 1 +
.../transforms/normaliser/NormaliserMetaTest.java | 167 +++++++++++++++-
.../transforms/normaliser/NormaliserTest.java | 219 +++++++++++++++++++++
.../src/test/resources/normaliser-transform.xml | 1 +
15 files changed, 919 insertions(+), 169 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/rownormaliser.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/rownormaliser.adoc
index 8c92709504..3fe51a73b3 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/rownormaliser.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/rownormaliser.adoc
@@ -31,6 +31,12 @@ For every row it reads, the transform writes one row per
type. Each input field
A type can fill each new field from only one input field: listing two fields
with the same type and the same new field is an error.
+== Normalising every input field
+
+When the **Fields table** is empty, the transform normalises every field it
receives. Each input field is a type of its own, named after the field, and its
value goes into the **Value field**. This works when the input fields are only
known when the pipeline runs, for example in a pipeline that profiles any file
or table it is given.
+
+Every input field is normalised this way, so no field is passed on unchanged:
the output has only the type field and the value field. When the input fields
have different data types the value field is a String, as described below. The
value field keeps a length or precision only when all input fields have the
same one. While you design a pipeline whose input fields are not known yet, the
transform shows the value field to the transforms after it as a String.
+
== Data types of the new fields
A new field is filled from a different input field on every row, so its data
type depends on all of them:
@@ -49,8 +55,9 @@ NOTE: Before Hop 2.20, a new field took the data type of the
first input field f
|Option|Description
|Transform name|Name of the transform, this name has to be unique in a single
pipeline.
|Typefield|The name of the type field (**key** in the example below).
+|Value field|The name of the field that holds the values when the **Fields
table** is empty and every input field is normalised. The default is **value**.
Give it a name other than the type field's: otherwise it is renamed in the
output, to **value_1** for example.
|Fields table|
-A list of the fields you want to normalize; you must set the following
properties for each selected field:
+A list of the fields you want to normalize. Leave it empty to normalise every
input field into the **Value field**. You must set the following properties for
each listed field:
* Fieldname: Name of the fields to normalize, as you get them from the input
transform.
* Type: Give a string to classify the field (you can use the same field names,
or input custom strings).
@@ -71,7 +78,7 @@ A list of the fields you want to normalize; you must set the
following propertie
|===
=== Normalized data (example 1)
-Set **Typefield** = "key" and use the **Get Fields** button to load all the
fields for normalization, set also **New field** = "value" in all rows. The
result is:
+Set **Typefield** = "key" and **Value field** = "value", and leave the
**Fields table** empty. The result is the same as using the **Get Fields**
button to list all the fields and setting **New field** = "value" in all rows,
but it follows the fields of the input, whatever they are:
[options="header"]
|===
@@ -91,7 +98,7 @@ Set **Typefield** = "key" and use the **Get Fields** button
to load all the fiel
|===
=== Normalized data (example 2)
-Similar to example 1, but remove the **RecordID** field from the **Fields
table**. The result is:
+Set **Typefield** = "key", use the **Get Fields** button to list all the
fields, remove the **RecordID** row from the **Fields table** and set **New
field** = "value" in the other rows. **RecordID** is not listed, so it is
passed on unchanged. The result is:
[options="header"]
|===
diff --git a/integration-tests/transforms/0020-row-normaliser-all-fields.hpl
b/integration-tests/transforms/0020-row-normaliser-all-fields.hpl
new file mode 100644
index 0000000000..9b0cf8a041
--- /dev/null
+++ b/integration-tests/transforms/0020-row-normaliser-all-fields.hpl
@@ -0,0 +1,159 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0020-row-normaliser-all-fields</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2021/05/31 11:24:13.758</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2021/05/31 11:24:13.758</modified_date>
+ <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Sample data</from>
+ <to>Normalise all fields</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Normalise all fields</from>
+ <to>Verify</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Normalise all fields</name>
+ <type>Normaliser</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <typefield>column</typefield>
+ <valuefield>content</valuefield>
+ <fields>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>272</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Sample data</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>name</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>age</name>
+ <type>Integer</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>city</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>Ann</item>
+ <item>31</item>
+ <item>Paris</item>
+ </line>
+ <line>
+ <item>Bob</item>
+ <item/>
+ <item>Rome</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>128</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Verify</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>416</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/transforms/datasets/golden-row-normaliser-all-fields.csv
b/integration-tests/transforms/datasets/golden-row-normaliser-all-fields.csv
new file mode 100644
index 0000000000..951775c0f2
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-row-normaliser-all-fields.csv
@@ -0,0 +1,7 @@
+column,content
+name,Ann
+age,31
+city,Paris
+name,Bob
+age,
+city,Rome
diff --git a/integration-tests/transforms/main-0020-row-normaliser.hwf
b/integration-tests/transforms/main-0020-row-normaliser.hwf
index bc6d6a3766..f0a0c7a785 100644
--- a/integration-tests/transforms/main-0020-row-normaliser.hwf
+++ b/integration-tests/transforms/main-0020-row-normaliser.hwf
@@ -63,6 +63,9 @@ limitations under the License.
<test_name>
<name>0020-row-normaliser-mixed-types UNIT</name>
</test_name>
+ <test_name>
+ <name>0020-row-normaliser-all-fields UNIT</name>
+ </test_name>
</test_names>
<parallel>N</parallel>
<xloc>272</xloc>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-row-normaliser-all-fields.json
b/integration-tests/transforms/metadata/dataset/golden-row-normaliser-all-fields.json
new file mode 100644
index 0000000000..0e62d690a6
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-row-normaliser-all-fields.json
@@ -0,0 +1,24 @@
+{
+ "base_filename": "golden-row-normaliser-all-fields.csv",
+ "name": "golden-row-normaliser-all-fields",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "column"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_format": "",
+ "field_name": "content"
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/unit-test/0020-row-normaliser-all-fields
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0020-row-normaliser-all-fields
UNIT.json
new file mode 100644
index 0000000000..9dda44bb9c
--- /dev/null
+++
b/integration-tests/transforms/metadata/unit-test/0020-row-normaliser-all-fields
UNIT.json
@@ -0,0 +1,30 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": true,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "column",
+ "data_set_field": "column"
+ },
+ {
+ "transform_field": "content",
+ "data_set_field": "content"
+ }
+ ],
+ "field_order": [],
+ "transform_name": "Verify",
+ "data_set_name": "golden-row-normaliser-all-fields"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0020-row-normaliser-all-fields UNIT",
+ "description": "",
+ "trans_test_tweaks": [],
+ "persist_filename": "",
+ "pipeline_filename": "./0020-row-normaliser-all-fields.hpl",
+ "test_type": "UNIT_TEST"
+}
\ No newline at end of file
diff --git
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/Normaliser.java
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/Normaliser.java
index dfda2c2768..4ff54446a5 100644
---
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/Normaliser.java
+++
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/Normaliser.java
@@ -18,9 +18,10 @@
package org.apache.hop.pipeline.transforms.normaliser;
import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
-import java.util.Set;
+import java.util.Map;
+import java.util.stream.IntStream;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.i18n.BaseMessages;
@@ -28,6 +29,7 @@ import org.apache.hop.pipeline.Pipeline;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransform;
import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.normaliser.NormaliserData.Placement;
/** Normalise de-normalised input data. */
public class Normaliser extends BaseTransform<NormaliserMeta, NormaliserData> {
@@ -53,121 +55,30 @@ public class Normaliser extends
BaseTransform<NormaliserMeta, NormaliserData> {
return false;
}
- if (first) { // INITIALISE
-
+ if (first) {
first = false;
-
- data.inputRowMeta = getInputRowMeta();
- data.outputRowMeta = data.inputRowMeta.clone();
- meta.getFields(data.outputRowMeta, getTransformName(), null, null, this,
metadataProvider);
- data.typeToPlacements = new HashMap<>();
- String typeValue;
- int dataFieldNr;
-
- // The normalised fields are the last ones of the output row, in this
order.
- //
- List<String> normNames = meta.getNormalisedFieldNames();
- int firstNormIndex = data.outputRowMeta.size() - normNames.size();
-
- String duplicate = meta.getDuplicateMapping();
- if (duplicate != null) {
- throw new HopException(duplicate);
- }
-
- // Get a unique list of occurrences...
- //
- data.type_occ = new ArrayList<>();
- data.maxlen = 0;
-
- for (NormaliserField field : meta.getNormaliserFields()) {
- typeValue = field.getValue();
- if (!data.type_occ.contains(typeValue)) {
- data.type_occ.add(typeValue);
- }
- if (typeValue.length() > data.maxlen) {
- data.maxlen = typeValue.length();
- }
-
- // For every type, the fields that get normalised under it and where
each one goes: into
- // the normalised field it names, whatever order the fields are listed
in.
- //
- dataFieldNr = data.inputRowMeta.indexOfValue(field.getName());
- if (dataFieldNr < 0) {
- logError(
- BaseMessages.getString(
- PKG, "Normaliser.Log.CouldNotFindFieldInRow",
field.getName()));
- setErrors(1);
- stopAll();
- return false;
- }
- int outputIndex = firstNormIndex + normNames.indexOf(field.getNorm());
- IValueMeta source = data.inputRowMeta.getValueMeta(dataFieldNr);
- IValueMeta target = data.outputRowMeta.getValueMeta(outputIndex);
- boolean copy =
- source.getType() == target.getType()
- && source.getStorageType() == target.getStorageType();
- data.typeToPlacements
- .computeIfAbsent(typeValue, k -> new ArrayList<>())
- .add(
- new NormaliserData.Placement(
- dataFieldNr, outputIndex, source, copy ? null : target));
- }
-
- // Which fields are not impacted? We can just copy these, leave them
alone.
- //
- data.copyFieldnrs = new ArrayList<>();
-
- Set<String> normaliserFields = meta.getFieldNames();
- int irmSize = data.inputRowMeta.size();
-
- for (int i = 0; i < irmSize; i++) {
- IValueMeta v = data.inputRowMeta.getValueMeta(i);
- // Backwards compatibility - old loop called Const.indexofstring which
uses equalsIgnoreCase
- if (!normaliserFields.contains(v.getName().toLowerCase())) {
- data.copyFieldnrs.add(i);
- }
+ if (!initialise()) {
+ return false;
}
}
- // Modest performance improvement over millions of rows - don't
recalculate on each loop
- // iteration something that doesn't change
- int typeOccSize = data.type_occ.size();
- int copyFldNrsSz = data.copyFieldnrs.size();
+ String[] types = data.types;
+ Placement[][] placements = data.placements;
+ int[] copyFieldNrs = data.copyFieldNrs;
int rowMetaSz = data.outputRowMeta.size();
- // Modest performance improvement (large memory improvement) - re-use
temporary objects instead
- // of re-creating them - better for GC over time
- String typeValue;
- Object[] outputRowData;
- int outputIndex, nr;
- Object value;
-
- // Now do the normalization
- // Loop over the unique occurrences of the different types.
+ // One output row per type: the fields that are kept, the type, then the
normalised fields.
//
- for (int e = 0; e < typeOccSize; e++) {
- typeValue = data.type_occ.get(e);
-
- // Create an output row per type
- //
- outputRowData = new Object[rowMetaSz];
- outputIndex = 0;
-
- // Copy the input row data, excluding the fields that are normalized...
- //
- for (int i = 0; i < copyFldNrsSz; i++) {
- nr = data.copyFieldnrs.get(i);
+ for (int t = 0; t < types.length; t++) {
+ Object[] outputRowData = new Object[rowMetaSz];
+ int outputIndex = 0;
+ for (int nr : copyFieldNrs) {
outputRowData[outputIndex++] = r[nr];
}
+ outputRowData[outputIndex] = types[t];
- // Add the typefield_value
- //
- outputRowData[outputIndex++] = typeValue;
-
- // Then add the normalized fields...
- //
- for (NormaliserData.Placement placement :
data.typeToPlacements.get(typeValue)) {
- value = r[placement.inputIndex()];
+ for (Placement placement : placements[t]) {
+ Object value = r[placement.inputIndex()];
IValueMeta target = placement.target();
if (target != null) {
// Either the same type in another storage, or text for a field of
mixed types.
@@ -179,8 +90,6 @@ public class Normaliser extends
BaseTransform<NormaliserMeta, NormaliserData> {
outputRowData[placement.outputIndex()] = value;
}
- // The row is constructed, now give it to the next transform(s)...
- //
putRow(data.outputRowMeta, outputRowData);
}
@@ -190,4 +99,64 @@ public class Normaliser extends
BaseTransform<NormaliserMeta, NormaliserData> {
return true;
}
+
+ /**
+ * Works out, once, which fields are kept and where every normalised value
goes.
+ *
+ * @return false when a field to normalise is missing from the input; the
transform has stopped
+ */
+ private boolean initialise() throws HopException {
+ data.inputRowMeta = getInputRowMeta();
+ data.outputRowMeta = data.inputRowMeta.clone();
+ meta.getFields(data.outputRowMeta, getTransformName(), null, null, this,
metadataProvider);
+
+ List<NormaliserField> fields =
meta.getFieldsToNormalise(data.inputRowMeta);
+
+ String duplicate = NormaliserMeta.getDuplicateMapping(fields);
+ if (duplicate != null) {
+ throw new HopException(duplicate);
+ }
+
+ // The normalised fields are the last ones of the output row, in this
order.
+ //
+ List<String> normNames = NormaliserMeta.getNormalisedFieldNames(fields);
+ int firstNormIndex = data.outputRowMeta.size() - normNames.size();
+
+ // For every type, in the order it first appears, the fields that get
normalised under it and
+ // where each one goes: into the normalised field it names, whatever order
they are listed in.
+ //
+ Map<String, List<Placement>> typeToPlacements = new LinkedHashMap<>();
+ boolean[] normalised = new boolean[data.inputRowMeta.size()];
+ for (NormaliserField field : fields) {
+ int dataFieldNr = data.inputRowMeta.indexOfValue(field.getName());
+ if (dataFieldNr < 0) {
+ logError(
+ BaseMessages.getString(PKG,
"Normaliser.Log.CouldNotFindFieldInRow", field.getName()));
+ setErrors(1);
+ stopAll();
+ return false;
+ }
+ normalised[dataFieldNr] = true;
+ int outputIndex = firstNormIndex + normNames.indexOf(field.getNorm());
+ IValueMeta source = data.inputRowMeta.getValueMeta(dataFieldNr);
+ IValueMeta target = data.outputRowMeta.getValueMeta(outputIndex);
+ boolean copy =
+ source.getType() == target.getType()
+ && source.getStorageType() == target.getStorageType();
+ typeToPlacements
+ .computeIfAbsent(field.getValue(), k -> new ArrayList<>())
+ .add(new Placement(dataFieldNr, outputIndex, source, copy ? null :
target));
+ }
+ data.types = typeToPlacements.keySet().toArray(new String[0]);
+ data.placements =
+ typeToPlacements.values().stream()
+ .map(list -> list.toArray(new Placement[0]))
+ .toArray(Placement[][]::new);
+
+ // The fields that are not normalised are copied as they are. Which ones
those are follows from
+ // the lookup above, the one getFields() uses to take the normalised
fields out of the layout.
+ //
+ data.copyFieldNrs = IntStream.range(0, normalised.length).filter(i ->
!normalised[i]).toArray();
+ return true;
+ }
}
diff --git
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserData.java
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserData.java
index 6bb2c777d8..1eee808880 100644
---
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserData.java
+++
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserData.java
@@ -17,8 +17,6 @@
package org.apache.hop.pipeline.transforms.normaliser;
-import java.util.List;
-import java.util.Map;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.pipeline.transform.BaseTransformData;
@@ -26,19 +24,17 @@ import org.apache.hop.pipeline.transform.ITransformData;
@SuppressWarnings("java:S1104")
public class NormaliserData extends BaseTransformData implements
ITransformData {
- public List<String> type_occ;
- public int maxlen;
- public List<Integer> copyFieldnrs;
- Map<String, List<Placement>> typeToPlacements;
+ /** The types, in the order their rows are written. */
+ String[] types;
- public IRowMeta inputRowMeta;
- public IRowMeta outputRowMeta;
+ /** Per type, in the order of {@link #types}: where its input fields go. */
+ Placement[][] placements;
- public NormaliserData() {
- super();
+ /** The input fields that are copied to every output row, in input order. */
+ int[] copyFieldNrs;
- type_occ = null;
- }
+ public IRowMeta inputRowMeta;
+ public IRowMeta outputRowMeta;
/**
* Where one input field goes on the output row of its type.
diff --git
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserDialog.java
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserDialog.java
index 65ef335092..16ee7ec63b 100644
---
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserDialog.java
+++
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserDialog.java
@@ -55,6 +55,10 @@ public class NormaliserDialog extends BaseTransformDialog {
private Text wTypefield;
+ private Label wlValuefield;
+
+ private Text wValuefield;
+
private TableView wFields;
private final NormaliserMeta input;
@@ -97,12 +101,32 @@ public class NormaliserDialog extends BaseTransformDialog {
fdTypefield.right = new FormAttachment(100, 0);
wTypefield.setLayoutData(fdTypefield);
+ // Value field line: used when no fields are listed
+ wlValuefield = new Label(shell, SWT.RIGHT);
+ wlValuefield.setText(BaseMessages.getString(PKG,
"NormaliserDialog.ValueField.Label"));
+ wlValuefield.setToolTipText(BaseMessages.getString(PKG,
"NormaliserDialog.ValueField.Tooltip"));
+ PropsUi.setLook(wlValuefield);
+ FormData fdlValuefield = new FormData();
+ fdlValuefield.left = new FormAttachment(0, 0);
+ fdlValuefield.right = new FormAttachment(middle, -margin);
+ fdlValuefield.top = new FormAttachment(wTypefield, margin);
+ wlValuefield.setLayoutData(fdlValuefield);
+ wValuefield = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
+ wValuefield.setToolTipText(BaseMessages.getString(PKG,
"NormaliserDialog.ValueField.Tooltip"));
+ PropsUi.setLook(wValuefield);
+ wValuefield.addModifyListener(lsMod);
+ FormData fdValuefield = new FormData();
+ fdValuefield.left = new FormAttachment(middle, 0);
+ fdValuefield.top = new FormAttachment(wTypefield, margin);
+ fdValuefield.right = new FormAttachment(100, 0);
+ wValuefield.setLayoutData(fdValuefield);
+
Label wlFields = new Label(shell, SWT.NONE);
wlFields.setText(BaseMessages.getString(PKG,
"NormaliserDialog.Fields.Label"));
PropsUi.setLook(wlFields);
FormData fdlFields = new FormData();
fdlFields.left = new FormAttachment(0, 0);
- fdlFields.top = new FormAttachment(wTypefield, margin);
+ fdlFields.top = new FormAttachment(wValuefield, margin);
wlFields.setLayoutData(fdlFields);
final int fieldsCols = 3;
@@ -126,6 +150,12 @@ public class NormaliserDialog extends BaseTransformDialog {
false);
colinf[2].setNamingSchemeType(NamingSchemeTypes.HOP_FIELD);
+ // The value field is only used while no fields are listed
+ ModifyListener lsFieldsMod =
+ e -> {
+ input.setChanged();
+ enableValueField();
+ };
wFields =
new TableView(
variables,
@@ -133,8 +163,9 @@ public class NormaliserDialog extends BaseTransformDialog {
SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI,
colinf,
fieldsRows,
- lsMod,
+ lsFieldsMod,
props);
+ wFields.setContentListener(e -> enableValueField());
FormData fdFields = new FormData();
fdFields.left = new FormAttachment(0, 0);
@@ -183,6 +214,7 @@ public class NormaliserDialog extends BaseTransformDialog {
if (input.getTypeField() != null) {
wTypefield.setText(input.getTypeField());
}
+ wValuefield.setText(input.getValueFieldName());
for (int i = 0; i < input.getNormaliserFields().size(); i++) {
TableItem item = wFields.table.getItem(i);
@@ -200,6 +232,14 @@ public class NormaliserDialog extends BaseTransformDialog {
wFields.setRowNums();
wFields.optWidth(true);
+ enableValueField();
+ }
+
+ /** The value field is only used when the fields table is empty. */
+ private void enableValueField() {
+ boolean noFieldsListed = wFields.nrNonEmpty() == 0;
+ wlValuefield.setEnabled(noFieldsListed);
+ wValuefield.setEnabled(noFieldsListed);
}
private void cancel() {
@@ -220,6 +260,10 @@ public class NormaliserDialog extends BaseTransformDialog {
int i;
int nrFields = wFields.nrNonEmpty();
+ if (nrFields == 0) {
+ // Listed fields leave the value field unused: keep what it was
+ input.setValueField(wValuefield.getText());
+ }
input.getNormaliserFields().clear();
for (i = 0; i < nrFields; i++) {
@@ -240,6 +284,7 @@ public class NormaliserDialog extends BaseTransformDialog {
if (r != null && !r.isEmpty()) {
BaseTransformDialog.getFieldsFromPrevious(
r, wFields, 1, new int[] {1, 2}, new int[] {}, -1, -1, null);
+ enableValueField();
}
} catch (HopException ke) {
new ErrorDialog(
diff --git
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMeta.java
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMeta.java
index 42469332e6..5b21328231 100644
---
a/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMeta.java
+++
b/plugins/transforms/normaliser/src/main/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMeta.java
@@ -18,10 +18,8 @@
package org.apache.hop.pipeline.transforms.normaliser;
import java.util.ArrayList;
-import java.util.HashSet;
import java.util.List;
import java.util.Objects;
-import java.util.Set;
import lombok.Getter;
import lombok.Setter;
import org.apache.hop.core.CheckResult;
@@ -32,6 +30,7 @@ import org.apache.hop.core.exception.HopTransformException;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.HopMetadataProperty;
@@ -77,6 +76,8 @@ DATE PRODUCT Sales Number
public class NormaliserMeta extends BaseTransformMeta<Normaliser,
NormaliserData> {
private static final Class<?> PKG = NormaliserMeta.class;
+ public static final String DEFAULT_VALUE_FIELD = "value";
+
/** Name of the new type-field */
@HopMetadataProperty(
key = "typefield",
@@ -91,32 +92,74 @@ public class NormaliserMeta extends
BaseTransformMeta<Normaliser, NormaliserData
injectionGroupDescription = "NormaliserMeta.Injection.FIELDS")
private List<NormaliserField> normaliserFields;
+ /** Name of the field that holds the values when every input field is
normalised. */
+ @HopMetadataProperty(
+ key = "valuefield",
+ injectionKey = "VALUEFIELD",
+ injectionKeyDescription = "NormaliserMeta.Injection.VALUEFIELD")
+ private String valueField;
+
public NormaliserMeta() {
super();
this.normaliserFields = new ArrayList<>();
}
- public Set<String> getFieldNames() {
- Set<String> fieldNames = new HashSet<>();
-
- for (NormaliserField field : normaliserFields) {
- if (field.getName() != null) {
- fieldNames.add(field.getName().toLowerCase());
- }
- }
- return fieldNames;
- }
-
@Override
public void setDefault() {
this.typeField = "typefield";
+ this.valueField = DEFAULT_VALUE_FIELD;
this.normaliserFields = new ArrayList<>();
}
- /** The names of the normalised fields, in output order: in the order they
first appear. */
- public List<String> getNormalisedFieldNames() {
+ /**
+ * @param normaliserFields the fields to normalise; null, from metadata
injection for example, is
+ * an empty list: every input field is normalised then
+ */
+ public void setNormaliserFields(List<NormaliserField> normaliserFields) {
+ this.normaliserFields = normaliserFields == null ? new ArrayList<>() :
normaliserFields;
+ }
+
+ /** True when no fields are listed: every input field is normalised then. */
+ public boolean isNormalisingAllFields() {
+ return normaliserFields == null || normaliserFields.isEmpty();
+ }
+
+ /** The name of the value field, or its default when none is set. */
+ public String getValueFieldName() {
+ return Utils.isEmpty(valueField) ? DEFAULT_VALUE_FIELD : valueField;
+ }
+
+ /**
+ * The fields to normalise: the listed fields, or with an empty list every
input field. An input
+ * field normalised that way gets its own name as type and puts its value in
the value field.
+ *
+ * @param inputRowMeta the fields entering the transform
+ * @return the fields to normalise
+ */
+ public List<NormaliserField> getFieldsToNormalise(IRowMeta inputRowMeta) {
+ if (!isNormalisingAllFields()) {
+ return normaliserFields;
+ }
+ String norm = getValueFieldName();
+ List<NormaliserField> fields = new ArrayList<>(inputRowMeta.size());
+ for (IValueMeta valueMeta : inputRowMeta.getValueMetaList()) {
+ NormaliserField field = new NormaliserField();
+ field.setName(valueMeta.getName());
+ field.setValue(valueMeta.getName());
+ field.setNorm(norm);
+ fields.add(field);
+ }
+ return fields;
+ }
+
+ /**
+ * The names of the normalised fields, in output order: in the order they
first appear.
+ *
+ * @param fields the fields to normalise, see {@link
#getFieldsToNormalise(IRowMeta)}
+ */
+ public static List<String> getNormalisedFieldNames(List<NormaliserField>
fields) {
List<String> names = new ArrayList<>();
- for (NormaliserField field : normaliserFields) {
+ for (NormaliserField field : fields) {
if (!names.contains(field.getNorm())) {
names.add(field.getNorm());
}
@@ -134,14 +177,28 @@ public class NormaliserMeta extends
BaseTransformMeta<Normaliser, NormaliserData
* transform that serializes or renders it. See issue #3636.
*
* @param inputRowMeta the fields entering the transform
- * @return one value metadata per name of {@link
#getNormalisedFieldNames()}, in the same order
+ * @return one value metadata per name of {@link
#getNormalisedFieldNames(List)}, in the same
+ * order
* @throws HopTransformException when the first input field of a normalised
field is missing
*/
public List<IValueMeta> getNormalisedValueMetas(IRowMeta inputRowMeta)
throws HopTransformException {
+ return getNormalisedValueMetas(inputRowMeta,
getFieldsToNormalise(inputRowMeta));
+ }
+
+ private List<IValueMeta> getNormalisedValueMetas(
+ IRowMeta inputRowMeta, List<NormaliserField> fieldsToNormalise) throws
HopTransformException {
+ boolean allFields = isNormalisingAllFields();
List<IValueMeta> valueMetas = new ArrayList<>();
- for (String normName : getNormalisedFieldNames()) {
- List<NormaliserField> fields = getFieldsOf(normName);
+ if (allFields && fieldsToNormalise.isEmpty()) {
+ // No input fields are known, while designing a pipeline whose input is
only known when it
+ // runs for example. The value field is there all the same, and it can
hold any value as text.
+ //
+ valueMetas.add(new ValueMetaString(getValueFieldName()));
+ return valueMetas;
+ }
+ for (String normName : getNormalisedFieldNames(fieldsToNormalise)) {
+ List<NormaliserField> fields = getFieldsOf(fieldsToNormalise, normName);
IValueMeta first = inputRowMeta.searchValueMeta(fields.get(0).getName());
if (first == null) {
throw new HopTransformException(
@@ -150,11 +207,15 @@ public class NormaliserMeta extends
BaseTransformMeta<Normaliser, NormaliserData
}
boolean sameType = true;
boolean sameStorage = true;
+ boolean sameLength = true;
+ boolean samePrecision = true;
for (NormaliserField field : fields) {
IValueMeta source = inputRowMeta.searchValueMeta(field.getName());
if (source != null) {
sameType &= source.getType() == first.getType();
sameStorage &= source.getStorageType() == first.getStorageType();
+ sameLength &= source.getLength() == first.getLength();
+ samePrecision &= source.getPrecision() == first.getPrecision();
}
}
@@ -167,6 +228,16 @@ public class NormaliserMeta extends
BaseTransformMeta<Normaliser, NormaliserData
v.setStorageType(IValueMeta.STORAGE_TYPE_NORMAL);
v.setStorageMetadata(null);
}
+ // With every input field normalised, which one comes first is up to
the input. The value
+ // field keeps only a length or precision they all share: the first
field's could be too
+ // short for the values of the others, in a database table or a fixed
width file.
+ //
+ if (allFields && !sameLength) {
+ v.setLength(-1);
+ }
+ if (allFields && !samePrecision) {
+ v.setPrecision(-1);
+ }
}
v.setName(normName);
valueMetas.add(v);
@@ -184,22 +255,34 @@ public class NormaliserMeta extends
BaseTransformMeta<Normaliser, NormaliserData
IHopMetadataProvider metadataProvider)
throws HopTransformException {
+ boolean allFields = isNormalisingAllFields();
+ List<NormaliserField> fields = getFieldsToNormalise(row);
+
int maxlen = 0;
- for (NormaliserField field : normaliserFields) {
+ for (NormaliserField field : fields) {
if (field.getValue().length() > maxlen) {
maxlen = field.getValue().length();
}
}
- // Take the normalised fields from the input before adding anything to it.
+ // Take the normalised fields from the input before changing it.
//
- List<IValueMeta> normalisedValueMetas = getNormalisedValueMetas(row);
+ List<IValueMeta> normalisedValueMetas = getNormalisedValueMetas(row,
fields);
+
+ // With every input field normalised, remove them before adding the new
fields: the value field
+ // may have the name of an input field, value for example, and keeps it.
+ //
+ if (allFields) {
+ removeFields(row, fields);
+ }
// Then add the type field!
//
IValueMeta typefieldValue = new ValueMetaString(typeField);
typefieldValue.setOrigin(name);
- typefieldValue.setLength(maxlen);
+ if (!fields.isEmpty()) {
+ typefieldValue.setLength(maxlen);
+ }
row.addValueMeta(typefieldValue);
// Add the new fields that need to be created.
@@ -209,9 +292,16 @@ public class NormaliserMeta extends
BaseTransformMeta<Normaliser, NormaliserData
row.addValueMeta(v);
}
- // Now remove all the normalized fields...
+ // Listed fields are removed after the new fields are added, as they
always were: a new field
+ // with the name of a listed field is called value_1 then, and pipelines
rely on that name.
//
- for (NormaliserField field : normaliserFields) {
+ if (!allFields) {
+ removeFields(row, fields);
+ }
+ }
+
+ private static void removeFields(IRowMeta row, List<NormaliserField> fields)
{
+ for (NormaliserField field : fields) {
int idx = row.indexOfValue(field.getName());
if (idx >= 0) {
row.removeValueMeta(idx);
@@ -244,11 +334,24 @@ public class NormaliserMeta extends
BaseTransformMeta<Normaliser, NormaliserData
transformMeta);
remarks.add(cr);
+ List<NormaliserField> fields = getFieldsToNormalise(prev);
+ if (isNormalisingAllFields()) {
+ remarks.add(
+ new CheckResult(
+ ICheckResult.TYPE_RESULT_OK,
+ BaseMessages.getString(
+ PKG,
+ "NormaliserMeta.CheckResult.AllInputFieldsNormalised",
+ prev.size() + "",
+ getValueFieldName()),
+ transformMeta));
+ }
+
boolean first = true;
errorMessage = "";
boolean errorFound = false;
- for (NormaliserField field : normaliserFields) {
+ for (NormaliserField field : fields) {
IValueMeta valueMeta = prev.searchValueMeta(field.getName());
if (valueMeta == null) {
if (first) {
@@ -271,8 +374,8 @@ public class NormaliserMeta extends
BaseTransformMeta<Normaliser, NormaliserData
}
remarks.add(cr);
- for (String normName : getNormalisedFieldNames()) {
- if (hasMixedTypes(prev, normName)) {
+ for (String normName : getNormalisedFieldNames(fields)) {
+ if (hasMixedTypes(prev, fields, normName)) {
remarks.add(
new CheckResult(
ICheckResult.TYPE_RESULT_WARNING,
@@ -289,11 +392,28 @@ public class NormaliserMeta extends
BaseTransformMeta<Normaliser, NormaliserData
remarks.add(cr);
}
- String duplicate = getDuplicateMapping();
+ String duplicate = getDuplicateMapping(normaliserFields);
if (duplicate != null) {
remarks.add(new CheckResult(ICheckResult.TYPE_RESULT_ERROR, duplicate,
transformMeta));
}
+ // The type field comes first, so a new field with its name is renamed, to
value_1 for example.
+ //
+ List<String> newFieldNames =
+ isNormalisingAllFields()
+ ? List.of(getValueFieldName())
+ : getNormalisedFieldNames(normaliserFields);
+ for (String newFieldName : newFieldNames) {
+ if (newFieldName != null && newFieldName.equalsIgnoreCase(typeField)) {
+ remarks.add(
+ new CheckResult(
+ ICheckResult.TYPE_RESULT_WARNING,
+ BaseMessages.getString(
+ PKG, "NormaliserMeta.CheckResult.SameNameAsTypeField",
newFieldName),
+ transformMeta));
+ }
+ }
+
// See if we have input streams leading to this transform!
if (input.length > 0) {
cr =
@@ -313,15 +433,14 @@ public class NormaliserMeta extends
BaseTransformMeta<Normaliser, NormaliserData
}
/** The fields filling a normalised field, in the order they are listed. */
- private List<NormaliserField> getFieldsOf(String normName) {
- return normaliserFields.stream()
- .filter(field -> Objects.equals(normName, field.getNorm()))
- .toList();
+ private static List<NormaliserField> getFieldsOf(List<NormaliserField>
fields, String normName) {
+ return fields.stream().filter(field -> Objects.equals(normName,
field.getNorm())).toList();
}
/** True when the input fields filling this normalised field are not all of
one type. */
- private boolean hasMixedTypes(IRowMeta inputRowMeta, String normName) {
- return getFieldsOf(normName).stream()
+ private static boolean hasMixedTypes(
+ IRowMeta inputRowMeta, List<NormaliserField> fields, String normName) {
+ return getFieldsOf(fields, normName).stream()
.map(field -> inputRowMeta.searchValueMeta(field.getName()))
.filter(Objects::nonNull)
.map(IValueMeta::getType)
@@ -334,13 +453,17 @@ public class NormaliserMeta extends
BaseTransformMeta<Normaliser, NormaliserData
* Two input fields filling the same normalised field on the same row leave
no room for one of
* them.
*
+ * @param fields the fields to normalise, see {@link
#getFieldsToNormalise(IRowMeta)}
* @return a description of the first such pair, or null when there is none
*/
- public String getDuplicateMapping() {
- for (int i = 0; i < normaliserFields.size(); i++) {
- NormaliserField one = normaliserFields.get(i);
- for (int j = i + 1; j < normaliserFields.size(); j++) {
- NormaliserField other = normaliserFields.get(j);
+ public static String getDuplicateMapping(List<NormaliserField> fields) {
+ if (fields == null) {
+ return null;
+ }
+ for (int i = 0; i < fields.size(); i++) {
+ NormaliserField one = fields.get(i);
+ for (int j = i + 1; j < fields.size(); j++) {
+ NormaliserField other = fields.get(j);
if (Objects.equals(one.getValue(), other.getValue())
&& Objects.equals(one.getNorm(), other.getNorm())) {
return BaseMessages.getString(
diff --git
a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_en_US.properties
b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_en_US.properties
index c0f43d493d..a5a8d7dabe 100644
---
a/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_en_US.properties
+++
b/plugins/transforms/normaliser/src/main/resources/org/apache/hop/pipeline/transforms/normaliser/messages/messages_en_US.properties
@@ -29,12 +29,16 @@ NormaliserDialog.GetFields.Button=\ &Get Fields
NormaliserDialog.Shell.Title=Row normaliser
NormaliserDialog.TransformName.Label=Transform name
NormaliserDialog.TypeField.Label=Type field
+NormaliserDialog.ValueField.Label=Value field
+NormaliserDialog.ValueField.Tooltip=Used when the fields table is empty: every
input field is normalised then, with its name in the type field and its value
in this field.
NormaliserMeta.CheckResult.AllFieldsFound=All fields to normalise are found in
the input stream.
+NormaliserMeta.CheckResult.AllInputFieldsNormalised=No fields are listed, so
all {0} input fields are normalised into field {1}.
NormaliserMeta.CheckResult.CouldNotReadFieldsFromPreviousTransform=Couldn''t
read fields from the previous transform.
NormaliserMeta.CheckResult.DuplicateMapping=Fields {0} and {1} both fill
normalised field {2} for type {3}. Every row has room for only one of them.
NormaliserMeta.CheckResult.FieldsNotFound=Fields to normalise, not found in
input stream:
NormaliserMeta.CheckResult.MixedTypes=Normalised field {0} is filled from
fields of different types, so it is a String holding the text of each value.
NormaliserMeta.CheckResult.NoInputReceivedError=No input received from other
transforms!
+NormaliserMeta.CheckResult.SameNameAsTypeField=Normalised field {0} has the
same name as the type field, so it is renamed in the output, to {0}_1 for
example.
NormaliserMeta.CheckResult.TransformReceivingFieldsOK=Transform is connected
to previous one, receiving {0} fields
NormaliserMeta.CheckResult.TransformReceivingInfoOK=Transform is receiving
info from other transforms.
NormaliserMeta.Exception.UnableToFindField=Unable to find field from upstream
transform: {0}
@@ -43,4 +47,5 @@ NormaliserMeta.Injection.NAME=Input field name
NormaliserMeta.Injection.NORMALISED=Normalised field name
NormaliserMeta.Injection.TYPEFIELD=The name of the type field
NormaliserMeta.Injection.VALUE=Type field value
+NormaliserMeta.Injection.VALUEFIELD=The name of the value field, used when no
fields are listed
NormaliserMeta.keyword=normalize,unpivot,denormalize,columns,reshape
diff --git
a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaInjectionTest.java
b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaInjectionTest.java
index bb73e9cc93..23efd9acc9 100644
---
a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaInjectionTest.java
+++
b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaInjectionTest.java
@@ -35,6 +35,7 @@ class NormaliserMetaInjectionTest extends
BaseMetadataInjectionTestJunit5<Normal
@Test
void test() throws Exception {
check("TYPEFIELD", () -> meta.getTypeField());
+ check("VALUEFIELD", () -> meta.getValueField());
check("NAME", () -> meta.getNormaliserFields().get(0).getName());
check("VALUE", () -> meta.getNormaliserFields().get(0).getValue());
check("NORMALISED", () -> meta.getNormaliserFields().get(0).getNorm());
diff --git
a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaTest.java
b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaTest.java
index be2131f6ea..1fe2c09fa8 100644
---
a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaTest.java
+++
b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserMetaTest.java
@@ -18,6 +18,7 @@ package org.apache.hop.pipeline.transforms.normaliser;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
@@ -60,9 +61,28 @@ class NormaliserMetaTest {
TransformSerializationTestUtil.testSerialization(
"/normaliser-transform.xml", NormaliserMeta.class);
- // assertEquals(2, meta.getFields().size());
- // assertEquals("fieldName", meta.getFields().get(0).getName());
- // assertEquals("two", meta.getFields().get(0).getValue());
+ assertEquals("type", meta.getTypeField());
+ assertEquals("content", meta.getValueField());
+ assertEquals(4, meta.getNormaliserFields().size());
+ NormaliserField last = meta.getNormaliserFields().get(3);
+ assertEquals("fieldD", last.getName());
+ assertEquals("D", last.getValue());
+ assertEquals("field", last.getNorm());
+ }
+
+ /** The type field is as long as the longest type it holds. */
+ @Test
+ void theTypeFieldIsAsLongAsTheLongestType() throws Exception {
+ IRowMeta row = new RowMeta();
+ row.addValueMeta(new ValueMetaInteger("i1"));
+ row.addValueMeta(new ValueMetaInteger("i2"));
+
+ meta(field("i1", "short", "value"), field("i2", "much longer", "value"))
+ .getFields(row, "normaliser", null, null, new Variables(), null);
+
+ IValueMeta type = row.searchValueMeta("typefield");
+ assertEquals(IValueMeta.TYPE_STRING, type.getType());
+ assertEquals("much longer".length(), type.getLength());
}
@Test
@@ -199,6 +219,147 @@ class NormaliserMetaTest {
remarks.toString());
}
+ /** Issue #9397: no listed fields means every input field, into the value
field. */
+ @Test
+ void withoutListedFieldsEveryInputFieldIsNormalisedIntoTheValueField()
throws Exception {
+ IRowMeta row = new RowMeta();
+ row.addValueMeta(new ValueMetaInteger("i"));
+ row.addValueMeta(new ValueMetaString("a much longer name"));
+
+ NormaliserMeta meta = meta();
+ meta.setValueField("content");
+ meta.getFields(row, "normaliser", null, null, new Variables(), null);
+
+ assertArrayEquals(new String[] {"typefield", "content"},
row.getFieldNames());
+ assertEquals("a much longer name".length(),
row.getValueMeta(0).getLength());
+ assertEquals(IValueMeta.TYPE_STRING, row.getValueMeta(1).getType());
+ }
+
+ /** A transform saved before the value field existed has none: it is called
value then. */
+ @Test
+ void aMissingValueFieldIsCalledValue() throws Exception {
+ IRowMeta row = new RowMeta();
+ row.addValueMeta(new ValueMetaInteger("i"));
+
+ NormaliserMeta meta = meta();
+ meta.setValueField(null);
+ meta.getFields(row, "normaliser", null, null, new Variables(), null);
+
+ assertArrayEquals(new String[] {"typefield", "value"},
row.getFieldNames());
+ assertEquals(IValueMeta.TYPE_INTEGER, row.getValueMeta(1).getType());
+ }
+
+ @Test
+ void checkSaysEveryInputFieldIsNormalisedWhenNoneAreListed() {
+ IRowMeta prev = new RowMeta();
+ prev.addValueMeta(new ValueMetaInteger("i1"));
+ prev.addValueMeta(new ValueMetaInteger("i2"));
+
+ List<ICheckResult> remarks = check(meta(), prev);
+
+ assertTrue(
+ remarks.stream()
+ .anyMatch(
+ r ->
+ r.getType() == ICheckResult.TYPE_RESULT_OK
+ && r.getText().contains("2")
+ && r.getText().contains("value")),
+ remarks.toString());
+ assertTrue(
+ remarks.stream().allMatch(r -> r.getType() ==
ICheckResult.TYPE_RESULT_OK),
+ remarks.toString());
+ }
+
+ /**
+ * Input fields that are not known yet, while designing a pipeline that gets
them at run time,
+ * still leave the value field for the transforms after this one.
+ */
+ @Test
+ void withoutListedFieldsTheValueFieldIsDeclaredWhileTheInputIsUnknown()
throws Exception {
+ IRowMeta row = new RowMeta();
+
+ meta().getFields(row, "normaliser", null, null, new Variables(), null);
+
+ assertArrayEquals(new String[] {"typefield", "value"},
row.getFieldNames());
+ assertEquals(IValueMeta.TYPE_STRING, row.getValueMeta(1).getType());
+ }
+
+ /**
+ * With every input field normalised, which one comes first is up to the
input: its length would
+ * be too short for the values of the others. The value field keeps only
what they all share.
+ */
+ @Test
+ void
withoutListedFieldsTheValueFieldKeepsOnlyALengthAndPrecisionAllFieldsShare()
+ throws Exception {
+ IRowMeta texts = new RowMeta();
+ texts.addValueMeta(new ValueMetaString("code", 10, -1));
+ texts.addValueMeta(new ValueMetaString("description", 500, -1));
+ meta().getFields(texts, "normaliser", null, null, new Variables(), null);
+ assertEquals(-1, texts.searchValueMeta("value").getLength());
+
+ IRowMeta amounts = new RowMeta();
+ amounts.addValueMeta(new ValueMetaNumber("net", 9, 2));
+ amounts.addValueMeta(new ValueMetaNumber("rate", 9, 4));
+ meta().getFields(amounts, "normaliser", null, null, new Variables(), null);
+ IValueMeta amount = amounts.searchValueMeta("value");
+ assertEquals(9, amount.getLength());
+ assertEquals(-1, amount.getPrecision());
+
+ IRowMeta same = new RowMeta();
+ same.addValueMeta(new ValueMetaNumber("q1", 9, 2));
+ same.addValueMeta(new ValueMetaNumber("q2", 9, 2));
+ meta().getFields(same, "normaliser", null, null, new Variables(), null);
+ IValueMeta quarter = same.searchValueMeta("value");
+ assertEquals(9, quarter.getLength());
+ assertEquals(2, quarter.getPrecision());
+ }
+
+ /** The type field comes first: a new field with its name is renamed, which
Verify points out. */
+ @Test
+ void checkWarnsAboutANewFieldWithTheNameOfTheTypeField() {
+ IRowMeta prev = new RowMeta();
+ prev.addValueMeta(new ValueMetaInteger("i1"));
+ prev.addValueMeta(new ValueMetaInteger("i2"));
+
+ NormaliserMeta allFields = meta();
+ allFields.setTypeField("Value");
+ assertTrue(
+ check(allFields, prev).stream()
+ .anyMatch(
+ r ->
+ r.getType() == ICheckResult.TYPE_RESULT_WARNING
+ && r.getText().contains("value")),
+ check(allFields, prev).toString());
+
+ NormaliserMeta listed = meta(field("i1", "A", "kind"), field("i2", "B",
"kind"));
+ listed.setTypeField("kind");
+ assertTrue(
+ check(listed, prev).stream()
+ .anyMatch(
+ r ->
+ r.getType() == ICheckResult.TYPE_RESULT_WARNING
+ && r.getText().contains("kind")),
+ check(listed, prev).toString());
+ }
+
+ /** No field list at all, from metadata injection for example, is an empty
one. */
+ @Test
+ void aMissingFieldListIsAnEmptyOne() {
+ IRowMeta prev = new RowMeta();
+ prev.addValueMeta(new ValueMetaInteger("i1"));
+
+ NormaliserMeta meta = meta();
+ meta.setNormaliserFields(null);
+
+ assertEquals(List.of(), meta.getNormaliserFields());
+ assertTrue(meta.isNormalisingAllFields());
+ assertNull(NormaliserMeta.getDuplicateMapping(null));
+ List<ICheckResult> remarks = check(meta, prev);
+ assertTrue(
+ remarks.stream().allMatch(r -> r.getType() ==
ICheckResult.TYPE_RESULT_OK),
+ remarks.toString());
+ }
+
private static List<ICheckResult> check(NormaliserMeta meta, IRowMeta prev) {
List<ICheckResult> remarks = new ArrayList<>();
meta.check(
diff --git
a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserTest.java
b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserTest.java
index ff4f2402b9..a91e3cc0cb 100644
---
a/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserTest.java
+++
b/plugins/transforms/normaliser/src/test/java/org/apache/hop/pipeline/transforms/normaliser/NormaliserTest.java
@@ -36,6 +36,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
+import java.util.Locale;
import org.apache.hop.core.BlockingRowSet;
import org.apache.hop.core.HopEnvironment;
import org.apache.hop.core.exception.HopException;
@@ -47,6 +48,7 @@ import org.apache.hop.core.row.value.ValueMetaDate;
import org.apache.hop.core.row.value.ValueMetaInteger;
import org.apache.hop.core.row.value.ValueMetaNumber;
import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.variables.Variables;
import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
import org.junit.jupiter.api.AfterEach;
@@ -312,6 +314,222 @@ class NormaliserTest {
assertEquals(List.of("a", "1", "b", "2"), output.rows.stream().map(row ->
row[1]).toList());
}
+ /** The fields that are not normalised keep their order, wherever they stand
in the input. */
+ @Test
+ void keptFieldsKeepTheirOrderAroundTheNormalisedOnes() throws Exception {
+ IRowMeta input = new RowMeta();
+ for (String name : new String[] {"id", "a", "name", "b", "city"}) {
+ input.addValueMeta(new ValueMetaString(name));
+ }
+
+ Output output =
+ normalise(
+ meta("type", field("a", "A", "value"), field("b", "B", "value")),
+ input,
+ "1",
+ "a",
+ "Ann",
+ "b",
+ "Paris");
+
+ assertArrayEquals(
+ new String[] {"id", "name", "city", "type", "value"},
output.rowMeta.getFieldNames());
+ assertArrayEquals(new Object[] {"1", "Ann", "Paris", "A", "a"},
output.rows.get(0));
+ assertArrayEquals(new Object[] {"1", "Ann", "Paris", "B", "b"},
output.rows.get(1));
+ }
+
+ /** A field is found in the input whatever the case it is listed in. */
+ @Test
+ void fieldNamesMatchWhateverTheirCase() throws Exception {
+ IRowMeta input = new RowMeta();
+ for (String name : new String[] {"ID", "Field_A", "FIELD_B"}) {
+ input.addValueMeta(new ValueMetaString(name));
+ }
+
+ Output output =
+ normalise(
+ meta("type", field("field_a", "A", "value"), field("field_b", "B",
"value")),
+ input,
+ "1",
+ "a",
+ "b");
+
+ assertArrayEquals(new String[] {"ID", "type", "value"},
output.rowMeta.getFieldNames());
+ assertArrayEquals(new Object[] {"1", "A", "a"}, output.rows.get(0));
+ assertArrayEquals(new Object[] {"1", "B", "b"}, output.rows.get(1));
+ }
+
+ /** The rows the transform writes have the layout it declares to the
transforms after it. */
+ @Test
+ void theRowsHaveTheLayoutTheTransformDeclares() throws Exception {
+ IRowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaString("id"));
+ input.addValueMeta(new ValueMetaInteger("i"));
+ input.addValueMeta(new ValueMetaString("kept"));
+ input.addValueMeta(new ValueMetaNumber("n"));
+ NormaliserMeta meta = meta("type", field("i", "I", "value"), field("n",
"N", "value"));
+
+ Output output = normalise(meta, input, "1", 2L, "k", 3.0);
+
+ IRowMeta declared = input.clone();
+ meta.getFields(declared, "Row normaliser", null, null, new Variables(),
null);
+ assertArrayEquals(declared.getFieldNames(),
output.rowMeta.getFieldNames());
+ for (int i = 0; i < declared.size(); i++) {
+ assertEquals(
+ declared.getValueMeta(i).getType(),
+ output.rowMeta.getValueMeta(i).getType(),
+ declared.getValueMeta(i).getName());
+ }
+ }
+
+ /**
+ * Issue #9397: without listed fields every input field is normalised, under
its own name, into
+ * the value field. Fields of different types make that a String holding the
text of each value.
+ */
+ @Test
+ void withoutListedFieldsEveryInputFieldIsNormalised() throws Exception {
+ IRowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaString("name"));
+ input.addValueMeta(new ValueMetaInteger("age"));
+ input.addValueMeta(new ValueMetaString("city"));
+
+ NormaliserMeta meta = meta("column");
+ meta.setValueField("content");
+
+ Output output =
+ normaliseRows(
+ meta, input, new Object[] {"Ann", 31L, "Paris"}, new Object[]
{"Bob", null, "Rome"});
+
+ assertArrayEquals(new String[] {"column", "content"},
output.rowMeta.getFieldNames());
+ assertEquals(IValueMeta.TYPE_STRING,
output.rowMeta.getValueMeta(1).getType());
+ assertEquals(6, output.rows.size());
+ assertArrayEquals(new Object[] {"name", "Ann"}, output.rows.get(0));
+ assertArrayEquals(new Object[] {"age", "31"}, output.rows.get(1));
+ assertArrayEquals(new Object[] {"city", "Paris"}, output.rows.get(2));
+ assertArrayEquals(new Object[] {"name", "Bob"}, output.rows.get(3));
+ assertArrayEquals(new Object[] {"age", null}, output.rows.get(4));
+ assertArrayEquals(new Object[] {"city", "Rome"}, output.rows.get(5));
+ }
+
+ /** Input fields of one type keep it, and their values go through untouched.
*/
+ @Test
+ void withoutListedFieldsFieldsOfOneTypeKeepThatType() throws Exception {
+ IRowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaInteger("q1"));
+ input.addValueMeta(new ValueMetaInteger("q2"));
+
+ Output output = normalise(meta("quarter"), input, 10L, 20L);
+
+ assertArrayEquals(new String[] {"quarter", "value"},
output.rowMeta.getFieldNames());
+ assertEquals(IValueMeta.TYPE_INTEGER,
output.rowMeta.getValueMeta(1).getType());
+ assertArrayEquals(new Object[] {"q1", 10L}, output.rows.get(0));
+ assertArrayEquals(new Object[] {"q2", 20L}, output.rows.get(1));
+ }
+
+ /**
+ * With every input field normalised, the input fields are removed before
the new fields are
+ * added: the type and value fields keep their names when input fields had
them.
+ */
+ @Test
+ void withoutListedFieldsTheNewFieldsKeepTheNamesOfInputFields() throws
Exception {
+ IRowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaString("type"));
+ input.addValueMeta(new ValueMetaString("value"));
+
+ Output output = normalise(meta("type"), input, "t", "v");
+
+ assertArrayEquals(new String[] {"type", "value"},
output.rowMeta.getFieldNames());
+ assertArrayEquals(new Object[] {"type", "t"}, output.rows.get(0));
+ assertArrayEquals(new Object[] {"value", "v"}, output.rows.get(1));
+ }
+
+ /**
+ * A new field with the name of a listed field is called value_1, as it
always was: the transforms
+ * after this one read it by that name.
+ */
+ @Test
+ void aNewFieldWithTheNameOfAListedFieldKeepsItsOldName() throws Exception {
+ IRowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaString("type"));
+ input.addValueMeta(new ValueMetaString("value"));
+
+ Output output =
+ normalise(
+ meta("kind", field("type", "T", "value"), field("value", "V",
"value")),
+ input,
+ "t",
+ "v");
+
+ assertArrayEquals(new String[] {"kind", "value_1"},
output.rowMeta.getFieldNames());
+ assertArrayEquals(new Object[] {"T", "t"}, output.rows.get(0));
+ assertArrayEquals(new Object[] {"V", "v"}, output.rows.get(1));
+ }
+
+ /**
+ * The fields that are copied are the ones the declared layout keeps, also
where lower case
+ * depends on the language: in Turkish, TITLE in lower case has a dotless i,
so it is not title.
+ */
+ @Test
+ void theCopiedFieldsAreTheOnesTheLayoutKeepsInATurkishLocale() throws
Exception {
+ IRowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaString("id"));
+ input.addValueMeta(new ValueMetaString("title"));
+
+ Locale locale = Locale.getDefault();
+ Locale.setDefault(Locale.forLanguageTag("tr-TR"));
+ try {
+ Output output = normalise(meta("type", field("TITLE", "T", "value")),
input, "1", "Dr");
+
+ assertArrayEquals(new String[] {"id", "type", "value"},
output.rowMeta.getFieldNames());
+ assertArrayEquals(new Object[] {"1", "T", "Dr"}, output.rows.get(0));
+ } finally {
+ Locale.setDefault(locale);
+ }
+ }
+
+ /** Every input field normalised: the declared layout matches the rows too.
*/
+ @Test
+ void withoutListedFieldsTheRowsHaveTheLayoutTheTransformDeclares() throws
Exception {
+ IRowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaString("s"));
+ input.addValueMeta(new ValueMetaNumber("n"));
+ NormaliserMeta meta = meta("type");
+
+ Output output = normalise(meta, input, "a", 1.5);
+
+ IRowMeta declared = input.clone();
+ meta.getFields(declared, "Row normaliser", null, null, new Variables(),
null);
+ assertArrayEquals(declared.getFieldNames(),
output.rowMeta.getFieldNames());
+ assertEquals(declared.getValueMeta(1).getType(),
output.rowMeta.getValueMeta(1).getType());
+ }
+
+ /** Without input rows the transform writes nothing and finishes without an
error. */
+ @Test
+ void noInputRowsWriteNothing() throws Exception {
+ IRowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaString("a"));
+ NormaliserMeta meta = meta("type", field("a", "A", "value"));
+ when(helper.transformMeta.getTransform()).thenReturn(meta);
+ Normaliser transform =
+ new Normaliser(
+ helper.transformMeta,
+ meta,
+ new NormaliserData(),
+ 0,
+ helper.pipelineMeta,
+ helper.pipeline);
+ transform.init();
+ transform.setInputRowMeta(input);
+ BlockingRowSet rowSet = new BlockingRowSet(10);
+ transform.setOutputRowSets(Collections.singletonList(rowSet));
+ Normaliser spied = spy(transform);
+ doReturn(null).when(spied).getRow();
+
+ assertFalse(spied.processRow());
+ assertNull(rowSet.getRowImmediate());
+ assertEquals(0, spied.getErrors());
+ }
+
/** A field that is not in the input stops the transform with an error. */
@Test
void aFieldMissingFromTheInputStopsTheTransform() throws Exception {
@@ -349,6 +567,7 @@ class NormaliserTest {
private static NormaliserMeta meta(String typeField, NormaliserField...
fields) {
NormaliserMeta meta = new NormaliserMeta();
+ meta.setDefault();
meta.setTypeField(typeField);
meta.setNormaliserFields(new ArrayList<>(List.of(fields)));
return meta;
diff --git
a/plugins/transforms/normaliser/src/test/resources/normaliser-transform.xml
b/plugins/transforms/normaliser/src/test/resources/normaliser-transform.xml
index 89733e17df..47598c3476 100644
--- a/plugins/transforms/normaliser/src/test/resources/normaliser-transform.xml
+++ b/plugins/transforms/normaliser/src/test/resources/normaliser-transform.xml
@@ -24,6 +24,7 @@
<schema_name />
</partitioning>
<typefield>type</typefield>
+ <valuefield>content</valuefield>
<fields>
<field>
<name>fieldA</name>