This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new d0709483b9 YAML input: quoted dates and booleans get the type their
field declar… (#9519)
d0709483b9 is described below
commit d0709483b91d9ac065d32417b980193a380d5d02
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Tue Nov 17 09:52:23 2026 +0100
YAML input: quoted dates and booleans get the type their field declar…
(#9519)
* YAML input: quoted dates and booleans get the type their field declares,
fixes #9510
* fix remarks
---
.../transforms/0088-yaml-input-validate.hpl | 24 +++++++++++++
.../transforms/datasets/golden-yaml-input.csv | 12 +++----
integration-tests/transforms/files/yaml-input.yaml | 2 ++
.../metadata/dataset/golden-yaml-input.json | 16 +++++++++
.../unit-test/0088-yaml-input-validate UNIT.json | 8 +++++
.../pipeline/transforms/yamlinput/YamlInput.java | 7 ++++
.../pipeline/transforms/yamlinput/YamlReader.java | 39 +++++++++++++++++++++-
.../transforms/yamlinput/YamlReaderTests.java | 26 +++++++++++++++
8 files changed, 127 insertions(+), 7 deletions(-)
diff --git a/integration-tests/transforms/0088-yaml-input-validate.hpl
b/integration-tests/transforms/0088-yaml-input-validate.hpl
index 8ccc00fff5..c4813695b2 100644
--- a/integration-tests/transforms/0088-yaml-input-validate.hpl
+++ b/integration-tests/transforms/0088-yaml-input-validate.hpl
@@ -119,6 +119,30 @@ limitations under the License.
<decimal/>
<group/>
</field>
+ <field>
+ <name>release-date</name>
+ <path>release-date</path>
+ <type>Date</type>
+ <length>-1</length>
+ <format>yyyy-MM-dd</format>
+ <trim_type>none</trim_type>
+ <precision>-1</precision>
+ <currency/>
+ <decimal/>
+ <group/>
+ </field>
+ <field>
+ <name>incubating</name>
+ <path>incubating</path>
+ <type>Boolean</type>
+ <length>-1</length>
+ <format/>
+ <trim_type>none</trim_type>
+ <precision>-1</precision>
+ <currency/>
+ <decimal/>
+ <group/>
+ </field>
<field>
<name>used-apache-projects</name>
<path>used-apache-projects</path>
diff --git a/integration-tests/transforms/datasets/golden-yaml-input.csv
b/integration-tests/transforms/datasets/golden-yaml-input.csv
index 84137e344f..8d8c0fc5ec 100644
--- a/integration-tests/transforms/datasets/golden-yaml-input.csv
+++ b/integration-tests/transforms/datasets/golden-yaml-input.csv
@@ -1,6 +1,6 @@
-organisation,acronym,project,javaFileCount,awesome,used-apache-projects,statistics
-The Apache Software Foundation,The ASF,Apache
Hop,4447,Y,Spark,"{number-of-libs: 1159,number-of-messages-bundles:
13025,integration-test-projects: {count: 39,pipelines: 1032,workflows:
498}}"
-The Apache Software Foundation,The ASF,Apache
Hop,4447,Y,Flink,"{number-of-libs: 1159,number-of-messages-bundles:
13025,integration-test-projects: {count: 39,pipelines: 1032,workflows:
498}}"
-The Apache Software Foundation,The ASF,Apache
Hop,4447,Y,Beam,"{number-of-libs: 1159,number-of-messages-bundles:
13025,integration-test-projects: {count: 39,pipelines: 1032,workflows:
498}}"
-The Apache Software Foundation,The ASF,Apache
Hop,4447,Y,Commons,"{number-of-libs: 1159,number-of-messages-bundles:
13025,integration-test-projects: {count: 39,pipelines: 1032,workflows:
498}}"
-The Apache Software Foundation,The ASF,Apache Hop,4447,Y,VFS,"{number-of-libs:
1159,number-of-messages-bundles: 13025,integration-test-projects: {count:
39,pipelines: 1032,workflows: 498}}"
+organisation,acronym,project,javaFileCount,awesome,used-apache-projects,statistics,release-date,incubating
+The Apache Software Foundation,The ASF,Apache
Hop,4447,Y,Spark,"{number-of-libs: 1159,number-of-messages-bundles:
13025,integration-test-projects: {count: 39,pipelines: 1032,workflows:
498}}",2020-02-28,N
+The Apache Software Foundation,The ASF,Apache
Hop,4447,Y,Flink,"{number-of-libs: 1159,number-of-messages-bundles:
13025,integration-test-projects: {count: 39,pipelines: 1032,workflows:
498}}",2020-02-28,N
+The Apache Software Foundation,The ASF,Apache
Hop,4447,Y,Beam,"{number-of-libs: 1159,number-of-messages-bundles:
13025,integration-test-projects: {count: 39,pipelines: 1032,workflows:
498}}",2020-02-28,N
+The Apache Software Foundation,The ASF,Apache
Hop,4447,Y,Commons,"{number-of-libs: 1159,number-of-messages-bundles:
13025,integration-test-projects: {count: 39,pipelines: 1032,workflows:
498}}",2020-02-28,N
+The Apache Software Foundation,The ASF,Apache Hop,4447,Y,VFS,"{number-of-libs:
1159,number-of-messages-bundles: 13025,integration-test-projects: {count:
39,pipelines: 1032,workflows: 498}}",2020-02-28,N
diff --git a/integration-tests/transforms/files/yaml-input.yaml
b/integration-tests/transforms/files/yaml-input.yaml
index 6b9bea2b92..6616f16f03 100644
--- a/integration-tests/transforms/files/yaml-input.yaml
+++ b/integration-tests/transforms/files/yaml-input.yaml
@@ -23,6 +23,8 @@
project: "Apache Hop"
javaFileCount: 4447
awesome: true
+ release-date: '2020-02-28'
+ incubating: 'N'
used-apache-projects:
- Spark
- Flink
diff --git
a/integration-tests/transforms/metadata/dataset/golden-yaml-input.json
b/integration-tests/transforms/metadata/dataset/golden-yaml-input.json
index a0c797199b..1d5dd2f3f0 100644
--- a/integration-tests/transforms/metadata/dataset/golden-yaml-input.json
+++ b/integration-tests/transforms/metadata/dataset/golden-yaml-input.json
@@ -58,6 +58,22 @@
"field_precision": -1,
"field_name": "statistics",
"field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 3,
+ "field_precision": -1,
+ "field_name": "release-date",
+ "field_format": "yyyy-MM-dd"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 4,
+ "field_precision": -1,
+ "field_name": "incubating",
+ "field_format": ""
}
],
"folder_name": ""
diff --git
a/integration-tests/transforms/metadata/unit-test/0088-yaml-input-validate
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0088-yaml-input-validate
UNIT.json
index 9d4c6b1262..622105fb7d 100644
--- a/integration-tests/transforms/metadata/unit-test/0088-yaml-input-validate
UNIT.json
+++ b/integration-tests/transforms/metadata/unit-test/0088-yaml-input-validate
UNIT.json
@@ -36,6 +36,14 @@
{
"transform_field": "statistics",
"data_set_field": "statistics"
+ },
+ {
+ "transform_field": "release-date",
+ "data_set_field": "release-date"
+ },
+ {
+ "transform_field": "incubating",
+ "data_set_field": "incubating"
}
],
"field_order": [
diff --git
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInput.java
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInput.java
index df807ed196..27baffdefa 100644
---
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInput.java
+++
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInput.java
@@ -430,6 +430,13 @@ public class YamlInput extends
BaseTransform<YamlInputMeta, YamlInputData> {
try {
IValueMeta valueMeta = ValueMetaFactory.createValueMeta(fieldName,
field.getType());
valueMeta.setTrimType(field.getTrimType());
+ // The field's format, as getFields() declares it: text in the YAML
is converted with it
+ valueMeta.setLength(field.getLength());
+ valueMeta.setPrecision(field.getPrecision());
+ valueMeta.setConversionMask(field.getFormat());
+ valueMeta.setDecimalSymbol(field.getDecimalSymbol());
+ valueMeta.setGroupingSymbol(field.getGroupSymbol());
+ valueMeta.setCurrencySymbol(field.getCurrencySymbol());
data.rowMeta.addValueMeta(valueMeta);
} catch (Exception e) {
logError("Unable to create value meta", e);
diff --git
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReader.java
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReader.java
index a531445867..79f9cbf64a 100644
---
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReader.java
+++
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReader.java
@@ -38,12 +38,14 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.vfs2.FileObject;
import org.apache.hop.core.Const;
import org.apache.hop.core.exception.HopPluginException;
+import org.apache.hop.core.exception.HopValueException;
import org.apache.hop.core.io.CountingInputStream;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.RowMeta;
import org.apache.hop.core.row.value.ValueMetaFactory;
import org.apache.hop.core.row.value.ValueMetaNone;
+import org.apache.hop.core.row.value.ValueMetaString;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.vfs.HopVfs;
import org.yaml.snakeyaml.Yaml;
@@ -437,7 +439,9 @@ public class YamlReader {
case IValueMeta.TYPE_INTEGER -> convertToInteger(normalizedValue);
case IValueMeta.TYPE_NUMBER -> convertToDouble(normalizedValue);
case IValueMeta.TYPE_BIGNUMBER -> convertToBigDecimal(normalizedValue);
- case IValueMeta.TYPE_BOOLEAN, IValueMeta.TYPE_DATE,
IValueMeta.TYPE_BINARY -> normalizedValue;
+ case IValueMeta.TYPE_DATE -> convertToDate(normalizedValue, valueMeta);
+ case IValueMeta.TYPE_BOOLEAN -> convertToBoolean(normalizedValue,
valueMeta);
+ case IValueMeta.TYPE_BINARY -> normalizedValue;
default -> convertToString(normalizedValue, valueMeta);
};
}
@@ -467,6 +471,39 @@ public class YamlReader {
return s.substring(0, end);
}
+ /**
+ * object -> date. SnakeYAML reads an unquoted ISO date as a {@link
java.util.Date}; a quoted one,
+ * or another format, stays text and is parsed with the field's format, so
the value matches the
+ * Date type the field declares.
+ */
+ private Object convertToDate(Object value, IValueMeta valueMeta) {
+ if (value instanceof java.util.Date) {
+ return value;
+ }
+ return convertText(value, valueMeta);
+ }
+
+ /** object -> boolean: YAML booleans stay, text (Y/N, true/false) is
converted. */
+ private Object convertToBoolean(Object value, IValueMeta valueMeta) {
+ if (value instanceof Boolean) {
+ return value;
+ }
+ return convertText(value, valueMeta);
+ }
+
+ /** Text to the field's type with its format, as Hop converts a String
field. */
+ private static Object convertText(Object value, IValueMeta valueMeta) {
+ ValueMetaString text = new ValueMetaString(valueMeta.getName());
+ text.setConversionMask(valueMeta.getConversionMask());
+ text.setDateFormatLenient(valueMeta.isDateFormatLenient());
+ try {
+ return valueMeta.convertData(text, value.toString());
+ } catch (HopValueException e) {
+ throw new IllegalArgumentException(
+ "Unable to convert '" + value + "' of field '" + valueMeta.getName()
+ "'", e);
+ }
+ }
+
/** object -> long */
@SuppressWarnings("javabugs:S2259") // the value is never null when this is
reached
private Long convertToInteger(Object value) {
diff --git
a/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReaderTests.java
b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReaderTests.java
index b3bbebdce5..02638392e0 100644
---
a/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReaderTests.java
+++
b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlReaderTests.java
@@ -33,7 +33,10 @@ import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopPluginException;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaBoolean;
+import org.apache.hop.core.row.value.ValueMetaDate;
import org.apache.hop.core.row.value.ValueMetaFactory;
+import org.apache.hop.core.row.value.ValueMetaInteger;
import org.apache.hop.core.vfs.HopVfs;
import org.apache.hop.junit.rules.RestoreHopEnvironmentExtension;
import org.junit.jupiter.api.BeforeAll;
@@ -102,6 +105,29 @@ class YamlReaderTests {
assertInstanceOf(Date.class, row[2]);
}
+ @Test
+ void aQuotedDateBecomesTheDateItsFieldDeclares() {
+ String yaml = """
+ order_id: 1
+ order_date: '2020-02-28'
+ shipped: 'Y'
+ """;
+
+ reader.loadString(yaml);
+
+ RowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaInteger("order_id"));
+ ValueMetaDate date = new ValueMetaDate("order_date");
+ date.setConversionMask("yyyy-MM-dd");
+ rowMeta.addValueMeta(date);
+ rowMeta.addValueMeta(new ValueMetaBoolean("shipped"));
+ Object[] row = reader.getRow(rowMeta);
+
+ assertEquals(1L, row[0]);
+ assertInstanceOf(Date.class, row[1]);
+ assertEquals(true, row[2]);
+ }
+
@Test
void testLoadString_ListMultipleValues() {
String yaml = """