This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 568416441e Issue 1951 : XML cleanup YAML Input (#6867)
568416441e is described below
commit 568416441eaa9bcc957c5b33d075bd926a2b603a
Author: Matt Casters <[email protected]>
AuthorDate: Thu Mar 26 20:06:12 2026 +0100
Issue 1951 : XML cleanup YAML Input (#6867)
* issue #1951 (basics)
* issue #1951 (XML cleanup YAML Input)
---
.../transforms/0088-yaml-input-validate.hpl | 189 +++++++
integration-tests/transforms/0088-yaml-input.hwf | 80 +++
.../transforms/datasets/golden-yaml-input.csv | 3 +
integration-tests/transforms/files/yaml-input.yaml | 38 ++
.../metadata/dataset/golden-yaml-input.json | 64 +++
.../unit-test/0088-yaml-input-validate UNIT.json | 58 ++
.../pipeline/transforms/yamlinput/YamlInput.java | 33 +-
.../transforms/yamlinput/YamlInputDialog.java | 170 +++---
.../transforms/yamlinput/YamlInputField.java | 215 ++------
.../transforms/yamlinput/YamlInputMeta.java | 591 +++++----------------
.../transforms/yamlinput/YamlInputMetaTest.java | 225 +++-----
.../yamlinput/src/test/resources/yaml-input.xml | 80 +++
12 files changed, 853 insertions(+), 893 deletions(-)
diff --git a/integration-tests/transforms/0088-yaml-input-validate.hpl
b/integration-tests/transforms/0088-yaml-input-validate.hpl
new file mode 100644
index 0000000000..1c70c26701
--- /dev/null
+++ b/integration-tests/transforms/0088-yaml-input-validate.hpl
@@ -0,0 +1,189 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0088-yaml-input-validate</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/03/24 20:36:32.371</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/03/24 20:36:32.371</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>files/yaml-input.yaml</from>
+ <to>validate</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>files/yaml-input.yaml</name>
+ <type>YamlInput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <include>N</include>
+ <include_field/>
+ <rownum>N</rownum>
+ <addresultfile>N</addresultfile>
+ <validating>N</validating>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <rownum_field/>
+ <encoding/>
+ <file>
+ <name>${PROJECT_HOME}/files/yaml-input.yaml</name>
+ <filemask/>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
+ </file>
+ <fields>
+ <field>
+ <name>organisation</name>
+ <path>organisation</path>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>acronym</name>
+ <path>acronym</path>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>project</name>
+ <path>project</path>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>javaFileCount</name>
+ <path>javaFileCount</path>
+ <type>Integer</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>awesome</name>
+ <path>awesome</path>
+ <type>Boolean</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>used-apache-projects</name>
+ <path>used-apache-projects</path>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>statistics</name>
+ <path>statistics</path>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ </fields>
+ <limit>0</limit>
+ <IsInFields>N</IsInFields>
+ <IsAFile>N</IsAFile>
+ <YamlField/>
+ <attributes/>
+ <GUI>
+ <xloc>80</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/transforms/0088-yaml-input.hwf
b/integration-tests/transforms/0088-yaml-input.hwf
new file mode 100644
index 0000000000..4440be7452
--- /dev/null
+++ b/integration-tests/transforms/0088-yaml-input.hwf
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>0088-yaml-input</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/03/24 20:36:14.720</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/03/24 20:36:14.720</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>80</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0088-yaml-input-validate UNIT</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0088-yaml-input-validate UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>272</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>0088-yaml-input-validate UNIT</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/transforms/datasets/golden-yaml-input.csv
b/integration-tests/transforms/datasets/golden-yaml-input.csv
new file mode 100644
index 0000000000..cd156b7622
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-yaml-input.csv
@@ -0,0 +1,3 @@
+organisation,acronym,project,javaFileCount,awesome,used-apache-projects,statistics
+The Apache Software Foundation,The ASF,Apache Hop,4447,Y,"[Spark, Flink, Beam,
Commons, VFS]
+","{number-of-libs: 1159,number-of-messages-bundles:
13025,integration-test-projects: {count: 39,pipelines: 1032,workflows:
498}}"
diff --git a/integration-tests/transforms/files/yaml-input.yaml
b/integration-tests/transforms/files/yaml-input.yaml
new file mode 100644
index 0000000000..6b9bea2b92
--- /dev/null
+++ b/integration-tests/transforms/files/yaml-input.yaml
@@ -0,0 +1,38 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+---
+ organisation: "The Apache Software Foundation"
+ acronym: "The ASF"
+ project: "Apache Hop"
+ javaFileCount: 4447
+ awesome: true
+ used-apache-projects:
+ - Spark
+ - Flink
+ - Beam
+ - Commons
+ - VFS
+ statistics:
+ number-of-libs: 1159
+ number-of-messages-bundles: 13025
+ integration-test-projects:
+ count: 39
+ pipelines: 1032
+ workflows: 498
diff --git
a/integration-tests/transforms/metadata/dataset/golden-yaml-input.json
b/integration-tests/transforms/metadata/dataset/golden-yaml-input.json
new file mode 100644
index 0000000000..a0c797199b
--- /dev/null
+++ b/integration-tests/transforms/metadata/dataset/golden-yaml-input.json
@@ -0,0 +1,64 @@
+{
+ "base_filename": "golden-yaml-input.csv",
+ "name": "golden-yaml-input",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "organisation",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "acronym",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "project",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "javaFileCount",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 4,
+ "field_precision": -1,
+ "field_name": "awesome",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "used-apache-projects",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "statistics",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/unit-test/0088-yaml-input-validate
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0088-yaml-input-validate
UNIT.json
new file mode 100644
index 0000000000..9d4c6b1262
--- /dev/null
+++ b/integration-tests/transforms/metadata/unit-test/0088-yaml-input-validate
UNIT.json
@@ -0,0 +1,58 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "${HOP_UNIT_TESTS_FOLDER}",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "organisation",
+ "data_set_field": "organisation"
+ },
+ {
+ "transform_field": "acronym",
+ "data_set_field": "acronym"
+ },
+ {
+ "transform_field": "project",
+ "data_set_field": "project"
+ },
+ {
+ "transform_field": "javaFileCount",
+ "data_set_field": "javaFileCount"
+ },
+ {
+ "transform_field": "awesome",
+ "data_set_field": "awesome"
+ },
+ {
+ "transform_field": "used-apache-projects",
+ "data_set_field": "used-apache-projects"
+ },
+ {
+ "transform_field": "statistics",
+ "data_set_field": "statistics"
+ }
+ ],
+ "field_order": [
+ "organisation",
+ "acronym",
+ "project",
+ "javaFileCount",
+ "awesome",
+ "used-apache-projects",
+ "statistics"
+ ],
+ "data_set_name": "golden-yaml-input",
+ "transform_name": "validate"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0088-yaml-input-validate UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0088-yaml-input-validate.hpl"
+}
\ No newline at end of file
diff --git
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInput.java
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInput.java
index bb90eba20c..34179a1978 100644
---
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInput.java
+++
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInput.java
@@ -77,7 +77,6 @@ public class YamlInput extends BaseTransform<YamlInputMeta,
YamlInputData> {
}
private boolean readNextString() {
-
try {
data.readrow = getRow(); // Grab another row ...
@@ -86,7 +85,7 @@ public class YamlInput extends BaseTransform<YamlInputMeta,
YamlInputData> {
if (isDetailed()) {
logDetailed(BaseMessages.getString(PKG,
"YamlInput.Log.FinishedProcessing"));
}
- return false;
+ return true;
}
if (first) {
@@ -98,7 +97,7 @@ public class YamlInput extends BaseTransform<YamlInputMeta,
YamlInputData> {
data.totalOutFields = data.totalPreviousFields + data.nrInputFields;
meta.getFields(data.outputRowMeta, getTransformName(), null, null,
this, metadataProvider);
- // Check is Yaml field is provided
+ // Check if YAML field is provided
if (Utils.isEmpty(meta.getYamlField())) {
logError(BaseMessages.getString(PKG, "YamlInput.Log.NoField"));
throw new HopException(BaseMessages.getString(PKG,
"YamlInput.Log.NoField"));
@@ -127,7 +126,7 @@ public class YamlInput extends BaseTransform<YamlInputMeta,
YamlInputData> {
PKG, "YamlInput.Log.YAMLStream", meta.getYamlField(),
fieldvalue));
}
- if (meta.getIsAFile()) {
+ if (meta.isSourceFile()) {
// source is a file.
@@ -147,13 +146,13 @@ public class YamlInput extends
BaseTransform<YamlInputMeta, YamlInputData> {
stopAll();
logError(Const.getStackTracker(e));
setErrors(1);
- return false;
+ return true;
}
- return true;
+ return false;
}
private void addFileToResultFilesname(FileObject file) {
- if (meta.addResultFile()) {
+ if (meta.isAddingResultFile()) {
// Add this to the result file names...
ResultFile resultFile =
new ResultFile(
@@ -178,7 +177,7 @@ public class YamlInput extends BaseTransform<YamlInputMeta,
YamlInputData> {
// Move file pointer ahead!
data.filenr++;
- if (meta.isIgnoreEmptyFile() && data.file.getContent().getSize() == 0) {
+ if (meta.isIgnoringEmptyFile() && data.file.getContent().getSize() == 0)
{
if (isBasic()) {
logBasic(
BaseMessages.getString(PKG, "YamlInput.Error.FileSizeZero",
data.file.getName()));
@@ -193,7 +192,7 @@ public class YamlInput extends BaseTransform<YamlInputMeta,
YamlInputData> {
}
// We have a file
- // define a Yaml reader and load file
+ // define a YAML reader and load file
data.yaml = new YamlReader();
data.yaml.loadFile(data.file);
dataVolumeIn =
@@ -229,7 +228,7 @@ public class YamlInput extends BaseTransform<YamlInputMeta,
YamlInputData> {
data.files = meta.getFiles(this);
- if (!meta.isdoNotFailIfNoFile() && data.files.nrOfFiles() == 0) {
+ if (!meta.isDoNotFailIfNoFile() && data.files.nrOfFiles() == 0) {
throw new HopException(BaseMessages.getString(PKG,
"YamlInput.Log.NoFiles"));
}
@@ -274,7 +273,7 @@ public class YamlInput extends BaseTransform<YamlInputMeta,
YamlInputData> {
boolean rowAvailable = false;
boolean fileOpened = false;
if (!meta.isInFields()) {
- while (data.file == null || (data.file != null && !fileOpened &&
!rowAvailable)) {
+ while (data.file == null || !fileOpened && !rowAvailable) {
if (data.file != null) {
// We have opened a file
// read one row
@@ -308,7 +307,7 @@ public class YamlInput extends BaseTransform<YamlInputMeta,
YamlInputData> {
row = getRowData();
if (row == null) {
// No row.. reader next row
- if (!readNextString()) {
+ if (readNextString()) {
return null;
}
fileOpened = true;
@@ -319,7 +318,7 @@ public class YamlInput extends BaseTransform<YamlInputMeta,
YamlInputData> {
} else {
// First time we get there
// We have to parse incoming Yaml value
- if (!readNextString()) {
+ if (readNextString()) {
return null;
}
fileOpened = true;
@@ -358,11 +357,11 @@ public class YamlInput extends
BaseTransform<YamlInputMeta, YamlInputData> {
int rowIndex = data.totalOutFields;
// See if we need to add the filename to the row...
- if (meta.includeFilename() && !Utils.isEmpty(meta.getFilenameField())) {
+ if (meta.isIncludeFilename() && !Utils.isEmpty(meta.getFilenameField()))
{
outputRowData[rowIndex++] = HopVfs.getFilename(data.file);
}
// See if we need to add the row number to the row...
- if (meta.includeRowNumber() && !Utils.isEmpty(meta.getRowNumberField()))
{
+ if (meta.isIncludeRowNumber() &&
!Utils.isEmpty(meta.getRowNumberField())) {
outputRowData[rowIndex++] = data.rownr;
}
@@ -394,11 +393,11 @@ public class YamlInput extends
BaseTransform<YamlInputMeta, YamlInputData> {
if (super.init()) {
data.rownr = 1L;
- data.nrInputFields = meta.getInputFields().length;
+ data.nrInputFields = meta.getInputFields().size();
data.rowMeta = new RowMeta();
for (int i = 0; i < data.nrInputFields; i++) {
- YamlInputField field = meta.getInputFields()[i];
+ YamlInputField field = meta.getInputFields().get(i);
String path = resolve(field.getPath());
try {
diff --git
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputDialog.java
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputDialog.java
index 73a7787c42..6ab30addf7 100644
---
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputDialog.java
+++
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputDialog.java
@@ -74,6 +74,15 @@ public class YamlInputDialog extends BaseTransformDialog {
public static final String
CONST_YAML_INPUT_DIALOG_ERROR_PARSING_DATA_DIALOG_MESSAGE =
"YamlInputDialog.ErrorParsingData.DialogMessage";
+ private static final String YES = "Y";
+
+ public static final String[] NO_YES_DEC =
+ new String[] {
+ BaseMessages.getString(PKG, "System.Combo.No"),
+ BaseMessages.getString(PKG, "System.Combo.Yes")
+ };
+ public static final String[] RequiredFilesCode = new String[] {"N", "Y"};
+
private Label wlFilename;
private Label wlYamlIsAFile;
private Button wbbFilename; // Browse: add file or directory
@@ -386,13 +395,13 @@ public class YamlInputDialog extends BaseTransformDialog {
new ColumnInfo(
BaseMessages.getString(PKG, "YamlInputDialog.Required.Column"),
ColumnInfo.COLUMN_TYPE_CCOMBO,
- YamlInputMeta.RequiredFilesDesc);
+ NO_YES_DEC);
colinfo[2].setToolTip(BaseMessages.getString(PKG,
"YamlInputDialog.Required.Tooltip"));
colinfo[3] =
new ColumnInfo(
BaseMessages.getString(PKG,
"YamlInputDialog.IncludeSubDirs.Column"),
ColumnInfo.COLUMN_TYPE_CCOMBO,
- YamlInputMeta.RequiredFilesDesc);
+ NO_YES_DEC);
colinfo[3].setToolTip(BaseMessages.getString(PKG,
"YamlInputDialog.IncludeSubDirs.Tooltip"));
wFilenameList =
@@ -689,7 +698,7 @@ public class YamlInputDialog extends BaseTransformDialog {
fdGet.bottom = new FormAttachment(100, 0);
wGet.setLayoutData(fdGet);
- final int FieldsRows = input.getInputFields().length;
+ final int FieldsRows = input.getInputFields().size();
ColumnInfo[] colinf =
new ColumnInfo[] {
@@ -1051,97 +1060,56 @@ public class YamlInputDialog extends
BaseTransformDialog {
* @param in The TextFileInputMeta object to obtain the data from.
*/
public void getData(YamlInputMeta in) {
- if (in.getFileName() != null) {
- wFilenameList.removeAll();
-
- for (int i = 0; i < in.getFileName().length; i++) {
- wFilenameList.add(
- new String[] {
- in.getFileName()[i],
- in.getFileMask()[i],
- in.getRequiredFilesDesc(in.getFileRequired()[i]),
- in.getRequiredFilesDesc(in.getIncludeSubFolders()[i])
- });
- }
- wFilenameList.removeEmptyRows();
- wFilenameList.setRowNums();
- wFilenameList.optWidth(true);
+ int index = 0;
+ for (YamlInputMeta.YamlFile file : in.getYamlFiles()) {
+ TableItem item = wFilenameList.table.getItem(index++);
+ item.setText(1, Const.NVL(file.getFilename(), ""));
+ item.setText(2, Const.NVL(file.getFileMask(), ""));
+ item.setText(3, file.isFileRequired() ? NO_YES_DEC[1] : NO_YES_DEC[0]);
+ item.setText(4, file.isIncludingSubFolders() ? NO_YES_DEC[1] :
NO_YES_DEC[0]);
}
- wInclFilename.setSelection(in.includeFilename());
- wInclRownum.setSelection(in.includeRowNumber());
- wAddResult.setSelection(in.addResultFile());
- wIgnoreEmptyFile.setSelection(in.isIgnoreEmptyFile());
- wDoNotFailIfNoFile.setSelection(in.isdoNotFailIfNoFile());
- wYAMLStreamField.setSelection(in.isInFields());
- wYAMLIsAFile.setSelection(in.getIsAFile());
+ wFilenameList.optimizeTableView();
- if (in.getYamlField() != null) {
- wYAMLLField.setText(in.getYamlField());
- }
-
- if (in.getFilenameField() != null) {
- wInclFilenameField.setText(in.getFilenameField());
- }
- if (in.getRowNumberField() != null) {
- wInclRownumField.setText(in.getRowNumberField());
- }
+ wInclFilename.setSelection(in.isIncludeFilename());
+ wInclRownum.setSelection(in.isIncludeRowNumber());
+ wAddResult.setSelection(in.isAddingResultFile());
+ wIgnoreEmptyFile.setSelection(in.isIgnoringEmptyFile());
+ wDoNotFailIfNoFile.setSelection(in.isDoNotFailIfNoFile());
+ wYAMLStreamField.setSelection(in.isInFields());
+ wYAMLIsAFile.setSelection(in.isSourceFile());
+ wYAMLLField.setText(Const.NVL(in.getYamlField(), ""));
+ wInclFilenameField.setText(Const.NVL(in.getFilenameField(), ""));
+ wInclRownumField.setText(Const.NVL(in.getRowNumberField(), ""));
wLimit.setText("" + in.getRowLimit());
if (isDebug()) {
logDebug(BaseMessages.getString(PKG,
"YamlInputDialog.Log.GettingFieldsInfo"));
}
- for (int i = 0; i < in.getInputFields().length; i++) {
- YamlInputField field = in.getInputFields()[i];
-
- if (field != null) {
- TableItem item = wFields.table.getItem(i);
- String name = field.getName();
- String path = field.getPath();
- String type = field.getTypeDesc();
- String format = field.getFormat();
- String length = "" + field.getLength();
- String prec = "" + field.getPrecision();
- String curr = field.getCurrencySymbol();
- String group = field.getGroupSymbol();
- String decim = field.getDecimalSymbol();
- String trim = field.getTrimTypeDesc();
-
- if (name != null) {
- item.setText(1, name);
- }
- if (path != null) {
- item.setText(2, path);
- }
- if (type != null) {
- item.setText(3, type);
- }
- if (format != null) {
- item.setText(4, format);
- }
- if (length != null && !"-1".equals(length)) {
- item.setText(5, length);
- }
- if (prec != null && !"-1".equals(prec)) {
- item.setText(6, prec);
- }
- if (curr != null) {
- item.setText(7, curr);
- }
- if (decim != null) {
- item.setText(8, decim);
- }
- if (group != null) {
- item.setText(9, group);
- }
- if (trim != null) {
- item.setText(10, trim);
- }
+ for (int i = 0; i < in.getInputFields().size(); i++) {
+ YamlInputField field = in.getInputFields().get(i);
+ TableItem item = wFields.table.getItem(i);
+ String length = "" + field.getLength();
+ String prec = "" + field.getPrecision();
+ String decim = field.getDecimalSymbol();
+
+ item.setText(1, Const.NVL(field.getName(), ""));
+ item.setText(2, Const.NVL(field.getPath(), ""));
+ item.setText(3, Const.NVL(field.getTypeDesc(), ""));
+ item.setText(4, Const.NVL(field.getFormat(), ""));
+ if (!"-1".equals(length)) {
+ item.setText(5, length);
+ }
+ if (!"-1".equals(prec)) {
+ item.setText(6, prec);
}
+ item.setText(7, Const.NVL(field.getCurrencySymbol(), ""));
+ if (decim != null) {
+ item.setText(8, decim);
+ }
+ item.setText(9, Const.NVL(field.getGroupSymbol(), ""));
+ item.setText(10, Const.NVL(field.getTrimTypeDesc(), ""));
}
-
- wFields.removeEmptyRows();
- wFields.setRowNums();
- wFields.optWidth(true);
+ wFields.optimizeTableView();
}
private void cancel() {
@@ -1170,30 +1138,30 @@ public class YamlInputDialog extends
BaseTransformDialog {
in.setRowLimit(Const.toLong(wLimit.getText(), 0L));
in.setFilenameField(wInclFilenameField.getText());
in.setRowNumberField(wInclRownumField.getText());
- in.setAddResultFile(wAddResult.getSelection());
+ in.setAddingResultFile(wAddResult.getSelection());
in.setIncludeFilename(wInclFilename.getSelection());
in.setIncludeRowNumber(wInclRownum.getSelection());
- in.setIgnoreEmptyFile(wIgnoreEmptyFile.getSelection());
- in.setdoNotFailIfNoFile(wDoNotFailIfNoFile.getSelection());
+ in.setIgnoringEmptyFile(wIgnoreEmptyFile.getSelection());
+ in.setDoNotFailIfNoFile(wDoNotFailIfNoFile.getSelection());
in.setInFields(wYAMLStreamField.getSelection());
- in.setIsAFile(wYAMLIsAFile.getSelection());
+ in.setSourceFile(wYAMLIsAFile.getSelection());
in.setYamlField(wYAMLLField.getText());
- int nrFiles = wFilenameList.getItemCount();
- int nrFields = wFields.nrNonEmpty();
-
- in.allocate(nrFiles, nrFields);
- in.setFileName(wFilenameList.getItems(0));
- in.setFileMask(wFilenameList.getItems(1));
- in.setFileRequired(wFilenameList.getItems(2));
- in.setIncludeSubFolders(wFilenameList.getItems(3));
+ in.getYamlFiles().clear();
+ for (TableItem item : wFilenameList.getNonEmptyItems()) {
+ YamlInputMeta.YamlFile file = new YamlInputMeta.YamlFile();
+ file.setFilename(item.getText(1));
+ file.setFileMask(item.getText(2));
+ file.setFileRequired(NO_YES_DEC[1].equalsIgnoreCase(item.getText(3)));
+
file.setIncludingSubFolders(NO_YES_DEC[1].equalsIgnoreCase(item.getText(4)));
+ in.getYamlFiles().add(file);
+ }
- for (int i = 0; i < nrFields; i++) {
+ in.getInputFields().clear();
+ for (TableItem item : wFields.getNonEmptyItems()) {
YamlInputField field = new YamlInputField();
- TableItem item = wFields.getNonEmpty(i);
-
field.setName(item.getText(1));
field.setPath(item.getText(2));
field.setType(ValueMetaFactory.getIdForValueMeta(item.getText(3)));
@@ -1205,7 +1173,7 @@ public class YamlInputDialog extends BaseTransformDialog {
field.setGroupSymbol(item.getText(9));
field.setTrimType(YamlInputField.getTrimTypeByDesc(item.getText(10)));
- in.getInputFields()[i] = field;
+ in.getInputFields().add(field);
}
}
diff --git
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputField.java
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputField.java
index 121484d152..c54827563b 100644
---
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputField.java
+++
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputField.java
@@ -17,17 +17,20 @@
package org.apache.hop.pipeline.transforms.yamlinput;
-import org.apache.hop.core.Const;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.value.ValueMetaBase;
import org.apache.hop.core.row.value.ValueMetaFactory;
-import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.i18n.BaseMessages;
-import org.w3c.dom.Node;
+import org.apache.hop.metadata.api.HopMetadataProperty;
/**
* Read YAML files, parse them and convert them to rows and writes these to
one or more output
* streams.
*/
+@Getter
+@Setter
public class YamlInputField implements Cloneable {
private static final Class<?> PKG = YamlInputMeta.class;
@@ -46,24 +49,45 @@ public class YamlInputField implements Cloneable {
};
public static final String CONST_SPACES = " ";
+ @HopMetadataProperty(key = "name")
private String name;
+
+ @HopMetadataProperty(key = "path")
private String path;
+
+ @HopMetadataProperty(key = "type", intCodeConverter =
ValueMetaBase.ValueTypeCodeConverter.class)
private int type;
+
+ @HopMetadataProperty(key = "length")
private int length;
+
+ @HopMetadataProperty(key = "format")
private String format;
- private int trimtype;
+
+ @HopMetadataProperty(
+ key = "trim_type",
+ intCodeConverter = ValueMetaBase.TrimTypeCodeConverter.class)
+ private int trimType;
+
+ @HopMetadataProperty(key = "precision")
private int precision;
+
+ @HopMetadataProperty(key = "currency")
private String currencySymbol;
+
+ @HopMetadataProperty(key = "decimal")
private String decimalSymbol;
+
+ @HopMetadataProperty(key = "group")
private String groupSymbol;
- public YamlInputField(String fieldname) {
- this.name = fieldname;
+ public YamlInputField(String fieldName) {
+ this.name = fieldName;
this.path = "";
this.length = -1;
this.type = IValueMeta.TYPE_STRING;
this.format = "";
- this.trimtype = TYPE_TRIM_NONE;
+ this.trimType = TYPE_TRIM_NONE;
this.groupSymbol = "";
this.decimalSymbol = "";
this.currencySymbol = "";
@@ -74,179 +98,38 @@ public class YamlInputField implements Cloneable {
this("");
}
- public String getXml() {
- StringBuilder retval = new StringBuilder(400);
-
- retval.append(" <field>").append(Const.CR);
- retval.append(CONST_SPACES).append(XmlHandler.addTagValue("name",
getName()));
- retval.append(CONST_SPACES).append(XmlHandler.addTagValue("path",
getPath()));
- retval.append(CONST_SPACES).append(XmlHandler.addTagValue("type",
getTypeDesc()));
- retval.append(CONST_SPACES).append(XmlHandler.addTagValue("format",
getFormat()));
- retval.append(CONST_SPACES).append(XmlHandler.addTagValue("currency",
getCurrencySymbol()));
- retval.append(CONST_SPACES).append(XmlHandler.addTagValue("decimal",
getDecimalSymbol()));
- retval.append(CONST_SPACES).append(XmlHandler.addTagValue("group",
getGroupSymbol()));
- retval.append(CONST_SPACES).append(XmlHandler.addTagValue("length",
getLength()));
- retval.append(CONST_SPACES).append(XmlHandler.addTagValue("precision",
getPrecision()));
- retval.append(CONST_SPACES).append(XmlHandler.addTagValue("trim_type",
getTrimTypeCode()));
-
- retval.append(" </field>").append(Const.CR);
-
- return retval.toString();
- }
-
- public YamlInputField(Node fnode) {
- setName(XmlHandler.getTagValue(fnode, "name"));
- setPath(XmlHandler.getTagValue(fnode, "path"));
- setType(ValueMetaFactory.getIdForValueMeta(XmlHandler.getTagValue(fnode,
"type")));
- setFormat(XmlHandler.getTagValue(fnode, "format"));
- setCurrencySymbol(XmlHandler.getTagValue(fnode, "currency"));
- setDecimalSymbol(XmlHandler.getTagValue(fnode, "decimal"));
- setGroupSymbol(XmlHandler.getTagValue(fnode, "group"));
- setLength(Const.toInt(XmlHandler.getTagValue(fnode, "length"), -1));
- setPrecision(Const.toInt(XmlHandler.getTagValue(fnode, "precision"), -1));
- setTrimType(getTrimTypeByCode(XmlHandler.getTagValue(fnode, "trim_type")));
- }
-
- public static final int getTrimTypeByCode(String tt) {
- if (tt == null) {
- return 0;
- }
-
- for (int i = 0; i < trimTypeCode.length; i++) {
- if (trimTypeCode[i].equalsIgnoreCase(tt)) {
- return i;
- }
- }
- return 0;
+ public YamlInputField(YamlInputField f) {
+ this();
+ this.name = name;
+ this.path = f.path;
+ this.type = f.type;
+ this.length = f.length;
+ this.format = f.format;
+ this.trimType = f.trimType;
+ this.precision = f.precision;
+ this.currencySymbol = f.currencySymbol;
+ this.decimalSymbol = f.decimalSymbol;
+ this.groupSymbol = f.groupSymbol;
}
- public static final int getTrimTypeByDesc(String tt) {
- if (tt == null) {
- return 0;
- }
-
- for (int i = 0; i < trimTypeDesc.length; i++) {
- if (trimTypeDesc[i].equalsIgnoreCase(tt)) {
- return i;
- }
- }
- return 0;
+ public static int getTrimTypeByDesc(String trimTypeDescription) {
+ return ValueMetaBase.getTrimTypeByDesc(trimTypeDescription);
}
- public static final String getTrimTypeCode(int i) {
- if (i < 0 || i >= trimTypeCode.length) {
- return trimTypeCode[0];
- }
- return trimTypeCode[i];
- }
-
- public static final String getTrimTypeDesc(int i) {
- if (i < 0 || i >= trimTypeDesc.length) {
- return trimTypeDesc[0];
- }
- return trimTypeDesc[i];
+ public static String getTrimTypeDesc(int trimType) {
+ return ValueMetaBase.getTrimTypeDesc(trimType);
}
@Override
public Object clone() {
- try {
- YamlInputField retval = (YamlInputField) super.clone();
-
- return retval;
- } catch (CloneNotSupportedException e) {
- return null;
- }
- }
-
- public int getLength() {
- return length;
- }
-
- public void setLength(int length) {
- this.length = length;
- }
-
- public String getName() {
- return name;
- }
-
- public String getPath() {
- return path;
- }
-
- public void setPath(String fieldpath) {
- this.path = fieldpath;
- }
-
- public void setName(String fieldname) {
- this.name = fieldname;
- }
-
- public int getType() {
- return type;
+ return new YamlInputField(this);
}
public String getTypeDesc() {
return ValueMetaFactory.getValueMetaName(type);
}
- public void setType(int type) {
- this.type = type;
- }
-
- public String getFormat() {
- return format;
- }
-
- public void setFormat(String format) {
- this.format = format;
- }
-
- public int getTrimType() {
- return trimtype;
- }
-
- public String getTrimTypeCode() {
- return getTrimTypeCode(trimtype);
- }
-
public String getTrimTypeDesc() {
- return getTrimTypeDesc(trimtype);
- }
-
- public void setTrimType(int trimtype) {
- this.trimtype = trimtype;
- }
-
- public String getGroupSymbol() {
- return groupSymbol;
- }
-
- public void setGroupSymbol(String groupSymbol) {
- this.groupSymbol = groupSymbol;
- }
-
- public String getDecimalSymbol() {
- return decimalSymbol;
- }
-
- public void setDecimalSymbol(String decimalSymbol) {
- this.decimalSymbol = decimalSymbol;
- }
-
- public String getCurrencySymbol() {
- return currencySymbol;
- }
-
- public void setCurrencySymbol(String currencySymbol) {
- this.currencySymbol = currencySymbol;
- }
-
- public int getPrecision() {
- return precision;
- }
-
- public void setPrecision(int precision) {
- this.precision = precision;
+ return getTrimTypeDesc(trimType);
}
}
diff --git
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputMeta.java
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputMeta.java
index 88ef39e7e7..0c02b3923b 100644
---
a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputMeta.java
+++
b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputMeta.java
@@ -20,16 +20,17 @@ package org.apache.hop.pipeline.transforms.yamlinput;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.commons.vfs2.FileObject;
import org.apache.hop.core.CheckResult;
-import org.apache.hop.core.Const;
import org.apache.hop.core.ICheckResult;
import org.apache.hop.core.annotations.Transform;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopPluginException;
import org.apache.hop.core.exception.HopTransformException;
-import org.apache.hop.core.exception.HopXmlException;
import org.apache.hop.core.fileinput.FileInputList;
+import org.apache.hop.core.fileinput.InputFile;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.value.ValueMetaFactory;
@@ -39,6 +40,7 @@ import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.HopMetadataProperty;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransformMeta;
@@ -55,490 +57,136 @@ import org.w3c.dom.Node;
categoryDescription =
"i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Input",
keywords = "i18n::YamlInputMeta.keyword",
documentationUrl = "/pipeline/transforms/yamlinput.html")
+@Getter
+@Setter
public class YamlInputMeta extends BaseTransformMeta<YamlInput, YamlInputData>
{
private static final Class<?> PKG = YamlInputMeta.class;
- private static final String YES = "Y";
+ @Getter
+ @Setter
+ public static class YamlFile {
+ @HopMetadataProperty(key = "name", injectionKey = "filename")
+ private String filename;
- public static final String[] RequiredFilesDesc =
- new String[] {
- BaseMessages.getString(PKG, "System.Combo.No"),
- BaseMessages.getString(PKG, "System.Combo.Yes")
- };
- public static final String[] RequiredFilesCode = new String[] {"N", "Y"};
- public static final String CONST_SPACES = " ";
- public static final String CONST_FIELD = "field";
+ @HopMetadataProperty(key = "filemask")
+ private String fileMask;
- /** Array of filenames */
- private String[] fileName;
+ @HopMetadataProperty(key = "file_required")
+ private boolean fileRequired;
- /** Wildcard or filemask (regular expression) */
- private String[] fileMask;
+ @HopMetadataProperty(key = "include_subfolders")
+ private boolean includingSubFolders;
- /** Array of boolean values as string, indicating if a file is required. */
- private String[] fileRequired;
+ public YamlFile() {
+ filename = "";
+ fileMask = "";
+ fileRequired = false;
+ }
+
+ public YamlFile(YamlFile f) {
+ this();
+ this.filename = f.filename;
+ this.fileMask = f.fileMask;
+ this.fileRequired = f.fileRequired;
+ this.includingSubFolders = f.includingSubFolders;
+ }
+ }
/** Flag indicating that we should include the filename in the output */
+ @HopMetadataProperty(key = "include")
private boolean includeFilename;
/** The name of the field in the output containing the filename */
+ @HopMetadataProperty(key = "include_field")
private String filenameField;
/** Flag indicating that a row number field should be included in the output
*/
+ @HopMetadataProperty(key = "rownum")
private boolean includeRowNumber;
/** The name of the field in the output containing the row number */
+ @HopMetadataProperty(key = "rownum_field")
private String rowNumberField;
/** The maximum number or lines to read */
+ @HopMetadataProperty(key = "limit")
private long rowLimit;
- /** The fields to import... */
- private YamlInputField[] inputFields;
-
/** The encoding to use for reading: null or empty string means system
default encoding */
+ @HopMetadataProperty(key = "encoding")
private String encoding;
/** Is In fields */
+ @HopMetadataProperty(key = "YamlField")
private String yamlField;
/** Is In fields */
+ @HopMetadataProperty(key = "IsInFields")
private boolean inFields;
/** Is a File */
- private boolean isAFile;
+ @HopMetadataProperty(key = "IsAFile")
+ private boolean sourceFile;
/** Flag: add result filename */
- private boolean addResultFile;
+ @HopMetadataProperty(key = "addresultfile")
+ private boolean addingResultFile;
/** Flag: set XML Validating */
+ @HopMetadataProperty(key = "validating")
private boolean validating;
/** Flag : do we ignore empty files */
- private boolean isIgnoreEmptyFile;
-
- /** Array of boolean values as string, indicating if we need to fetch sub
folders. */
- private String[] includeSubFolders;
+ @HopMetadataProperty(key = "IsIgnoreEmptyFile")
+ private boolean ignoringEmptyFile;
/** Flag : do not fail if no file */
+ @HopMetadataProperty(key = "doNotFailIfNoFile")
private boolean doNotFailIfNoFile;
- public YamlInputMeta() {
- super(); // allocate BaseTransformMeta
- }
-
- /**
- * @return the add result filesname flag
- */
- public boolean addResultFile() {
- return addResultFile;
- }
-
- /**
- * @return the validating flag
- */
- public boolean isValidating() {
- return validating;
- }
-
- /**
- * @param validating the validating flag to set
- */
- public void setValidating(boolean validating) {
- this.validating = validating;
- }
-
- public void setAddResultFile(boolean addResultFile) {
- this.addResultFile = addResultFile;
- }
-
- /**
- * @return Returns the input fields.
- */
- public YamlInputField[] getInputFields() {
- return inputFields;
- }
-
- /**
- * @param inputFields The input fields to set.
- */
- public void setInputFields(YamlInputField[] inputFields) {
- this.inputFields = inputFields;
- }
-
- /** Get XML field. */
- public String getYamlField() {
- return yamlField;
- }
-
- /** Set XML field. */
- public void setYamlField(String yamlField) {
- this.yamlField = yamlField;
- }
-
- /** Get the IsInFields. */
- public boolean isInFields() {
- return inFields;
- }
-
- /**
- * @param inFields set the inFields.
- */
- public void setInFields(boolean inFields) {
- this.inFields = inFields;
- }
-
- /**
- * @return Returns the fileMask.
- */
- public String[] getFileMask() {
- return fileMask;
- }
-
- /**
- * @param fileMask The fileMask to set.
- */
- public void setFileMask(String[] fileMask) {
- this.fileMask = fileMask;
- }
-
- public String[] getFileRequired() {
- return fileRequired;
- }
-
- public void setFileRequired(String[] fileRequiredin) {
- for (int i = 0; i < fileRequiredin.length; i++) {
- this.fileRequired[i] = getRequiredFilesCode(fileRequiredin[i]);
- }
- }
-
- public void setIncludeSubFolders(String[] includeSubFoldersin) {
- for (int i = 0; i < includeSubFoldersin.length; i++) {
- this.includeSubFolders[i] = getRequiredFilesCode(includeSubFoldersin[i]);
- }
- }
-
- /**
- * @return Returns the fileName.
- */
- public String[] getFileName() {
- return fileName;
- }
-
- /**
- * @param fileName The fileName to set.
- */
- public void setFileName(String[] fileName) {
- this.fileName = fileName;
- }
-
- /**
- * @return Returns the filenameField.
- */
- public String getFilenameField() {
- return filenameField;
- }
-
- /**
- * @param filenameField The filenameField to set.
- */
- public void setFilenameField(String filenameField) {
- this.filenameField = filenameField;
- }
-
- /**
- * @return Returns the includeFilename.
- */
- public boolean includeFilename() {
- return includeFilename;
- }
+ @HopMetadataProperty(key = "file", groupKey = "files")
+ private List<YamlFile> yamlFiles;
- /**
- * @param includeFilename The includeFilename to set.
- */
- public void setIncludeFilename(boolean includeFilename) {
- this.includeFilename = includeFilename;
- }
-
- /**
- * @return Returns the includeRowNumber.
- */
- public boolean includeRowNumber() {
- return includeRowNumber;
- }
-
- /**
- * @param includeRowNumber The includeRowNumber to set.
- */
- public void setIncludeRowNumber(boolean includeRowNumber) {
- this.includeRowNumber = includeRowNumber;
- }
-
- /**
- * @return Returns the rowLimit.
- */
- public long getRowLimit() {
- return rowLimit;
- }
-
- /**
- * @param rowLimit The rowLimit to set.
- */
- public void setRowLimit(long rowLimit) {
- this.rowLimit = rowLimit;
- }
-
- /**
- * @return the IsIgnoreEmptyFile flag
- */
- public boolean isIgnoreEmptyFile() {
- return isIgnoreEmptyFile;
- }
-
- /**
- * @param isIgnoreEmptyFile the isIgnoreEmptyFile to set
- */
- public void setIgnoreEmptyFile(boolean isIgnoreEmptyFile) {
- this.isIgnoreEmptyFile = isIgnoreEmptyFile;
- }
-
- /**
- * @return the doNotFailIfNoFile flag
- */
- public boolean isdoNotFailIfNoFile() {
- return doNotFailIfNoFile;
- }
-
- /**
- * @param doNotFailIfNoFile the doNotFailIfNoFile to set
- */
- public void setdoNotFailIfNoFile(boolean doNotFailIfNoFile) {
- this.doNotFailIfNoFile = doNotFailIfNoFile;
- }
-
- /**
- * @return Returns the rowNumberField.
- */
- public String getRowNumberField() {
- return rowNumberField;
- }
-
- /**
- * @param rowNumberField The rowNumberField to set.
- */
- public void setRowNumberField(String rowNumberField) {
- this.rowNumberField = rowNumberField;
- }
-
- /**
- * @return the encoding
- */
- public String getEncoding() {
- return encoding;
- }
-
- /**
- * @param encoding the encoding to set
- */
- public void setEncoding(String encoding) {
- this.encoding = encoding;
- }
-
- public boolean getIsAFile() {
- return isAFile;
- }
-
- public void setIsAFile(boolean isAFile) {
- this.isAFile = isAFile;
- }
-
- public String[] getIncludeSubFolders() {
- return includeSubFolders;
- }
-
- @Override
- public void loadXml(Node transformNode, IHopMetadataProvider
metadataProvider)
- throws HopXmlException {
- readData(transformNode);
- }
-
- @Override
- public Object clone() {
- YamlInputMeta retval = (YamlInputMeta) super.clone();
-
- int nrFiles = fileName.length;
- int nrFields = inputFields.length;
-
- retval.allocate(nrFiles, nrFields);
-
- System.arraycopy(fileName, 0, retval.fileName, 0, nrFiles);
- System.arraycopy(fileMask, 0, retval.fileMask, 0, nrFiles);
- System.arraycopy(fileRequired, 0, retval.fileRequired, 0, nrFiles);
- System.arraycopy(includeSubFolders, 0, retval.includeSubFolders, 0,
nrFiles);
-
- for (int i = 0; i < nrFields; i++) {
- if (inputFields[i] != null) {
- retval.inputFields[i] = (YamlInputField) inputFields[i].clone();
- }
- }
- return retval;
- }
-
- @Override
- public String getXml() {
- StringBuilder retval = new StringBuilder(400);
-
- retval.append(" ").append(XmlHandler.addTagValue("include",
includeFilename));
- retval.append(" ").append(XmlHandler.addTagValue("include_field",
filenameField));
- retval.append(" ").append(XmlHandler.addTagValue("rownum",
includeRowNumber));
- retval.append(" ").append(XmlHandler.addTagValue("addresultfile",
addResultFile));
- retval.append(" ").append(XmlHandler.addTagValue("validating",
validating));
- retval.append(" " + XmlHandler.addTagValue("IsIgnoreEmptyFile",
isIgnoreEmptyFile));
- retval.append(" " + XmlHandler.addTagValue("doNotFailIfNoFile",
doNotFailIfNoFile));
-
- retval.append(" ").append(XmlHandler.addTagValue("rownum_field",
rowNumberField));
- retval.append(" ").append(XmlHandler.addTagValue("encoding", encoding));
-
- retval.append(" <file>").append(Const.CR);
- for (int i = 0; i < fileName.length; i++) {
- retval.append(CONST_SPACES).append(XmlHandler.addTagValue("name",
fileName[i]));
- retval.append(CONST_SPACES).append(XmlHandler.addTagValue("filemask",
fileMask[i]));
-
retval.append(CONST_SPACES).append(XmlHandler.addTagValue("file_required",
fileRequired[i]));
- retval
- .append(CONST_SPACES)
- .append(XmlHandler.addTagValue("include_subfolders",
includeSubFolders[i]));
- }
- retval.append(" </file>").append(Const.CR);
-
- retval.append(" <fields>").append(Const.CR);
- for (YamlInputField field : inputFields) {
- retval.append(field.getXml());
- }
- retval.append(" </fields>").append(Const.CR);
-
- retval.append(" ").append(XmlHandler.addTagValue("limit", rowLimit));
- retval.append(" ").append(XmlHandler.addTagValue("IsInFields",
inFields));
- retval.append(" ").append(XmlHandler.addTagValue("IsAFile", isAFile));
- retval.append(" ").append(XmlHandler.addTagValue("YamlField",
yamlField));
-
- return retval.toString();
- }
-
- public String getRequiredFilesDesc(String tt) {
- if (Utils.isEmpty(tt)) {
- return RequiredFilesDesc[0];
- }
- if (tt.equalsIgnoreCase(RequiredFilesCode[1])) {
- return RequiredFilesDesc[1];
- } else {
- return RequiredFilesDesc[0];
- }
- }
-
- public String getRequiredFilesCode(String tt) {
- if (tt == null) {
- return RequiredFilesCode[0];
- }
- if (tt.equals(RequiredFilesDesc[1])) {
- return RequiredFilesCode[1];
- } else {
- return RequiredFilesCode[0];
- }
- }
-
- private void readData(Node transformNode) throws HopXmlException {
- try {
- includeFilename =
"Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode, "include"));
- filenameField = XmlHandler.getTagValue(transformNode, "include_field");
-
- addResultFile =
"Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode, "addresultfile"));
- validating = "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode,
"validating"));
- isIgnoreEmptyFile =
- "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode,
"IsIgnoreEmptyFile"));
- doNotFailIfNoFile =
- "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode,
"doNotFailIfNoFile"));
-
- includeRowNumber =
"Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode, "rownum"));
- rowNumberField = XmlHandler.getTagValue(transformNode, "rownum_field");
- encoding = XmlHandler.getTagValue(transformNode, "encoding");
-
- Node filenode = XmlHandler.getSubNode(transformNode, "file");
- Node fields = XmlHandler.getSubNode(transformNode, "fields");
- int nrFiles = XmlHandler.countNodes(filenode, "name");
- int nrFields = XmlHandler.countNodes(fields, CONST_FIELD);
-
- allocate(nrFiles, nrFields);
-
- for (int i = 0; i < nrFiles; i++) {
- Node filenamenode = XmlHandler.getSubNodeByNr(filenode, "name", i);
- Node filemasknode = XmlHandler.getSubNodeByNr(filenode, "filemask", i);
- Node fileRequirednode = XmlHandler.getSubNodeByNr(filenode,
"file_required", i);
- Node includeSubFoldersnode = XmlHandler.getSubNodeByNr(filenode,
"include_subfolders", i);
- fileName[i] = XmlHandler.getNodeValue(filenamenode);
- fileMask[i] = XmlHandler.getNodeValue(filemasknode);
- fileRequired[i] = XmlHandler.getNodeValue(fileRequirednode);
- includeSubFolders[i] = XmlHandler.getNodeValue(includeSubFoldersnode);
- }
-
- for (int i = 0; i < nrFields; i++) {
- Node fnode = XmlHandler.getSubNodeByNr(fields, CONST_FIELD, i);
- YamlInputField field = new YamlInputField(fnode);
- inputFields[i] = field;
- }
-
- // Is there a limit on the number of rows we process?
- rowLimit = Const.toLong(XmlHandler.getTagValue(transformNode, "limit"),
0L);
- inFields = "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode,
"IsInFields"));
- isAFile = "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode,
"IsAFile"));
- yamlField = XmlHandler.getTagValue(transformNode, "YamlField");
- } catch (Exception e) {
- throw new HopXmlException(
- BaseMessages.getString(PKG,
"YamlInputMeta.Exception.ErrorLoadingXml", e.toString()));
- }
- }
+ /** The fields to import... */
+ @HopMetadataProperty(key = "field", groupKey = "fields")
+ private List<YamlInputField> inputFields;
- public void allocate(int nrfiles, int nrFields) {
- fileName = new String[nrfiles];
- fileMask = new String[nrfiles];
- fileRequired = new String[nrfiles];
- includeSubFolders = new String[nrfiles];
- inputFields = new YamlInputField[nrFields];
- }
+ public YamlInputMeta() {
+ super();
+ this.yamlFiles = new ArrayList<>();
+ this.inputFields = new ArrayList<>();
- @Override
- public void setDefault() {
- isIgnoreEmptyFile = false;
doNotFailIfNoFile = true;
- includeFilename = false;
filenameField = "";
- includeRowNumber = false;
rowNumberField = "";
- isAFile = false;
- addResultFile = false;
- validating = false;
-
- int nrFiles = 0;
- int nrFields = 0;
-
- allocate(nrFiles, nrFields);
-
- for (int i = 0; i < nrFiles; i++) {
- fileName[i] = "filename" + (i + 1);
- fileMask[i] = "";
- fileRequired[i] = RequiredFilesCode[0];
- includeSubFolders[i] = RequiredFilesCode[0];
- }
-
- for (int i = 0; i < nrFields; i++) {
- inputFields[i] = new YamlInputField(CONST_FIELD + (i + 1));
- }
-
rowLimit = 0;
-
- inFields = false;
yamlField = "";
}
+ public YamlInputMeta(YamlInputMeta m) {
+ this();
+ this.includeFilename = m.includeFilename;
+ this.filenameField = m.filenameField;
+ this.includeRowNumber = m.includeRowNumber;
+ this.rowNumberField = m.rowNumberField;
+ this.rowLimit = m.rowLimit;
+ this.encoding = m.encoding;
+ this.yamlField = m.yamlField;
+ this.inFields = m.inFields;
+ this.sourceFile = m.sourceFile;
+ this.addingResultFile = m.addingResultFile;
+ this.validating = m.validating;
+ this.ignoringEmptyFile = m.ignoringEmptyFile;
+ this.doNotFailIfNoFile = m.doNotFailIfNoFile;
+ m.yamlFiles.forEach(y -> this.yamlFiles.add(new YamlFile(y)));
+ m.inputFields.forEach(f -> this.inputFields.add(new YamlInputField(f)));
+ }
+
+ @Override
+ public YamlInputMeta clone() {
+ return new YamlInputMeta(this);
+ }
+
@Override
public void getFields(
IRowMeta r,
@@ -549,8 +197,8 @@ public class YamlInputMeta extends
BaseTransformMeta<YamlInput, YamlInputData> {
IHopMetadataProvider metadataProvider)
throws HopTransformException {
int i;
- for (i = 0; i < inputFields.length; i++) {
- YamlInputField field = inputFields[i];
+ for (i = 0; i < inputFields.size(); i++) {
+ YamlInputField field = inputFields.get(i);
int type = field.getType();
if (type == IValueMeta.TYPE_NONE) {
@@ -590,19 +238,17 @@ public class YamlInputMeta extends
BaseTransformMeta<YamlInput, YamlInputData> {
}
public FileInputList getFiles(IVariables variables) {
- return FileInputList.createFileList(
- variables,
- FileInputList.buildInputFiles(
- fileName, fileMask, null, fileRequired, includeSubFolderBoolean(),
null));
- }
-
- private boolean[] includeSubFolderBoolean() {
- int len = fileName.length;
- boolean[] includeSubFolderBoolean = new boolean[len];
- for (int i = 0; i < len; i++) {
- includeSubFolderBoolean[i] = YES.equalsIgnoreCase(includeSubFolders[i]);
+ List<InputFile> inputFiles = new ArrayList<>();
+ for (YamlFile file : yamlFiles) {
+ InputFile inputFile = new InputFile();
+ inputFile.setFileName(file.getFilename());
+ inputFile.setFileMask(file.getFileMask());
+ inputFile.setIncludeSubFolders(file.isIncludingSubFolders());
+ inputFile.setFileRequired(file.isFileRequired());
+ inputFile.setExcludeFileMask(null); // Not provided in this transform
+ inputFiles.add(inputFile);
}
- return includeSubFolderBoolean;
+ return FileInputList.createFileList(variables, inputFiles, null);
}
@Override
@@ -635,7 +281,7 @@ public class YamlInputMeta extends
BaseTransformMeta<YamlInput, YamlInputData> {
remarks.add(cr);
}
- if (getInputFields().length <= 0) {
+ if (getInputFields().size() <= 0) {
cr =
new CheckResult(
ICheckResult.TYPE_RESULT_ERROR,
@@ -693,8 +339,8 @@ public class YamlInputMeta extends
BaseTransformMeta<YamlInput, YamlInputData> {
* pray that the file is on a shared drive or something like that.
*
* @param variables the variable variables to use
- * @param definitions
- * @param iResourceNaming
+ * @param definitions The definitions to use.
+ * @param iResourceNaming The resource naming method.
* @param metadataProvider the metadataProvider in which non-hop metadata
could reside.
* @return the filename of the exported resource
*/
@@ -710,7 +356,7 @@ public class YamlInputMeta extends
BaseTransformMeta<YamlInput, YamlInputData> {
// So let's change the filename from relative to absolute by grabbing
the file object...
// In case the name of the file comes from previous transforms, forget
about this!
//
- List<String> newFilenames = new ArrayList<>();
+ List<YamlFile> newFiles = new ArrayList<>();
if (!isInFields()) {
FileInputList fileList = getFiles(variables);
@@ -721,18 +367,15 @@ public class YamlInputMeta extends
BaseTransformMeta<YamlInput, YamlInputData> {
if (fileObject.exists()) {
// Convert to an absolute path and add it to the list.
//
- newFilenames.add(fileObject.getName().getPath());
+ YamlFile newFile = new YamlFile();
+ newFile.setFilename(fileObject.getName().getPath());
+ newFiles.add(newFile);
}
}
- // Still here: set a new list of absolute filenames!
+ // Replace with the absolute paths
//
- fileName = newFilenames.toArray(new String[newFilenames.size()]);
- fileMask = new String[newFilenames.size()]; // all null since
converted to absolute path.
- fileRequired = new String[newFilenames.size()]; // all null, turn to
"Y" :
- for (int i = 0; i < newFilenames.size(); i++) {
- fileRequired[i] = "Y";
- }
+ yamlFiles = newFiles;
}
}
return null;
@@ -740,4 +383,36 @@ public class YamlInputMeta extends
BaseTransformMeta<YamlInput, YamlInputData> {
throw new HopException(e);
}
}
+
+ @Override
+ public void convertLegacyXml(Node node) throws HopException {
+ Node fileNode = XmlHandler.getSubNode(node, "file");
+ int count = XmlHandler.countNodes(fileNode, "name");
+ if (fileNode == null || count == 0) {
+ // This is already using the new files/file structure.
+ return;
+ }
+
+ yamlFiles.clear();
+ for (int i = 0; i < count; i++) {
+ YamlFile inputFile = new YamlFile();
+ String fileName =
XmlHandler.getNodeValue(XmlHandler.getSubNodeByNr(fileNode, "name", i));
+ String fileMask =
XmlHandler.getNodeValue(XmlHandler.getSubNodeByNr(fileNode, "filemask", i));
+ boolean fileRequired =
+ "Y"
+ .equalsIgnoreCase(
+ XmlHandler.getNodeValue(XmlHandler.getSubNodeByNr(fileNode,
"file_required", i)));
+ boolean includeSubFolders =
+ "Y"
+ .equalsIgnoreCase(
+ XmlHandler.getNodeValue(
+ XmlHandler.getSubNodeByNr(fileNode,
"include_subfolders", i)));
+
+ inputFile.setFilename(fileName);
+ inputFile.setFileMask(fileMask);
+ inputFile.setFileRequired(fileRequired);
+ inputFile.setIncludingSubFolders(includeSubFolders);
+ yamlFiles.add(inputFile);
+ }
+ }
}
diff --git
a/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputMetaTest.java
b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputMetaTest.java
index c49188c444..42360c4236 100644
---
a/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputMetaTest.java
+++
b/plugins/transforms/yamlinput/src/test/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputMetaTest.java
@@ -16,168 +16,91 @@
*/
package org.apache.hop.pipeline.transforms.yamlinput;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.UUID;
-import org.apache.hop.core.HopEnvironment;
-import org.apache.hop.core.exception.HopException;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
import org.apache.hop.core.plugins.PluginRegistry;
-import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
-import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester;
-import org.apache.hop.pipeline.transforms.loadsave.initializer.IInitializer;
-import
org.apache.hop.pipeline.transforms.loadsave.validator.ArrayLoadSaveValidator;
-import
org.apache.hop.pipeline.transforms.loadsave.validator.IFieldLoadSaveValidator;
-import
org.apache.hop.pipeline.transforms.loadsave.validator.StringLoadSaveValidator;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.value.ValueMetaDate;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaJson;
+import org.apache.hop.core.row.value.ValueMetaNumber;
+import org.apache.hop.core.row.value.ValueMetaPlugin;
+import org.apache.hop.core.row.value.ValueMetaPluginType;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.pipeline.transform.TransformSerializationTestUtil;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-class YamlInputMetaTest implements IInitializer<YamlInputMeta> {
- LoadSaveTester<YamlInputMeta> loadSaveTester;
- Class<YamlInputMeta> testMetaClass = YamlInputMeta.class;
-
- @RegisterExtension
- static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
+class YamlInputMetaTest {
@BeforeEach
- void setUpLoadSave() throws Exception {
- HopEnvironment.init();
- PluginRegistry.init();
- List<String> attributes =
- Arrays.asList(
- "includeFilename",
- "filenameField",
- "includeRowNumber",
- "rowNumberField",
- "rowLimit",
- "encoding",
- "yamlField",
- "inFields",
- "IsAFile",
- "addResultFile",
- "validating",
- "IsIgnoreEmptyFile",
- "doNotFailIfNoFile",
- "fileName",
- "fileMask",
- "fileRequired",
- "includeSubFolders",
- "inputFields");
-
- Map<String, String> getterMap =
- new HashMap<>() {
- {
- put("includeFilename", "includeFilename");
- put("filenameField", "getFilenameField");
- put("includeRowNumber", "includeRowNumber");
- put("rowNumberField", "getRowNumberField");
- put("rowLimit", "getRowLimit");
- put("encoding", "getEncoding");
- put("yamlField", "getYamlField");
- put("inFields", "isInFields");
- put("IsAFile", "getIsAFile");
- put("addResultFile", "addResultFile");
- put("validating", "isValidating");
- put("IsIgnoreEmptyFile", "isIgnoreEmptyFile");
- put("doNotFailIfNoFile", "isdoNotFailIfNoFile");
- put("fileName", "getFileName");
- put("fileMask", "getFileMask");
- put("fileRequired", "getFileRequired");
- put("includeSubFolders", "getIncludeSubFolders");
- put("inputFields", "getInputFields");
- }
- };
- Map<String, String> setterMap =
- new HashMap<>() {
- {
- put("includeFilename", "setIncludeFilename");
- put("filenameField", "setFilenameField");
- put("includeRowNumber", "setIncludeRowNumber");
- put("rowNumberField", "setRowNumberField");
- put("rowLimit", "setRowLimit");
- put("encoding", "setEncoding");
- put("yamlField", "setYamlField");
- put("inFields", "setInFields");
- put("IsAFile", "setIsAFile");
- put("addResultFile", "setAddResultFile");
- put("validating", "setValidating");
- put("IsIgnoreEmptyFile", "setIgnoreEmptyFile");
- put("doNotFailIfNoFile", "setdoNotFailIfNoFile");
- put("fileName", "setFileName");
- put("fileMask", "setFileMask");
- put("fileRequired", "setFileRequired");
- put("includeSubFolders", "setIncludeSubFolders");
- put("inputFields", "setInputFields");
- }
- };
- IFieldLoadSaveValidator<String[]> stringArrayLoadSaveValidator =
- new ArrayLoadSaveValidator<>(new StringLoadSaveValidator(), 5);
- IFieldLoadSaveValidator<YamlInputField[]>
yamlInputFieldArrayLoadSaveValidator =
- new ArrayLoadSaveValidator<>(new YamlInputFieldLoadSaveValidator(), 5);
-
- Map<String, IFieldLoadSaveValidator<?>> attrValidatorMap = new HashMap<>();
- attrValidatorMap.put("fileName", stringArrayLoadSaveValidator);
- attrValidatorMap.put("fileRequired", stringArrayLoadSaveValidator);
- attrValidatorMap.put("fileMask", stringArrayLoadSaveValidator);
- attrValidatorMap.put("includeSubFolders", stringArrayLoadSaveValidator);
- attrValidatorMap.put("inputFields", yamlInputFieldArrayLoadSaveValidator);
-
- Map<String, IFieldLoadSaveValidator<?>> typeValidatorMap = new HashMap<>();
-
- loadSaveTester =
- new LoadSaveTester(
- testMetaClass,
- attributes,
- getterMap,
- setterMap,
- attrValidatorMap,
- typeValidatorMap,
- this);
- }
-
- // Call the allocate method on the LoadSaveTester meta class
- @Override
- public void modify(YamlInputMeta someMeta) {
- if (someMeta instanceof YamlInputMeta) {
- ((YamlInputMeta) someMeta).allocate(5, 5);
+ void beforeEach() throws Exception {
+ PluginRegistry registry = PluginRegistry.getInstance();
+ String[] classNames = {
+ ValueMetaString.class.getName(),
+ ValueMetaInteger.class.getName(),
+ ValueMetaDate.class.getName(),
+ ValueMetaNumber.class.getName(),
+ ValueMetaJson.class.getName()
+ };
+ for (String className : classNames) {
+ registry.registerPluginClass(className, ValueMetaPluginType.class,
ValueMetaPlugin.class);
}
}
@Test
- void testSerialization() throws HopException {
- loadSaveTester.testSerialization();
- }
+ void testSerializationRoundTrip() throws Exception {
+ YamlInputMeta meta =
+ TransformSerializationTestUtil.testSerialization("/yaml-input.xml",
YamlInputMeta.class);
- // YamlInputFieldLoadSaveValidator
- public class YamlInputFieldLoadSaveValidator implements
IFieldLoadSaveValidator<YamlInputField> {
- final Random rand = new Random();
+ assertTrue(meta.isIncludeFilename());
+ assertEquals("includeField", meta.getFilenameField());
+ assertTrue(meta.isIncludeRowNumber());
+ assertEquals("rowNumField", meta.getRowNumberField());
+ assertTrue(meta.isValidating());
+ assertTrue(meta.isAddingResultFile());
+ assertTrue(meta.isIgnoringEmptyFile());
+ assertTrue(meta.isDoNotFailIfNoFile());
+ assertEquals("rowNumField", meta.getRowNumberField());
+ assertEquals("UTF-8", meta.getEncoding());
+ assertEquals(999L, meta.getRowLimit());
+ assertTrue(meta.isSourceFile());
+ assertTrue(meta.isInFields());
+ assertEquals("filenameField", meta.getYamlField());
- @Override
- public YamlInputField getTestObject() {
- YamlInputField rtn = new YamlInputField();
- rtn.setCurrencySymbol(UUID.randomUUID().toString());
- rtn.setDecimalSymbol(UUID.randomUUID().toString());
- rtn.setFormat(UUID.randomUUID().toString());
- rtn.setGroupSymbol(UUID.randomUUID().toString());
- rtn.setName(UUID.randomUUID().toString());
- rtn.setTrimType(rand.nextInt(4));
- rtn.setPrecision(rand.nextInt(9));
- rtn.setLength(rand.nextInt(50));
- rtn.setPath(UUID.randomUUID().toString());
- rtn.setType(rand.nextInt(8));
- return rtn;
- }
+ assertEquals(2, meta.getYamlFiles().size());
+ YamlInputMeta.YamlFile file = meta.getYamlFiles().getFirst();
+ assertEquals("folder1/", file.getFilename());
+ assertEquals(".*\\.yaml", file.getFileMask());
+ assertTrue(file.isFileRequired());
+ assertTrue(file.isIncludingSubFolders());
+ file = meta.getYamlFiles().get(1);
+ assertEquals("folder1/", file.getFilename());
+ assertEquals(".*\\.yml", file.getFileMask());
+ assertTrue(file.isFileRequired());
+ assertTrue(file.isIncludingSubFolders());
- @Override
- public boolean validateTestObject(YamlInputField testObject, Object
actual) {
- if (!(actual instanceof YamlInputField)) {
- return false;
- }
- YamlInputField actualInput = (YamlInputField) actual;
- return (testObject.getXml().equals(actualInput.getXml()));
- }
+ assertEquals(3, meta.getInputFields().size());
+ YamlInputField field = meta.getInputFields().getFirst();
+ assertEquals("field1", field.getName());
+ assertEquals("key1", field.getPath());
+ assertEquals(IValueMeta.TYPE_STRING, field.getType());
+ assertEquals(100, field.getLength());
+ assertEquals(-1, field.getPrecision());
+ assertEquals(IValueMeta.TRIM_TYPE_LEFT, field.getTrimType());
+ field = meta.getInputFields().get(1);
+ assertEquals("field2", field.getName());
+ assertEquals("key2", field.getPath());
+ assertEquals(IValueMeta.TYPE_INTEGER, field.getType());
+ assertEquals(7, field.getLength());
+ assertEquals(0, field.getPrecision());
+ assertEquals(IValueMeta.TRIM_TYPE_RIGHT, field.getTrimType());
+ field = meta.getInputFields().getLast();
+ assertEquals("field3", field.getName());
+ assertEquals("key3", field.getPath());
+ assertEquals(IValueMeta.TYPE_NUMBER, field.getType());
+ assertEquals(9, field.getLength());
+ assertEquals(2, field.getPrecision());
+ assertEquals(IValueMeta.TRIM_TYPE_BOTH, field.getTrimType());
}
}
diff --git a/plugins/transforms/yamlinput/src/test/resources/yaml-input.xml
b/plugins/transforms/yamlinput/src/test/resources/yaml-input.xml
new file mode 100644
index 0000000000..9a8761cef5
--- /dev/null
+++ b/plugins/transforms/yamlinput/src/test/resources/yaml-input.xml
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<transform>
+ <include>Y</include>
+ <include_field>includeField</include_field>
+ <rownum>Y</rownum>
+ <addresultfile>Y</addresultfile>
+ <validating>Y</validating>
+ <IsIgnoreEmptyFile>Y</IsIgnoreEmptyFile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <rownum_field>rowNumField</rownum_field>
+ <encoding>UTF-8</encoding>
+ <file>
+ <name>folder1/</name>
+ <filemask>.*\.yaml</filemask>
+ <file_required>Y</file_required>
+ <include_subfolders>Y</include_subfolders>
+ <name>folder1/</name>
+ <filemask>.*\.yml</filemask>
+ <file_required>Y</file_required>
+ <include_subfolders>Y</include_subfolders>
+ </file>
+ <fields>
+ <field>
+ <name>field1</name>
+ <path>key1</path>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>100</length>
+ <precision>-1</precision>
+ <trim_type>left</trim_type>
+ </field>
+ <field>
+ <name>field2</name>
+ <path>key2</path>
+ <type>Integer</type>
+ <format>#</format>
+ <currency/>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>7</length>
+ <precision>0</precision>
+ <trim_type>right</trim_type>
+ </field>
+ <field>
+ <name>field3</name>
+ <path>key3</path>
+ <type>Number</type>
+ <format>#.#</format>
+ <currency/>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>9</length>
+ <precision>2</precision>
+ <trim_type>both</trim_type>
+ </field>
+ </fields>
+ <limit>999</limit>
+ <IsInFields>Y</IsInFields>
+ <IsAFile>Y</IsAFile>
+ <YamlField>filenameField</YamlField>
+</transform>
