This is an automated email from the ASF dual-hosted git repository.
mcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 36af1f6f19 Issue 1932 (XML cleanup Sort Rows) (#6872)
36af1f6f19 is described below
commit 36af1f6f19e29da6099e27337260ecf9304de720
Author: Matt Casters <[email protected]>
AuthorDate: Fri Mar 27 13:00:10 2026 +0100
Issue 1932 (XML cleanup Sort Rows) (#6872)
* issue #1932 (XML cleanup Sort Rows, extra unit test, Unit Testing fix for
not sorting results)
* issue #1932 (test fix and spotless)
---
integration-tests/transforms/0090-sort-rows.hpl | 333 +++++++++++++++++
.../transforms/datasets/data-set-filename.csv | 8 +-
.../datasets/golden-sort-rows-asc-insensitive.csv | 5 +
.../datasets/golden-sort-rows-asc-sensitive.csv | 5 +
.../datasets/golden-sort-rows-desc-sensitive.csv | 5 +
.../transforms/main-0090-sort-rows.hwf | 80 +++++
.../dataset/golden-sort-rows-asc-insensitive.json | 24 ++
.../dataset/golden-sort-rows-asc-sensitive.json | 24 ++
.../dataset/golden-sort-rows-desc-insensitive.json | 24 ++
.../dataset/golden-sort-rows-desc-sensitive.json | 24 ++
.../metadata/unit-test/0090-sort-rows UNIT.json | 81 +++++
.../org/apache/hop/testing/DataSetCsvUtil.java | 7 +-
.../apache/hop/testing/gui/TestingGuiPlugin.java | 14 +-
.../org/apache/hop/testing/util/DataSetConst.java | 400 ++++++++++-----------
.../hop/pipeline/transforms/sort/SortRows.java | 170 ++++-----
.../pipeline/transforms/sort/SortRowsField.java | 77 ++--
.../hop/pipeline/transforms/sort/SortRowsMeta.java | 177 ++-------
17 files changed, 925 insertions(+), 533 deletions(-)
diff --git a/integration-tests/transforms/0090-sort-rows.hpl
b/integration-tests/transforms/0090-sort-rows.hpl
new file mode 100644
index 0000000000..b40000aaa7
--- /dev/null
+++ b/integration-tests/transforms/0090-sort-rows.hpl
@@ -0,0 +1,333 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0090-sort-rows</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/03/25 13:53:26.203</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/03/25 13:53:26.203</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>sample-data</from>
+ <to>name ASC i</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>sample-data</from>
+ <to>name ASC s</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>name ASC i</from>
+ <to>validate ASC i</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>name ASC s</from>
+ <to>validate ASC s</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>name DESC i</from>
+ <to>validate DESC i</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>name DESC s</from>
+ <to>validate DESC s</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>sample-data</from>
+ <to>name DESC i</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>sample-data</from>
+ <to>name DESC s</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>name ASC i</name>
+ <type>SortRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <compress>N</compress>
+ <directory>${java.io.tmpdir}</directory>
+ <fields>
+ <field>
+ <ascending>Y</ascending>
+ <case_sensitive>N</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <name>name</name>
+ <presorted>N</presorted>
+ </field>
+ </fields>
+ <sort_prefix>out</sort_prefix>
+ <sort_size>1000000</sort_size>
+ <unique_rows>N</unique_rows>
+ <attributes/>
+ <GUI>
+ <xloc>224</xloc>
+ <yloc>32</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>name ASC s</name>
+ <type>SortRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <compress>N</compress>
+ <directory>${java.io.tmpdir}</directory>
+ <fields>
+ <field>
+ <ascending>Y</ascending>
+ <case_sensitive>Y</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <name>name</name>
+ <presorted>N</presorted>
+ </field>
+ </fields>
+ <sort_prefix>out</sort_prefix>
+ <sort_size>1000000</sort_size>
+ <unique_rows>N</unique_rows>
+ <attributes/>
+ <GUI>
+ <xloc>224</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>name DESC i</name>
+ <type>SortRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <compress>N</compress>
+ <directory>${java.io.tmpdir}</directory>
+ <fields>
+ <field>
+ <ascending>N</ascending>
+ <case_sensitive>N</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <name>name</name>
+ <presorted>N</presorted>
+ </field>
+ </fields>
+ <sort_prefix>out</sort_prefix>
+ <sort_size>1000000</sort_size>
+ <unique_rows>N</unique_rows>
+ <attributes/>
+ <GUI>
+ <xloc>224</xloc>
+ <yloc>192</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>name DESC s</name>
+ <type>SortRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <compress>N</compress>
+ <directory>${java.io.tmpdir}</directory>
+ <fields>
+ <field>
+ <ascending>N</ascending>
+ <case_sensitive>Y</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <name>name</name>
+ <presorted>N</presorted>
+ </field>
+ </fields>
+ <sort_prefix>out</sort_prefix>
+ <sort_size>1000000</sort_size>
+ <unique_rows>N</unique_rows>
+ <attributes/>
+ <GUI>
+ <xloc>224</xloc>
+ <yloc>272</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>sample-data</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>N</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>3</item>
+ <item>Hop</item>
+ </line>
+ <line>
+ <item>2</item>
+ <item>Beam</item>
+ </line>
+ <line>
+ <item>1</item>
+ <item>Spark</item>
+ </line>
+ <line>
+ <item>0</item>
+ <item>flink</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>3</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>id</name>
+ <format>#</format>
+ <type>Integer</type>
+ </field>
+ <field>
+ <length>100</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>name</name>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate ASC i</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>352</xloc>
+ <yloc>32</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate ASC s</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>352</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate DESC i</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>352</xloc>
+ <yloc>192</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate DESC s</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>352</xloc>
+ <yloc>272</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/transforms/datasets/data-set-filename.csv
b/integration-tests/transforms/datasets/data-set-filename.csv
index 3d52d4e002..6859795977 100644
--- a/integration-tests/transforms/datasets/data-set-filename.csv
+++ b/integration-tests/transforms/datasets/data-set-filename.csv
@@ -1,3 +1,5 @@
-field1,field2,field3,target1,target2
-Z,Y,X,One,Two
-W,V,U,Three,Four
+id,name
+1,Spark
+3,Hop
+0,flink
+2,Beam
diff --git
a/integration-tests/transforms/datasets/golden-sort-rows-asc-insensitive.csv
b/integration-tests/transforms/datasets/golden-sort-rows-asc-insensitive.csv
new file mode 100644
index 0000000000..939297ae6e
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-sort-rows-asc-insensitive.csv
@@ -0,0 +1,5 @@
+id,name
+2,Beam
+0,flink
+3,Hop
+1,Spark
diff --git
a/integration-tests/transforms/datasets/golden-sort-rows-asc-sensitive.csv
b/integration-tests/transforms/datasets/golden-sort-rows-asc-sensitive.csv
new file mode 100644
index 0000000000..86adfe02c8
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-sort-rows-asc-sensitive.csv
@@ -0,0 +1,5 @@
+id,name
+2,Beam
+3,Hop
+1,Spark
+0,flink
diff --git
a/integration-tests/transforms/datasets/golden-sort-rows-desc-sensitive.csv
b/integration-tests/transforms/datasets/golden-sort-rows-desc-sensitive.csv
new file mode 100644
index 0000000000..c63a8e1d88
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-sort-rows-desc-sensitive.csv
@@ -0,0 +1,5 @@
+id,name
+0,flink
+1,Spark
+3,Hop
+2,Beam
diff --git a/integration-tests/transforms/main-0090-sort-rows.hwf
b/integration-tests/transforms/main-0090-sort-rows.hwf
new file mode 100644
index 0000000000..a2318e98e9
--- /dev/null
+++ b/integration-tests/transforms/main-0090-sort-rows.hwf
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0090-sort-rows</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/03/25 13:53:08.752</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/03/25 13:53:08.752</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>96</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0090-sort-rows UNIT</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0090-sort-rows UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>288</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>0090-sort-rows UNIT</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-sort-rows-asc-insensitive.json
b/integration-tests/transforms/metadata/dataset/golden-sort-rows-asc-insensitive.json
new file mode 100644
index 0000000000..1b933126c9
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-sort-rows-asc-insensitive.json
@@ -0,0 +1,24 @@
+{
+ "base_filename": "golden-sort-rows-asc-insensitive.csv",
+ "name": "golden-sort-rows-asc-insensitive",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": 3,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "id",
+ "field_format": "#"
+ },
+ {
+ "field_comment": "",
+ "field_length": 100,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "name",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/dataset/golden-sort-rows-asc-sensitive.json
b/integration-tests/transforms/metadata/dataset/golden-sort-rows-asc-sensitive.json
new file mode 100644
index 0000000000..ff786650ac
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-sort-rows-asc-sensitive.json
@@ -0,0 +1,24 @@
+{
+ "base_filename": "golden-sort-rows-asc-sensitive.csv",
+ "name": "golden-sort-rows-asc-sensitive",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": 3,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "id",
+ "field_format": "#"
+ },
+ {
+ "field_comment": "",
+ "field_length": 100,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "name",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/dataset/golden-sort-rows-desc-insensitive.json
b/integration-tests/transforms/metadata/dataset/golden-sort-rows-desc-insensitive.json
new file mode 100644
index 0000000000..a3dc99105d
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-sort-rows-desc-insensitive.json
@@ -0,0 +1,24 @@
+{
+ "base_filename": "data-set-filename.csv",
+ "name": "golden-sort-rows-desc-insensitive",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": 3,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "id",
+ "field_format": "#"
+ },
+ {
+ "field_comment": "",
+ "field_length": 100,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "name",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/dataset/golden-sort-rows-desc-sensitive.json
b/integration-tests/transforms/metadata/dataset/golden-sort-rows-desc-sensitive.json
new file mode 100644
index 0000000000..b3669dc410
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-sort-rows-desc-sensitive.json
@@ -0,0 +1,24 @@
+{
+ "base_filename": "golden-sort-rows-desc-sensitive.csv",
+ "name": "golden-sort-rows-desc-sensitive",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": 3,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "id",
+ "field_format": "#"
+ },
+ {
+ "field_comment": "",
+ "field_length": 100,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "name",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git a/integration-tests/transforms/metadata/unit-test/0090-sort-rows
UNIT.json b/integration-tests/transforms/metadata/unit-test/0090-sort-rows
UNIT.json
new file mode 100644
index 0000000000..033a85125d
--- /dev/null
+++ b/integration-tests/transforms/metadata/unit-test/0090-sort-rows UNIT.json
@@ -0,0 +1,81 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "${HOP_UNIT_TESTS_FOLDER}",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ },
+ {
+ "transform_field": "name",
+ "data_set_field": "name"
+ }
+ ],
+ "field_order": [
+ ""
+ ],
+ "data_set_name": "golden-sort-rows-asc-insensitive",
+ "transform_name": "validate ASC i"
+ },
+ {
+ "field_mappings": [
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ },
+ {
+ "transform_field": "name",
+ "data_set_field": "name"
+ }
+ ],
+ "field_order": [
+ ""
+ ],
+ "data_set_name": "golden-sort-rows-asc-sensitive",
+ "transform_name": "validate ASC s"
+ },
+ {
+ "field_mappings": [
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ },
+ {
+ "transform_field": "name",
+ "data_set_field": "name"
+ }
+ ],
+ "field_order": [
+ ""
+ ],
+ "data_set_name": "golden-sort-rows-desc-insensitive",
+ "transform_name": "validate DESC i"
+ },
+ {
+ "field_mappings": [
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ },
+ {
+ "transform_field": "name",
+ "data_set_field": "name"
+ }
+ ],
+ "field_order": [],
+ "data_set_name": "golden-sort-rows-desc-sensitive",
+ "transform_name": "validate DESC s"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0090-sort-rows UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0090-sort-rows.hpl"
+}
\ No newline at end of file
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvUtil.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvUtil.java
index bf6e56ad66..15e08420d3 100644
---
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvUtil.java
+++
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvUtil.java
@@ -25,7 +25,6 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
@@ -196,12 +195,10 @@ public class DataSetCsvUtil {
return new ArrayList<>();
}
- if (!sortFields.isEmpty()) {
-
+ if (!sortFields.isEmpty() &&
StringUtils.isNotEmpty(sortFields.getFirst())) {
// Sort the rows...
//
- Collections.sort(
- rows,
+ rows.sort(
(o1, o2) -> {
try {
return outputRowMeta.compare(o1, o2, sortIndexes);
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/gui/TestingGuiPlugin.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/gui/TestingGuiPlugin.java
index e696fe79a0..a87214494b 100644
---
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/gui/TestingGuiPlugin.java
+++
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/gui/TestingGuiPlugin.java
@@ -19,6 +19,7 @@ package org.apache.hop.testing.gui;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
@@ -398,7 +399,7 @@ public class TestingGuiPlugin {
EnterSelectionDialog esd =
new EnterSelectionDialog(
hopGui.getShell(),
- setNames.toArray(new String[setNames.size()]),
+ setNames.toArray(new String[0]),
BaseMessages.getString(PKG,
"TestingGuiPlugin.ContextAction.SetGoldenDataset.Header"),
BaseMessages.getString(
PKG,
"TestingGuiPlugin.ContextAction.SetGoldenDataset.Message"));
@@ -478,6 +479,17 @@ public class TestingGuiPlugin {
return false;
}
+ // Remove empty values
+ //
+ Iterator<Object[]> orderMappingsIterator = orderMappings.iterator();
+ if (orderMappingsIterator.hasNext()) {
+ Object[] orderFieldLine = orderMappingsIterator.next();
+ String orderField = sortMeta.getString(orderFieldLine, 0);
+ if (StringUtils.isEmpty(orderField)) {
+ orderMappingsIterator.remove();
+ }
+ }
+
// Modify the test
//
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/util/DataSetConst.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/util/DataSetConst.java
index c4fb6d5089..892e11597b 100644
---
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/util/DataSetConst.java
+++
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/util/DataSetConst.java
@@ -35,7 +35,6 @@ import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.engine.IPipelineEngine;
import org.apache.hop.testing.DataSet;
-import org.apache.hop.testing.PipelineTweak;
import org.apache.hop.testing.PipelineUnitTest;
import org.apache.hop.testing.PipelineUnitTestFieldMapping;
import org.apache.hop.testing.PipelineUnitTestSetLocation;
@@ -95,7 +94,7 @@ public class DataSetConst {
* @return The nr of errors, 0 if no errors found
* @throws HopException In case there was an error loading data or metadata.
*/
- public static final int validateTransformResultAgainstUnitTest(
+ public static int validateTransformResultAgainstUnitTest(
IPipelineEngine<PipelineMeta> pipeline,
PipelineUnitTest unitTest,
IHopMetadataProvider metadataProvider,
@@ -198,93 +197,99 @@ public class DataSetConst {
comment));
nrLocationErrors++;
} else {
-
// To compare the 2 data sets they need to be explicitly sorted on the
same keys
// The added problem is that the user provided a field mapping.
// So for every "location field order" we need to find the transform
source field
//
// Sort transform result rows
//
- final int[] resultFieldIndexes = new
int[location.getFieldOrder().size()];
- for (int i = 0; i < resultFieldIndexes.length; i++) {
- String dataSetOrderField = location.getFieldOrder().get(i);
- String transformOrderField =
location.findTransformField(dataSetOrderField);
- if (transformOrderField == null) {
- throw new HopException(
- "There is no transform field provided in the mappings so I
don't know which field to use to sort '"
- + dataSetOrderField
- + "'");
+ boolean hasSortOrder =
+ !location.getFieldOrder().isEmpty()
+ && StringUtils.isNotEmpty(location.getFieldOrder().getFirst());
+ if (hasSortOrder) {
+ final int[] resultFieldIndexes = new
int[location.getFieldOrder().size()];
+ for (int i = 0; i < resultFieldIndexes.length; i++) {
+ String dataSetOrderField = location.getFieldOrder().get(i);
+ String transformOrderField =
location.findTransformField(dataSetOrderField);
+ if (transformOrderField == null) {
+ throw new HopException(
+ "There is no transform field provided in the mappings so I
don't know which field to use to sort '"
+ + dataSetOrderField
+ + "'");
+ }
+ resultFieldIndexes[i] =
resultRowMeta.indexOfValue(transformOrderField);
+ if (resultFieldIndexes[i] < 0) {
+ throw new HopException(
+ "Unable to find sort field '"
+ + transformOrderField
+ + "' in transform results : "
+ + Arrays.toString(resultRowMeta.getFieldNames()));
+ }
}
- resultFieldIndexes[i] =
resultRowMeta.indexOfValue(transformOrderField);
- if (resultFieldIndexes[i] < 0) {
+ try {
+ log.logDetailed(
+ "Sorting result rows collection on fields: " +
location.getFieldOrder());
+ resultCollection
+ .getRows()
+ .sort(
+ (row1, row2) -> {
+ try {
+ return resultRowMeta.compare(row1, row2,
resultFieldIndexes);
+ } catch (HopValueException e) {
+ throw new RuntimeException("Error comparing golden
data result rows", e);
+ }
+ });
+ } catch (RuntimeException e) {
throw new HopException(
- "Unable to find sort field '"
- + transformOrderField
- + "' in transform results : "
- + Arrays.toString(resultRowMeta.getFieldNames()));
+ "Error sorting result rows for golden data set '" +
location.getDataSetName() + "'",
+ e);
}
- }
- try {
- log.logDetailed("Sorting result rows collection on fields: " +
location.getFieldOrder());
- resultCollection
- .getRows()
- .sort(
- (row1, row2) -> {
- try {
- return resultRowMeta.compare(row1, row2,
resultFieldIndexes);
- } catch (HopValueException e) {
- throw new RuntimeException("Error comparing golden data
result rows", e);
- }
- });
- } catch (RuntimeException e) {
- throw new HopException(
- "Error sorting result rows for golden data set '" +
location.getDataSetName() + "'",
- e);
- }
- // Print the first 10 result rows
- //
- if (log.isDebug()) {
- for (int i = 0; i < 10 && i < resultCollection.getRows().size();
i++) {
- log.logDetailed(
- "Result row #"
- + (i + 1)
- + " : "
- +
resultRowMeta.getString(resultCollection.getRows().get(i)));
+ // Print the first 10 result rows
+ //
+ if (log.isDebug()) {
+ for (int i = 0; i < 10 && i < resultCollection.getRows().size();
i++) {
+ log.logDetailed(
+ "Result row #"
+ + (i + 1)
+ + " : "
+ +
resultRowMeta.getString(resultCollection.getRows().get(i)));
+ }
}
- }
- // Golden rows
- //
- final int[] goldenFieldIndexes = new
int[location.getFieldOrder().size()];
- for (int i = 0; i < goldenFieldIndexes.length; i++) {
- goldenFieldIndexes[i] =
goldenRowMeta.indexOfValue(location.getFieldOrder().get(i));
- if (goldenFieldIndexes[i] < 0) {
+ // Golden rows
+ //
+ final int[] goldenFieldIndexes = new
int[location.getFieldOrder().size()];
+ for (int i = 0; i < goldenFieldIndexes.length; i++) {
+ goldenFieldIndexes[i] =
goldenRowMeta.indexOfValue(location.getFieldOrder().get(i));
+ if (goldenFieldIndexes[i] < 0) {
+ throw new HopException(
+ "Unable to find sort field '"
+ + location.getFieldOrder().get(i)
+ + "' in golden rows : "
+ + Arrays.toString(goldenRowMeta.getFieldNames()));
+ }
+ }
+ try {
+ log.logDetailed(
+ "Sorting golden rows collection on fields: " +
location.getFieldOrder());
+
+ goldenRows.sort(
+ (row1, row2) -> {
+ try {
+ return goldenRowMeta.compare(row1, row2,
goldenFieldIndexes);
+ } catch (HopValueException e) {
+ throw new RuntimeException("Error comparing golden data
set rows", e);
+ }
+ });
+ } catch (RuntimeException e) {
throw new HopException(
- "Unable to find sort field '"
- + location.getFieldOrder().get(i)
- + "' in golden rows : "
- + Arrays.toString(goldenRowMeta.getFieldNames()));
+ "Error sorting golden data rows for golden data set '"
+ + location.getDataSetName()
+ + "'",
+ e);
}
}
- try {
- log.logDetailed("Sorting golden rows collection on fields: " +
location.getFieldOrder());
-
- goldenRows.sort(
- (row1, row2) -> {
- try {
- return goldenRowMeta.compare(row1, row2, goldenFieldIndexes);
- } catch (HopValueException e) {
- throw new RuntimeException("Error comparing golden data set
rows", e);
- }
- });
- } catch (RuntimeException e) {
- throw new HopException(
- "Error sorting golden data rows for golden data set '"
- + location.getDataSetName()
- + "'",
- e);
- }
// Print the first 10 golden rows
//
@@ -295,124 +300,121 @@ public class DataSetConst {
}
}
- if (nrLocationErrors == 0) {
- final int[] transformFieldIndices = new
int[location.getFieldMappings().size()];
- final int[] goldenIndices = new
int[location.getFieldMappings().size()];
- for (int i = 0; i < location.getFieldMappings().size(); i++) {
- PipelineUnitTestFieldMapping fieldMapping =
location.getFieldMappings().get(i);
+ final int[] transformFieldIndices = new
int[location.getFieldMappings().size()];
+ final int[] goldenIndices = new
int[location.getFieldMappings().size()];
+ for (int i = 0; i < location.getFieldMappings().size(); i++) {
+ PipelineUnitTestFieldMapping fieldMapping =
location.getFieldMappings().get(i);
- transformFieldIndices[i] =
-
resultRowMeta.indexOfValue(fieldMapping.getTransformFieldName());
- if (transformFieldIndices[i] < 0) {
- throw new HopException(
- "Unable to find output field '"
- + fieldMapping.getTransformFieldName()
- + "' while testing output of transform '"
- + location.getTransformName()
- + "'");
- }
- goldenIndices[i] =
goldenRowMeta.indexOfValue(fieldMapping.getDataSetFieldName());
- if (goldenIndices[i] < 0) {
- throw new HopException(
- "Unable to find golden data set field '"
- + fieldMapping.getDataSetFieldName()
- + "' while testing output of transform '"
- + location.getTransformName()
- + "'");
- }
- log.logDetailed(
- "Field to compare #"
- + i
- + " found on transform index : "
- + transformFieldIndices[i]
- + ", golden index : "
- + goldenIndices[i]);
+ transformFieldIndices[i] =
+ resultRowMeta.indexOfValue(fieldMapping.getTransformFieldName());
+ if (transformFieldIndices[i] < 0) {
+ throw new HopException(
+ "Unable to find output field '"
+ + fieldMapping.getTransformFieldName()
+ + "' while testing output of transform '"
+ + location.getTransformName()
+ + "'");
+ }
+ goldenIndices[i] =
goldenRowMeta.indexOfValue(fieldMapping.getDataSetFieldName());
+ if (goldenIndices[i] < 0) {
+ throw new HopException(
+ "Unable to find golden data set field '"
+ + fieldMapping.getDataSetFieldName()
+ + "' while testing output of transform '"
+ + location.getTransformName()
+ + "'");
}
+ log.logDetailed(
+ "Field to compare #"
+ + i
+ + " found on transform index : "
+ + transformFieldIndices[i]
+ + ", golden index : "
+ + goldenIndices[i]);
+ }
- for (int rowNumber = 0; rowNumber < resultRows.size(); rowNumber++) {
- Object[] resultRow = resultRows.get(rowNumber);
- Object[] goldenRow = goldenRows.get(rowNumber);
+ for (int rowNumber = 0; rowNumber < resultRows.size(); rowNumber++) {
+ Object[] resultRow = resultRows.get(rowNumber);
+ Object[] goldenRow = goldenRows.get(rowNumber);
- // Now compare the input to the golden row
- //
- for (int i = 0; i < location.getFieldMappings().size(); i++) {
- IValueMeta transformValueMeta =
-
resultCollection.getRowMeta().getValueMeta(transformFieldIndices[i]);
- Object transformValue = resultRow[transformFieldIndices[i]];
-
- IValueMeta goldenValueMeta =
goldenRowMeta.getValueMeta(goldenIndices[i]);
- Object goldenValue = goldenRow[goldenIndices[i]];
-
- if (log.isDetailed()) {
- log.logDebug(
- "Comparing Meta '"
- + transformValueMeta.toString()
- + "' with '"
- + goldenValueMeta.toString()
- + "'");
- log.logDebug("Comparing Value '" + transformValue + "' with '"
+ goldenValue + "'");
- }
+ // Now compare the input to the golden row
+ //
+ for (int i = 0; i < location.getFieldMappings().size(); i++) {
+ IValueMeta transformValueMeta =
+
resultCollection.getRowMeta().getValueMeta(transformFieldIndices[i]);
+ Object transformValue = resultRow[transformFieldIndices[i]];
+
+ IValueMeta goldenValueMeta =
goldenRowMeta.getValueMeta(goldenIndices[i]);
+ Object goldenValue = goldenRow[goldenIndices[i]];
+
+ if (log.isDetailed()) {
+ log.logDebug(
+ "Comparing Meta '"
+ + transformValueMeta.toString()
+ + "' with '"
+ + goldenValueMeta.toString()
+ + "'");
+ log.logDebug("Comparing Value '" + transformValue + "' with '" +
goldenValue + "'");
+ }
- Object goldenValueConverted;
+ Object goldenValueConverted;
- // sometimes there are data conversion issues because of the the
database...
- //
- if (goldenValueMeta.getType() == transformValueMeta.getType()) {
- goldenValueConverted = goldenValue;
- } else {
- goldenValueConverted =
transformValueMeta.convertData(goldenValueMeta, goldenValue);
- }
+ // sometimes there are data conversion issues because of the the
database...
+ //
+ if (goldenValueMeta.getType() == transformValueMeta.getType()) {
+ goldenValueConverted = goldenValue;
+ } else {
+ goldenValueConverted =
transformValueMeta.convertData(goldenValueMeta, goldenValue);
+ }
- try {
- int cmp =
- transformValueMeta.compare(
- transformValue, goldenValueMeta, goldenValueConverted);
- if (cmp != 0
- && transformValueMeta.isNumber()
- && !transformValueMeta.isNull(transformValue)
- && !transformValueMeta.isNull(goldenValueConverted)) {
-
- // See if it's a floating point issue...
- // Convert to an epsilon of 1 millionth.
- Double d1 = transformValueMeta.getNumber(transformValue);
- Double d2 =
transformValueMeta.getNumber(goldenValueConverted);
-
- if (DoubleMath.fuzzyEquals(d1, d2, 0.000001d)) {
- cmp = 0;
- }
+ try {
+ int cmp =
+ transformValueMeta.compare(transformValue, goldenValueMeta,
goldenValueConverted);
+ if (cmp != 0
+ && transformValueMeta.isNumber()
+ && !transformValueMeta.isNull(transformValue)
+ && !transformValueMeta.isNull(goldenValueConverted)) {
+
+ // See if it's a floating point issue...
+ // Convert to an epsilon of 1 millionth.
+ Double d1 = transformValueMeta.getNumber(transformValue);
+ Double d2 = transformValueMeta.getNumber(goldenValueConverted);
+
+ if (DoubleMath.fuzzyEquals(d1, d2, 0.000001d)) {
+ cmp = 0;
}
- if (cmp != 0) {
- if (log.isDebug()) {
- log.logDebug(
- "Unit test failure: '" + transformValue + "' <> '" +
goldenValue + "'");
- }
- String comment =
- "Validation against golden data failed for row number "
- + (rowNumber + 1)
- + ", field "
- + transformValueMeta.getName()
- + " : transform value ["
- + transformValueMeta.getString(transformValue)
- + "] does not correspond to data set value ["
- + goldenValueMeta.getString(goldenValue)
- + "]";
- results.add(
- new UnitTestResult(
- pipeline.getPipelineMeta().getName(),
- unitTest.getName(),
- location.getDataSetName(),
- location.getTransformName(),
- true,
- comment));
- nrLocationErrors++;
+ }
+ if (cmp != 0) {
+ if (log.isDebug()) {
+ log.logDebug(
+ "Unit test failure: '" + transformValue + "' <> '" +
goldenValue + "'");
}
- } catch (HopValueException e) {
- throw new HopException(
- "Unable to compare transform data against golden data set
'"
- + location.getDataSetName()
- + "'",
- e);
+ String comment =
+ "Validation against golden data failed for row number "
+ + (rowNumber + 1)
+ + ", field "
+ + transformValueMeta.getName()
+ + " : transform value ["
+ + transformValueMeta.getString(transformValue)
+ + "] does not correspond to data set value ["
+ + goldenValueMeta.getString(goldenValue)
+ + "]";
+ results.add(
+ new UnitTestResult(
+ pipeline.getPipelineMeta().getName(),
+ unitTest.getName(),
+ location.getDataSetName(),
+ location.getTransformName(),
+ true,
+ comment));
+ nrLocationErrors++;
}
+ } catch (HopValueException e) {
+ throw new HopException(
+ "Unable to compare transform data against golden data set '"
+ + location.getDataSetName()
+ + "'",
+ e);
}
}
}
@@ -447,7 +449,7 @@ public class DataSetConst {
return nrErrors;
}
- public static final String getDirectoryFromPath(String path) {
+ public static String getDirectoryFromPath(String path) {
int lastSlashIndex = path.lastIndexOf('/');
if (lastSlashIndex >= 0) {
return path.substring(0, lastSlashIndex);
@@ -456,15 +458,6 @@ public class DataSetConst {
}
}
- public static final String getNameFromPath(String path) {
- int lastSlashIndex = path.lastIndexOf('/');
- if (lastSlashIndex >= 0) {
- return path.substring(lastSlashIndex + 1);
- } else {
- return path;
- }
- }
-
public static IRowMeta getTransformOutputFields(
DataSet dataSet, PipelineUnitTestSetLocation inputLocation) throws
HopException {
IRowMeta dataSetRowMeta = dataSet.getSetRowMeta();
@@ -491,24 +484,7 @@ public class DataSetConst {
return outputRowMeta;
}
- /**
- * Get the PipelineTweak for a tweak description (from the dialog)
- *
- * @param tweakDescription The description to look for
- * @return the tweak or NONE if nothing matched
- */
- public PipelineTweak getTweakForDescription(String tweakDescription) {
- if (StringUtils.isEmpty(tweakDescription)) {
- return PipelineTweak.NONE;
- }
- int index = Const.indexOfString(tweakDescription, tweakDesc);
- if (index < 0) {
- return PipelineTweak.NONE;
- }
- return PipelineTweak.values()[index];
- }
-
- public static final String getTestTypeDescription(TestType testType) {
+ public static String getTestTypeDescription(TestType testType) {
int index = 0; // DEVELOPMENT
if (testType != null) {
TestType[] testTypes = TestType.values();
@@ -529,7 +505,7 @@ public class DataSetConst {
* @param testTypeDescription The description to look for
* @return the test type or NONE if nothing matched
*/
- public static final TestType getTestTypeForDescription(String
testTypeDescription) {
+ public static TestType getTestTypeForDescription(String testTypeDescription)
{
if (StringUtils.isEmpty(testTypeDescription)) {
return TestType.DEVELOPMENT;
}
@@ -540,7 +516,7 @@ public class DataSetConst {
return TestType.values()[index];
}
- public static final String[] getTestTypeDescriptions() {
+ public static String[] getTestTypeDescriptions() {
return testTypeDesc;
}
}
diff --git
a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java
b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java
index 45db5ea7b5..2cfe9bb03a 100644
---
a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java
+++
b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java
@@ -255,12 +255,9 @@ public class SortRows extends BaseTransform<SortRowsMeta,
SortRowsData> {
if (data.compressFiles) {
di =
getDataInputStream(
- new GZIPInputStream(
- new BufferedInputStream(data.fis.get(data.fis.size() -
1))));
+ new GZIPInputStream(new
BufferedInputStream(data.fis.getLast())));
} else {
- di =
- new DataInputStream(
- new BufferedInputStream(data.fis.get(data.fis.size() - 1),
50000));
+ di = new DataInputStream(new
BufferedInputStream(data.fis.getLast(), 50000));
}
data.dis.add(di);
@@ -281,7 +278,7 @@ public class SortRows extends BaseTransform<SortRowsMeta,
SortRowsData> {
}
// Sort the data row buffer
- Collections.sort(data.tempRows, data.comparator);
+ data.tempRows.sort(data.comparator);
} catch (Exception e) {
logError(BaseMessages.getString(PKG,
"SortRows.Error.ErrorReadingBackTempFiles"), e);
}
@@ -313,7 +310,7 @@ public class SortRows extends BaseTransform<SortRowsMeta,
SortRowsData> {
}
}
- RowTempFile rowTempFile = data.tempRows.remove(0);
+ RowTempFile rowTempFile = data.tempRows.removeFirst();
retval = rowTempFile.row;
int smallest = rowTempFile.fileNumber;
@@ -385,83 +382,15 @@ public class SortRows extends BaseTransform<SortRowsMeta,
SortRowsData> {
@Override
public boolean processRow() throws HopException {
-
- // wait for first for is available
Object[] r = getRow();
- List<SortRowsField> groupFields = null;
-
if (first) {
this.first = false;
- // do we have any row at start processing?
- if (r == null) {
- // seems that we don't
- this.setOutputDone();
+ if (processRowFirstCall(r)) {
return false;
}
-
- IRowMeta inputRowMeta = getInputRowMeta();
-
- // do we have group numbers?
- if (meta.isGroupSortEnabled()) {
- data.newBatch = true;
-
- // we do set exact list instead of null
- groupFields = meta.getGroupFields();
- data.groupnrs = new int[groupFields.size()];
-
- for (int i = 0; i < data.groupnrs.length; i++) {}
-
- for (int i = 0; i < groupFields.size(); i++) {
- data.groupnrs[i] =
inputRowMeta.indexOfValue(groupFields.get(i).getFieldName());
- if (data.groupnrs[i] < 0) {
- logError(
- BaseMessages.getString(
- PKG, "SortRows.Error.PresortedFieldNotFound",
groupFields.get(i)));
- setErrors(1);
- stopAll();
- return false;
- }
- }
- }
-
- String[] fieldNames = new String[meta.getSortFields().size()];
- for (int i = 0; i < fieldNames.length; i++) {
- fieldNames[i] = meta.getSortFields().get(i).getFieldName();
- }
- data.fieldnrs = new int[fieldNames.length];
- List<Integer> toConvert = new ArrayList<>();
-
- // Metadata
- data.outputRowMeta = inputRowMeta.clone();
- meta.getFields(data.outputRowMeta, getTransformName(), null, null, this,
metadataProvider);
- data.comparator = new RowTemapFileComparator(data.outputRowMeta,
data.fieldnrs);
-
- for (int i = 0; i < meta.getSortFields().size(); i++) {
- data.fieldnrs[i] =
inputRowMeta.indexOfValue(meta.getSortFields().get(i).getFieldName());
- if (data.fieldnrs[i] < 0) {
- throw new HopException(
- BaseMessages.getString(
- PKG,
- "SortRowsMeta.CheckResult.TransformFieldNotInInputStream",
- meta.getSortFields().get(i).getFieldName(),
- getTransformName()));
- }
- // do we need binary conversion for this type?
- if
(inputRowMeta.getValueMeta(data.fieldnrs[i]).isStorageBinaryString()) {
- toConvert.add(data.fieldnrs[i]);
- }
- }
-
- data.convertKeysToNative = toConvert.isEmpty() ? null : new
int[toConvert.size()];
- int i = 0;
- for (Integer in : toConvert) {
- data.convertKeysToNative[i] = in;
- i++;
- }
- data.rowComparator = new RowObjectArrayComparator(data.outputRowMeta,
data.fieldnrs);
- } // end if first
+ }
// it is not first row and it is null
if (r == null) {
@@ -472,34 +401,26 @@ public class SortRows extends BaseTransform<SortRowsMeta,
SortRowsData> {
return false;
}
- // if Group Sort is not enabled then do the normal sort.
- if (!meta.isGroupSortEnabled()) {
+ if (data.newBatch) {
+ data.newBatch = false;
+ setPrevious(r);
+ // this enables Sort stuff to initialize its state.
this.addBuffer(getInputRowMeta(), r);
} else {
- // Otherwise do grouping sort
- if (data.newBatch) {
- data.newBatch = false;
- setPrevious(r);
- // this enables Sort stuff to initialize it's state.
+ if (this.sameGroup(data.previous, r)) {
+ // this performs normal row collection functionality.
this.addBuffer(getInputRowMeta(), r);
} else {
- if (this.sameGroup(data.previous, r)) {
- // setPrevious( r ); // we are not need to set it every time
+ this.preSortBeforeFlush();
- // this performs SortRows normal row collection functionality.
- this.addBuffer(getInputRowMeta(), r);
- } else {
- this.preSortBeforeFlush();
-
- // flush sorted block to next transform:
- this.passBuffer();
+ // flush sorted block to next transform:
+ this.passBuffer();
- // new sorted block beginning
- setPrevious(r);
- data.newBatch = true;
+ // new sorted block beginning
+ setPrevious(r);
+ data.newBatch = true;
- this.addBuffer(getInputRowMeta(), r);
- }
+ this.addBuffer(getInputRowMeta(), r);
}
}
@@ -510,6 +431,50 @@ public class SortRows extends BaseTransform<SortRowsMeta,
SortRowsData> {
return true;
}
+ private boolean processRowFirstCall(Object[] r) throws HopException {
+ // do we have any row at start processing?
+ if (r == null) {
+ // seems that we don't
+ this.setOutputDone();
+ return true;
+ }
+
+ IRowMeta inputRowMeta = getInputRowMeta();
+
+ data.fieldnrs = new int[meta.getSortFields().size()];
+ List<Integer> toConvert = new ArrayList<>();
+
+ // Metadata
+ data.outputRowMeta = inputRowMeta.clone();
+ meta.getFields(data.outputRowMeta, getTransformName(), null, null, this,
metadataProvider);
+ data.comparator = new RowTemapFileComparator(data.outputRowMeta,
data.fieldnrs);
+
+ for (int i = 0; i < meta.getSortFields().size(); i++) {
+ data.fieldnrs[i] =
inputRowMeta.indexOfValue(meta.getSortFields().get(i).getFieldName());
+ if (data.fieldnrs[i] < 0) {
+ throw new HopException(
+ BaseMessages.getString(
+ PKG,
+ "SortRowsMeta.CheckResult.TransformFieldNotInInputStream",
+ meta.getSortFields().get(i).getFieldName(),
+ getTransformName()));
+ }
+ // do we need binary conversion for this type?
+ if (inputRowMeta.getValueMeta(data.fieldnrs[i]).isStorageBinaryString())
{
+ toConvert.add(data.fieldnrs[i]);
+ }
+ }
+
+ data.convertKeysToNative = toConvert.isEmpty() ? null : new
int[toConvert.size()];
+ int i = 0;
+ for (Integer in : toConvert) {
+ data.convertKeysToNative[i] = in;
+ i++;
+ }
+ data.rowComparator = new RowObjectArrayComparator(data.outputRowMeta,
data.fieldnrs);
+ return false;
+ }
+
/**
* This method passes all rows in the buffer to the next transforms. Usually
call to this method
* indicates that this particular transform finishing processing.
@@ -639,7 +604,7 @@ public class SortRows extends BaseTransform<SortRowsMeta,
SortRowsData> {
/** Sort the entire vector, if it is not empty. */
void quickSort(List<Object[]> elements) {
if (CollectionUtils.isNotEmpty(elements)) {
- Collections.sort(elements, data.rowComparator);
+ elements.sort(data.rowComparator);
long nrConversions = 0L;
for (IValueMeta valueMeta : data.outputRowMeta.getValueMetaList()) {
@@ -654,11 +619,6 @@ public class SortRows extends BaseTransform<SortRowsMeta,
SortRowsData> {
}
}
- @Override
- public void startBundle() throws HopException {
- // Do nothing
- }
-
/**
* Calling this method will alert the transform that we finished passing
records to the transform.
* Specifically for transforms like "Sort Rows" it means that the buffered
rows can be sorted and
@@ -708,7 +668,7 @@ public class SortRows extends BaseTransform<SortRowsMeta,
SortRowsData> {
}
}
- private class SortRowsComparator {
+ private static class SortRowsComparator {
protected IRowMeta rowMeta;
protected int[] fieldNrs;
diff --git
a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsField.java
b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsField.java
index 673517435a..be7291228c 100644
---
a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsField.java
+++
b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsField.java
@@ -17,27 +17,13 @@
package org.apache.hop.pipeline.transforms.sort;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.hop.metadata.api.HopMetadataProperty;
+@Getter
+@Setter
public class SortRowsField {
-
- public SortRowsField() {}
-
- public SortRowsField(
- String fieldName,
- boolean ascending,
- boolean caseSensitive,
- boolean collatorEnabled,
- int collatorStrength,
- boolean preSortedField) {
- this.fieldName = fieldName;
- this.ascending = ascending;
- this.caseSensitive = caseSensitive;
- this.collatorEnabled = collatorEnabled;
- this.collatorStrength = collatorStrength;
- this.preSortedField = preSortedField;
- }
-
/** order by which fields? */
@HopMetadataProperty(key = "name", injectionKey = "NAME")
private String fieldName;
@@ -62,51 +48,30 @@ public class SortRowsField {
@HopMetadataProperty(key = "presorted", injectionKey = "PRESORTED")
private boolean preSortedField = false;
- public String getFieldName() {
- return fieldName;
- }
+ public SortRowsField() {}
- public void setFieldName(String fieldName) {
+ public SortRowsField(
+ String fieldName,
+ boolean ascending,
+ boolean caseSensitive,
+ boolean collatorEnabled,
+ int collatorStrength,
+ boolean preSortedField) {
this.fieldName = fieldName;
- }
-
- public boolean isAscending() {
- return ascending;
- }
-
- public void setAscending(boolean ascending) {
this.ascending = ascending;
- }
-
- public boolean isCaseSensitive() {
- return caseSensitive;
- }
-
- public void setCaseSensitive(boolean caseSensitive) {
this.caseSensitive = caseSensitive;
- }
-
- public boolean isCollatorEnabled() {
- return collatorEnabled;
- }
-
- public void setCollatorEnabled(boolean collatorEnabled) {
this.collatorEnabled = collatorEnabled;
- }
-
- public int getCollatorStrength() {
- return collatorStrength;
- }
-
- public void setCollatorStrength(int collatorStrength) {
this.collatorStrength = collatorStrength;
+ this.preSortedField = preSortedField;
}
- public boolean isPreSortedField() {
- return preSortedField;
- }
-
- public void setPreSortedField(boolean preSortedField) {
- this.preSortedField = preSortedField;
+ public SortRowsField(SortRowsField f) {
+ this();
+ this.fieldName = f.fieldName;
+ this.ascending = f.ascending;
+ this.caseSensitive = f.caseSensitive;
+ this.collatorEnabled = f.collatorEnabled;
+ this.collatorStrength = f.collatorStrength;
+ this.preSortedField = f.preSortedField;
}
}
diff --git
a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsMeta.java
b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsMeta.java
index 3d64e1073a..02507087dd 100644
---
a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsMeta.java
+++
b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsMeta.java
@@ -24,6 +24,8 @@ import java.text.Collator;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.hop.core.CheckResult;
import org.apache.hop.core.Const;
import org.apache.hop.core.ICheckResult;
@@ -47,6 +49,8 @@ import org.apache.hop.pipeline.transform.TransformMeta;
categoryDescription =
"i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Transform",
keywords = "i18n::SortRowsMeta.keyword",
documentationUrl = "/pipeline/transforms/sort.html")
+@Getter
+@Setter
public class SortRowsMeta extends BaseTransformMeta<SortRows, SortRowsData>
implements Serializable {
@Serial private static final long serialVersionUID = -9075883720765645655L;
@@ -56,7 +60,7 @@ public class SortRowsMeta extends BaseTransformMeta<SortRows,
SortRowsData>
private static final String CONST_FIELD = "field";
@HopMetadataProperty(groupKey = "fields", key = "field", injectionGroupKey =
"FIELDS")
- private List<SortRowsField> sortFields = new ArrayList<>();
+ private List<SortRowsField> sortFields;
/** Directory to store the temp files */
@HopMetadataProperty(key = "directory", injectionKey = "SORT_DIRECTORY")
@@ -92,57 +96,9 @@ public class SortRowsMeta extends
BaseTransformMeta<SortRows, SortRowsData>
@HopMetadataProperty(key = "compress_variables", injectionKey =
"COMPRESS_VARIABLE")
private String compressFilesVariable;
- private List<SortRowsField> groupFields;
-
public SortRowsMeta() {
- super(); // allocate BaseTransformMeta
- }
-
- /**
- * @return Returns the directory.
- */
- public String getDirectory() {
- return directory;
- }
-
- /**
- * @param directory The directory to set.
- */
- public void setDirectory(String directory) {
- this.directory = directory;
- }
-
- /**
- * @return Returns the prefix.
- */
- public String getPrefix() {
- return prefix;
- }
-
- /**
- * @param prefix The prefix to set.
- */
- public void setPrefix(String prefix) {
- this.prefix = prefix;
- }
-
- public List<SortRowsField> getSortFields() {
- return sortFields;
- }
-
- public void setSortFields(List<SortRowsField> sortFields) {
- this.sortFields = sortFields;
- }
-
- @Override
- public Object clone() {
- SortRowsMeta retval = (SortRowsMeta) super.clone();
-
- return retval;
- }
-
- @Override
- public void setDefault() {
+ super();
+ sortFields = new ArrayList<>();
directory = "${java.io.tmpdir}";
prefix = "out";
sortSize = "1000000";
@@ -150,18 +106,29 @@ public class SortRowsMeta extends
BaseTransformMeta<SortRows, SortRowsData>
compressFiles = false;
compressFilesVariable = null;
onlyPassingUniqueRows = false;
+ }
- int nrFields = 0;
+ public SortRowsMeta(SortRowsMeta m) {
+ this();
+ this.directory = m.directory;
+ this.prefix = m.prefix;
+ this.sortSize = m.sortSize;
+ this.freeMemoryLimit = m.freeMemoryLimit;
+ this.onlyPassingUniqueRows = m.onlyPassingUniqueRows;
+ this.compressFiles = m.compressFiles;
+ this.compressFilesVariable = m.compressFilesVariable;
+ m.sortFields.forEach(f -> sortFields.add(new SortRowsField(f)));
}
- // Returns the default collation strength based on the users' default locale.
- // Package protected for testing purposes
- int getDefaultCollationStrength() {
- return getDefaultCollationStrength(Locale.getDefault());
+ @Override
+ public Object clone() {
+ return new SortRowsMeta(this);
}
- // Returns the collation strength based on the passed in locale.
- // Package protected for testing purposes
+ /**
+ * Returns the collation strength based on the passed in locale. Package
protected for testing
+ * purposes
+ */
int getDefaultCollationStrength(Locale aLocale) {
int defaultStrength = Collator.IDENTICAL;
if (aLocale != null) {
@@ -195,6 +162,7 @@ public class SortRowsMeta extends
BaseTransformMeta<SortRows, SortRowsData>
valueMeta.setCaseInsensitive(!field.isCaseSensitive());
valueMeta.setCollatorDisabled(!field.isCollatorEnabled());
valueMeta.setCollatorStrength(field.getCollatorStrength());
+
// Also see if lazy conversion is active on these key fields.
// If so we want to automatically convert them to the normal storage
type.
// This will improve performance
@@ -320,97 +288,4 @@ public class SortRowsMeta extends
BaseTransformMeta<SortRows, SortRowsData>
remarks.add(cr);
}
}
-
- /**
- * @return Returns the sortSize.
- */
- public String getSortSize() {
- return sortSize;
- }
-
- /**
- * @param sortSize The sortSize to set.
- */
- public void setSortSize(String sortSize) {
- this.sortSize = sortSize;
- }
-
- /**
- * @return Returns whether temporary files should be compressed
- */
- public boolean isCompressFiles() {
- return compressFiles;
- }
-
- /**
- * @param compressFiles Whether to compress temporary files created during
sorting
- */
- public void setCompressFiles(boolean compressFiles) {
- this.compressFiles = compressFiles;
- }
-
- /**
- * @return the onlyPassingUniqueRows
- */
- public boolean isOnlyPassingUniqueRows() {
- return onlyPassingUniqueRows;
- }
-
- /**
- * @param onlyPassingUniqueRows the onlyPassingUniqueRows to set
- */
- public void setOnlyPassingUniqueRows(boolean onlyPassingUniqueRows) {
- this.onlyPassingUniqueRows = onlyPassingUniqueRows;
- }
-
- /**
- * @return the compressFilesVariable
- */
- public String getCompressFilesVariable() {
- return compressFilesVariable;
- }
-
- /**
- * @param compressFilesVariable the compressFilesVariable to set
- */
- public void setCompressFilesVariable(String compressFilesVariable) {
- this.compressFilesVariable = compressFilesVariable;
- }
-
- /**
- * @return the freeMemoryLimit
- */
- public String getFreeMemoryLimit() {
- return freeMemoryLimit;
- }
-
- /**
- * @param freeMemoryLimit the freeMemoryLimit to set
- */
- public void setFreeMemoryLimit(String freeMemoryLimit) {
- this.freeMemoryLimit = freeMemoryLimit;
- }
-
- public boolean isGroupSortEnabled() {
- return this.getSortFields() != null;
- }
-
- public void setGroupFields(List<SortRowsField> groupFields) {
- this.groupFields = groupFields;
- }
-
- public List<SortRowsField> getGroupFields() {
- if (this.groupFields == null) {
- groupFields = new ArrayList<>();
- for (int i = 0; i < getSortFields().size(); i++) {
- if (getSortFields().get(i).isPreSortedField()) {
- // if (groupFields == null) {
- // groupFields = new ArrayList<>();
- // }
- groupFields.add(getSortFields().get(i));
- }
- }
- }
- return groupFields;
- }
}
