This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 1310a3dd9f Sorted merge merges in the direction it is set to, and
stops at input… (#9529)
1310a3dd9f is described below
commit 1310a3dd9f6944ed8977f71324ad3d7fc388b900
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Tue Nov 17 11:06:26 2026 +0100
Sorted merge merges in the direction it is set to, and stops at input…
(#9529)
* Sorted merge merges in the direction it is set to, and stops at input out
of that order, fixes #9528
* fix remarks
---
.../pages/pipeline/transforms/sortedmerge.adoc | 2 +
.../transforms/0129-sorted-merge-descending.hpl | 441 +++++++++++++++++++++
.../golden-0129-sorted-merge-descending.csv | 7 +
.../main-0129-sorted-merge-descending.hwf | 80 ++++
.../golden-0129-sorted-merge-descending.json | 24 ++
.../0129-sorted-merge-descending UNIT.json | 49 +++
.../transforms/sortedmerge/SortedMerge.java | 89 ++++-
.../transforms/sortedmerge/SortedMergeDialog.java | 4 +-
.../transforms/sortedmerge/SortedMergeMeta.java | 38 ++
.../sortedmerge/messages/messages_en_US.properties | 5 +
.../sortedmerge/SortedMergeMetaTest.java | 41 ++
.../transforms/sortedmerge/SortedMergeTest.java | 72 ++++
12 files changed, 850 insertions(+), 2 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/sortedmerge.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/sortedmerge.adoc
index f0ab958af4..e6e5069302 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/sortedmerge.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/sortedmerge.adoc
@@ -25,6 +25,8 @@ under the License.
The Sorted Merge transform merges rows coming from multiple input transforms
providing these rows are sorted themselves on the given key fields.
+Every input must be sorted on the key fields in the direction set for each
field in the Fields table. The transform merges in that direction, even when a
transform before it sorted the rows the other way. When a row of an input sorts
before the previous row of the same input, the transform stops with an error
naming the input and the two rows, rather than merging the rows in the wrong
place. Verify pipeline warns when a previous transform, such as Sort rows,
sorts a key field descending [...]
+
== Options
[options="header"]
diff --git a/integration-tests/transforms/0129-sorted-merge-descending.hpl
b/integration-tests/transforms/0129-sorted-merge-descending.hpl
new file mode 100644
index 0000000000..efbabfe680
--- /dev/null
+++ b/integration-tests/transforms/0129-sorted-merge-descending.hpl
@@ -0,0 +1,441 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0129-sorted-merge-descending</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Sorted merge on a descending field merges descending, whether
or not a Sort rows before it marked the field descending (#9528)</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/16 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/16 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>descending a</from>
+ <to>merge descending input</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>descending b</from>
+ <to>merge descending input</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>merge descending input</from>
+ <to>number descending input</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>number descending input</from>
+ <to>validate descending input</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>unsorted c</from>
+ <to>sort c descending</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>unsorted d</from>
+ <to>sort d descending</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>sort c descending</from>
+ <to>merge sorted input</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>sort d descending</from>
+ <to>merge sorted input</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>merge sorted input</from>
+ <to>number sorted input</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>number sorted input</from>
+ <to>validate sorted input</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>descending a</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>id</name>
+ <precision>-1</precision>
+ <type>Integer</type>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>5</item>
+ </line>
+ <line>
+ <item>3</item>
+ </line>
+ <line>
+ <item>1</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>descending b</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>id</name>
+ <precision>-1</precision>
+ <type>Integer</type>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>6</item>
+ </line>
+ <line>
+ <item>4</item>
+ </line>
+ <line>
+ <item>2</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>160</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>merge descending input</name>
+ <type>SortedMerge</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>id</name>
+ <ascending>N</ascending>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>number descending input</name>
+ <type>Sequence</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <increment_by>1</increment_by>
+ <max_value>999999999</max_value>
+ <seqname>SEQ_</seqname>
+ <counter_name>SEQ_DESCENDING_INPUT</counter_name>
+ <start_at>1</start_at>
+ <use_counter>Y</use_counter>
+ <use_database>N</use_database>
+ <valuename>nr</valuename>
+ <attributes/>
+ <GUI>
+ <xloc>480</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate descending input</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>672</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>unsorted c</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>id</name>
+ <precision>-1</precision>
+ <type>Integer</type>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>1</item>
+ </line>
+ <line>
+ <item>5</item>
+ </line>
+ <line>
+ <item>3</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>288</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>unsorted d</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>id</name>
+ <precision>-1</precision>
+ <type>Integer</type>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>4</item>
+ </line>
+ <line>
+ <item>2</item>
+ </line>
+ <line>
+ <item>6</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>384</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>sort c descending</name>
+ <type>SortRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <directory>${java.io.tmpdir}</directory>
+ <prefix>out</prefix>
+ <sort_size>1000000</sort_size>
+ <free_memory/>
+ <compress>N</compress>
+ <compress_variable/>
+ <unique_rows>N</unique_rows>
+ <fields>
+ <field>
+ <name>id</name>
+ <ascending>N</ascending>
+ <case_sensitive>Y</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <presorted>N</presorted>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>288</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>sort d descending</name>
+ <type>SortRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <directory>${java.io.tmpdir}</directory>
+ <prefix>out</prefix>
+ <sort_size>1000000</sort_size>
+ <free_memory/>
+ <compress>N</compress>
+ <compress_variable/>
+ <unique_rows>N</unique_rows>
+ <fields>
+ <field>
+ <name>id</name>
+ <ascending>N</ascending>
+ <case_sensitive>Y</case_sensitive>
+ <collator_enabled>N</collator_enabled>
+ <collator_strength>0</collator_strength>
+ <presorted>N</presorted>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>384</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>merge sorted input</name>
+ <type>SortedMerge</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>id</name>
+ <ascending>N</ascending>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>480</xloc>
+ <yloc>336</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>number sorted input</name>
+ <type>Sequence</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <increment_by>1</increment_by>
+ <max_value>999999999</max_value>
+ <seqname>SEQ_</seqname>
+ <counter_name>SEQ_SORTED_INPUT</counter_name>
+ <start_at>1</start_at>
+ <use_counter>Y</use_counter>
+ <use_database>N</use_database>
+ <valuename>nr</valuename>
+ <attributes/>
+ <GUI>
+ <xloc>672</xloc>
+ <yloc>336</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate sorted input</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>864</xloc>
+ <yloc>336</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/transforms/datasets/golden-0129-sorted-merge-descending.csv
b/integration-tests/transforms/datasets/golden-0129-sorted-merge-descending.csv
new file mode 100644
index 0000000000..154dc2e7fe
--- /dev/null
+++
b/integration-tests/transforms/datasets/golden-0129-sorted-merge-descending.csv
@@ -0,0 +1,7 @@
+nr,id
+1,6
+2,5
+3,4
+4,3
+5,2
+6,1
diff --git a/integration-tests/transforms/main-0129-sorted-merge-descending.hwf
b/integration-tests/transforms/main-0129-sorted-merge-descending.hwf
new file mode 100644
index 0000000000..b1d5872634
--- /dev/null
+++ b/integration-tests/transforms/main-0129-sorted-merge-descending.hwf
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0129-sorted-merge-descending</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/06/03 20:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/06/03 20:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>96</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0129-sorted-merge-descending UNIT</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0129-sorted-merge-descending UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>288</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>0129-sorted-merge-descending UNIT</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-0129-sorted-merge-descending.json
b/integration-tests/transforms/metadata/dataset/golden-0129-sorted-merge-descending.json
new file mode 100644
index 0000000000..6f55e8fe66
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-0129-sorted-merge-descending.json
@@ -0,0 +1,24 @@
+{
+ "base_filename": "golden-0129-sorted-merge-descending.csv",
+ "name": "golden-0129-sorted-merge-descending",
+ "description": "Six ids merged descending, numbered in the order they come
out",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": -1,
+ "field_name": "nr",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": -1,
+ "field_name": "id",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
diff --git
a/integration-tests/transforms/metadata/unit-test/0129-sorted-merge-descending
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0129-sorted-merge-descending
UNIT.json
new file mode 100644
index 0000000000..72311b93fb
--- /dev/null
+++
b/integration-tests/transforms/metadata/unit-test/0129-sorted-merge-descending
UNIT.json
@@ -0,0 +1,49 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "${HOP_UNIT_TESTS_FOLDER}",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "nr",
+ "data_set_field": "nr"
+ },
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ }
+ ],
+ "field_order": [
+ "nr"
+ ],
+ "data_set_name": "golden-0129-sorted-merge-descending",
+ "transform_name": "validate descending input"
+ },
+ {
+ "field_mappings": [
+ {
+ "transform_field": "nr",
+ "data_set_field": "nr"
+ },
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ }
+ ],
+ "field_order": [
+ "nr"
+ ],
+ "data_set_name": "golden-0129-sorted-merge-descending",
+ "transform_name": "validate sorted input"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0129-sorted-merge-descending UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0129-sorted-merge-descending.hpl"
+}
diff --git
a/plugins/transforms/sortedmerge/src/main/java/org/apache/hop/pipeline/transforms/sortedmerge/SortedMerge.java
b/plugins/transforms/sortedmerge/src/main/java/org/apache/hop/pipeline/transforms/sortedmerge/SortedMerge.java
index e149aa26ee..d1637fe375 100644
---
a/plugins/transforms/sortedmerge/src/main/java/org/apache/hop/pipeline/transforms/sortedmerge/SortedMerge.java
+++
b/plugins/transforms/sortedmerge/src/main/java/org/apache/hop/pipeline/transforms/sortedmerge/SortedMerge.java
@@ -20,11 +20,13 @@ package org.apache.hop.pipeline.transforms.sortedmerge;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import org.apache.hop.core.Const;
import org.apache.hop.core.IRowSet;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopRuntimeException;
import org.apache.hop.core.exception.HopTransformException;
import org.apache.hop.core.exception.HopValueException;
+import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.pipeline.Pipeline;
import org.apache.hop.pipeline.PipelineMeta;
@@ -82,6 +84,8 @@ public class SortedMerge extends
BaseTransform<SortedMergeMeta, SortedMergeData>
// Add it to the sorted buffer in the right position...
//
if (extraRow != null) {
+ verifyMergeOrder(smallestRow, extraRow);
+
// Add this one to the sortedBuffer
//
RowSetRow add =
@@ -160,10 +164,12 @@ public class SortedMerge extends
BaseTransform<SortedMergeMeta, SortedMergeData>
}
}
+ // Compare with the metadata that carries the merge settings: the
metadata of the rows only
+ // says how they are sorted when the transform before this one says so
data.comparator =
(o1, o2) -> {
try {
- return o1.getRowMeta().compare(o1.getRowData(), o2.getRowData(),
data.fieldIndices);
+ return data.rowMeta.compare(o1.getRowData(), o2.getRowData(),
data.fieldIndices);
} catch (HopValueException e) {
throw new HopRuntimeException(
BaseMessages.getString(PKG,
"SortedMerge.Exception.ErrorComparingRows"), e);
@@ -176,6 +182,87 @@ public class SortedMerge extends
BaseTransform<SortedMergeMeta, SortedMergeData>
}
}
+ /**
+ * Every input must give its rows in the merge order. A row that sorts
before the previous row of
+ * the same input would be merged in the wrong place, without an error, so
the merge stops there.
+ *
+ * @param previous the previous row of the input, the row just merged
+ * @param row the next row of that input
+ */
+ private void verifyMergeOrder(RowSetRow previous, Object[] row) throws
HopTransformException {
+ int order;
+ try {
+ order = data.rowMeta.compare(row, previous.getRowData(),
data.fieldIndices);
+ } catch (HopValueException e) {
+ throw new HopTransformException(
+ BaseMessages.getString(PKG,
"SortedMerge.Exception.ErrorComparingRows"), e);
+ }
+ if (order >= 0) {
+ return;
+ }
+
+ IRowSet rowSet = previous.getRowSet();
+ StringBuilder mergeOrder = new StringBuilder();
+ StringBuilder hint = new StringBuilder();
+ for (int f = 0; f < data.fieldIndices.length; f++) {
+ SortedMergeMeta.MergeField mergeField = meta.getMergeFields().get(f);
+ if (f > 0) {
+ mergeOrder.append(", ");
+ }
+ mergeOrder
+ .append(mergeField.getFieldName())
+ .append(' ')
+ .append(
+ BaseMessages.getString(
+ PKG,
+ mergeField.isAscending()
+ ? "SortedMerge.Exception.Ascending"
+ : "SortedMerge.Exception.Descending"));
+
+ // The transform before this one can say the rows are sorted the other
way
+ IValueMeta declared =
+ rowSet.getRowMeta() == null
+ ? null
+ : rowSet.getRowMeta().searchValueMeta(mergeField.getFieldName());
+ if (mergeField.isAscending() && declared != null &&
declared.isSortedDescending()) {
+ hint.append(Const.CR)
+ .append(
+ BaseMessages.getString(
+ PKG,
+ "SortedMerge.Exception.InputSortedDescending",
+ rowSet.getOriginTransformName(),
+ mergeField.getFieldName()));
+ }
+ }
+ throw new HopTransformException(
+ BaseMessages.getString(
+ PKG,
+ "SortedMerge.Exception.InputNotInMergeOrder",
+ rowSet.getOriginTransformName(),
+ mergeValues(row),
+ mergeValues(previous.getRowData()),
+ mergeOrder.toString())
+ + hint);
+ }
+
+ /** The values of the merge fields of a row, for a message. */
+ private String mergeValues(Object[] row) {
+ StringBuilder values = new StringBuilder();
+ for (int index : data.fieldIndices) {
+ IValueMeta valueMeta = data.rowMeta.getValueMeta(index);
+ if (!values.isEmpty()) {
+ values.append(", ");
+ }
+ values.append(valueMeta.getName()).append('=');
+ try {
+ values.append(valueMeta.getString(row[index]));
+ } catch (HopValueException e) {
+ values.append(row[index]);
+ }
+ }
+ return values.toString();
+ }
+
@Override
public boolean processRow() throws HopException {
// get row, sorted
diff --git
a/plugins/transforms/sortedmerge/src/main/java/org/apache/hop/pipeline/transforms/sortedmerge/SortedMergeDialog.java
b/plugins/transforms/sortedmerge/src/main/java/org/apache/hop/pipeline/transforms/sortedmerge/SortedMergeDialog.java
index c294863c24..ea428ff0fc 100644
---
a/plugins/transforms/sortedmerge/src/main/java/org/apache/hop/pipeline/transforms/sortedmerge/SortedMergeDialog.java
+++
b/plugins/transforms/sortedmerge/src/main/java/org/apache/hop/pipeline/transforms/sortedmerge/SortedMergeDialog.java
@@ -215,7 +215,9 @@ public class SortedMergeDialog extends BaseTransformDialog {
-1,
-1,
(tableItem, v) -> {
- tableItem.setText(2, "Y");
+ // Merge in the order the previous transform gives, as far as it
says so (Sort rows
+ // does): a field sorted descending there is merged descending
here
+ tableItem.setText(2, v.isSortedDescending() ? "N" : "Y");
return true;
});
}
diff --git
a/plugins/transforms/sortedmerge/src/main/java/org/apache/hop/pipeline/transforms/sortedmerge/SortedMergeMeta.java
b/plugins/transforms/sortedmerge/src/main/java/org/apache/hop/pipeline/transforms/sortedmerge/SortedMergeMeta.java
index 14db79b7c0..172bed11bd 100644
---
a/plugins/transforms/sortedmerge/src/main/java/org/apache/hop/pipeline/transforms/sortedmerge/SortedMergeMeta.java
+++
b/plugins/transforms/sortedmerge/src/main/java/org/apache/hop/pipeline/transforms/sortedmerge/SortedMergeMeta.java
@@ -129,6 +129,7 @@ public class SortedMergeMeta extends
BaseTransformMeta<SortedMerge, SortedMergeD
BaseMessages.getString(PKG,
"SortedMergeMeta.CheckResult.AllSortKeysFound"),
transformMeta);
remarks.add(cr);
+ checkInputSortOrder(remarks, pipelineMeta, transformMeta, variables);
} else {
cr =
new CheckResult(
@@ -165,6 +166,43 @@ public class SortedMergeMeta extends
BaseTransformMeta<SortedMerge, SortedMergeD
}
}
+ /**
+ * Warns about an input that says its rows are sorted descending on a field
the merge sorts
+ * ascending: the merge would stop at its first row out of order.
+ */
+ private void checkInputSortOrder(
+ List<ICheckResult> remarks,
+ PipelineMeta pipelineMeta,
+ TransformMeta transformMeta,
+ IVariables variables) {
+ if (pipelineMeta == null) {
+ return;
+ }
+ for (TransformMeta previous :
pipelineMeta.findPreviousTransforms(transformMeta)) {
+ IRowMeta fields;
+ try {
+ fields = pipelineMeta.getTransformFields(variables, previous);
+ } catch (HopTransformException e) {
+ // The missing fields are reported above
+ continue;
+ }
+ for (MergeField field : mergeFields) {
+ IValueMeta valueMeta = fields.searchValueMeta(field.getFieldName());
+ if (field.isAscending() && valueMeta != null &&
valueMeta.isSortedDescending()) {
+ remarks.add(
+ new CheckResult(
+ ICheckResult.TYPE_RESULT_WARNING,
+ BaseMessages.getString(
+ PKG,
+ "SortedMergeMeta.CheckResult.InputSortedDescending",
+ previous.getName(),
+ field.getFieldName()),
+ transformMeta));
+ }
+ }
+ }
+ }
+
@Override
public PipelineType[] getSupportedPipelineTypes() {
return new PipelineType[] {
diff --git
a/plugins/transforms/sortedmerge/src/main/resources/org/apache/hop/pipeline/transforms/sortedmerge/messages/messages_en_US.properties
b/plugins/transforms/sortedmerge/src/main/resources/org/apache/hop/pipeline/transforms/sortedmerge/messages/messages_en_US.properties
index 7dc4172771..3d0e48adc9 100644
---
a/plugins/transforms/sortedmerge/src/main/resources/org/apache/hop/pipeline/transforms/sortedmerge/messages/messages_en_US.properties
+++
b/plugins/transforms/sortedmerge/src/main/resources/org/apache/hop/pipeline/transforms/sortedmerge/messages/messages_en_US.properties
@@ -20,6 +20,10 @@ SortedMerge.Injection.ASCENDING=Specify if the fields are
ascending or not (Y/N)
SortedMerge.Injection.FIELD_NAME=The name of the input field used for sorting.
SortedMerge.Injection.FIELDS=The names of the input fields that are being
merged.
SortedMerge.Exception.ErrorComparingRows=Error comparing rows during sorted
merge
+SortedMerge.Exception.Ascending=ascending
+SortedMerge.Exception.Descending=descending
+SortedMerge.Exception.InputNotInMergeOrder=The rows from {0} are not in the
merge order ({3}): {1} comes after {2}. Every input of a Sorted merge must be
sorted on the merge fields, in the order set for each field.
+SortedMerge.Exception.InputSortedDescending={0} gives its rows sorted
descending on {1}: set Ascending to N for {1}.
SortedMerge.Log.LineNumber=Linenr
SortedMerge.Name=Sorted merge
SortedMergeDialog.Ascending.Column=Ascending
@@ -37,6 +41,7 @@ SortedMergeMeta.CheckResult.AllSortKeysFound=All sort keys
are found in the inpu
SortedMergeMeta.CheckResult.ExpectedInputError=No input received from other
transforms!
SortedMergeMeta.CheckResult.ExpectedInputOk=Transform is receiving info from
other transforms.
SortedMergeMeta.CheckResult.FieldsReceived=Transform is connected to previous
one, receiving {0} fields
+SortedMergeMeta.CheckResult.InputSortedDescending={0} gives its rows sorted
descending on {1}, but the merge sorts {1} ascending: set Ascending to N for
{1}.
SortedMergeMeta.CheckResult.NoFields=Couldn''t find fields from previous
transforms, check the hops...!
SortedMergeMeta.CheckResult.NoSortKeysEntered=No sort keys are entered.
SortedMergeMeta.CheckResult.SortKeysNotFound=Sort keys that were not found in
input stream:\n\n{0}
diff --git
a/plugins/transforms/sortedmerge/src/test/java/org/apache/hop/pipeline/transforms/sortedmerge/SortedMergeMetaTest.java
b/plugins/transforms/sortedmerge/src/test/java/org/apache/hop/pipeline/transforms/sortedmerge/SortedMergeMetaTest.java
index 125640025c..ab4c76b4a8 100644
---
a/plugins/transforms/sortedmerge/src/test/java/org/apache/hop/pipeline/transforms/sortedmerge/SortedMergeMetaTest.java
+++
b/plugins/transforms/sortedmerge/src/test/java/org/apache/hop/pipeline/transforms/sortedmerge/SortedMergeMetaTest.java
@@ -20,7 +20,10 @@ package org.apache.hop.pipeline.transforms.sortedmerge;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.List;
@@ -29,6 +32,7 @@ import org.apache.hop.core.ICheckResult;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.RowMeta;
import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.variables.Variables;
import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
import org.apache.hop.pipeline.PipelineMeta;
@@ -170,6 +174,43 @@ class SortedMergeMetaTest {
return remarks;
}
+ /** An input sorted descending on a field the merge sorts ascending: Verify
pipeline warns. */
+ @Test
+ void checkWarnsWhenAnInputIsSortedDescendingOnAnAscendingMergeField() throws
Exception {
+ SortedMergeMeta meta = new SortedMergeMeta();
+ meta.getMergeFields().add(new MergeField("key", true));
+ TransformMeta transformMeta = mock(TransformMeta.class);
+ TransformMeta sort = new TransformMeta();
+ sort.setName("Sort descending");
+ IRowMeta sortedDescending = stringMeta("key");
+ sortedDescending.getValueMeta(0).setSortedDescending(true);
+ PipelineMeta pipelineMeta = mock(PipelineMeta.class);
+
when(pipelineMeta.findPreviousTransforms(transformMeta)).thenReturn(List.of(sort));
+ when(pipelineMeta.getTransformFields(any(IVariables.class), eq(sort)))
+ .thenReturn(sortedDescending);
+
+ List<ICheckResult> remarks = new ArrayList<>();
+ meta.check(
+ remarks,
+ pipelineMeta,
+ transformMeta,
+ stringMeta("key"),
+ new String[] {"Sort descending"},
+ new String[0],
+ null,
+ new Variables(),
+ null);
+
+ assertTrue(
+ remarks.stream()
+ .anyMatch(
+ r ->
+ r.getType() == ICheckResult.TYPE_RESULT_WARNING
+ && r.getText().contains("Sort descending")
+ && r.getText().contains("set Ascending to N for key")),
+ remarks.toString());
+ }
+
private static IRowMeta stringMeta(String... names) {
RowMeta rowMeta = new RowMeta();
for (String name : names) {
diff --git
a/plugins/transforms/sortedmerge/src/test/java/org/apache/hop/pipeline/transforms/sortedmerge/SortedMergeTest.java
b/plugins/transforms/sortedmerge/src/test/java/org/apache/hop/pipeline/transforms/sortedmerge/SortedMergeTest.java
index 54f489da24..b5b4a50202 100644
---
a/plugins/transforms/sortedmerge/src/test/java/org/apache/hop/pipeline/transforms/sortedmerge/SortedMergeTest.java
+++
b/plugins/transforms/sortedmerge/src/test/java/org/apache/hop/pipeline/transforms/sortedmerge/SortedMergeTest.java
@@ -23,6 +23,7 @@ import static
org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
@@ -109,6 +110,69 @@ class SortedMergeTest {
assertTrue(drain(transform.getOutputRowSets().getFirst()).isEmpty());
}
+ /** Rows sorted descending, without a transform before saying so, merge
descending. */
+ @Test
+ void processRowMergesDescendingStreams() throws Exception {
+ SortedMerge transform =
+ createTransform(
+ descendingKey(),
+ stringKeyMeta(),
+ new Object[][] {{"echo"}, {"charlie"}, {"alpha"}},
+ new Object[][] {{"foxtrot"}, {"delta"}, {"bravo"}});
+
+ assertEquals(
+ List.of("foxtrot", "echo", "delta", "charlie", "bravo", "alpha"),
+ keys(runToCompletion(transform)));
+ }
+
+ /** Rows a Sort rows marked descending merge descending when the merge says
so too. */
+ @Test
+ void processRowMergesStreamsMarkedDescending() throws Exception {
+ IRowMeta rowMeta = stringKeyMeta();
+ rowMeta.getValueMeta(0).setSortedDescending(true);
+ SortedMerge transform =
+ createTransform(
+ descendingKey(),
+ rowMeta,
+ new Object[][] {{"echo"}, {"charlie"}, {"alpha"}},
+ new Object[][] {{"foxtrot"}, {"delta"}, {"bravo"}});
+
+ assertEquals(
+ List.of("foxtrot", "echo", "delta", "charlie", "bravo", "alpha"),
+ keys(runToCompletion(transform)));
+ }
+
+ /** An input that is not in the merge order would be merged wrongly: the
merge stops there. */
+ @Test
+ void processRowFailsWhenAnInputIsNotInTheMergeOrder() {
+ SortedMerge transform =
+ createTransform(
+ stringKeyMeta(), new Object[][] {{"bravo"}, {"alpha"}}, new
Object[][] {{"charlie"}});
+
+ HopTransformException thrown =
+ assertThrows(HopTransformException.class, () ->
runToCompletion(transform));
+ assertTrue(
+ thrown.getMessage().contains("key=alpha comes after key=bravo"),
thrown.getMessage());
+ assertFalse(thrown.getMessage().contains("set Ascending to N"),
thrown.getMessage());
+ }
+
+ /**
+ * Rows a Sort rows marked descending, merged ascending: the setting
decides, and the error says
+ * what to change.
+ */
+ @Test
+ void processRowFailsWithAHintWhenAnInputIsMarkedDescending() {
+ IRowMeta rowMeta = stringKeyMeta();
+ rowMeta.getValueMeta(0).setSortedDescending(true);
+ SortedMerge transform =
+ createTransform(
+ rowMeta, new Object[][] {{"bravo"}, {"alpha"}}, new Object[][]
{{"charlie"}});
+
+ HopTransformException thrown =
+ assertThrows(HopTransformException.class, () ->
runToCompletion(transform));
+ assertTrue(thrown.getMessage().contains("set Ascending to N for key"),
thrown.getMessage());
+ }
+
@Test
void processRowFailsWhenSortKeyIsMissing() {
IRowMeta rowMeta = stringKeyMeta();
@@ -128,6 +192,8 @@ class SortedMergeTest {
doThrow(new HopValueException("compare failed"))
.when(rowMeta)
.compare(any(Object[].class), any(Object[].class), any(int[].class));
+ // The merge compares with its own copy of the input's metadata
+ doReturn(rowMeta).when(rowMeta).clone();
SortedMerge transform =
createTransform(rowMeta, new Object[][] {{"left"}}, new Object[][]
{{"right"}});
@@ -179,6 +245,12 @@ class SortedMergeTest {
return transform;
}
+ private static SortedMergeMeta descendingKey() {
+ SortedMergeMeta meta = new SortedMergeMeta();
+ meta.getMergeFields().add(new SortedMergeMeta.MergeField("key", false));
+ return meta;
+ }
+
private static IRowMeta stringKeyMeta() {
RowMeta rowMeta = new RowMeta();
rowMeta.addValueMeta(new ValueMetaString("key"));