This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new bbe2961a9a Group By: a nearest-rank percentile of a group without
values is null… (#9521)
bbe2961a9a is described below
commit bbe2961a9aacfda6ef529a0d129091f6c865c304
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Tue Nov 17 10:45:40 2026 +0100
Group By: a nearest-rank percentile of a group without values is null…
(#9521)
* Group By: a nearest-rank percentile of a group without values is null,
fixes #9512
* fix remark
---
.../0006-groupby-alwaysgiveback-result.hpl | 74 ++++++++++++++++++++--
integration-tests/transforms/main-0006-groupby.hwf | 14 ++--
.../hop/pipeline/transforms/groupby/GroupBy.java | 4 +-
.../pipeline/transforms/groupby/GroupByTest.java | 44 +++++++++++++
4 files changed, 123 insertions(+), 13 deletions(-)
diff --git
a/integration-tests/transforms/0006-groupby-alwaysgiveback-result.hpl
b/integration-tests/transforms/0006-groupby-alwaysgiveback-result.hpl
index 581687b0d2..d057304d0e 100644
--- a/integration-tests/transforms/0006-groupby-alwaysgiveback-result.hpl
+++ b/integration-tests/transforms/0006-groupby-alwaysgiveback-result.hpl
@@ -53,11 +53,6 @@ limitations under the License.
<to>Count = 0</to>
<enabled>Y</enabled>
</hop>
- <hop>
- <from>Count = 0</from>
- <to>Success</to>
- <enabled>Y</enabled>
- </hop>
<hop>
<from>Group by</from>
<to>Detect empty stream</to>
@@ -68,6 +63,21 @@ limitations under the License.
<to>Abort</to>
<enabled>Y</enabled>
</hop>
+ <hop>
+ <from>Count = 0</from>
+ <to>Percentile is null</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Percentile is null</from>
+ <to>Success</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Percentile is null</from>
+ <to>Percentile not null - Abort</to>
+ <enabled>Y</enabled>
+ </hop>
</order>
<transform>
<name>Abort</name>
@@ -237,6 +247,12 @@ limitations under the License.
<type>COUNT_ANY</type>
<valuefield/>
</field>
+ <field>
+ <aggregate>Percentile</aggregate>
+ <subject>ColA</subject>
+ <type>PERCENTILE_NEAREST_RANK</type>
+ <valuefield>50</valuefield>
+ </field>
</fields>
<give_back_row>Y</give_back_row>
<group>
@@ -250,6 +266,54 @@ limitations under the License.
<yloc>272</yloc>
</GUI>
</transform>
+ <transform>
+ <name>Percentile is null</name>
+ <type>FilterRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <send_true_to>Success</send_true_to>
+ <send_false_to>Percentile not null - Abort</send_false_to>
+ <compare>
+ <condition>
+ <negated>N</negated>
+ <leftvalue>Percentile</leftvalue>
+ <function>IS NULL</function>
+ <rightvalue/>
+ </condition>
+ </compare>
+ <attributes/>
+ <GUI>
+ <xloc>864</xloc>
+ <yloc>160</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Percentile not null - Abort</name>
+ <type>Abort</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <abort_option>ABORT_WITH_ERROR</abort_option>
+ <always_log_rows>Y</always_log_rows>
+ <message>The percentile of a group without values is not null</message>
+ <row_threshold>0</row_threshold>
+ <attributes/>
+ <GUI>
+ <xloc>1088</xloc>
+ <yloc>160</yloc>
+ </GUI>
+ </transform>
<transform>
<name>Success</name>
<type>Dummy</type>
diff --git a/integration-tests/transforms/main-0006-groupby.hwf
b/integration-tests/transforms/main-0006-groupby.hwf
index 138c2606e1..739e43c46d 100644
--- a/integration-tests/transforms/main-0006-groupby.hwf
+++ b/integration-tests/transforms/main-0006-groupby.hwf
@@ -109,7 +109,7 @@ limitations under the License.
<attributes_hac/>
</action>
<action>
-
<filename>${PROJECT_HOME}/0006-groupby-alwaysgiveback-result.hpl</filename>
+
<filename>${PROJECT_HOME}/0006-groupby-alwaysgiveback-result-false.hpl</filename>
<params_from_previous>N</params_from_previous>
<exec_per_row>N</exec_per_row>
<clear_rows>N</clear_rows>
@@ -127,7 +127,7 @@ limitations under the License.
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<run_configuration>local</run_configuration>
- <name>0006-groupby-alwaysgiveback-result.hpl 2</name>
+ <name>0006-groupby-alwaysgiveback-result-false.hpl</name>
<description/>
<type>PIPELINE</type>
<attributes/>
@@ -147,16 +147,16 @@ limitations under the License.
</hop>
<hop>
<from>0006-groupby-alwaysgiveback-result.hpl</from>
- <to>0006-groupby-alwaysgiveback-result.hpl 2</to>
- <evaluation>N</evaluation>
- <unconditional>Y</unconditional>
+ <to>0006-groupby-alwaysgiveback-result-false.hpl</to>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
<enabled>Y</enabled>
</hop>
<hop>
- <from>0006-groupby-alwaysgiveback-result.hpl 2</from>
+ <from>0006-groupby-alwaysgiveback-result-false.hpl</from>
<to>Run Group By tests</to>
<evaluation>Y</evaluation>
- <unconditional>Y</unconditional>
+ <unconditional>N</unconditional>
<enabled>Y</enabled>
</hop>
</hops>
diff --git
a/plugins/transforms/groupby/src/main/java/org/apache/hop/pipeline/transforms/groupby/GroupBy.java
b/plugins/transforms/groupby/src/main/java/org/apache/hop/pipeline/transforms/groupby/GroupBy.java
index 59ea72612c..21f23aea34 100644
---
a/plugins/transforms/groupby/src/main/java/org/apache/hop/pipeline/transforms/groupby/GroupBy.java
+++
b/plugins/transforms/groupby/src/main/java/org/apache/hop/pipeline/transforms/groupby/GroupBy.java
@@ -928,7 +928,9 @@ public class GroupBy extends BaseTransform<GroupByMeta,
GroupByData> {
Double[] latencies = new Double[latenciesList.size()];
latencies = latenciesList.toArray(latencies);
int index = (int) Math.ceil((percentileValue / 100) *
latencies.length);
- ag = latencies[index - 1];
+ // No values in the group (all null, or no rows at all): no
percentile. The 0th
+ // percentile is the smallest value.
+ ag = latencies.length == 0 ? null : latencies[Math.max(index, 1) -
1];
break;
case Aggregation.TYPE_GROUP_COUNT_ANY,
Aggregation.TYPE_GROUP_COUNT_ALL:
ag = data.counts[i];
diff --git
a/plugins/transforms/groupby/src/test/java/org/apache/hop/pipeline/transforms/groupby/GroupByTest.java
b/plugins/transforms/groupby/src/test/java/org/apache/hop/pipeline/transforms/groupby/GroupByTest.java
index c5c929b406..cf66238911 100644
---
a/plugins/transforms/groupby/src/test/java/org/apache/hop/pipeline/transforms/groupby/GroupByTest.java
+++
b/plugins/transforms/groupby/src/test/java/org/apache/hop/pipeline/transforms/groupby/GroupByTest.java
@@ -20,6 +20,7 @@ package org.apache.hop.pipeline.transforms.groupby;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
@@ -269,6 +270,49 @@ class GroupByTest {
assertEquals(2L, ((Number) result[1]).longValue());
}
+ @Test
+ void percentileNearestRankOfAGroupWithoutValuesIsNull() throws Exception {
+ data.inputRowMeta = inputRowMeta;
+ data.subjectnrs = new int[] {1, 1};
+ data.counts = new long[2];
+ data.mean = new double[2];
+ data.previousSums = new Object[0];
+ data.previousAvgSum = new Object[0];
+ data.previousAvgCount = new long[0];
+ data.movingAvgWindows = new ArrayDeque[2];
+
+ meta.getAggregations().clear();
+ meta.getAggregations()
+ .add(
+ new Aggregation(
+ "p75",
+ "amount",
+
Aggregation.getTypeDescLongFromCode(Aggregation.TYPE_GROUP_PERCENTILE_NEAREST_RANK),
+ "75"));
+ meta.getAggregations()
+ .add(
+ new Aggregation(
+ "p0",
+ "amount",
+
Aggregation.getTypeDescLongFromCode(Aggregation.TYPE_GROUP_PERCENTILE_NEAREST_RANK),
+ "0"));
+
+ // Only null amounts: no values to rank
+ groupBy.newAggregate(new Object[] {"A", null});
+ groupBy.calcAggregate(new Object[] {"A", null});
+ Object[] result = groupBy.getAggregateResult();
+ assertNull(result[0]);
+ assertNull(result[1]);
+
+ // A 0th percentile is the smallest value, not an index before the first
one
+ groupBy.newAggregate(new Object[] {"A", 4.0});
+ groupBy.calcAggregate(new Object[] {"A", 4.0});
+ groupBy.calcAggregate(new Object[] {"A", 2.0});
+ result = groupBy.getAggregateResult();
+ assertEquals(4.0, (Double) result[0], 1e-9);
+ assertEquals(2.0, (Double) result[1], 1e-9);
+ }
+
@Test
void batchCompleteFlushesCurrentGroup() throws Exception {
QueueRowSet input = new QueueRowSet();