This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new cd3ce1a1eb Date A + B" calculations fail the transform when the date
or the amou… (#9260)
cd3ce1a1eb is described below
commit cd3ce1a1eb086219a99f35ff8847f5358877b75f
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Wed Nov 4 17:37:25 2026 +0100
Date A + B" calculations fail the transform when the date or the amou…
(#9260)
* Date A + B" calculations fail the transform when the date or the amount
is null, fixes #9259
* calculation fixes
---
.../org/apache/hop/core/row/ValueDataUtil.java | 36 ++++--
.../hop/core/row/ValueDataUtilDateNullTest.java | 65 +++++++++++
.../hop/core/row/ValueDataUtilPrecisionTest.java | 125 +++++++++++++++++++++
3 files changed, 217 insertions(+), 9 deletions(-)
diff --git a/core/src/main/java/org/apache/hop/core/row/ValueDataUtil.java
b/core/src/main/java/org/apache/hop/core/row/ValueDataUtil.java
index 3aff35e742..700c023729 100644
--- a/core/src/main/java/org/apache/hop/core/row/ValueDataUtil.java
+++ b/core/src/main/java/org/apache/hop/core/row/ValueDataUtil.java
@@ -57,6 +57,7 @@ public class ValueDataUtil {
private static final Log log = LogFactory.getLog(ValueDataUtil.class);
private static final String CONST_ROUND_ONLY_WORK_ON_NUMERIC =
"The 'round' function only works on numeric data";
+ private static final BigDecimal HALF = new BigDecimal("0.5");
/**
* System property sets rounding mode of calculator's function ROUND(A,B)
@@ -548,7 +549,7 @@ public class ValueDataUtil {
case IValueMeta.TYPE_INTEGER ->
metaA.getInteger(dataA) + metaB.getInteger(dataB) +
metaC.getInteger(dataC);
case IValueMeta.TYPE_BOOLEAN ->
- metaA.getBoolean(dataA) || metaB.getBoolean(dataB) ||
metaB.getBoolean(dataC);
+ metaA.getBoolean(dataA) || metaB.getBoolean(dataB) ||
metaC.getBoolean(dataC);
case IValueMeta.TYPE_BIGNUMBER ->
metaA.getBigNumber(dataA).add(metaB.getBigNumber(dataB).add(metaC.getBigNumber(dataC)));
default ->
@@ -949,7 +950,9 @@ public class ValueDataUtil {
return switch (metaA.getType()) {
case IValueMeta.TYPE_NUMBER -> (double)
Math.round(metaA.getNumber(dataA));
case IValueMeta.TYPE_INTEGER -> metaA.getInteger(dataA);
- case IValueMeta.TYPE_BIGNUMBER -> new
BigDecimal(Math.round(metaA.getNumber(dataA)));
+ // Math.round rounds half up towards positive infinity: floor(x + 0.5)
+ case IValueMeta.TYPE_BIGNUMBER ->
+ metaA.getBigNumber(dataA).add(HALF).setScale(0, RoundingMode.FLOOR);
default -> throw new HopValueException(CONST_ROUND_ONLY_WORK_ON_NUMERIC);
};
}
@@ -1065,7 +1068,9 @@ public class ValueDataUtil {
return switch (metaA.getType()) {
case IValueMeta.TYPE_NUMBER -> Math.ceil(metaA.getNumber(dataA));
case IValueMeta.TYPE_INTEGER -> metaA.getInteger(dataA);
- case IValueMeta.TYPE_BIGNUMBER ->
BigDecimal.valueOf(Math.ceil(metaA.getNumber(dataA)));
+ // scale 1, as the result always had when it went through
BigDecimal.valueOf(double)
+ case IValueMeta.TYPE_BIGNUMBER ->
+ metaA.getBigNumber(dataA).setScale(0,
RoundingMode.CEILING).setScale(1);
default -> throw new HopValueException("The 'ceil' function only works
on numeric data");
};
}
@@ -1078,7 +1083,9 @@ public class ValueDataUtil {
return switch (metaA.getType()) {
case IValueMeta.TYPE_NUMBER -> Math.floor(metaA.getNumber(dataA));
case IValueMeta.TYPE_INTEGER -> metaA.getInteger(dataA);
- case IValueMeta.TYPE_BIGNUMBER ->
BigDecimal.valueOf(Math.floor(metaA.getNumber(dataA)));
+ // scale 1, as the result always had when it went through
BigDecimal.valueOf(double)
+ case IValueMeta.TYPE_BIGNUMBER ->
+ metaA.getBigNumber(dataA).setScale(0,
RoundingMode.FLOOR).setScale(1);
default -> throw new HopValueException("The 'floor' function only works
on numeric data");
};
}
@@ -1091,9 +1098,8 @@ public class ValueDataUtil {
return switch (metaA.getType()) {
case IValueMeta.TYPE_NUMBER -> Math.abs(metaA.getNumber(dataA));
- case IValueMeta.TYPE_INTEGER ->
- metaA.getInteger(Math.abs(metaA.getNumber(dataA).longValue()));
- case IValueMeta.TYPE_BIGNUMBER ->
BigDecimal.valueOf(Math.abs(metaA.getNumber(dataA)));
+ case IValueMeta.TYPE_INTEGER -> Math.abs(metaA.getInteger(dataA));
+ case IValueMeta.TYPE_BIGNUMBER -> metaA.getBigNumber(dataA).abs();
default -> throw new HopValueException("The 'abs' function only works on
numeric data");
};
}
@@ -1121,7 +1127,7 @@ public class ValueDataUtil {
return metaA.getInteger(dataA) % metaB.getInteger(dataB);
case IValueMeta.TYPE_BIGNUMBER:
BigDecimal aValue = metaA.getBigNumber(dataA);
- BigDecimal bValue = metaA.getBigNumber(dataB);
+ BigDecimal bValue = metaB.getBigNumber(dataB);
BigDecimal result =
aValue.remainder(
bValue, new MathContext(getMaxPrecision(aValue, bValue),
RoundingMode.HALF_EVEN));
@@ -1231,6 +1237,9 @@ public class ValueDataUtil {
public static Object addDays(IValueMeta metaA, Object dataA, IValueMeta
metaB, Object dataB)
throws HopValueException {
+ if (dataA == null || dataB == null) {
+ return null;
+ }
Calendar cal = Calendar.getInstance();
cal.setTime(metaA.getDate(dataA));
@@ -1241,6 +1250,9 @@ public class ValueDataUtil {
public static Object addHours(IValueMeta metaA, Object dataA, IValueMeta
metaB, Object dataB)
throws HopValueException {
+ if (dataA == null || dataB == null) {
+ return null;
+ }
Calendar cal = Calendar.getInstance();
cal.setTime(metaA.getDate(dataA));
@@ -1251,6 +1263,9 @@ public class ValueDataUtil {
public static Object addMinutes(IValueMeta metaA, Object dataA, IValueMeta
metaB, Object dataB)
throws HopValueException {
+ if (dataA == null || dataB == null) {
+ return null;
+ }
Calendar cal = Calendar.getInstance();
cal.setTime(metaA.getDate(dataA));
@@ -1261,6 +1276,9 @@ public class ValueDataUtil {
public static Object addSeconds(IValueMeta metaA, Object dataA, IValueMeta
metaB, Object dataB)
throws HopValueException {
+ if (dataA == null || dataB == null) {
+ return null;
+ }
Calendar cal = Calendar.getInstance();
cal.setTime(metaA.getDate(dataA));
@@ -1294,7 +1312,7 @@ public class ValueDataUtil {
return (cal.getTime());
} else {
- throw new HopValueException("Unable to add months with a null value");
+ return null;
}
}
diff --git
a/core/src/test/java/org/apache/hop/core/row/ValueDataUtilDateNullTest.java
b/core/src/test/java/org/apache/hop/core/row/ValueDataUtilDateNullTest.java
new file mode 100644
index 0000000000..ad413b1a3f
--- /dev/null
+++ b/core/src/test/java/org/apache/hop/core/row/ValueDataUtilDateNullTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.row;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.util.Calendar;
+import java.util.Date;
+import org.apache.hop.core.row.value.ValueMetaDate;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.junit.jupiter.api.Test;
+
+/**
+ * The Calculator's "Date A + B" calculations return null for a null date or
amount, like the other
+ * calculations. Issue #9259.
+ */
+class ValueDataUtilDateNullTest {
+
+ private final IValueMeta date = new ValueMetaDate("a");
+ private final IValueMeta amount = new ValueMetaInteger("b");
+ private final Date someDate = new Date(1_700_000_000_000L);
+
+ @Test
+ void nullDateGivesNull() throws Exception {
+ assertNull(ValueDataUtil.addDays(date, null, amount, 1L));
+ assertNull(ValueDataUtil.addHours(date, null, amount, 1L));
+ assertNull(ValueDataUtil.addMinutes(date, null, amount, 1L));
+ assertNull(ValueDataUtil.addSeconds(date, null, amount, 1L));
+ assertNull(ValueDataUtil.addMonths(date, null, amount, 1L));
+ }
+
+ @Test
+ void nullAmountGivesNull() throws Exception {
+ assertNull(ValueDataUtil.addDays(date, someDate, amount, null));
+ assertNull(ValueDataUtil.addHours(date, someDate, amount, null));
+ assertNull(ValueDataUtil.addMinutes(date, someDate, amount, null));
+ assertNull(ValueDataUtil.addSeconds(date, someDate, amount, null));
+ assertNull(ValueDataUtil.addMonths(date, someDate, amount, null));
+ }
+
+ @Test
+ void valuesStillAdd() throws Exception {
+ Calendar expected = Calendar.getInstance();
+ expected.setTime(someDate);
+ expected.add(Calendar.DAY_OF_YEAR, 2);
+
+ assertEquals(expected.getTime(), ValueDataUtil.addDays(date, someDate,
amount, 2L));
+ }
+}
diff --git
a/core/src/test/java/org/apache/hop/core/row/ValueDataUtilPrecisionTest.java
b/core/src/test/java/org/apache/hop/core/row/ValueDataUtilPrecisionTest.java
new file mode 100644
index 0000000000..2907797f32
--- /dev/null
+++ b/core/src/test/java/org/apache/hop/core/row/ValueDataUtilPrecisionTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.row;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.math.BigDecimal;
+import org.apache.hop.core.row.value.ValueMetaBigNumber;
+import org.apache.hop.core.row.value.ValueMetaBoolean;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Calculator functions read every value with its own metadata and keep
BigNumber and Integer
+ * precision. Issue #9259.
+ */
+class ValueDataUtilPrecisionTest {
+
+ private final IValueMeta big = new ValueMetaBigNumber("b");
+ private final IValueMeta integer = new ValueMetaInteger("i");
+
+ @Test
+ void plus3ReadsTheThirdValueWithItsOwnMetadata() throws Exception {
+ IValueMeta bool = new ValueMetaBoolean("a");
+ IValueMeta string = new ValueMetaString("c");
+
+ // dataC is "Y" described by a String meta; reading it through the Boolean
meta of B fails
+ assertEquals(Boolean.TRUE, ValueDataUtil.plus3(bool, false, bool, false,
string, "Y"));
+ }
+
+ @Test
+ void remainderReadsTheDivisorWithItsOwnMetadata() throws Exception {
+ IValueMeta divisorAsString = new ValueMetaString("s");
+
+ Object result = ValueDataUtil.remainder(big, new BigDecimal("10"),
divisorAsString, "3");
+
+ assertEquals(0, BigDecimal.ONE.compareTo((BigDecimal) result));
+ }
+
+ @Test
+ void bigNumberFunctionsKeepPrecision() throws Exception {
+ // 2^53 + 1 cannot be represented as a double
+ BigDecimal huge = new BigDecimal("9007199254740993.4");
+
+ assertEquals(new BigDecimal("9007199254740993"), ValueDataUtil.round(big,
huge));
+ assertEquals(new BigDecimal("9007199254740994.0"), ValueDataUtil.ceil(big,
huge));
+ assertEquals(new BigDecimal("9007199254740993.0"),
ValueDataUtil.floor(big, huge));
+ assertEquals(huge, ValueDataUtil.abs(big, huge.negate()));
+ }
+
+ @Test
+ void bigNumberRoundKeepsMathRoundSemantics() throws Exception {
+ assertEquals(new BigDecimal("3"), ValueDataUtil.round(big, new
BigDecimal("2.5")));
+ assertEquals(new BigDecimal("-2"), ValueDataUtil.round(big, new
BigDecimal("-2.5")));
+ assertEquals(new BigDecimal("-3"), ValueDataUtil.round(big, new
BigDecimal("-2.6")));
+ }
+
+ @Test
+ void bigNumberCeilAndFloorKeepTheirScale() throws Exception {
+ // the result has always had one decimal, e.g. 6.0, which shows in JSON
and other native output
+ assertEquals(new BigDecimal("6.0"), ValueDataUtil.ceil(big, new
BigDecimal("5.2")));
+ assertEquals(new BigDecimal("5.0"), ValueDataUtil.floor(big, new
BigDecimal("5.8")));
+ }
+
+ @Test
+ void manyDecimalsGiveTheSameResultAsTheDoubleFunctions() throws Exception {
+ String[] values = {
+ "2.567",
+ "-2.567",
+ "5.001",
+ "-5.001",
+ "5.999",
+ "-5.999",
+ "0.4999",
+ "-0.5001",
+ "7.5000001",
+ "-7.4999999",
+ "123.456789",
+ "-0.0001"
+ };
+ for (String text : values) {
+ BigDecimal value = new BigDecimal(text);
+ double d = value.doubleValue();
+
+ assertEquals(
+ 0,
+ BigDecimal.valueOf(Math.round(d)).compareTo((BigDecimal)
ValueDataUtil.round(big, value)),
+ "round " + text);
+ assertEquals(
+ 0,
+ BigDecimal.valueOf(Math.ceil(d)).compareTo((BigDecimal)
ValueDataUtil.ceil(big, value)),
+ "ceil " + text);
+ assertEquals(
+ 0,
+ BigDecimal.valueOf(Math.floor(d)).compareTo((BigDecimal)
ValueDataUtil.floor(big, value)),
+ "floor " + text);
+ assertEquals(
+ 0, value.abs().compareTo((BigDecimal) ValueDataUtil.abs(big,
value)), "abs " + text);
+ assertEquals(1, ((BigDecimal) ValueDataUtil.ceil(big, value)).scale(),
"ceil scale " + text);
+ assertEquals(
+ 1, ((BigDecimal) ValueDataUtil.floor(big, value)).scale(), "floor
scale " + text);
+ }
+ }
+
+ @Test
+ void integerAbsKeepsPrecision() throws Exception {
+ assertEquals(9007199254740993L, ValueDataUtil.abs(integer,
-9007199254740993L));
+ }
+}