This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 69b93a814c use a large format mask to not lose decimal digits, fixes
#3970 (#8580)
69b93a814c is described below
commit 69b93a814c8cd22e661493beecc359f0b9f7f0f3
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Fri Sep 25 11:27:59 2026 +0200
use a large format mask to not lose decimal digits, fixes #3970 (#8580)
* use a large format mask to not lose decimal digits, fixes #3970
* fix remarks
---
.../org/apache/hop/testing/DataSetCsvUtil.java | 18 ++-
.../apache/hop/testing/DataSetCsvWriterTest.java | 128 +++++++++++++++++++++
2 files changed, 144 insertions(+), 2 deletions(-)
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvUtil.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvUtil.java
index 29070d6e9f..c0e5e1c05a 100644
---
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvUtil.java
+++
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvUtil.java
@@ -46,6 +46,16 @@ import org.apache.hop.core.vfs.HopVfs;
* file defined by the tableName in the data set
*/
public class DataSetCsvUtil {
+ /**
+ * Storage mask for Number and BigNumber fields without a format.
DecimalFormat prints the
+ * shortest decimal representation of a double, so 3.14 is written as "3.14"
while no digits are
+ * ever dropped (340 is the maximum number of fraction digits DecimalFormat
honours for a double).
+ * Without it a BigNumber falls back to the default mask, which keeps 19
fraction digits, or to a
+ * pattern built from its length and precision, which rounds and zero-pads.
Declared precision is
+ * applied when comparing against golden data, not when storing.
+ */
+ static final String NUMBER_STORAGE_MASK = "0." + "#".repeat(340);
+
public static void setValueFormats(IRowMeta rowMeta) {
for (IValueMeta valueMeta : rowMeta.getValueMetaList()) {
if (StringUtils.isEmpty(valueMeta.getConversionMask())) {
@@ -53,8 +63,8 @@ public class DataSetCsvUtil {
case IValueMeta.TYPE_INTEGER:
valueMeta.setConversionMask("0");
break;
- case IValueMeta.TYPE_NUMBER:
- valueMeta.setConversionMask("0.#");
+ case IValueMeta.TYPE_NUMBER, IValueMeta.TYPE_BIGNUMBER:
+ valueMeta.setConversionMask(NUMBER_STORAGE_MASK);
break;
case IValueMeta.TYPE_DATE:
valueMeta.setConversionMask("yyyyMMdd-HHmmss.SSS");
@@ -236,6 +246,10 @@ public class DataSetCsvUtil {
if (valueMeta.isBinary()) {
return decodeHex(value, valueMeta.getName());
}
+ if (value != null && (valueMeta.isNumber() || valueMeta.isBigNumber())) {
+ // Earlier versions padded a BigNumber with a length to " 00001234.57"
+ value = value.trim();
+ }
return valueMeta.convertData(stringMeta, value);
}
diff --git
a/plugins/misc/testing/src/test/java/org/apache/hop/testing/DataSetCsvWriterTest.java
b/plugins/misc/testing/src/test/java/org/apache/hop/testing/DataSetCsvWriterTest.java
index a1c633b53d..e357ad6516 100644
---
a/plugins/misc/testing/src/test/java/org/apache/hop/testing/DataSetCsvWriterTest.java
+++
b/plugins/misc/testing/src/test/java/org/apache/hop/testing/DataSetCsvWriterTest.java
@@ -22,9 +22,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import java.math.BigDecimal;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
+import java.util.Locale;
import org.apache.hop.core.HopEnvironment;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.row.IRowMeta;
@@ -134,6 +136,132 @@ class DataSetCsvWriterTest {
assertTrue(Files.exists(nested.resolve("nested.csv")));
}
+ @Test
+ void numberWithoutFormatKeepsAllDigits() throws Exception {
+ Variables variables = new Variables();
+ variables.setVariable(DataSet.VARIABLE_HOP_DATASETS_FOLDER,
tempDir.toString());
+
+ DataSet dataSet = new DataSet();
+ dataSet.setName("numbers");
+ dataSet.setBaseFilename("numbers.csv");
+ dataSet.setFields(
+ List.of(
+ new DataSetField("id", IValueMeta.TYPE_INTEGER, -1, 0, "", "0"),
+ new DataSetField("pi", IValueMeta.TYPE_NUMBER, -1, -1, "", "")));
+
+ double[] values = {3.14, -1234.5678, 0.1 + 0.2, 1e-20, 12345678901.25,
7.0};
+ IRowMeta rowMeta = dataSet.getSetRowMeta();
+ try (DataSetCsvWriter writer = new DataSetCsvWriter(variables, dataSet,
rowMeta)) {
+ for (int i = 0; i < values.length; i++) {
+ writer.writeRow(new Object[] {(long) i, values[i]});
+ }
+ writer.writeRow(new Object[] {(long) values.length, null});
+ }
+
+ // The caller's row metadata must not be touched
+ assertTrue(rowMeta.getValueMeta(1).getConversionMask().isEmpty());
+
+ String csv = Files.readString(tempDir.resolve("numbers.csv"));
+ assertTrue(csv.contains("0,3.14"), csv);
+ // The previous "0.#" mask wrote -1234.6
+ assertTrue(csv.contains("1,-1234.5678"), csv);
+
+ List<Object[]> rows = DataSetCsvUtil.getAllRows(variables, dataSet);
+ assertEquals(values.length + 1, rows.size());
+ for (int i = 0; i < values.length; i++) {
+ assertEquals(values[i], (Double) rows.get(i)[1]);
+ }
+ assertNull(rows.get(values.length)[1]);
+ }
+
+ @Test
+ void bigNumberWithoutFormatKeepsAllDigits() throws Exception {
+ Locale defaultLocale = Locale.getDefault();
+ // A locale with a comma decimal separator and a dot grouping separator
+ Locale.setDefault(Locale.GERMANY);
+ try {
+ Variables variables = new Variables();
+ variables.setVariable(DataSet.VARIABLE_HOP_DATASETS_FOLDER,
tempDir.toString());
+
+ DataSet dataSet = new DataSet();
+ dataSet.setName("big-numbers");
+ dataSet.setBaseFilename("big-numbers.csv");
+ dataSet.setFields(
+ List.of(
+ new DataSetField("id", IValueMeta.TYPE_INTEGER, -1, 0, "", "0"),
+ new DataSetField("unsized", IValueMeta.TYPE_BIGNUMBER, -1, -1,
"", ""),
+ new DataSetField("sized", IValueMeta.TYPE_BIGNUMBER, 10, 2, "",
"")));
+
+ BigDecimal[] values = {
+ new BigDecimal("1234.56789"),
+ new BigDecimal("-1234567.5"),
+ new BigDecimal("0.1234567890123456789012345"),
+ new BigDecimal("12345678901234567890.5")
+ };
+ try (DataSetCsvWriter writer =
+ new DataSetCsvWriter(variables, dataSet, dataSet.getSetRowMeta())) {
+ for (int i = 0; i < values.length; i++) {
+ writer.writeRow(new Object[] {(long) i, values[i], values[i]});
+ }
+ writer.writeRow(new Object[] {(long) values.length, null, null});
+ }
+
+ String csv = Files.readString(tempDir.resolve("big-numbers.csv"));
+ // No grouping, a dot decimal separator and no zero-padding to the
declared length
+ assertTrue(csv.contains("0,1234.56789,1234.56789"), csv);
+ assertTrue(csv.contains("1,-1234567.5,-1234567.5"), csv);
+
+ List<Object[]> rows = DataSetCsvUtil.getAllRows(variables, dataSet);
+ assertEquals(values.length + 1, rows.size());
+ for (int i = 0; i < values.length; i++) {
+ assertEquals(0, values[i].compareTo((BigDecimal) rows.get(i)[1]), csv);
+ assertEquals(0, values[i].compareTo((BigDecimal) rows.get(i)[2]), csv);
+ }
+ assertNull(rows.get(values.length)[1]);
+ assertNull(rows.get(values.length)[2]);
+ } finally {
+ Locale.setDefault(defaultLocale);
+ }
+ }
+
+ @Test
+ void bigNumberWrittenZeroPaddedIsStillRead() throws Exception {
+ Variables variables = new Variables();
+ variables.setVariable(DataSet.VARIABLE_HOP_DATASETS_FOLDER,
tempDir.toString());
+
+ DataSet dataSet = new DataSet();
+ dataSet.setName("padded");
+ dataSet.setBaseFilename("padded.csv");
+ dataSet.setFields(List.of(new DataSetField("sized",
IValueMeta.TYPE_BIGNUMBER, 10, 2, "", "")));
+
+ // What earlier versions wrote for a BigNumber(10,2) field without a format
+ Files.writeString(tempDir.resolve("padded.csv"), "sized\n\"
00001234.57\"\n-01234567.50\n");
+
+ List<Object[]> rows = DataSetCsvUtil.getAllRows(variables, dataSet);
+ assertEquals(2, rows.size());
+ assertEquals(0, new BigDecimal("1234.57").compareTo((BigDecimal)
rows.get(0)[0]));
+ assertEquals(0, new BigDecimal("-1234567.5").compareTo((BigDecimal)
rows.get(1)[0]));
+ }
+
+ @Test
+ void numberWithFormatUsesThatFormat() throws Exception {
+ Variables variables = new Variables();
+ variables.setVariable(DataSet.VARIABLE_HOP_DATASETS_FOLDER,
tempDir.toString());
+
+ DataSet dataSet = new DataSet();
+ dataSet.setName("formatted");
+ dataSet.setBaseFilename("formatted.csv");
+ dataSet.setFields(List.of(new DataSetField("pi", IValueMeta.TYPE_NUMBER,
-1, -1, "", "0.00")));
+
+ try (DataSetCsvWriter writer =
+ new DataSetCsvWriter(variables, dataSet, dataSet.getSetRowMeta())) {
+ writer.writeRow(new Object[] {3.14159});
+ }
+
+ List<Object[]> rows = DataSetCsvUtil.getAllRows(variables, dataSet);
+ assertEquals(3.14, (Double) rows.get(0)[0]);
+ }
+
@Test
void binaryValuesRoundTripAsLowercaseHex() throws Exception {
Variables variables = new Variables();
