This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new d8dd72fc66 File Metadata: keep the enclosure with quotes inside fields
and keep all decimals, fixes #5609 (#8555)
d8dd72fc66 is described below
commit d8dd72fc66616ef2262d31c6652387fea05694da
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Thu Sep 24 10:08:14 2026 +0200
File Metadata: keep the enclosure with quotes inside fields and keep all
decimals, fixes #5609 (#8555)
---
.../org/apache/hop/core/util/StringEvaluator.java | 110 +++++++++++++++--
.../apache/hop/core/util/StringEvaluatorTest.java | 91 ++++++++++++++
.../pages/pipeline/transforms/filemetadata.adoc | 23 +++-
...055-file-metadata-embedded-enclosure-verify.hpl | 102 ++++++++++++++++
.../golden-file-metadata-embedded-enclosure.csv | 12 ++
.../files/file-metadata-embedded-enclosure.csv | 21 ++++
.../transforms/main-0055-file-metadata.hwf | 3 +
.../golden-file-metadata-embedded-enclosure.json | 120 ++++++++++++++++++
...le-metadata-embedded-enclosure-verify UNIT.json | 93 ++++++++++++++
.../util/delimiters/DelimiterDetector.java | 59 +++++----
.../filemetadata/DelimiterDetectorTest.java | 104 ++++++++++++++++
.../FileMetadataEmbeddedEnclosureTest.java | 134 +++++++++++++++++++++
.../filemetadata/delimited/embedded-enclosure.csv | 21 ++++
.../fileinput/text/TextFileCsvImportScanTest.java | 31 ++++-
14 files changed, 887 insertions(+), 37 deletions(-)
diff --git a/core/src/main/java/org/apache/hop/core/util/StringEvaluator.java
b/core/src/main/java/org/apache/hop/core/util/StringEvaluator.java
index 606c89a8d5..cb28bf5be3 100644
--- a/core/src/main/java/org/apache/hop/core/util/StringEvaluator.java
+++ b/core/src/main/java/org/apache/hop/core/util/StringEvaluator.java
@@ -67,6 +67,8 @@ public class StringEvaluator {
protected static final Pattern PRECISION_PATTERN =
Pattern.compile("[^0-9#]");
+ private static final String NUMBER_CURRENCY = "number-currency";
+
public StringEvaluator() {
this(true);
}
@@ -245,7 +247,11 @@ public class StringEvaluator {
}
private void evaluatePrecision(String value) {
- int p = determinePrecision(value);
+ char decimalSymbol =
+ ((DecimalFormat) NumberFormat.getInstance())
+ .getDecimalFormatSymbols()
+ .getDecimalSeparator();
+ int p = determinePrecision(value, decimalSymbol);
if (p > maxPrecision) {
maxPrecision = p;
}
@@ -367,17 +373,96 @@ public class StringEvaluator {
StringEvaluationResult result = evaluationResults.get(0);
IValueMeta conversionMeta = result.getConversionMeta();
- if (conversionMeta.isNumber() && conversionMeta.getCurrencySymbol() ==
null) {
- conversionMeta.setPrecision(maxPrecision);
- if (maxPrecision > 0 && maxLength > 0) {
- conversionMeta.setLength(maxLength);
+ if (conversionMeta.isNumber() && !isCurrencyResult(result)) {
+ // Every decimal mask parses every decimal value, so the shortest mask
(#.#) always wins
+ // the sort. Its precision would round away the decimals the data
actually has.
+ int dataPrecision = determineDataPrecision(conversionMeta);
+ result = findMaskKeepingPrecision(result, dataPrecision);
+ IValueMeta advised = result.getConversionMeta();
+ if (determinePrecision(advised.getConversionMask()) < dataPrecision) {
+ advised.setConversionMask(widenMask(advised.getConversionMask(),
dataPrecision));
}
+ advised.setPrecision(dataPrecision);
}
return result;
}
}
+ private static boolean isCurrencyResult(StringEvaluationResult result) {
+ return NUMBER_CURRENCY.equals(result.getConversionMeta().getName());
+ }
+
+ /** The largest number of decimals in the evaluated values, read with the
mask's symbols. */
+ private int determineDataPrecision(IValueMeta conversionMeta) {
+ char decimalSymbol = conversionMeta.getDecimalSymbol().charAt(0);
+ int precision = 0;
+ for (String value : values) {
+ if (value != null) {
+ precision = Math.max(precision, determinePrecision(value.trim(),
decimalSymbol));
+ }
+ }
+ return precision;
+ }
+
+ /**
+ * Picks the shortest mask with the same decimal symbol as the given one
that keeps all the
+ * decimals. When no mask keeps them all, the one keeping the most decimals
is used.
+ */
+ private StringEvaluationResult findMaskKeepingPrecision(
+ StringEvaluationResult shortest, int dataPrecision) {
+ IValueMeta shortestMeta = shortest.getConversionMeta();
+ StringEvaluationResult best = shortest;
+ int bestPrecision = determinePrecision(shortestMeta.getConversionMask());
+ // the results are sorted on mask length, so the first one that keeps the
decimals wins
+ for (StringEvaluationResult candidate : evaluationResults) {
+ if (bestPrecision >= dataPrecision) {
+ break;
+ }
+ IValueMeta meta = candidate.getConversionMeta();
+ if (meta.isNumber()
+ && !isCurrencyResult(candidate)
+ && meta.getTrimType() == shortestMeta.getTrimType()
+ && meta.getDecimalSymbol().equals(shortestMeta.getDecimalSymbol())) {
+ int maskPrecision = determinePrecision(meta.getConversionMask());
+ if (maskPrecision > bestPrecision) {
+ best = candidate;
+ bestPrecision = maskPrecision;
+ }
+ }
+ }
+ return best;
+ }
+
+ /**
+ * Gives a mask the given number of decimals. The mask parsed all the values
already, the number
+ * of decimals in a mask only changes how values are formatted.
+ */
+ static String widenMask(String mask, int precision) {
+ // a mask can have a positive and a negative pattern
+ StringBuilder widened = new StringBuilder();
+ for (String pattern : mask.split(";", -1)) {
+ if (!widened.isEmpty()) {
+ widened.append(';');
+ }
+ int dot = pattern.lastIndexOf('.');
+ if (dot < 0) {
+ widened.append(pattern).append('.').append("0".repeat(precision));
+ } else {
+ int end = pattern.length();
+ Matcher m = PRECISION_PATTERN.matcher(pattern.substring(dot + 1));
+ if (m.find()) {
+ end = dot + 1 + m.start();
+ }
+ widened
+ .append(pattern, 0, dot + 1)
+ .append("0".repeat(precision))
+ .append(pattern.substring(end));
+ }
+ }
+ return widened.toString();
+ }
+
public String[] getDateFormats() {
return dateFormats;
}
@@ -430,7 +515,7 @@ public class StringEvaluator {
// Try the locale's Currency
DecimalFormat currencyFormat = ((DecimalFormat)
NumberFormat.getCurrencyInstance());
- IValueMeta conversionMeta = new ValueMetaNumber("number-currency");
+ IValueMeta conversionMeta = new ValueMetaNumber(NUMBER_CURRENCY);
// replace the universal currency symbol with the locale's currency
symbol for user
// recognition
String currencyMask =
@@ -497,12 +582,17 @@ public class StringEvaluator {
}
}
+ /**
+ * The number of decimals a number format mask describes. Masks are
non-localized {@link
+ * DecimalFormat} patterns, so the decimal separator is always a dot,
whatever the locale.
+ */
protected static int determinePrecision(String numericFormat) {
+ return determinePrecision(numericFormat, '.');
+ }
+
+ /** The number of digits (or mask digits) after the last decimal symbol. */
+ protected static int determinePrecision(String numericFormat, char
decimalSymbol) {
if (numericFormat != null) {
- char decimalSymbol =
- ((DecimalFormat) NumberFormat.getInstance())
- .getDecimalFormatSymbols()
- .getDecimalSeparator();
int loc = numericFormat.lastIndexOf(decimalSymbol);
if (loc >= 0 && loc < numericFormat.length()) {
Matcher m = PRECISION_PATTERN.matcher(numericFormat.substring(loc +
1));
diff --git
a/core/src/test/java/org/apache/hop/core/util/StringEvaluatorTest.java
b/core/src/test/java/org/apache/hop/core/util/StringEvaluatorTest.java
index 9d69e64fcd..81d73bf5ca 100644
--- a/core/src/test/java/org/apache/hop/core/util/StringEvaluatorTest.java
+++ b/core/src/test/java/org/apache/hop/core/util/StringEvaluatorTest.java
@@ -30,6 +30,7 @@ import java.util.List;
import java.util.Locale;
import org.apache.hop.core.HopClientEnvironment;
import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.row.IValueMeta;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
@@ -187,6 +188,96 @@ class StringEvaluatorTest {
assertTrue(evaluator.getAdvicedResult().getConversionMeta().isString());
}
+ /** #5609: the shortest mask (#.#) used to win and round the amounts to one
decimal. */
+ @Test
+ void testNumberKeepsAllDecimals() {
+ for (String string : Arrays.asList("123.58", "90524.10", "9872.52",
"130011.63")) {
+ evaluator.evaluateString(string);
+ }
+ IValueMeta advised = evaluator.getAdvicedResult().getConversionMeta();
+ assertTrue(advised.isNumber());
+ assertEquals("#.00", advised.getConversionMask());
+ assertEquals(".", advised.getDecimalSymbol());
+ assertEquals(2, advised.getPrecision());
+ }
+
+ @Test
+ void testNumberPrecisionFollowsTheMostDecimals() {
+ for (String string : Arrays.asList("1.5", "2.125", "3")) {
+ evaluator.evaluateString(string);
+ }
+ IValueMeta advised = evaluator.getAdvicedResult().getConversionMeta();
+ assertTrue(advised.isNumber());
+ assertEquals("#.000", advised.getConversionMask());
+ assertEquals(3, advised.getPrecision());
+ }
+
+ @Test
+ void testNumberWithMoreDecimalsThanAnyMask() {
+ evaluator.evaluateString("1.123456789");
+ evaluator.evaluateString(null);
+ IValueMeta advised = evaluator.getAdvicedResult().getConversionMeta();
+ assertTrue(advised.isNumber());
+ assertEquals("#.000000000", advised.getConversionMask());
+ assertEquals(9, advised.getPrecision());
+ }
+
+ /** Grouped values only parse with a grouped mask, none of which has three
decimals. */
+ @Test
+ void testGroupedNumberKeepsAllDecimals() {
+ evaluator.evaluateString("1,234.567");
+ IValueMeta advised = evaluator.getAdvicedResult().getConversionMeta();
+ assertTrue(advised.isNumber());
+ assertEquals("#,##0.000", advised.getConversionMask());
+ assertEquals(3, advised.getPrecision());
+ }
+
+ /** A currency mask keeps the precision of the currency. */
+ @Test
+ void testCurrencyKeepsItsMask() {
+ StringEvaluator usEvaluator = new StringEvaluator();
+ usEvaluator.evaluateString("$1,234.5");
+ usEvaluator.evaluateString("$12.50");
+ IValueMeta advised = usEvaluator.getAdvicedResult().getConversionMeta();
+ assertTrue(advised.isNumber());
+ assertEquals("$#,##0.00", advised.getConversionMask());
+ assertEquals(2, advised.getPrecision());
+ }
+
+ @Test
+ void testWidenMask() {
+ assertEquals("#.000", StringEvaluator.widenMask("#.#", 3));
+ assertEquals("#,##0.000", StringEvaluator.widenMask("#,##0.00", 3));
+ assertEquals(" #.0000", StringEvaluator.widenMask(" #.0#", 4));
+ assertEquals("#,##0.00;-#,##0.00",
StringEvaluator.widenMask("#,##0.0;-#,##0.0", 2));
+ assertEquals("#.00", StringEvaluator.widenMask("#", 2));
+ assertEquals("#.00 EUR", StringEvaluator.widenMask("#.0 EUR", 2));
+ }
+
+ /** Masks always use a dot, the locale's decimal separator must not change
the precision. */
+ @Test
+ void testNumberPrecisionDoesNotDependOnLocale() {
+ Locale.setDefault(Locale.GERMANY);
+ StringEvaluator germanEvaluator = new StringEvaluator();
+ for (String string : Arrays.asList("123.58", "9872.52")) {
+ germanEvaluator.evaluateString(string);
+ }
+ IValueMeta advised =
germanEvaluator.getAdvicedResult().getConversionMeta();
+ assertTrue(advised.isNumber());
+ assertEquals("#.00", advised.getConversionMask());
+ assertEquals(2, advised.getPrecision());
+ }
+
+ @Test
+ void testDeterminePrecision() {
+ assertEquals(2, StringEvaluator.determinePrecision("#.00"));
+ assertEquals(2, StringEvaluator.determinePrecision(" #.0#"));
+ assertEquals(0, StringEvaluator.determinePrecision("#"));
+ assertEquals(0, StringEvaluator.determinePrecision(null));
+ assertEquals(3, StringEvaluator.determinePrecision("1.234,567", ','));
+ assertEquals(0, StringEvaluator.determinePrecision("1.234", ','));
+ }
+
private void testNumber(String mask, String... strings) {
for (String string : strings) {
evaluator.evaluateString(string);
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/filemetadata.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/filemetadata.adoc
index c2fe0d928d..b5a8a9cdf9 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/filemetadata.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/filemetadata.adoc
@@ -48,16 +48,37 @@ Use 0 rows to scan the entire file.
|Fallback charset|Charset to use while scanning the file
|Delimiter candidates|List of delimiters to try while detecting the file
layout.
Tab, semicolon, comma are provided by default.
-|Enclosure candidates|List of delimiters to try while detecting the file
layout.
+|Enclosure candidates|List of enclosure characters to try while detecting the
file layout.
Double and single quote are provided by default.
|===
+== Detection rules
+
+=== Enclosures
+
+An enclosure opens a field when it is the first character of the field.
+It closes the field when it is followed by the delimiter or the end of the
line.
+Inside an enclosed field:
+
+* delimiters are part of the value: `"Smith, Bob"` is one field.
+* a doubled enclosure is an escaped enclosure: `"BOB ""ROBERT"" SMITH"`.
+* any other enclosure is part of the value as well: `"BOB "ROBERT" SMITH"` is
one field.
+
+An enclosure in the middle of a field that is not enclosed, such as `5" pipe`,
does not open a field, and the line does not fit that enclosure.
+The same goes for a line that ends inside an enclosed field: fields spanning
several lines are not supported.
+
+=== Numbers
+
+A number column gets the precision of the value with the most decimals, read
with the detected decimal symbol.
+The suggested mask keeps all of those decimals: `123.58` and `90524.10` give
the mask `#.00` and a precision of 2.
+
== Output Fields
The fields returned by this transform are
* charset
* delimiter
+* enclosure
* field_count
* skip_header_lines
* skip_footer_lines
diff --git
a/integration-tests/transforms/0055-file-metadata-embedded-enclosure-verify.hpl
b/integration-tests/transforms/0055-file-metadata-embedded-enclosure-verify.hpl
new file mode 100644
index 0000000000..b40cc36deb
--- /dev/null
+++
b/integration-tests/transforms/0055-file-metadata-embedded-enclosure-verify.hpl
@@ -0,0 +1,102 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0055-file-metadata-embedded-enclosure-verify</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2022/11/28 12:13:32.297</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2022/11/28 12:13:32.297</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>files/file-metadata-embedded-enclosure.csv</from>
+ <to>Verify</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Verify</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>432</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>files/file-metadata-embedded-enclosure.csv</name>
+ <type>FileMetadataPlugin</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+
<fileName>${PROJECT_HOME}/files/file-metadata-embedded-enclosure.csv</fileName>
+ <limitRows>10000</limitRows>
+ <defaultCharset>ISO-8859-1</defaultCharset>
+ <delimiterCandidate>
+ <candidate> </candidate>
+ </delimiterCandidate>
+ <delimiterCandidate>
+ <candidate>;</candidate>
+ </delimiterCandidate>
+ <delimiterCandidate>
+ <candidate>,</candidate>
+ </delimiterCandidate>
+ <enclosureCandidate>
+ <candidate>"</candidate>
+ </enclosureCandidate>
+ <enclosureCandidate>
+ <candidate>'</candidate>
+ </enclosureCandidate>
+ <attributes/>
+ <GUI>
+ <xloc>128</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/transforms/datasets/golden-file-metadata-embedded-enclosure.csv
b/integration-tests/transforms/datasets/golden-file-metadata-embedded-enclosure.csv
new file mode 100644
index 0000000000..7218aac9c9
--- /dev/null
+++
b/integration-tests/transforms/datasets/golden-file-metadata-embedded-enclosure.csv
@@ -0,0 +1,12 @@
+charset,delimiter,enclosure,field_count,skip_header_lines,skip_footer_lines,header_line_present,name,type,length,precision,mask,decimal_symbol,grouping_symbol
+US-ASCII,",","""",11,0,0,Y,FY_DC,Integer,15,0,#,".",","
+US-ASCII,",","""",11,0,0,Y,DEPT_CD,String,3,,,".",","
+US-ASCII,",","""",11,0,0,Y,FUND_CD,Integer,15,0,#,".",","
+US-ASCII,",","""",11,0,0,Y,UNIT_CD,Integer,15,0,#,".",","
+US-ASCII,",","""",11,0,0,Y,UNIT_NM,String,28,,,".",","
+US-ASCII,",","""",11,0,0,Y,VEND_CUST_CD,String,13,,,".",","
+US-ASCII,",","""",11,0,0,Y,EDU_VENDOR,Boolean,,,,".",","
+US-ASCII,",","""",11,0,0,Y,LGL_NM_UP,String,18,,,".",","
+US-ASCII,",","""",11,0,0,Y,OBJ_CD,Integer,15,0,#,".",","
+US-ASCII,",","""",11,0,0,Y,OBJ_NM,String,16,,,".",","
+US-ASCII,",","""",11,0,0,Y,SUM_PSTNG_AM,Number,15,2,#.00,".",","
diff --git
a/integration-tests/transforms/files/file-metadata-embedded-enclosure.csv
b/integration-tests/transforms/files/file-metadata-embedded-enclosure.csv
new file mode 100644
index 0000000000..e65316b88f
--- /dev/null
+++ b/integration-tests/transforms/files/file-metadata-embedded-enclosure.csv
@@ -0,0 +1,21 @@
+"FY_DC","DEPT_CD","FUND_CD","UNIT_CD","UNIT_NM","VEND_CUST_CD","EDU_VENDOR","LGL_NM_UP","OBJ_CD","OBJ_NM",SUM_PSTNG_AM
+"2023","99B","999","1111","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","ENTITY one","9876","GRANTS FOR TEST1","123.58"
+"2023","99B","999","2222","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","ENTITY two","8765","GRANTS FOR TEST2","90524.10"
+"2023","99B","999","3333","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","ENTITY three","7654","GRANTS FOR TEST3","9872.52"
+"2023","99B","999","4444","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","ENTITY four","6543","GRANTS FOR TEST4","130011.63"
+"2023","99B","999","5555","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","ENTITY asides","5432","GRANTS FOR TEST5","20514.57"
+"2023","99B","999","6666","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","BOB "ROBERT" SMITH","5433","GRANTS FOR BOB","123456.78"
+"2023","99B","999","6666","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","BOB "ROBERT" SMITH","5433","GRANTS FOR BOB","123456.78"
+"2023","99B","999","6666","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","BOB "ROBERT" SMITH","5433","GRANTS FOR BOB","123456.78"
+"2023","99B","999","6666","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","BOB "ROBERT" SMITH","5433","GRANTS FOR BOB","123456.78"
+"2023","99B","999","6666","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","BOB "ROBERT" SMITH","5433","GRANTS FOR BOB","123456.78"
+"2023","99B","999","6666","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","BOB "ROBERT" SMITH","5433","GRANTS FOR BOB","123456.78"
+"2023","99B","999","6666","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","BOB "ROBERT" SMITH","5433","GRANTS FOR BOB","123456.78"
+"2023","99B","999","6666","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","BOB "ROBERT" SMITH","5433","GRANTS FOR BOB","123456.78"
+"2023","99B","999","6666","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","BOB "ROBERT" SMITH","5433","GRANTS FOR BOB","123456.78"
+"2023","99B","999","6666","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","BOB "ROBERT" SMITH","5433","GRANTS FOR BOB","123456.78"
+"2023","99B","999","1111","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","ENTITY one","9876","GRANTS FOR TEST1","123.58"
+"2023","99B","999","2222","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","ENTITY two","8765","GRANTS FOR TEST2","90524.10"
+"2023","99B","999","3333","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","ENTITY three","7654","GRANTS FOR TEST3","9872.52"
+"2023","99B","999","4444","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","ENTITY four","6543","GRANTS FOR TEST4","130011.63"
+"2023","99B","999","5555","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","ENTITY asides","5432","GRANTS FOR TEST5","20514.57"
\ No newline at end of file
diff --git a/integration-tests/transforms/main-0055-file-metadata.hwf
b/integration-tests/transforms/main-0055-file-metadata.hwf
index e9e24572f9..8f91faed89 100644
--- a/integration-tests/transforms/main-0055-file-metadata.hwf
+++ b/integration-tests/transforms/main-0055-file-metadata.hwf
@@ -60,6 +60,9 @@ limitations under the License.
<test_name>
<name>0055-file-metadata-filename-fromfield-verify UNIT</name>
</test_name>
+ <test_name>
+ <name>0055-file-metadata-embedded-enclosure-verify UNIT</name>
+ </test_name>
</test_names>
<parallel>N</parallel>
<xloc>336</xloc>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-file-metadata-embedded-enclosure.json
b/integration-tests/transforms/metadata/dataset/golden-file-metadata-embedded-enclosure.json
new file mode 100644
index 0000000000..667541cf41
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-file-metadata-embedded-enclosure.json
@@ -0,0 +1,120 @@
+{
+ "base_filename": "golden-file-metadata-embedded-enclosure.csv",
+ "name": "golden-file-metadata-embedded-enclosure",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "charset",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "delimiter",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "enclosure",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "field_count",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "skip_header_lines",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "skip_footer_lines",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 4,
+ "field_precision": -1,
+ "field_name": "header_line_present",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "name",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "type",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "length",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "precision",
+ "field_format": "####0;-####0"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "mask",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "decimal_symbol",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "grouping_symbol",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git
a/integration-tests/transforms/metadata/unit-test/0055-file-metadata-embedded-enclosure-verify
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0055-file-metadata-embedded-enclosure-verify
UNIT.json
new file mode 100644
index 0000000000..ba5a98efc9
--- /dev/null
+++
b/integration-tests/transforms/metadata/unit-test/0055-file-metadata-embedded-enclosure-verify
UNIT.json
@@ -0,0 +1,93 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": true,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "charset",
+ "data_set_field": "charset"
+ },
+ {
+ "transform_field": "delimiter",
+ "data_set_field": "delimiter"
+ },
+ {
+ "transform_field": "enclosure",
+ "data_set_field": "enclosure"
+ },
+ {
+ "transform_field": "field_count",
+ "data_set_field": "field_count"
+ },
+ {
+ "transform_field": "skip_header_lines",
+ "data_set_field": "skip_header_lines"
+ },
+ {
+ "transform_field": "skip_footer_lines",
+ "data_set_field": "skip_footer_lines"
+ },
+ {
+ "transform_field": "header_line_present",
+ "data_set_field": "header_line_present"
+ },
+ {
+ "transform_field": "name",
+ "data_set_field": "name"
+ },
+ {
+ "transform_field": "type",
+ "data_set_field": "type"
+ },
+ {
+ "transform_field": "length",
+ "data_set_field": "length"
+ },
+ {
+ "transform_field": "precision",
+ "data_set_field": "precision"
+ },
+ {
+ "transform_field": "mask",
+ "data_set_field": "mask"
+ },
+ {
+ "transform_field": "decimal_symbol",
+ "data_set_field": "decimal_symbol"
+ },
+ {
+ "transform_field": "grouping_symbol",
+ "data_set_field": "grouping_symbol"
+ }
+ ],
+ "field_order": [
+ "charset",
+ "delimiter",
+ "enclosure",
+ "field_count",
+ "skip_header_lines",
+ "skip_footer_lines",
+ "header_line_present",
+ "name",
+ "type",
+ "length",
+ "precision",
+ "mask",
+ "decimal_symbol",
+ "grouping_symbol"
+ ],
+ "data_set_name": "golden-file-metadata-embedded-enclosure",
+ "transform_name": "Verify"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0055-file-metadata-embedded-enclosure-verify UNIT",
+ "description": "",
+ "persist_filename": "",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0055-file-metadata-embedded-enclosure-verify.hpl",
+ "test_type": "UNIT_TEST"
+}
\ No newline at end of file
diff --git
a/plugins/transforms/filemetadata/src/main/java/org/apache/hop/pipeline/transforms/filemetadata/util/delimiters/DelimiterDetector.java
b/plugins/transforms/filemetadata/src/main/java/org/apache/hop/pipeline/transforms/filemetadata/util/delimiters/DelimiterDetector.java
index 2c433c918c..9ec366e361 100644
---
a/plugins/transforms/filemetadata/src/main/java/org/apache/hop/pipeline/transforms/filemetadata/util/delimiters/DelimiterDetector.java
+++
b/plugins/transforms/filemetadata/src/main/java/org/apache/hop/pipeline/transforms/filemetadata/util/delimiters/DelimiterDetector.java
@@ -238,6 +238,8 @@ public class DelimiterDetector {
boolean[] enclosureOpen = new boolean[potentialResults.size()];
boolean[] enclosureSeen = new boolean[potentialResults.size()];
boolean[] enclosureConsistent = new boolean[potentialResults.size()];
+ boolean[] fieldStart = new boolean[potentialResults.size()];
+ boolean[] skipNext = new boolean[potentialResults.size()];
String s = "";
try {
@@ -247,11 +249,14 @@ public class DelimiterDetector {
int remainingResults = potentialResults.size();
- // clear occurrences for each char
+ // clear the per-line state for each result
for (int j = 0; j < remainingResults; j++) {
frequencies[j] = 0;
enclosureOpen[j] = false;
+ enclosureSeen[j] = false;
enclosureConsistent[j] = true;
+ fieldStart[j] = true;
+ skipNext[j] = false;
}
// find occurrences for each char
@@ -262,37 +267,41 @@ public class DelimiterDetector {
char c = detectionResult.getDelimiter();
Character enc = detectionResult.getEnclosure();
- boolean hasEnclosure = enc != null;
- // if enclosure is involved, ignore enclosed delimiters
- if (hasEnclosure) {
-
- if (!enclosureOpen[j] && sc == c) {
+ if (enc == null) {
+ // no enclosure logic, just delimiters
+ if (sc == c) {
frequencies[j] += 1;
}
-
+ } else if (skipNext[j]) {
+ // the second enclosure of an escaped pair
+ skipNext[j] = false;
+ } else if (enclosureOpen[j]) {
+ // enclosed delimiters are data. The enclosure only closes the
field when followed
+ // by a delimiter or the end of the line, a doubled one is
escaped. Any other
+ // enclosure in the field (BOB "ROBERT" SMITH) is data too.
if (sc == enc) {
- enclosureSeen[j] = true;
- enclosureConsistent[j] =
- enclosureConsistent[j]
- && (i == 0 && !enclosureOpen[j]
- || i == s.length() - 1 && enclosureOpen[j]
- || i > 0 && s.charAt(i - 1) == c &&
!enclosureOpen[j]
- || i > 0 && s.charAt(i - 1) == enc &&
!enclosureOpen[j]
- || s.length() > i + 1 && s.charAt(i + 1) == c &&
enclosureOpen[j]
- || s.length() > i + 1 && s.charAt(i + 1) == enc &&
enclosureOpen[j]);
-
- enclosureOpen[j] = !enclosureOpen[j];
+ boolean lastChar = i == s.length() - 1;
+ if (lastChar || s.charAt(i + 1) == c) {
+ enclosureOpen[j] = false;
+ } else if (s.charAt(i + 1) == enc) {
+ skipNext[j] = true;
+ }
}
-
- }
- // no enclosure logic, just delimiters
- else {
-
- if (sc == c) {
- frequencies[j] += 1;
+ } else if (sc == c) {
+ frequencies[j] += 1;
+ fieldStart[j] = true;
+ continue;
+ } else if (sc == enc) {
+ // an enclosure opens a field, anywhere else in an unenclosed
field it is suspect
+ enclosureSeen[j] = true;
+ if (fieldStart[j]) {
+ enclosureOpen[j] = true;
+ } else {
+ enclosureConsistent[j] = false;
}
}
+ fieldStart[j] = false;
}
}
diff --git
a/plugins/transforms/filemetadata/src/test/java/org/apache/hop/pipeline/transforms/filemetadata/DelimiterDetectorTest.java
b/plugins/transforms/filemetadata/src/test/java/org/apache/hop/pipeline/transforms/filemetadata/DelimiterDetectorTest.java
index 7ff84f798c..3c41a36462 100644
---
a/plugins/transforms/filemetadata/src/test/java/org/apache/hop/pipeline/transforms/filemetadata/DelimiterDetectorTest.java
+++
b/plugins/transforms/filemetadata/src/test/java/org/apache/hop/pipeline/transforms/filemetadata/DelimiterDetectorTest.java
@@ -24,8 +24,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import com.google.common.base.Charsets;
import java.io.BufferedReader;
+import java.io.StringReader;
import java.nio.file.Files;
import java.nio.file.Paths;
+import java.util.Arrays;
import
org.apache.hop.pipeline.transforms.filemetadata.util.delimiters.DelimiterDetector;
import
org.apache.hop.pipeline.transforms.filemetadata.util.delimiters.DelimiterDetectorBuilder;
import org.junit.jupiter.api.Test;
@@ -432,4 +434,106 @@ class DelimiterDetectorTest {
assertEquals(2, result.getDataLineFrequency());
}
}
+
+ /**
+ * #5609: a quote inside an enclosed field (BOB "ROBERT" SMITH) is data.
Once more than 10 such
+ * lines were scanned, the enclosure was dropped and every value kept its
quotes.
+ */
+ @Test
+ void keepsEnclosureWithEnclosuresInsideFields() throws Exception {
+
+ try (BufferedReader f =
+ Files.newBufferedReader(
+ Paths.get(
+ getClass()
+ .getResource(
+ '/'
+ +
this.getClass().getPackage().getName().replace('.', '/')
+ + "/delimited/embedded-enclosure.csv")
+ .toURI()),
+ Charsets.UTF_8)) {
+ DelimiterDetector detector =
+ new DelimiterDetectorBuilder()
+ .withDelimiterCandidates('\t', ';', ',')
+ .withEnclosureCandidates('"', '\'')
+ .withInput(f)
+ .withRowLimit(10000)
+ .build();
+ DelimiterDetector.DetectionResult result = detector.detectDelimiters();
+ assertNotNull(result);
+ assertEquals(',', (char) result.getDelimiter());
+ assertEquals('"', (char) result.getEnclosure());
+ assertTrue(result.isConsistentEnclosure());
+ assertEquals(21, result.getDataLines());
+ assertEquals(0, result.getBadHeaders());
+ assertEquals(0, result.getBadFooters());
+ assertEquals(10, result.getDataLineFrequency());
+ }
+ }
+
+ /** Every such line used to be a streak of its own, a large file dropped the
enclosure early. */
+ @Test
+ void keepsEnclosureWithManyEnclosuresInsideFields() throws Exception {
+ StringBuilder csv = new StringBuilder("\"id\",\"name\",\"amount\"\n");
+ for (int i = 0; i < 1000; i++) {
+ csv.append('"')
+ .append(i)
+ .append("\",\"BOB \"ROBERT\" SMITH\",\"")
+ .append(i)
+ .append(".25\"\n");
+ }
+
+ DelimiterDetector.DetectionResult result = detect(csv.toString(), ',',
'"');
+
+ assertNotNull(result);
+ assertEquals('"', (char) result.getEnclosure());
+ assertEquals(1001, result.getDataLines());
+ assertEquals(0, result.getBadFooters());
+ assertEquals(2, result.getDataLineFrequency());
+ }
+
+ @Test
+ void keepsEnclosureWithEscapedEnclosures() throws Exception {
+ String csv =
+ "\"id\",\"name\",\"note\"\n"
+ + "\"1\",\"BOB \"\"ROBERT\"\" SMITH\",\"a, b\"\n"
+ + "\"2\",\"\"\"quoted\"\"\",\"\"\n"
+ + "\"3\",\"ends with \"\"\",\"x\"\n";
+
+ DelimiterDetector.DetectionResult result = detect(csv, ',', '"');
+
+ assertNotNull(result);
+ assertEquals('"', (char) result.getEnclosure());
+ assertTrue(result.isConsistentEnclosure());
+ assertEquals(4, result.getDataLines());
+ assertEquals(2, result.getDataLineFrequency());
+ }
+
+ /** Another delimiter candidate between the enclosures inside a field is
data. */
+ @Test
+ void countsOnlyDelimitersOutsideEnclosedFields() throws Exception {
+ String csv = "\"id\",\"name\"\n" + "\"1\",\"a \"b; c\" d\"\n" +
"\"2\",\"e\"\n";
+
+ DelimiterDetector.DetectionResult result = detect(csv, ';', ',', '"');
+
+ assertNotNull(result);
+ assertEquals(',', (char) result.getDelimiter());
+ assertEquals('"', (char) result.getEnclosure());
+ assertEquals(3, result.getDataLines());
+ assertEquals(1, result.getDataLineFrequency());
+ }
+
+ private static DelimiterDetector.DetectionResult detect(String csv, char...
candidates)
+ throws Exception {
+ // the last candidate is the enclosure
+ char[] delimiters = Arrays.copyOf(candidates, candidates.length - 1);
+ try (BufferedReader reader = new BufferedReader(new StringReader(csv))) {
+ return new DelimiterDetectorBuilder()
+ .withDelimiterCandidates(delimiters)
+ .withEnclosureCandidates(candidates[candidates.length - 1])
+ .withInput(reader)
+ .build()
+ .detectDelimiters();
+ }
+ }
}
diff --git
a/plugins/transforms/filemetadata/src/test/java/org/apache/hop/pipeline/transforms/filemetadata/FileMetadataEmbeddedEnclosureTest.java
b/plugins/transforms/filemetadata/src/test/java/org/apache/hop/pipeline/transforms/filemetadata/FileMetadataEmbeddedEnclosureTest.java
new file mode 100644
index 0000000000..d2850e1750
--- /dev/null
+++
b/plugins/transforms/filemetadata/src/test/java/org/apache/hop/pipeline/transforms/filemetadata/FileMetadataEmbeddedEnclosureTest.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.filemetadata;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.hop.core.BlockingRowSet;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+/**
+ * #5609: a file with enclosures inside enclosed fields (BOB "ROBERT" SMITH)
and amounts with two
+ * decimals. The enclosure was dropped once more than 15 rows were scanned,
and the amounts were
+ * given a precision of 1.
+ */
+class FileMetadataEmbeddedEnclosureTest {
+
+ @RegisterExtension
+ static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
+
+ private static final int ENCLOSURE = 2;
+ private static final int FIELD_COUNT = 3;
+ private static final int SKIP_FOOTER_LINES = 5;
+ private static final int NAME = 7;
+ private static final int TYPE = 8;
+ private static final int LENGTH = 9;
+ private static final int PRECISION = 10;
+ private static final int MASK = 11;
+
+ private TransformMockHelper<FileMetadataMeta, FileMetadataData> mockHelper;
+
+ @BeforeEach
+ void setUp() {
+ mockHelper =
+ new TransformMockHelper<>("FileMetadata", FileMetadataMeta.class,
FileMetadataData.class);
+ when(mockHelper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+ .thenReturn(mockHelper.iLogChannel);
+ when(mockHelper.pipeline.isRunning()).thenReturn(true);
+ }
+
+ @AfterEach
+ void tearDown() {
+ mockHelper.cleanUp();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"15", "16", "10000", ""})
+ void detectsEnclosureAndDecimals(String limitRows) throws Exception {
+ List<Object[]> rows = detect(limitRows);
+
+ assertEquals(11, rows.size(), "one row per column");
+ Object[] first = rows.get(0);
+ assertEquals("\"", first[ENCLOSURE]);
+ assertEquals(11L, first[FIELD_COUNT]);
+ assertEquals(0L, first[SKIP_FOOTER_LINES]);
+
+ assertEquals("FY_DC", first[NAME]);
+ assertEquals("Integer", first[TYPE]);
+
+ Object[] name = rows.get(7);
+ assertEquals("LGL_NM_UP", name[NAME]);
+ assertEquals("String", name[TYPE]);
+ // BOB "ROBERT" SMITH is read as one value, enclosures inside included
+ assertEquals(18L, name[LENGTH]);
+
+ Object[] amount = rows.get(10);
+ assertEquals("SUM_PSTNG_AM", amount[NAME]);
+ assertEquals("Number", amount[TYPE]);
+ assertEquals(2L, amount[PRECISION]);
+ assertEquals("#.00", amount[MASK]);
+ }
+
+ private List<Object[]> detect(String limitRows) throws Exception {
+ FileMetadataMeta meta = new FileMetadataMeta();
+ meta.setDefault();
+ meta.setLimitRows(limitRows);
+ meta.setFileName(
+ getClass()
+ .getResource(
+ '/'
+ + getClass().getPackage().getName().replace('.', '/')
+ + "/delimited/embedded-enclosure.csv")
+ .toURI()
+ .toString());
+
+ when(mockHelper.transformMeta.getTransform()).thenReturn(meta);
+ FileMetadata transform =
+ new FileMetadata(
+ mockHelper.transformMeta,
+ meta,
+ new FileMetadataData(),
+ 0,
+ mockHelper.pipelineMeta,
+ mockHelper.pipeline);
+ BlockingRowSet output = new BlockingRowSet(100);
+ transform.addRowSetToOutputRowSets(output);
+
+ assertFalse(transform.processRow(), "the transform generates its rows and
is then done");
+
+ List<Object[]> rows = new ArrayList<>();
+ Object[] row;
+ while ((row = output.getRowWait(100, TimeUnit.MILLISECONDS)) != null) {
+ rows.add(row);
+ }
+ return rows;
+ }
+}
diff --git
a/plugins/transforms/filemetadata/src/test/resources/org/apache/hop/pipeline/transforms/filemetadata/delimited/embedded-enclosure.csv
b/plugins/transforms/filemetadata/src/test/resources/org/apache/hop/pipeline/transforms/filemetadata/delimited/embedded-enclosure.csv
new file mode 100644
index 0000000000..e65316b88f
--- /dev/null
+++
b/plugins/transforms/filemetadata/src/test/resources/org/apache/hop/pipeline/transforms/filemetadata/delimited/embedded-enclosure.csv
@@ -0,0 +1,21 @@
+"FY_DC","DEPT_CD","FUND_CD","UNIT_CD","UNIT_NM","VEND_CUST_CD","EDU_VENDOR","LGL_NM_UP","OBJ_CD","OBJ_NM",SUM_PSTNG_AM
+"2023","99B","999","1111","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","ENTITY one","9876","GRANTS FOR TEST1","123.58"
+"2023","99B","999","2222","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","ENTITY two","8765","GRANTS FOR TEST2","90524.10"
+"2023","99B","999","3333","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","ENTITY three","7654","GRANTS FOR TEST3","9872.52"
+"2023","99B","999","4444","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","ENTITY four","6543","GRANTS FOR TEST4","130011.63"
+"2023","99B","999","5555","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","ENTITY asides","5432","GRANTS FOR TEST5","20514.57"
+"2023","99B","999","6666","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","BOB "ROBERT" SMITH","5433","GRANTS FOR BOB","123456.78"
+"2023","99B","999","6666","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","BOB "ROBERT" SMITH","5433","GRANTS FOR BOB","123456.78"
+"2023","99B","999","6666","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","BOB "ROBERT" SMITH","5433","GRANTS FOR BOB","123456.78"
+"2023","99B","999","6666","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","BOB "ROBERT" SMITH","5433","GRANTS FOR BOB","123456.78"
+"2023","99B","999","6666","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","BOB "ROBERT" SMITH","5433","GRANTS FOR BOB","123456.78"
+"2023","99B","999","6666","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","BOB "ROBERT" SMITH","5433","GRANTS FOR BOB","123456.78"
+"2023","99B","999","6666","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","BOB "ROBERT" SMITH","5433","GRANTS FOR BOB","123456.78"
+"2023","99B","999","6666","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","BOB "ROBERT" SMITH","5433","GRANTS FOR BOB","123456.78"
+"2023","99B","999","6666","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","BOB "ROBERT" SMITH","5433","GRANTS FOR BOB","123456.78"
+"2023","99B","999","6666","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","BOB "ROBERT" SMITH","5433","GRANTS FOR BOB","123456.78"
+"2023","99B","999","1111","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","ENTITY one","9876","GRANTS FOR TEST1","123.58"
+"2023","99B","999","2222","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","ENTITY two","8765","GRANTS FOR TEST2","90524.10"
+"2023","99B","999","3333","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","ENTITY three","7654","GRANTS FOR TEST3","9872.52"
+"2023","99B","999","4444","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","ENTITY four","6543","GRANTS FOR TEST4","130011.63"
+"2023","99B","999","5555","TEST DATA TO BE ENTERED HERE","AB10000000001","Y
","ENTITY asides","5432","GRANTS FOR TEST5","20514.57"
\ No newline at end of file
diff --git
a/plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileCsvImportScanTest.java
b/plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileCsvImportScanTest.java
index 4dd95df5fb..da5f9bfad1 100644
---
a/plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileCsvImportScanTest.java
+++
b/plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileCsvImportScanTest.java
@@ -142,6 +142,31 @@ class TextFileCsvImportScanTest {
}
}
+ /** #5609: the amounts were suggested with the mask #.# and a precision of
1. */
+ @Test
+ void suggestsAMaskKeepingAllDecimals() throws Exception {
+ Path file = Files.createTempFile("amounts", ".csv");
+ Files.writeString(file, "id;amount\n1;123.58\n2;90524.10\n3;9872.52\n",
StandardCharsets.UTF_8);
+ TextFileInputMeta meta = newMeta("mixed", List.of("id", "amount"));
+ InputFile inputFile = new InputFile();
+ inputFile.setFileName(file.toAbsolutePath().toString());
+ inputFile.setFileRequired(true);
+ meta.getFileInput().getInputFiles().add(inputFile);
+ Variables variables = new Variables();
+ variables.initializeFrom(null);
+ try (InputStreamReader reader =
+ new InputStreamReader(Files.newInputStream(file),
StandardCharsets.UTF_8)) {
+ new CountingDialog(variables, meta, reader, 100)
+ .doScan(new ProgressNullMonitorListener(), false);
+ }
+
+ TextFileInputField amount = field(meta, "amount");
+ assertEquals(IValueMeta.TYPE_NUMBER, amount.getType());
+ assertEquals("#.00", amount.getFormat());
+ assertEquals(2, amount.getPrecision());
+ assertEquals(".", amount.getDecimalSymbol());
+ }
+
@Test
void missingFileFailsOnceInsteadOfPerLine() throws Exception {
TextFileInputMeta meta = newMeta("mixed");
@@ -176,6 +201,10 @@ class TextFileCsvImportScanTest {
}
private static TextFileInputMeta newMeta(String format) {
+ return newMeta(format, FIELD_NAMES);
+ }
+
+ private static TextFileInputMeta newMeta(String format, List<String>
fieldNames) {
TextFileInputMeta meta = new TextFileInputMeta();
meta.getContent().setFileType("CSV");
meta.getContent().setFileFormat(format);
@@ -185,7 +214,7 @@ class TextFileCsvImportScanTest {
meta.getContent().setHeader(true);
meta.getContent().setNrHeaderLines(1);
meta.getContent().setNoEmptyLines(true);
- for (String name : FIELD_NAMES) {
+ for (String name : fieldNames) {
TextFileInputField field = new TextFileInputField();
field.setName(name);
meta.getInputFields().add(field);
