This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 163281a443 Issue #8404 : Return sampled field metadata from Get Fields
(#8524)
163281a443 is described below
commit 163281a443ec5fecb5c0ef3d83c2a402555402d3
Author: Matt Casters <[email protected]>
AuthorDate: Tue Sep 22 10:26:14 2026 +0200
Issue #8404 : Return sampled field metadata from Get Fields (#8524)
* Issue #8404 : Return sampled field metadata from Get Fields
The scan derived types, lengths and masks, then closed the progress
dialog before that result was published. The fields grid kept the
header names. Text File Input and CSV Input share this dialog.
Resolve the sample file once, skip blank lines that are not data, and
report the StringEvaluator result instead of the old date sentinels.
* Issue #8404 : Scan the customers fixture as CRLF after checkout
Git stores the fixture with LF. The test rewrites it to CRLF so Unix
mode still sees the blank line that follows every row.
---
.../transforms/common/ICsvInputAwareMeta.java | 9 +
.../text/TextFileCSVImportProgressDialog.java | 417 +++++++++------------
.../fileinput/text/TextFileInputMeta.java | 5 +
.../fileinput/text/TextFileCsvImportScanTest.java | 241 ++++++++++++
.../fileinput/text/files/customers-crlf.csv | 131 +++++++
.../common/IGetFieldsCapableTransformDialog.java | 16 +-
6 files changed, 587 insertions(+), 232 deletions(-)
diff --git
a/engine/src/main/java/org/apache/hop/pipeline/transforms/common/ICsvInputAwareMeta.java
b/engine/src/main/java/org/apache/hop/pipeline/transforms/common/ICsvInputAwareMeta.java
index 1cc4d19b8b..0081a938a6 100644
---
a/engine/src/main/java/org/apache/hop/pipeline/transforms/common/ICsvInputAwareMeta.java
+++
b/engine/src/main/java/org/apache/hop/pipeline/transforms/common/ICsvInputAwareMeta.java
@@ -66,6 +66,15 @@ public interface ICsvInputAwareMeta<T extends
ITextFileInputField> {
boolean hasHeader();
+ /**
+ * When true, a blank line is not one of the rows used to guess field types.
Text file input
+ * follows its "No empty lines" option. Blank lines must not use up the
sample size: a DOS file
+ * read as Unix yields an empty line after every row, and counting those
would hide the data.
+ */
+ default boolean skipEmptyLines() {
+ return false;
+ }
+
int getNrHeaderLines();
ICsvInputAwareMeta clone();
diff --git
a/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileCSVImportProgressDialog.java
b/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileCSVImportProgressDialog.java
index 9f6094a0fd..eb941558ed 100644
---
a/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileCSVImportProgressDialog.java
+++
b/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileCSVImportProgressDialog.java
@@ -22,10 +22,8 @@ import java.lang.reflect.InvocationTargetException;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.text.NumberFormat;
-import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Date;
import java.util.List;
import org.apache.hop.core.Const;
import org.apache.hop.core.IProgressMonitor;
@@ -151,12 +149,21 @@ public class TextFileCSVImportProgressDialog<T extends
ITextFileInputField>
return message;
}
- private String doScan(IProgressMonitor monitor) throws HopException {
- return doScan(monitor, true);
+ /**
+ * File name stored on each sampled row. Resolved once: {@link
FileInputList#createFilePathList}
+ * stats the file, and on a remote URL that is several HTTP calls. Doing it
per sample line is
+ * what made "Get fields" on a large CSV take forever.
+ */
+ protected String resolveSampleFileName() throws HopException {
+ String[] paths = FileInputList.createFilePathList(variables,
meta.getInputFiles());
+ if (paths.length == 0) {
+ throw new HopException(
+ BaseMessages.getString(PKG,
"TextFileInputDialog.NoValidFile.DialogMessage"));
+ }
+ return paths[0];
}
- private String doScan(IProgressMonitor monitor, final boolean
failOnParseError)
- throws HopException {
+ String doScan(IProgressMonitor monitor, final boolean failOnParseError)
throws HopException {
if (samples > 0) {
monitor.beginTask(
BaseMessages.getString(PKG,
"TextFileCSVImportProgressDialog.Task.ScanningFile"),
@@ -173,32 +180,6 @@ public class TextFileCSVImportProgressDialog<T extends
ITextFileInputField>
int nrFields = meta.getInputFields().size();
- IRowMeta outputRowMeta = new RowMeta();
- meta.getFields(outputRowMeta, null, null, null, variables, null);
-
- // Remove the storage meta-data (don't go for lazy conversion during scan)
- for (IValueMeta valueMeta : outputRowMeta.getValueMetaList()) {
- valueMeta.setStorageMetadata(null);
- valueMeta.setStorageType(IValueMeta.STORAGE_TYPE_NORMAL);
- }
-
- IRowMeta convertRowMeta =
outputRowMeta.cloneToType(IValueMeta.TYPE_STRING);
-
- // How many null values?
- int[] nullCounts = new int[nrFields]; // How many times null value?
-
- // String info
- String[] minStrings = new String[nrFields]; // min string
- String[] maxStrings = new String[nrFields]; // max string
-
- // Date info
- int[] dateFormatCount = new int[nrFields]; // How many date formats work?
- boolean[][] dateFormat =
- new boolean[nrFields][Const.getDateFormats().length]; // What are the
date formats that
- // work?
- Date[][] minDate = new Date[nrFields][Const.getDateFormats().length]; //
min date value
- Date[][] maxDate = new Date[nrFields][Const.getDateFormats().length]; //
max date value
-
for (int i = 0; i < nrFields; i++) {
T field = meta.getInputFields().get(i);
if (replaceMeta) { // Clear previous info...
@@ -211,25 +192,28 @@ public class TextFileCSVImportProgressDialog<T extends
ITextFileInputField>
field.setNullString("-");
field.setTrimType(IValueMeta.TRIM_TYPE_NONE);
}
+ }
- nullCounts[i] = 0;
- minStrings[i] = "";
- maxStrings[i] = "";
-
- // Init data guess
- for (int j = 0; j < Const.getDateFormats().length; j++) {
- dateFormat[i][j] = true;
- minDate[i][j] = Const.MAX_DATE;
- maxDate[i][j] = Const.MIN_DATE;
- }
- dateFormatCount[i] = Const.getDateFormats().length;
+ // Row layout once. Repeating getFields per line repeated a remote file
listing when "prepend
+ // file name" was on, and it does not change while we sample.
+ IRowMeta outputRowMeta = new RowMeta();
+ meta.getFields(outputRowMeta, "transformName", null, null, variables,
null);
+ for (IValueMeta valueMeta : outputRowMeta.getValueMetaList()) {
+ valueMeta.setStorageMetadata(null);
+ valueMeta.setStorageType(IValueMeta.STORAGE_TYPE_NORMAL);
}
+ IRowMeta convertRowMeta =
outputRowMeta.cloneToType(IValueMeta.TYPE_STRING);
ICsvInputAwareMeta<T> inputMeta = meta.clone();
for (int i = 0; i < nrFields; i++) {
inputMeta.getInputFields().get(i).setType(IValueMeta.TYPE_STRING);
}
+ String sampleFileName = resolveSampleFileName();
+ String delimiter = variables.resolve(meta.getDelimiter());
+ String enclosure = variables.resolve(meta.getEnclosure());
+ String escapeCharacter = variables.resolve(meta.getEscapeCharacter());
+
// Sample <samples> rows...
debug = "get first line";
@@ -240,31 +224,13 @@ public class TextFileCSVImportProgressDialog<T extends
ITextFileInputField>
// However, if it doesn't have a header, take a new line
//
- line =
- TextFileLineUtil.getLine(
- log,
- reader,
- encodingType,
- fileFormatType,
- lineBuffer,
- meta.getEnclosure(),
- meta.getEscapeCharacter(),
- meta.isBreakInEnclosureAllowed());
+ line = readSampleLine(lineBuffer, fileFormatType);
fileLineNumber++;
if (meta.hasHeader()) {
int skipped = 0;
while (line != null && skipped < meta.getNrHeaderLines()) {
- line =
- TextFileLineUtil.getLine(
- log,
- reader,
- encodingType,
- fileFormatType,
- lineBuffer,
- meta.getEnclosure(),
- meta.getEscapeCharacter(),
- meta.isBreakInEnclosureAllowed());
+ line = readSampleLine(lineBuffer, fileFormatType);
skipped++;
fileLineNumber++;
}
@@ -273,16 +239,22 @@ public class TextFileCSVImportProgressDialog<T extends
ITextFileInputField>
List<StringEvaluator> evaluators = new ArrayList<>();
- // Allocate number and date parsers
DecimalFormat df2 = (DecimalFormat) NumberFormat.getInstance();
DecimalFormatSymbols dfs2 = new DecimalFormatSymbols();
- SimpleDateFormat daf2 = new SimpleDateFormat();
boolean errorFound = false;
while (!errorFound
&& line != null
&& (linenr <= samples || samples == 0)
&& !monitor.isCanceled()) {
+ // A blank line is not a sample row. Unix mode on a CRLF file produces
one after every row;
+ // counting it would make the first sample empty and burn the sample
budget.
+ if (meta.skipEmptyLines() && line.isEmpty()) {
+ fileLineNumber++;
+ line = readSampleLine(lineBuffer, fileFormatType);
+ continue;
+ }
+
monitor.subTask(
BaseMessages.getString(
PKG, "TextFileCSVImportProgressDialog.Task.ScanningLine", "" +
linenr));
@@ -293,17 +265,6 @@ public class TextFileCSVImportProgressDialog<T extends
ITextFileInputField>
if (log.isDebug()) {
debug = "convert line #" + linenr + " to row";
}
- IRowMeta rowMeta = new RowMeta();
- meta.getFields(rowMeta, "transformName", null, null, variables, null);
- // Remove the storage meta-data (don't go for lazy conversion during
scan)
- for (IValueMeta valueMeta : rowMeta.getValueMetaList()) {
- valueMeta.setStorageMetadata(null);
- valueMeta.setStorageType(IValueMeta.STORAGE_TYPE_NORMAL);
- }
-
- String delimiter = variables.resolve(meta.getDelimiter());
- String enclosure = variables.resolve(meta.getEnclosure());
- String escapeCharacter = variables.resolve(meta.getEscapeCharacter());
Object[] r =
TextFileInputUtils.convertLineToRow(
log,
@@ -313,7 +274,7 @@ public class TextFileCSVImportProgressDialog<T extends
ITextFileInputField>
0,
outputRowMeta,
convertRowMeta,
- FileInputList.createFilePathList(variables,
meta.getInputFiles())[0],
+ sampleFileName,
rowNumber,
delimiter,
enclosure,
@@ -343,25 +304,14 @@ public class TextFileCSVImportProgressDialog<T extends
ITextFileInputField>
evaluator = evaluators.get(i);
}
- String string = getStringFromRow(rowMeta, r, i, failOnParseError);
+ String string = getStringFromRow(outputRowMeta, r, i,
failOnParseError);
evaluator.evaluateString(string);
}
fileLineNumber++;
linenr++;
- // Grab another line...
- //
- line =
- TextFileLineUtil.getLine(
- log,
- reader,
- encodingType,
- fileFormatType,
- lineBuffer,
- meta.getEnclosure(),
- meta.getEscapeCharacter(),
- meta.isBreakInEnclosureAllowed());
+ line = readSampleLine(lineBuffer, fileFormatType);
}
monitor.worked(1);
@@ -380,20 +330,23 @@ public class TextFileCSVImportProgressDialog<T extends
ITextFileInputField>
for (int i = 0; i < nrFields; i++) {
T field = meta.getInputFields().get(i);
StringEvaluator evaluator = i < evaluators.size() ? evaluators.get(i) :
null;
+ // Copy successful masks before getAdvicedResult(), which drops the ones
it does not keep.
List<StringEvaluationResult> evaluationResults =
evaluator == null ? Collections.emptyList() :
evaluator.getStringEvaluationResults();
- // If we didn't find any matching result, it's a String...
- //
- if (evaluationResults.isEmpty()) {
+ StringEvaluationResult advised = null;
+ if (evaluator == null || evaluationResults.isEmpty()) {
+ // Nothing converted: the column is a string. Still ask the evaluator
for min/max, or the
+ // summary prints an empty value under the field name.
field.setType(IValueMeta.TYPE_STRING);
field.setLength(evaluator == null ? -1 : evaluator.getMaxLength());
+ if (evaluator != null) {
+ advised = evaluator.getAdvicedResult();
+ }
} else {
- StringEvaluationResult result = evaluator.getAdvicedResult();
- if (result != null) {
- // Take the first option we find, list the others below...
- //
- IValueMeta conversionMeta = result.getConversionMeta();
+ advised = evaluator.getAdvicedResult();
+ if (advised != null) {
+ IValueMeta conversionMeta = advised.getConversionMeta();
field.setType(conversionMeta.getType());
field.setTrimType(conversionMeta.getTrimType());
field.setFormat(conversionMeta.getConversionMask());
@@ -401,17 +354,21 @@ public class TextFileCSVImportProgressDialog<T extends
ITextFileInputField>
field.setGroupSymbol(conversionMeta.getGroupingSymbol());
field.setLength(conversionMeta.getLength());
field.setPrecision(conversionMeta.getPrecision());
-
- nullCounts[i] = result.getNrNull();
- minStrings[i] = result.getMin() == null ? "" :
result.getMin().toString();
- maxStrings[i] = result.getMax() == null ? "" :
result.getMax().toString();
+ // An integer guess leaves precision at -1. The fields grid hides
-1, so Get Fields
+ // would show no precision for a column that sampled as a whole
number.
+ if (field.getType() == IValueMeta.TYPE_INTEGER &&
field.getPrecision() < 0) {
+ field.setPrecision(0);
+ }
}
}
+ String minValue = valueText(advised == null ? null : advised.getMin());
+ String maxValue = valueText(advised == null ? null : advised.getMax());
+ int nullCount = advised == null ? 0 : advised.getNrNull();
+
resultsMessage.append(
BaseMessages.getString(
PKG, "TextFileCSVImportProgressDialog.Info.FieldNumber", "" + (i
+ 1)));
-
resultsMessage.append(
BaseMessages.getString(
PKG, "TextFileCSVImportProgressDialog.Info.FieldName",
field.getName()));
@@ -421,77 +378,8 @@ public class TextFileCSVImportProgressDialog<T extends
ITextFileInputField>
switch (field.getType()) {
case IValueMeta.TYPE_NUMBER:
- resultsMessage.append(
- BaseMessages.getString(
- PKG,
- "TextFileCSVImportProgressDialog.Info.EstimatedLength",
- (field.getLength() < 0 ? "-" : "" + field.getLength())));
- resultsMessage.append(
- BaseMessages.getString(
- PKG,
- "TextFileCSVImportProgressDialog.Info.EstimatedPrecision",
- field.getPrecision() < 0 ? "-" : "" + field.getPrecision()));
- resultsMessage.append(
- BaseMessages.getString(
- PKG, "TextFileCSVImportProgressDialog.Info.NumberFormat",
field.getFormat()));
-
- if (!evaluationResults.isEmpty()) {
- if (evaluationResults.size() > 1) {
- resultsMessage.append(
- BaseMessages.getString(
- PKG,
"TextFileCSVImportProgressDialog.Info.WarnNumberFormat"));
- }
-
- for (StringEvaluationResult seResult : evaluationResults) {
- String mask = seResult.getConversionMeta().getConversionMask();
-
- resultsMessage.append(
- BaseMessages.getString(
- PKG,
"TextFileCSVImportProgressDialog.Info.NumberFormat2", mask));
- resultsMessage.append(
- BaseMessages.getString(
- PKG,
- "TextFileCSVImportProgressDialog.Info.TrimType",
- seResult.getConversionMeta().getTrimType()));
- resultsMessage.append(
- BaseMessages.getString(
- PKG,
- "TextFileCSVImportProgressDialog.Info.NumberMinValue",
- seResult.getMin()));
- resultsMessage.append(
- BaseMessages.getString(
- PKG,
- "TextFileCSVImportProgressDialog.Info.NumberMaxValue",
- seResult.getMax()));
-
- try {
- df2.applyPattern(mask);
- df2.setDecimalFormatSymbols(dfs2);
- double mn =
df2.parse(seResult.getMin().toString()).doubleValue();
- resultsMessage.append(
- BaseMessages.getString(
- PKG,
- "TextFileCSVImportProgressDialog.Info.NumberExample",
- mask,
- seResult.getMin(),
- Double.toString(mn)));
- } catch (Exception e) {
- if (log.isDetailed()) {
- log.logDetailed(
- "This is unexpected: parsing ["
- + seResult.getMin()
- + "] with format ["
- + mask
- + "] did not work.");
- }
- }
- }
- }
- resultsMessage.append(
- BaseMessages.getString(
- PKG,
- "TextFileCSVImportProgressDialog.Info.NumberNrNullValues",
- "" + nullCounts[i]));
+ case IValueMeta.TYPE_INTEGER:
+ appendNumericSummary(resultsMessage, field, evaluationResults,
nullCount, df2, dfs2);
break;
case IValueMeta.TYPE_STRING:
resultsMessage.append(
@@ -501,15 +389,13 @@ public class TextFileCSVImportProgressDialog<T extends
ITextFileInputField>
"" + field.getLength()));
resultsMessage.append(
BaseMessages.getString(
- PKG, "TextFileCSVImportProgressDialog.Info.StringMinValue",
minStrings[i]));
+ PKG, "TextFileCSVImportProgressDialog.Info.StringMinValue",
minValue));
resultsMessage.append(
BaseMessages.getString(
- PKG, "TextFileCSVImportProgressDialog.Info.StringMaxValue",
maxStrings[i]));
+ PKG, "TextFileCSVImportProgressDialog.Info.StringMaxValue",
maxValue));
resultsMessage.append(
BaseMessages.getString(
- PKG,
- "TextFileCSVImportProgressDialog.Info.StringNrNullValues",
- "" + nullCounts[i]));
+ PKG,
"TextFileCSVImportProgressDialog.Info.StringNrNullValues", "" + nullCount));
break;
case IValueMeta.TYPE_DATE:
resultsMessage.append(
@@ -520,64 +406,34 @@ public class TextFileCSVImportProgressDialog<T extends
ITextFileInputField>
resultsMessage.append(
BaseMessages.getString(
PKG, "TextFileCSVImportProgressDialog.Info.DateFormat",
field.getFormat()));
- if (dateFormatCount[i] > 1) {
- resultsMessage.append(
- BaseMessages.getString(PKG,
"TextFileCSVImportProgressDialog.Info.WarnDateFormat"));
- }
- if (!Utils.isEmpty(minStrings[i])) {
- for (int x = 0; x < Const.getDateFormats().length; x++) {
- if (dateFormat[i][x]) {
- resultsMessage.append(
- BaseMessages.getString(
- PKG,
- "TextFileCSVImportProgressDialog.Info.DateFormat2",
- Const.getDateFormats()[x]));
- Date mindate = minDate[i][x];
- Date maxdate = maxDate[i][x];
- resultsMessage.append(
- BaseMessages.getString(
- PKG,
- "TextFileCSVImportProgressDialog.Info.DateMinValue",
- mindate.toString()));
- resultsMessage.append(
- BaseMessages.getString(
- PKG,
- "TextFileCSVImportProgressDialog.Info.DateMaxValue",
- maxdate.toString()));
-
- daf2.applyPattern(Const.getDateFormats()[x]);
- try {
- Date md = daf2.parse(minStrings[i]);
- resultsMessage.append(
- BaseMessages.getString(
- PKG,
- "TextFileCSVImportProgressDialog.Info.DateExample",
- Const.getDateFormats()[x],
- minStrings[i],
- md.toString()));
- } catch (Exception e) {
- if (log.isDetailed()) {
- log.logDetailed(
- "This is unexpected: parsing ["
- + minStrings[i]
- + "] with format ["
- + Const.getDateFormats()[x]
- + "] did not work.");
- }
- }
- }
+ for (StringEvaluationResult seResult : evaluationResults) {
+ if (!seResult.getConversionMeta().isDate()) {
+ continue;
}
+ resultsMessage.append(
+ BaseMessages.getString(
+ PKG,
+ "TextFileCSVImportProgressDialog.Info.DateFormat2",
+ seResult.getConversionMeta().getConversionMask()));
+ resultsMessage.append(
+ BaseMessages.getString(
+ PKG,
+ "TextFileCSVImportProgressDialog.Info.DateMinValue",
+ valueText(seResult.getMin())));
+ resultsMessage.append(
+ BaseMessages.getString(
+ PKG,
+ "TextFileCSVImportProgressDialog.Info.DateMaxValue",
+ valueText(seResult.getMax())));
}
resultsMessage.append(
BaseMessages.getString(
- PKG,
- "TextFileCSVImportProgressDialog.Info.DateNrNullValues",
- "" + nullCounts[i]));
+ PKG,
"TextFileCSVImportProgressDialog.Info.DateNrNullValues", "" + nullCount));
break;
default:
break;
}
- if (nullCounts[i] == linenr - 1) {
+ if (nullCount > 0 && nullCount == linenr - 1) {
resultsMessage.append(
BaseMessages.getString(PKG,
"TextFileCSVImportProgressDialog.Info.AllNullValues"));
}
@@ -585,8 +441,107 @@ public class TextFileCSVImportProgressDialog<T extends
ITextFileInputField>
}
monitor.worked(1);
- monitor.done();
+ // Do not call monitor.done() here. That disposes the progress dialog from
inside this call,
+ // and open() then returns before it can publish the result. The grid
would keep the header
+ // names and never receive the sampled types, lengths and masks. The
dialog closes itself
+ // after open() has stored the message.
+ String result = resultsMessage.toString();
+ message = result;
+ return result;
+ }
+
+ private void appendNumericSummary(
+ StringBuilder resultsMessage,
+ T field,
+ List<StringEvaluationResult> evaluationResults,
+ int nullCount,
+ DecimalFormat df2,
+ DecimalFormatSymbols dfs2) {
+ resultsMessage.append(
+ BaseMessages.getString(
+ PKG,
+ "TextFileCSVImportProgressDialog.Info.EstimatedLength",
+ (field.getLength() < 0 ? "-" : "" + field.getLength())));
+ resultsMessage.append(
+ BaseMessages.getString(
+ PKG,
+ "TextFileCSVImportProgressDialog.Info.EstimatedPrecision",
+ field.getPrecision() < 0 ? "-" : "" + field.getPrecision()));
+ resultsMessage.append(
+ BaseMessages.getString(
+ PKG, "TextFileCSVImportProgressDialog.Info.NumberFormat",
field.getFormat()));
+
+ if (!evaluationResults.isEmpty()) {
+ if (evaluationResults.size() > 1) {
+ resultsMessage.append(
+ BaseMessages.getString(PKG,
"TextFileCSVImportProgressDialog.Info.WarnNumberFormat"));
+ }
+ for (StringEvaluationResult seResult : evaluationResults) {
+ if (!seResult.getConversionMeta().isNumeric()) {
+ continue;
+ }
+ String mask = seResult.getConversionMeta().getConversionMask();
+ resultsMessage.append(
+ BaseMessages.getString(
+ PKG, "TextFileCSVImportProgressDialog.Info.NumberFormat2",
mask));
+ resultsMessage.append(
+ BaseMessages.getString(
+ PKG,
+ "TextFileCSVImportProgressDialog.Info.TrimType",
+ seResult.getConversionMeta().getTrimType()));
+ resultsMessage.append(
+ BaseMessages.getString(
+ PKG,
+ "TextFileCSVImportProgressDialog.Info.NumberMinValue",
+ valueText(seResult.getMin())));
+ resultsMessage.append(
+ BaseMessages.getString(
+ PKG,
+ "TextFileCSVImportProgressDialog.Info.NumberMaxValue",
+ valueText(seResult.getMax())));
+ if (seResult.getMin() != null && !Utils.isEmpty(mask)) {
+ try {
+ df2.applyPattern(mask);
+ df2.setDecimalFormatSymbols(dfs2);
+ double mn = df2.parse(seResult.getMin().toString()).doubleValue();
+ resultsMessage.append(
+ BaseMessages.getString(
+ PKG,
+ "TextFileCSVImportProgressDialog.Info.NumberExample",
+ mask,
+ seResult.getMin(),
+ Double.toString(mn)));
+ } catch (Exception e) {
+ if (log.isDetailed()) {
+ log.logDetailed(
+ "This is unexpected: parsing ["
+ + seResult.getMin()
+ + "] with format ["
+ + mask
+ + "] did not work.");
+ }
+ }
+ }
+ }
+ }
+ resultsMessage.append(
+ BaseMessages.getString(
+ PKG, "TextFileCSVImportProgressDialog.Info.NumberNrNullValues", ""
+ nullCount));
+ }
+
+ private static String valueText(Object value) {
+ return value == null ? "" : value.toString();
+ }
- return resultsMessage.toString();
+ private String readSampleLine(StringBuilder lineBuffer, int fileFormatType)
throws HopException {
+ return TextFileLineUtil.getLine(
+ log,
+ reader,
+ encodingType,
+ fileFormatType,
+ lineBuffer,
+ meta.getEnclosure(),
+ meta.getEscapeCharacter(),
+ meta.isBreakInEnclosureAllowed());
}
}
diff --git
a/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileInputMeta.java
b/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileInputMeta.java
index ecb6db27a8..4bd2b108a4 100644
---
a/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileInputMeta.java
+++
b/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileInputMeta.java
@@ -885,6 +885,11 @@ public class TextFileInputMeta
return content != null && content.header;
}
+ @Override
+ public boolean skipEmptyLines() {
+ return content != null && content.isNoEmptyLines();
+ }
+
@Override
public String getEscapeCharacter() {
return content == null ? null : content.escapeCharacter;
diff --git
a/plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileCsvImportScanTest.java
b/plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileCsvImportScanTest.java
new file mode 100644
index 0000000000..4dd95df5fb
--- /dev/null
+++
b/plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileCsvImportScanTest.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.fileinput.text;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.IProgressMonitor;
+import org.apache.hop.core.ProgressNullMonitorListener;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.file.TextFileInputField;
+import org.apache.hop.core.fileinput.InputFile;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/**
+ * Get Fields samples a CSV once and reports the types it actually found.
These guard the two ways
+ * that scan used to go wrong on a CRLF customers file: a remote file listing
on every sample line,
+ * and a summary that still printed the old date-format guess (empty string
min/max, and 2199/1900
+ * sentinels).
+ */
+class TextFileCsvImportScanTest {
+ private static final List<String> FIELD_NAMES =
+ List.of(
+ "id",
+ "name",
+ "firstname",
+ "zip",
+ "city",
+ "birthdate",
+ "street",
+ "housenr",
+ "stateCode",
+ "state");
+
+ @RegisterExtension
+ static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
+
+ @BeforeAll
+ static void setUp() throws Exception {
+ HopEnvironment.init();
+ }
+
+ @Test
+ void samplesTheRequestedRowsAndResolvesTheFileOnce() throws Exception {
+ Scan scan = scan(fixture(), "mixed", 100);
+ assertEquals(1, scan.dialog.resolves);
+ assertTrue(
+ scan.message.contains("Result after scanning 100 lines."),
+ () -> "expected to stop at 100 data rows, was:\n" + scan.message);
+ assertFalse(scan.message.contains("2199"), scan.message);
+ assertFalse(scan.message.contains("More then 1 date format"),
scan.message);
+ assertTrue(scan.message.contains("name-0001"), scan.message);
+
+ assertEquals(IValueMeta.TYPE_INTEGER, field(scan.meta, "id").getType());
+ assertEquals(" #", field(scan.meta, "id").getFormat());
+ assertEquals(15, field(scan.meta, "id").getLength());
+ assertEquals(0, field(scan.meta, "id").getPrecision());
+ assertEquals(IValueMeta.TYPE_STRING, field(scan.meta, "name").getType());
+ assertEquals(9, field(scan.meta, "name").getLength());
+ assertEquals(IValueMeta.TYPE_DATE, field(scan.meta,
"birthdate").getType());
+ assertEquals("yyyy/MM/dd", field(scan.meta, "birthdate").getFormat());
+ }
+
+ @Test
+ void firstSampleIsTheFirstDataRowForDosUnixAndMixed() throws Exception {
+ Path file = fixture();
+ for (String format : List.of("DOS", "unix", "mixed")) {
+ Scan scan = scan(file, format, 1);
+ assertTrue(
+ scan.message.contains("Result after scanning 1 lines."),
+ () -> format + " scanned more than the first data row:\n" +
scan.message);
+ // "name-0001" is the first data row. The header would be the
4-character word "name", and a
+ // blank Unix leftover line would be length 0.
+ assertEquals(IValueMeta.TYPE_INTEGER, field(scan.meta, "id").getType(),
format);
+ assertEquals(9, field(scan.meta, "name").getLength(), format + "\n" +
scan.message);
+ assertTrue(scan.message.contains("name-0001"), format + "\n" +
scan.message);
+ }
+ }
+
+ /**
+ * The progress dialog closes itself when the monitor is done. If the scan
does that before
+ * returning, the caller sees a null result and leaves the grid on the
header names.
+ */
+ @Test
+ void resultIsReturnedBeforeTheProgressDialogCloses() throws Exception {
+ TextFileInputMeta meta = newMeta("mixed");
+ InputFile inputFile = new InputFile();
+ inputFile.setFileName(fixture().toAbsolutePath().toString());
+ inputFile.setFileRequired(true);
+ meta.getFileInput().getInputFiles().add(inputFile);
+ Variables variables = new Variables();
+ variables.initializeFrom(null);
+ try (InputStreamReader reader =
+ new InputStreamReader(Files.newInputStream(fixture()),
StandardCharsets.UTF_8)) {
+ CountingDialog dialog = new CountingDialog(variables, meta, reader, 20);
+ IProgressMonitor monitor =
+ new ProgressNullMonitorListener() {
+ @Override
+ public void done() {
+ throw new AssertionError(
+ "closing the dialog from the scan drops the result and keeps
only header names");
+ }
+ };
+ String message = dialog.doScan(monitor, false);
+ assertTrue(message.contains("Result after scanning 20 lines."));
+ assertEquals(IValueMeta.TYPE_INTEGER, field(meta, "id").getType());
+ assertEquals(" #", field(meta, "id").getFormat());
+ assertEquals(15, field(meta, "id").getLength());
+ assertEquals(0, field(meta, "id").getPrecision());
+ assertEquals("yyyy/MM/dd", field(meta, "birthdate").getFormat());
+ assertEquals(9, field(meta, "name").getLength());
+ }
+ }
+
+ @Test
+ void missingFileFailsOnceInsteadOfPerLine() throws Exception {
+ TextFileInputMeta meta = newMeta("mixed");
+ Variables variables = new Variables();
+ variables.initializeFrom(null);
+ try (InputStreamReader reader =
+ new InputStreamReader(new ByteArrayInputStream(new byte[0]),
StandardCharsets.UTF_8)) {
+ CountingDialog dialog = new CountingDialog(variables, meta, reader, 100);
+ HopException exception =
+ assertThrows(
+ HopException.class, () -> dialog.doScan(new
ProgressNullMonitorListener(), false));
+ assertTrue(exception.getMessage().contains("valid file"),
exception.getMessage());
+ assertEquals(1, dialog.resolves);
+ }
+ }
+
+ private static Scan scan(Path file, String format, int samples) throws
Exception {
+ TextFileInputMeta meta = newMeta(format);
+ InputFile inputFile = new InputFile();
+ inputFile.setFileName(file.toAbsolutePath().toString());
+ inputFile.setFileRequired(true);
+ meta.getFileInput().getInputFiles().add(inputFile);
+
+ Variables variables = new Variables();
+ variables.initializeFrom(null);
+ try (InputStreamReader reader =
+ new InputStreamReader(Files.newInputStream(file),
StandardCharsets.UTF_8)) {
+ CountingDialog dialog = new CountingDialog(variables, meta, reader,
samples);
+ String message = dialog.doScan(new ProgressNullMonitorListener(), false);
+ return new Scan(meta, dialog, message);
+ }
+ }
+
+ private static TextFileInputMeta newMeta(String format) {
+ TextFileInputMeta meta = new TextFileInputMeta();
+ meta.getContent().setFileType("CSV");
+ meta.getContent().setFileFormat(format);
+ meta.getContent().setSeparator(";");
+ meta.getContent().setEnclosure("\"");
+ meta.getContent().setEscapeCharacter("\\");
+ meta.getContent().setHeader(true);
+ meta.getContent().setNrHeaderLines(1);
+ meta.getContent().setNoEmptyLines(true);
+ for (String name : FIELD_NAMES) {
+ TextFileInputField field = new TextFileInputField();
+ field.setName(name);
+ meta.getInputFields().add(field);
+ }
+ return meta;
+ }
+
+ private static TextFileInputField field(TextFileInputMeta meta, String name)
{
+ return meta.getInputFields().stream()
+ .filter(candidate -> name.equals(candidate.getName()))
+ .findFirst()
+ .orElseThrow();
+ }
+
+ /**
+ * Checkout may store this fixture with LF (core.autocrlf). The scan under
test has to see CRLF,
+ * because Unix mode then leaves a blank line after every row.
+ */
+ private static Path fixture() throws Exception {
+ var resource =
TextFileCsvImportScanTest.class.getResource("files/customers-crlf.csv");
+ if (resource == null) {
+ throw new IllegalStateException("customers-crlf.csv fixture is missing");
+ }
+ String text =
+ Files.readString(Path.of(resource.toURI()), StandardCharsets.UTF_8)
+ .replace("\r\n", "\n")
+ .replace("\n", "\r\n");
+ if (!text.contains("\r\n")) {
+ throw new IllegalStateException("customers fixture has no lines to
scan");
+ }
+ Path copy = Files.createTempFile("customers-crlf", ".csv");
+ Files.writeString(copy, text, StandardCharsets.UTF_8);
+ return copy;
+ }
+
+ private record Scan(TextFileInputMeta meta, CountingDialog dialog, String
message) {}
+
+ private static final class CountingDialog
+ extends TextFileCSVImportProgressDialog<TextFileInputField> {
+ private int resolves;
+
+ private CountingDialog(
+ Variables variables, TextFileInputMeta meta, InputStreamReader reader,
int samples) {
+ super(null, variables, meta, new PipelineMeta(), reader, samples, true);
+ }
+
+ @Override
+ protected String resolveSampleFileName() throws HopException {
+ resolves++;
+ return super.resolveSampleFileName();
+ }
+ }
+}
diff --git
a/plugins/transforms/textfile/src/test/resources/org/apache/hop/pipeline/transforms/fileinput/text/files/customers-crlf.csv
b/plugins/transforms/textfile/src/test/resources/org/apache/hop/pipeline/transforms/fileinput/text/files/customers-crlf.csv
new file mode 100644
index 0000000000..075b226c99
--- /dev/null
+++
b/plugins/transforms/textfile/src/test/resources/org/apache/hop/pipeline/transforms/fileinput/text/files/customers-crlf.csv
@@ -0,0 +1,131 @@
+id;name;firstname;zip;city;birthdate;street;housenr;stateCode;state
+ 1;name-0001;fn-0001; 10001;city;1954/02/07;street; 1;AK;ALASKA
+ 2;name-0002;fn-0002; 10002;city;1954/02/07;street; 2;AK;ALASKA
+ 3;name-0003;fn-0003; 10003;city;1954/02/07;street; 3;AK;ALASKA
+ 4;name-0004;fn-0004; 10004;city;1954/02/07;street; 4;AK;ALASKA
+ 5;name-0005;fn-0005; 10005;city;1954/02/07;street; 5;AK;ALASKA
+ 6;name-0006;fn-0006; 10006;city;1954/02/07;street; 6;AK;ALASKA
+ 7;name-0007;fn-0007; 10007;city;1954/02/07;street; 7;AK;ALASKA
+ 8;name-0008;fn-0008; 10008;city;1954/02/07;street; 8;AK;ALASKA
+ 9;name-0009;fn-0009; 10009;city;1954/02/07;street; 9;AK;ALASKA
+ 10;name-0010;fn-0010; 10010;city;1954/02/07;street; 10;AK;ALASKA
+ 11;name-0011;fn-0011; 10011;city;1954/02/07;street; 11;AK;ALASKA
+ 12;name-0012;fn-0012; 10012;city;1954/02/07;street; 12;AK;ALASKA
+ 13;name-0013;fn-0013; 10013;city;1954/02/07;street; 13;AK;ALASKA
+ 14;name-0014;fn-0014; 10014;city;1954/02/07;street; 14;AK;ALASKA
+ 15;name-0015;fn-0015; 10015;city;1954/02/07;street; 15;AK;ALASKA
+ 16;name-0016;fn-0016; 10016;city;1954/02/07;street; 16;AK;ALASKA
+ 17;name-0017;fn-0017; 10017;city;1954/02/07;street; 17;AK;ALASKA
+ 18;name-0018;fn-0018; 10018;city;1954/02/07;street; 18;AK;ALASKA
+ 19;name-0019;fn-0019; 10019;city;1954/02/07;street; 19;AK;ALASKA
+ 20;name-0020;fn-0020; 10020;city;1954/02/07;street; 20;AK;ALASKA
+ 21;name-0021;fn-0021; 10021;city;1954/02/07;street; 21;AK;ALASKA
+ 22;name-0022;fn-0022; 10022;city;1954/02/07;street; 22;AK;ALASKA
+ 23;name-0023;fn-0023; 10023;city;1954/02/07;street; 23;AK;ALASKA
+ 24;name-0024;fn-0024; 10024;city;1954/02/07;street; 24;AK;ALASKA
+ 25;name-0025;fn-0025; 10025;city;1954/02/07;street; 25;AK;ALASKA
+ 26;name-0026;fn-0026; 10026;city;1954/02/07;street; 26;AK;ALASKA
+ 27;name-0027;fn-0027; 10027;city;1954/02/07;street; 27;AK;ALASKA
+ 28;name-0028;fn-0028; 10028;city;1954/02/07;street; 28;AK;ALASKA
+ 29;name-0029;fn-0029; 10029;city;1954/02/07;street; 29;AK;ALASKA
+ 30;name-0030;fn-0030; 10030;city;1954/02/07;street; 30;AK;ALASKA
+ 31;name-0031;fn-0031; 10031;city;1954/02/07;street; 31;AK;ALASKA
+ 32;name-0032;fn-0032; 10032;city;1954/02/07;street; 32;AK;ALASKA
+ 33;name-0033;fn-0033; 10033;city;1954/02/07;street; 33;AK;ALASKA
+ 34;name-0034;fn-0034; 10034;city;1954/02/07;street; 34;AK;ALASKA
+ 35;name-0035;fn-0035; 10035;city;1954/02/07;street; 35;AK;ALASKA
+ 36;name-0036;fn-0036; 10036;city;1954/02/07;street; 36;AK;ALASKA
+ 37;name-0037;fn-0037; 10037;city;1954/02/07;street; 37;AK;ALASKA
+ 38;name-0038;fn-0038; 10038;city;1954/02/07;street; 38;undefined;undefined
+ 39;name-0039;fn-0039; 10039;city;1954/02/07;street; 39;AK;ALASKA
+ 40;name-0040;fn-0040; 10040;city;1954/02/07;street; 40;AK;ALASKA
+ 41;name-0041;fn-0041; 10041;city;1954/02/07;street; 41;AK;ALASKA
+ 42;name-0042;fn-0042; 10042;city;1954/02/07;street; 42;AK;ALASKA
+ 43;name-0043;fn-0043; 10043;city;1954/02/07;street; 43;AK;ALASKA
+ 44;name-0044;fn-0044; 10044;city;1954/02/07;street; 44;AK;ALASKA
+ 45;name-0045;fn-0045; 10045;city;1954/02/07;street; 45;AK;ALASKA
+ 46;name-0046;fn-0046; 10046;city;1954/02/07;street; 46;AK;ALASKA
+ 47;name-0047;fn-0047; 10047;city;1954/02/07;street; 47;AK;ALASKA
+ 48;name-0048;fn-0048; 10048;city;1954/02/07;street; 48;AK;ALASKA
+ 49;name-0049;fn-0049; 10049;city;1954/02/07;street; 49;AK;ALASKA
+ 50;name-0050;fn-0050; 10050;city;1954/02/07;street; 50;AK;ALASKA
+ 51;name-0051;fn-0051; 10051;city;1954/02/07;street; 51;AK;ALASKA
+ 52;name-0052;fn-0052; 10052;city;1954/02/07;street; 52;AK;ALASKA
+ 53;name-0053;fn-0053; 10053;city;1954/02/07;street; 53;AK;ALASKA
+ 54;name-0054;fn-0054; 10054;city;1954/02/07;street; 54;AK;ALASKA
+ 55;name-0055;fn-0055; 10055;city;1954/02/07;street; 55;AK;ALASKA
+ 56;name-0056;fn-0056; 10056;city;1954/02/07;street; 56;AK;ALASKA
+ 57;name-0057;fn-0057; 10057;city;1954/02/07;street; 57;AK;ALASKA
+ 58;name-0058;fn-0058; 10058;city;1954/02/07;street; 58;AK;ALASKA
+ 59;name-0059;fn-0059; 10059;city;1954/02/07;street; 59;AK;ALASKA
+ 60;name-0060;fn-0060; 10060;city;1954/02/07;street; 60;AK;ALASKA
+ 61;name-0061;fn-0061; 10061;city;1954/02/07;street; 61;AK;ALASKA
+ 62;name-0062;fn-0062; 10062;city;1954/02/07;street; 62;AK;ALASKA
+ 63;name-0063;fn-0063; 10063;city;1954/02/07;street; 63;AK;ALASKA
+ 64;name-0064;fn-0064; 10064;city;1954/02/07;street; 64;AK;ALASKA
+ 65;name-0065;fn-0065; 10065;city;1954/02/07;street; 65;AK;ALASKA
+ 66;name-0066;fn-0066; 10066;city;1954/02/07;street; 66;AK;ALASKA
+ 67;name-0067;fn-0067; 10067;city;1954/02/07;street; 67;AK;ALASKA
+ 68;name-0068;fn-0068; 10068;city;1954/02/07;street; 68;AK;ALASKA
+ 69;name-0069;fn-0069; 10069;city;1954/02/07;street; 69;AK;ALASKA
+ 70;name-0070;fn-0070; 10070;city;1954/02/07;street; 70;AK;ALASKA
+ 71;name-0071;fn-0071; 10071;city;1954/02/07;street; 71;AK;ALASKA
+ 72;name-0072;fn-0072; 10072;city;1954/02/07;street; 72;AK;ALASKA
+ 73;name-0073;fn-0073; 10073;city;1954/02/07;street; 73;AK;ALASKA
+ 74;name-0074;fn-0074; 10074;city;1954/02/07;street; 74;AK;ALASKA
+ 75;name-0075;fn-0075; 10075;city;1954/02/07;street; 75;AK;ALASKA
+ 76;name-0076;fn-0076; 10076;city;1954/02/07;street; 76;AK;ALASKA
+ 77;name-0077;fn-0077; 10077;city;1954/02/07;street; 77;AK;ALASKA
+ 78;name-0078;fn-0078; 10078;city;1954/02/07;street; 78;AK;ALASKA
+ 79;name-0079;fn-0079; 10079;city;1954/02/07;street; 79;AK;ALASKA
+ 80;name-0080;fn-0080; 10080;city;1954/02/07;street; 80;AK;ALASKA
+ 81;name-0081;fn-0081; 10081;city;1954/02/07;street; 81;AK;ALASKA
+ 82;name-0082;fn-0082; 10082;city;1954/02/07;street; 82;AK;ALASKA
+ 83;name-0083;fn-0083; 10083;city;1954/02/07;street; 83;AK;ALASKA
+ 84;name-0084;fn-0084; 10084;city;1954/02/07;street; 84;AK;ALASKA
+ 85;name-0085;fn-0085; 10085;city;1954/02/07;street; 85;AK;ALASKA
+ 86;name-0086;fn-0086; 10086;city;1954/02/07;street; 86;AK;ALASKA
+ 87;name-0087;fn-0087; 10087;city;1954/02/07;street; 87;AK;ALASKA
+ 88;name-0088;fn-0088; 10088;city;1954/02/07;street; 88;AK;ALASKA
+ 89;name-0089;fn-0089; 10089;city;1954/02/07;street; 89;AK;ALASKA
+ 90;name-0090;fn-0090; 10090;city;1954/02/07;street; 90;AK;ALASKA
+ 91;name-0091;fn-0091; 10091;city;1954/02/07;street; 91;AK;ALASKA
+ 92;name-0092;fn-0092; 10092;city;1954/02/07;street; 92;AK;ALASKA
+ 93;name-0093;fn-0093; 10093;city;1954/02/07;street; 93;AK;ALASKA
+ 94;name-0094;fn-0094; 10094;city;1954/02/07;street; 94;AK;ALASKA
+ 95;name-0095;fn-0095; 10095;city;1954/02/07;street; 95;AK;ALASKA
+ 96;name-0096;fn-0096; 10096;city;1954/02/07;street; 96;AK;ALASKA
+ 97;name-0097;fn-0097; 10097;city;1954/02/07;street; 97;AK;ALASKA
+ 98;name-0098;fn-0098; 10098;city;1954/02/07;street; 98;AK;ALASKA
+ 99;name-0099;fn-0099; 10099;city;1954/02/07;street; 99;AK;ALASKA
+ 100;name-0100;fn-0100; 10100;city;1954/02/07;street; 100;AK;ALASKA
+ 101;name-0101;fn-0101; 10101;city;1954/02/07;street; 101;AK;ALASKA
+ 102;name-0102;fn-0102; 10102;city;1954/02/07;street; 102;AK;ALASKA
+ 103;name-0103;fn-0103; 10103;city;1954/02/07;street; 103;AK;ALASKA
+ 104;name-0104;fn-0104; 10104;city;1954/02/07;street; 104;AK;ALASKA
+ 105;name-0105;fn-0105; 10105;city;1954/02/07;street; 105;AK;ALASKA
+ 106;name-0106;fn-0106; 10106;city;1954/02/07;street; 106;AK;ALASKA
+ 107;name-0107;fn-0107; 10107;city;1954/02/07;street; 107;AK;ALASKA
+ 108;name-0108;fn-0108; 10108;city;1954/02/07;street; 108;AK;ALASKA
+ 109;name-0109;fn-0109; 10109;city;1954/02/07;street; 109;AK;ALASKA
+ 110;name-0110;fn-0110; 10110;city;1954/02/07;street; 110;AK;ALASKA
+ 111;name-0111;fn-0111; 10111;city;1954/02/07;street; 111;AK;ALASKA
+ 112;name-0112;fn-0112; 10112;city;1954/02/07;street; 112;AK;ALASKA
+ 113;name-0113;fn-0113; 10113;city;1954/02/07;street; 113;AK;ALASKA
+ 114;name-0114;fn-0114; 10114;city;1954/02/07;street; 114;AK;ALASKA
+ 115;name-0115;fn-0115; 10115;city;1954/02/07;street; 115;AK;ALASKA
+ 116;name-0116;fn-0116; 10116;city;1954/02/07;street; 116;AK;ALASKA
+ 117;name-0117;fn-0117; 10117;city;1954/02/07;street; 117;AK;ALASKA
+ 118;name-0118;fn-0118; 10118;city;1954/02/07;street; 118;AK;ALASKA
+ 119;name-0119;fn-0119; 10119;city;1954/02/07;street; 119;AK;ALASKA
+ 120;name-0120;fn-0120; 10120;city;1954/02/07;street; 120;AK;ALASKA
+ 121;name-0121;fn-0121; 10121;city;1954/02/07;street; 121;AK;ALASKA
+ 122;name-0122;fn-0122; 10122;city;1954/02/07;street; 122;AK;ALASKA
+ 123;name-0123;fn-0123; 10123;city;1954/02/07;street; 123;AK;ALASKA
+ 124;name-0124;fn-0124; 10124;city;1954/02/07;street; 124;AK;ALASKA
+ 125;name-0125;fn-0125; 10125;city;1954/02/07;street; 125;AK;ALASKA
+ 126;name-0126;fn-0126; 10126;city;1954/02/07;street; 126;AK;ALASKA
+ 127;name-0127;fn-0127; 10127;city;1954/02/07;street; 127;AK;ALASKA
+ 128;name-0128;fn-0128; 10128;city;1954/02/07;street; 128;AK;ALASKA
+ 129;name-0129;fn-0129; 10129;city;1954/02/07;street; 129;AK;ALASKA
+ 130;name-0130;fn-0130; 10130;city;1954/02/07;street; 130;AK;ALASKA
diff --git
a/ui/src/main/java/org/apache/hop/ui/pipeline/transform/common/IGetFieldsCapableTransformDialog.java
b/ui/src/main/java/org/apache/hop/ui/pipeline/transform/common/IGetFieldsCapableTransformDialog.java
index 399247545c..995bb23cec 100644
---
a/ui/src/main/java/org/apache/hop/ui/pipeline/transform/common/IGetFieldsCapableTransformDialog.java
+++
b/ui/src/main/java/org/apache/hop/ui/pipeline/transform/common/IGetFieldsCapableTransformDialog.java
@@ -234,6 +234,11 @@ public interface
IGetFieldsCapableTransformDialog<TransformMetaType extends Base
// ...repopulate the field values in the correct order, keeping track of
new incoming fields
final List<String> newFieldNames = repopulateFields(meta, fieldValues,
reloadAllFields);
+ // removeAll/removeEmptyRows leave one blank row. Drop it so it is not a
field, including when
+ // the scan fails and getData never runs.
+ getFieldsTable().removeEmptyRows();
+ getFieldsTable().setRowNums();
+ getFieldsTable().optWidth(true);
populateMeta(meta);
final String message = loadFieldsImpl(meta, samples);
@@ -241,8 +246,17 @@ public interface
IGetFieldsCapableTransformDialog<TransformMetaType extends Base
if (reloadAllFields) {
getFieldsTable().removeAll();
}
- // OK, what's the result of our search?
+ // Copy the sampled type, mask, length and precision onto the grid.
Without this the
+ // table stays on the header names written above.
getData(meta, false, reloadAllFields, newFieldNames);
+ // removeAll leaves one blank placeholder above the sampled rows.
+ getFieldsTable().removeEmptyRows();
+ getFieldsTable().setRowNums();
+ getFieldsTable().optWidth(true);
+ } else {
+ getFieldsTable().removeEmptyRows();
+ getFieldsTable().setRowNums();
+ getFieldsTable().optWidth(true);
}
NamingSchemeColumnApplierRegistry.getInstance()
.applyAnnotatedColumns(
