This is an automated email from the ASF dual-hosted git repository.
mcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 0868c90386 issues #6381, #6947 (data set as variable) (#6949)
0868c90386 is described below
commit 0868c903865e6df2ae55a8faf2402b26f91d5020
Author: Matt Casters <[email protected]>
AuthorDate: Fri Apr 3 15:20:22 2026 +0200
issues #6381, #6947 (data set as variable) (#6949)
* issue #6381 (CSV Input Dialog fix for get fields dialog)
* issue #6381 (Pipeline Unit Test fails to execute when Input Dataset is
defined as a variable)
* issue #6950 (When clicking on a dataset in a pipeline extra windows show)
---
.../java/org/apache/hop/core/gui/BasePainter.java | 4 +
.../transforms/common/ICsvInputAwareMeta.java | 3 +-
.../org/apache/hop/testing/PipelineUnitTest.java | 7 +-
.../org/apache/hop/testing/util/DataSetConst.java | 4 +-
.../org/apache/hop/testing/util/UnitTestUtil.java | 2 +-
.../xp/LocationMouseDoubleClickExtensionPoint.java | 1 +
.../xp/ValidatePipelineUnitTestExtensionPoint.java | 2 +-
.../transforms/csvinput/CsvInputField.java | 53 +++++++++-
.../pipeline/transforms/csvinput/CsvInputMeta.java | 111 ++++++++++++++++++++-
.../common/IGetFieldsCapableTransformDialog.java | 1 -
10 files changed, 173 insertions(+), 15 deletions(-)
diff --git a/engine/src/main/java/org/apache/hop/core/gui/BasePainter.java
b/engine/src/main/java/org/apache/hop/core/gui/BasePainter.java
index 012b3bd1cc..b7c88214c4 100644
--- a/engine/src/main/java/org/apache/hop/core/gui/BasePainter.java
+++ b/engine/src/main/java/org/apache/hop/core/gui/BasePainter.java
@@ -33,6 +33,7 @@ import org.apache.hop.core.gui.IGc.ELineStyle;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.pipeline.transform.stream.StreamIcon;
@Getter
@@ -86,6 +87,9 @@ public abstract class BasePainter<Hop extends BaseHopMeta<?>,
Part extends IBase
*/
protected boolean showOriginBoundary;
+ /** In case we want to use metadata objects to help with drawing on the
pipeline or workflow */
+ protected IHopMetadataProvider metadataProvider;
+
public BasePainter(
IGc gc,
IVariables variables,
diff --git
a/engine/src/main/java/org/apache/hop/pipeline/transforms/common/ICsvInputAwareMeta.java
b/engine/src/main/java/org/apache/hop/pipeline/transforms/common/ICsvInputAwareMeta.java
index f1eb43c56a..99ae4a5b72 100644
---
a/engine/src/main/java/org/apache/hop/pipeline/transforms/common/ICsvInputAwareMeta.java
+++
b/engine/src/main/java/org/apache/hop/pipeline/transforms/common/ICsvInputAwareMeta.java
@@ -19,6 +19,7 @@ package org.apache.hop.pipeline.transforms.common;
import java.util.List;
import org.apache.commons.vfs2.FileObject;
+import org.apache.hop.core.exception.HopFileException;
import org.apache.hop.core.exception.HopTransformException;
import org.apache.hop.core.fileinput.InputFile;
import org.apache.hop.core.gui.ITextFileInputField;
@@ -66,7 +67,7 @@ public interface ICsvInputAwareMeta<T extends
ITextFileInputField> {
* @param variables the {@link PipelineMeta}
* @return null if the {@link FileObject} cannot be created.
*/
- FileObject getHeaderFileObject(final IVariables variables);
+ FileObject getHeaderFileObject(final IVariables variables) throws
HopFileException;
/**
* Gets the fields.
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/PipelineUnitTest.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/PipelineUnitTest.java
index da22825da5..b4634ca3e5 100644
---
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/PipelineUnitTest.java
+++
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/PipelineUnitTest.java
@@ -143,11 +143,14 @@ public class PipelineUnitTest extends HopMetadataBase
implements Cloneable, IHop
* @throws HopException
*/
public DataSet getGoldenDataSet(
- ILogChannel log, IHopMetadataProvider metadataProvider,
PipelineUnitTestSetLocation location)
+ ILogChannel log,
+ IHopMetadataProvider metadataProvider,
+ IVariables variables,
+ PipelineUnitTestSetLocation location)
throws HopException {
String transformName = location.getTransformName();
- String goldenDataSetName = location.getDataSetName();
+ String goldenDataSetName = variables.resolve(location.getDataSetName());
try {
// Look in the golden data sets list for the mentioned transform name
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/util/DataSetConst.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/util/DataSetConst.java
index 54f3ace28d..4340b18954 100644
---
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/util/DataSetConst.java
+++
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/util/DataSetConst.java
@@ -31,6 +31,7 @@ import org.apache.hop.core.logging.ILogChannel;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.pipeline.PipelineMeta;
@@ -99,6 +100,7 @@ public class DataSetConst {
IPipelineEngine<PipelineMeta> pipeline,
PipelineUnitTest unitTest,
IHopMetadataProvider metadataProvider,
+ IVariables variables,
List<UnitTestResult> results)
throws HopException {
int nrErrors = 0;
@@ -166,7 +168,7 @@ public class DataSetConst {
+ "', fields: "
+ resultRowMeta.toString());
- DataSet goldenDataSet = unitTest.getGoldenDataSet(log, metadataProvider,
location);
+ DataSet goldenDataSet = unitTest.getGoldenDataSet(log, metadataProvider,
variables, location);
List<Object[]> goldenRows = goldenDataSet.getAllRows(pipeline, log,
location);
IRowMeta goldenRowMeta =
goldenDataSet.getMappedDataSetFieldsRowMeta(location);
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/util/UnitTestUtil.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/util/UnitTestUtil.java
index 6977bf063e..16c31432c1 100644
---
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/util/UnitTestUtil.java
+++
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/util/UnitTestUtil.java
@@ -120,7 +120,7 @@ public class UnitTestUtil {
List<UnitTestResult> testResults = new ArrayList<>();
DataSetConst.validateTransformResultAgainstUnitTest(
- testPipeline, test, metadataProvider, testResults);
+ testPipeline, test, metadataProvider, variables, testResults);
testResultsEvaluator.evaluateTestResults(testPipeline, testResults);
} catch (HopException e) {
exceptionEvaluator.evaluateTestException(test, testPipelineMeta, e);
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/LocationMouseDoubleClickExtensionPoint.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/LocationMouseDoubleClickExtensionPoint.java
index acff5b569a..e816c46299 100644
---
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/LocationMouseDoubleClickExtensionPoint.java
+++
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/LocationMouseDoubleClickExtensionPoint.java
@@ -59,6 +59,7 @@ public class LocationMouseDoubleClickExtensionPoint
throws HopException {
HopGuiPipelineGraph pipelineGraph =
pipelineGraphExtension.getPipelineGraph();
PipelineMeta pipelineMeta = pipelineGraph.getPipelineMeta();
+ pipelineGraphExtension.setPreventingDefault(true);
PipelineUnitTest unitTest =
TestingGuiPlugin.getCurrentUnitTest(pipelineMeta);
if (unitTest == null) {
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/ValidatePipelineUnitTestExtensionPoint.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/ValidatePipelineUnitTestExtensionPoint.java
index 8fea276cb9..586def91f8 100644
---
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/ValidatePipelineUnitTestExtensionPoint.java
+++
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/ValidatePipelineUnitTestExtensionPoint.java
@@ -83,7 +83,7 @@ public class ValidatePipelineUnitTestExtensionPoint
//
int errors =
DataSetConst.validateTransformResultAgainstUnitTest(
- pipeline, unitTest, metadataProvider, results);
+ pipeline, unitTest, metadataProvider, variables, results);
if (errors == 0) {
log.logBasic("Unit test '" + unitTest.getName() + "' passed
successfully");
} else {
diff --git
a/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputField.java
b/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputField.java
index 09f39b39e7..14b3a8ae8d 100644
---
a/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputField.java
+++
b/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputField.java
@@ -28,16 +28,18 @@ import lombok.Setter;
import org.apache.commons.lang3.StringUtils;
import org.apache.hop.core.Const;
import org.apache.hop.core.exception.HopPluginException;
+import org.apache.hop.core.gui.ITextFileInputField;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.value.ValueMetaBase;
import org.apache.hop.core.row.value.ValueMetaFactory;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.jspecify.annotations.NonNull;
/** Describes a single field in a text file */
@Getter
@Setter
-public class CsvInputField {
+public class CsvInputField implements ITextFileInputField {
@HopMetadataProperty(key = "name", injectionKey = "INPUT_NAME")
private String name;
@@ -96,8 +98,8 @@ public class CsvInputField {
this(null, -9, -9);
}
- public CsvInputField(String fieldname, int position, int length) {
- this.name = fieldname;
+ public CsvInputField(String fieldName, int position, int length) {
+ this.name = fieldName;
this.length = length;
this.type = IValueMeta.TYPE_STRING;
this.format = "";
@@ -426,8 +428,18 @@ public class CsvInputField {
return -1;
}
- public CsvInputField createNewInstance(String newFieldname, int x, int
newlength) {
- return new CsvInputField(newFieldname, x, newlength);
+ @Override
+ public int getPosition() {
+ return -1;
+ }
+
+ public CsvInputField createNewInstance(String newFieldName, int position,
int newLength) {
+ return new CsvInputField(newFieldName, position, newLength);
+ }
+
+ @Override
+ public void setNullString(String nullString) {
+ // Ignore this option
}
public IValueMeta toValueMeta(String fieldOriginTransformName, IVariables
variables)
@@ -449,4 +461,35 @@ public class CsvInputField {
v.setTrimType(getTrimType());
return v;
}
+
+ @Override
+ public boolean isRepeated() {
+ return false;
+ }
+
+ @Override
+ public String getNullString() {
+ return "";
+ }
+
+ @Override
+ public String getIfNullValue() {
+ return "";
+ }
+
+ @Override
+ public boolean isIgnored() {
+ return false;
+ }
+
+ @Override
+ public int compareTo(@NonNull ITextFileInputField iTextFileInputField) {
+ if (!(iTextFileInputField instanceof CsvInputField field)) {
+ return -9;
+ }
+ if (name == null) {
+ return 9;
+ }
+ return name.compareTo(field.getName());
+ }
}
diff --git
a/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputMeta.java
b/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputMeta.java
index d38d1bc7b3..f15d4a4560 100644
---
a/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputMeta.java
+++
b/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputMeta.java
@@ -22,13 +22,16 @@ import java.util.List;
import java.util.Map;
import lombok.Getter;
import lombok.Setter;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.vfs2.FileObject;
import org.apache.hop.core.CheckResult;
import org.apache.hop.core.ICheckResult;
import org.apache.hop.core.annotations.Transform;
import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.exception.HopFileException;
import org.apache.hop.core.exception.HopPluginException;
import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.fileinput.InputFile;
import org.apache.hop.core.injection.Injection;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
@@ -45,6 +48,9 @@ import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransformMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.common.ICsvInputAwareMeta;
+import org.apache.hop.pipeline.transforms.file.BaseFileErrorHandling;
+import org.apache.hop.pipeline.transforms.fileinput.text.TextFileInputMeta;
import org.apache.hop.resource.IResourceNaming;
import org.apache.hop.resource.ResourceDefinition;
import org.apache.hop.resource.ResourceEntry;
@@ -63,7 +69,8 @@ import org.apache.hop.staticschema.util.SchemaDefinitionUtil;
documentationUrl = "/pipeline/transforms/csvinput.html")
@Getter
@Setter
-public class CsvInputMeta extends BaseTransformMeta<CsvInput, CsvInputData> {
+public class CsvInputMeta extends BaseTransformMeta<CsvInput, CsvInputData>
+ implements ICsvInputAwareMeta<CsvInputField> {
private static final Class<?> PKG = CsvInput.class;
@@ -320,6 +327,46 @@ public class CsvInputMeta extends
BaseTransformMeta<CsvInput, CsvInputData> {
}
}
+ @Override
+ public BaseFileErrorHandling getErrorHandling() {
+ return null;
+ }
+
+ @Override
+ public String getErrorCountField() {
+ return "";
+ }
+
+ @Override
+ public String getErrorFieldsField() {
+ return "";
+ }
+
+ @Override
+ public String getErrorTextField() {
+ return "";
+ }
+
+ @Override
+ public boolean isErrorLineSkipped() {
+ return false;
+ }
+
+ @Override
+ public boolean isIncludeFilename() {
+ return false;
+ }
+
+ @Override
+ public boolean isIncludeRowNumber() {
+ return false;
+ }
+
+ @Override
+ public String getLength() {
+ return "";
+ }
+
@Override
public void check(
List<ICheckResult> remarks,
@@ -392,8 +439,8 @@ public class CsvInputMeta extends
BaseTransformMeta<CsvInput, CsvInputData> {
/**
* @param variables the variable variables to use
- * @param definitions
- * @param iResourceNaming
+ * @param definitions The definitions
+ * @param iResourceNaming The resource naming to use
* @param metadataProvider the metadataProvider in which non-hop metadata
could reside.
* @return the filename of the exported resource
*/
@@ -435,4 +482,62 @@ public class CsvInputMeta extends
BaseTransformMeta<CsvInput, CsvInputData> {
public boolean supportsErrorHandling() {
return true;
}
+
+ @Override
+ public String getFileType() {
+ return "CSV";
+ }
+
+ @Override
+ public List<InputFile> getInputFiles() {
+ if (StringUtils.isNotEmpty(filenameField) ||
StringUtils.isEmpty(filename)) {
+ return List.of();
+ }
+ List<InputFile> inputFiles = new ArrayList<>();
+ InputFile inputFile = new InputFile();
+ inputFiles.add(inputFile);
+ inputFile.setFileName(filename);
+ return inputFiles;
+ }
+
+ @Override
+ public String getEscapeCharacter() {
+ return "";
+ }
+
+ @Override
+ public boolean isBreakInEnclosureAllowed() {
+ return false;
+ }
+
+ @Override
+ public int getFileFormatTypeNr() {
+ return TextFileInputMeta.FILE_FORMAT_MIXED;
+ }
+
+ @Override
+ public boolean hasHeader() {
+ return isHeaderPresent();
+ }
+
+ @Override
+ public int getNrHeaderLines() {
+ return isHeaderPresent() ? 1 : 0;
+ }
+
+ /**
+ * This is the file to get the header from for the "Get Fields" button.
+ *
+ * @param variables the {@link PipelineMeta}
+ * @return The header file object
+ */
+ @Override
+ public FileObject getHeaderFileObject(IVariables variables) throws
HopFileException {
+ List<InputFile> inputFiles = getInputFiles();
+ if (inputFiles.isEmpty()) {
+ return null;
+ }
+ String headerFileName =
variables.resolve(inputFiles.getFirst().getFileName());
+ return HopVfs.getFileObject(headerFileName, variables);
+ }
}
diff --git
a/ui/src/main/java/org/apache/hop/ui/pipeline/transform/common/IGetFieldsCapableTransformDialog.java
b/ui/src/main/java/org/apache/hop/ui/pipeline/transform/common/IGetFieldsCapableTransformDialog.java
index fca7507576..0beb3caa80 100644
---
a/ui/src/main/java/org/apache/hop/ui/pipeline/transform/common/IGetFieldsCapableTransformDialog.java
+++
b/ui/src/main/java/org/apache/hop/ui/pipeline/transform/common/IGetFieldsCapableTransformDialog.java
@@ -107,7 +107,6 @@ public interface
IGetFieldsCapableTransformDialog<TransformMetaType extends Base
}
default void getFields(final TransformMetaType meta) {
-
final String[] incomingFieldNames = getFieldNames(meta);
final List<String> newFieldNames = getNewFieldNames(incomingFieldNames);
