This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 145fa2450b Issue #2723 : Json Input should not fail when no files if
that option is set (#8186)
145fa2450b is described below
commit 145fa2450b6ca2411d33cc124f5b4547cc23b685
Author: Matt Casters <[email protected]>
AuthorDate: Mon Aug 31 09:41:49 2026 +0200
Issue #2723 : Json Input should not fail when no files if that option is
set (#8186)
Honor Content tab "Do not raise an error if no files" in onNewFile so
missing or invalid paths are skipped instead of incrementing errors.
Existing listed files already resolve before the empty-list check; tests
cover A/C/E from the issue plus the onNewFile path.
fixes #2723
---
.../pipeline/transforms/jsoninput/JsonInput.java | 10 ++
.../transforms/jsoninput/JsonInputData.java | 5 +-
.../transforms/jsoninput/JsonInputTest.java | 185 +++++++++++++++++++++
3 files changed, 198 insertions(+), 2 deletions(-)
diff --git
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInput.java
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInput.java
index 178aab2ef3..c52a12df88 100644
---
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInput.java
+++
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInput.java
@@ -283,6 +283,16 @@ public class JsonInput extends
BaseFileInputTransform<JsonInputMeta, JsonInputDa
inputError(errMsg);
return false;
} else if (!file.exists()) {
+ if (meta.isDoNotFailIfNoFile()) {
+ if (isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG, "JsonInput.Log.IsNotAFile",
file.getName().getFriendlyURI()));
+ }
+ // Skip this missing file; InputsReader will advance to the next one.
+ data.skipEmptyFile = true;
+ return false;
+ }
String errMsg =
BaseMessages.getString(PKG, "JsonInput.Log.IsNotAFile",
file.getName().getFriendlyURI());
logError(errMsg);
diff --git
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputData.java
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputData.java
index 77c82ef72f..043f09d072 100644
---
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputData.java
+++
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputData.java
@@ -58,8 +58,9 @@ public class JsonInputData extends BaseFileInputTransformData
implements ITransf
public BitSet repeatedFields;
/**
- * When {@link JsonInput#onNewFile} rejects an empty file with "ignore empty
file" enabled, the
- * file iterator loops to the next file instead of substituting placeholder
JSON.
+ * When {@link JsonInput#onNewFile} rejects a file that should be skipped
(empty with "ignore
+ * empty file", or missing with "do not fail if no files"), the file
iterator loops to the next
+ * file instead of substituting placeholder JSON.
*/
public boolean skipEmptyFile;
diff --git
a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputTest.java
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputTest.java
index 0ba02a6eca..e60e59bed1 100644
---
a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputTest.java
+++
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputTest.java
@@ -18,6 +18,7 @@
package org.apache.hop.pipeline.transforms.jsoninput;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
@@ -47,6 +48,7 @@ import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopFileException;
import org.apache.hop.core.exception.HopPluginException;
import org.apache.hop.core.fileinput.FileInputList;
+import org.apache.hop.core.fileinput.InputFile;
import org.apache.hop.core.json.HopJson;
import org.apache.hop.core.logging.ILoggingObject;
import org.apache.hop.core.logging.LogLevel;
@@ -985,6 +987,121 @@ class JsonInputTest {
assertTrue(errMsgs.contains("No file(s) specified!"), errMsgs);
}
+ /**
+ * Issue #2723 A/C: listed files that exist must not emit "No file(s)
specified" or finish with
+ * errors, regardless of Required and "Do not raise an error if no files".
+ */
+ @Test
+ void testExistingListedFilesDoNotEmitNoFilesError() throws Exception {
+ for (boolean fileRequired : new boolean[] {false, true}) {
+ for (boolean doNotFailIfNoFile : new boolean[] {false, true}) {
+ assertExistingListedFilesProcessed(fileRequired, doNotFailIfNoFile);
+ }
+ }
+ }
+
+ /** Issue #2723 E: no files and checkbox unchecked still errors. */
+ @Test
+ void testMissingListedFilesFailWhenDoNotFailUnchecked() throws Exception {
+ ByteArrayOutputStream err = new ByteArrayOutputStream();
+ helper.redirectLog(err, LogLevel.ERROR);
+
+ JsonInputMeta meta = createListedFilesMeta(false, BASE_RAM_DIR +
"does-not-exist.json");
+ meta.setDoNotFailIfNoFile(false);
+ meta.getInputFields().add(priceField());
+
+ try (LocaleChange enUS = new LocaleChange(Locale.US)) {
+ JsonInput jsonInput = createJsonInput(meta);
+ processRows(jsonInput, 1);
+ disposeJsonInput(jsonInput);
+ String errMsgs = err.toString();
+ assertTrue(errMsgs.contains("No file(s) specified!"), errMsgs);
+ assertTrue(jsonInput.getErrors() > 0, "expected errors when no files and
checkbox off");
+ }
+ }
+
+ /** Invalid / missing path with the checkbox on must not fail the transform.
*/
+ @Test
+ void testInvalidListedPathDoesNotFailWhenDoNotFailChecked() throws Exception
{
+ ByteArrayOutputStream err = new ByteArrayOutputStream();
+ helper.redirectLog(err, LogLevel.ERROR);
+
+ JsonInputMeta meta = createListedFilesMeta(false, BASE_RAM_DIR +
"does-not-exist.json");
+ meta.setDoNotFailIfNoFile(true);
+ meta.getInputFields().add(priceField());
+
+ try (LocaleChange enUS = new LocaleChange(Locale.US)) {
+ JsonInput jsonInput = createJsonInput(meta);
+ processRows(jsonInput, 1);
+ disposeJsonInput(jsonInput);
+ String errMsgs = err.toString();
+ assertFalse(errMsgs.contains("No file(s) specified!"), errMsgs);
+ assertEquals(0, jsonInput.getErrors(), errMsgs);
+ }
+ }
+
+ /**
+ * {@link JsonInput#onNewFile} must honor "Do not raise an error if no
files" when a listed
+ * FileObject does not exist (the checkbox used to only guard the empty-list
check).
+ */
+ @Test
+ void testOnNewFileMissingFileHonorsDoNotFailIfNoFile() throws Exception {
+ ByteArrayOutputStream err = new ByteArrayOutputStream();
+ helper.redirectLog(err, LogLevel.ERROR);
+
+ try (FileObject missing = HopVfs.getFileObject(BASE_RAM_DIR +
"missing.json");
+ FileObject good = HopVfs.getFileObject(BASE_RAM_DIR + "good.json");
+ LocaleChange enUS = new LocaleChange(Locale.US)) {
+ try (OutputStream out = good.getContent().getOutputStream()) {
+ out.write(getBasicTestJson().getBytes(StandardCharsets.UTF_8));
+ }
+
+ JsonInputMeta meta = createFileListMeta(List.of(missing, good));
+ meta.getInputFields().add(priceField());
+ meta.setDoNotFailIfNoFile(true);
+
+ JsonInput jsonInput = createJsonInput(meta);
+ jsonInput.addRowListener(
+ new RowComparatorListener(
+ new Object[] {8.95d},
+ new Object[] {12.99d},
+ new Object[] {8.99d},
+ new Object[] {22.99d}));
+ processRows(jsonInput, 8);
+ disposeJsonInput(jsonInput);
+
+ String errMsgs = err.toString();
+ assertFalse(errMsgs.contains("is not a file"), errMsgs);
+ assertEquals(0, jsonInput.getErrors(), errMsgs);
+ assertEquals(4, jsonInput.getLinesWritten(), "rows written");
+ } finally {
+ deleteFiles();
+ }
+ }
+
+ @Test
+ void testOnNewFileMissingFileFailsWhenDoNotFailUnchecked() throws Exception {
+ ByteArrayOutputStream err = new ByteArrayOutputStream();
+ helper.redirectLog(err, LogLevel.ERROR);
+
+ try (FileObject missing = HopVfs.getFileObject(BASE_RAM_DIR +
"missing.json");
+ LocaleChange enUS = new LocaleChange(Locale.US)) {
+ JsonInputMeta meta = createFileListMeta(List.of(missing));
+ meta.getInputFields().add(priceField());
+ meta.setDoNotFailIfNoFile(false);
+
+ JsonInput jsonInput = createJsonInput(meta);
+ processRows(jsonInput, 3);
+ disposeJsonInput(jsonInput);
+
+ String errMsgs = err.toString();
+ assertTrue(errMsgs.contains("is not a file"), errMsgs);
+ assertTrue(jsonInput.getErrors() > 0, errMsgs);
+ } finally {
+ deleteFiles();
+ }
+ }
+
@Test
void testZipFileInput() throws Exception {
ByteArrayOutputStream err = new ByteArrayOutputStream();
@@ -1428,6 +1545,74 @@ class JsonInputTest {
return meta;
}
+ /**
+ * Meta that lists files through {@link
JsonInputMeta#getFileInputList(IVariables)} (the GUI
+ * path), not a stubbed list.
+ */
+ private JsonInputMeta createListedFilesMeta(boolean fileRequired, String...
fileNames) {
+ JsonInputMeta meta = new JsonInputMeta();
+ meta.setInFields(false);
+ meta.setIgnoringMissingPath(false);
+ for (String fileName : fileNames) {
+ InputFile inputFile = new InputFile();
+ inputFile.setFileName(fileName);
+ inputFile.setFileRequired(fileRequired);
+ meta.getFileInput().getInputFiles().add(inputFile);
+ }
+ return meta;
+ }
+
+ private JsonInputField priceField() {
+ JsonInputField price = new JsonInputField();
+ price.setName("price");
+ price.setType(IValueMeta.TYPE_NUMBER);
+ price.setPath("$..book[*].price");
+ return price;
+ }
+
+ private void assertExistingListedFilesProcessed(boolean fileRequired,
boolean doNotFailIfNoFile)
+ throws Exception {
+ ByteArrayOutputStream err = new ByteArrayOutputStream();
+ helper.redirectLog(err, LogLevel.ERROR);
+ final String input1 = getBasicTestJson();
+ final String input2 = "{ \"store\": { \"book\": [ { \"price\": 9.99 } ] }
}";
+ String flags = "required=" + fileRequired + ", doNotFailIfNoFile=" +
doNotFailIfNoFile;
+ try (FileObject fileObj1 = HopVfs.getFileObject(BASE_RAM_DIR +
"test1.json");
+ FileObject fileObj2 = HopVfs.getFileObject(BASE_RAM_DIR +
"test2.json");
+ LocaleChange enUS = new LocaleChange(Locale.US)) {
+ try (OutputStream out = fileObj1.getContent().getOutputStream()) {
+ out.write(input1.getBytes(StandardCharsets.UTF_8));
+ }
+ try (OutputStream out = fileObj2.getContent().getOutputStream()) {
+ out.write(input2.getBytes(StandardCharsets.UTF_8));
+ }
+
+ JsonInputMeta meta =
+ createListedFilesMeta(
+ fileRequired, BASE_RAM_DIR + "test1.json", BASE_RAM_DIR +
"test2.json");
+ meta.setDoNotFailIfNoFile(doNotFailIfNoFile);
+ meta.getInputFields().add(priceField());
+
+ JsonInput jsonInput = createJsonInput(meta);
+ jsonInput.addRowListener(
+ new RowComparatorListener(
+ new Object[] {8.95d},
+ new Object[] {12.99d},
+ new Object[] {8.99d},
+ new Object[] {22.99d},
+ new Object[] {9.99d}));
+ processRows(jsonInput, 8);
+ disposeJsonInput(jsonInput);
+
+ String errMsgs = err.toString();
+ assertFalse(errMsgs.contains("No file(s) specified!"), flags + " " +
errMsgs);
+ assertEquals(0, jsonInput.getErrors(), flags + " " + errMsgs);
+ assertEquals(5, jsonInput.getLinesWritten(), flags);
+ } finally {
+ deleteFiles();
+ }
+ }
+
protected void testSimpleJsonPath(
String jsonPath, IValueMeta outputMeta, Object[][] inputRows, Object[][]
outputRows)
throws Exception {
