This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new d986f4001d Issue #8711 : Correct JSON Input and Enhanced JSON Output
(#8715)
d986f4001d is described below
commit d986f4001ddd86a2ecc79cb3b6a3d802016a7ce7
Author: Matt Casters <[email protected]>
AuthorDate: Sat Oct 10 09:40:31 2026 +0200
Issue #8711 : Correct JSON Input and Enhanced JSON Output (#8715)
* Issue #8711 : Document and verify JSON-typed JSON Input sources
* Issue #8711 : Characterize microsecond timestamp conversion for issue 4907
* Issue #8711 : Verify missing timestamp offset fails cleanly
* Issue #8711 : Correct JSON Input timestamp format examples
* Issue #8711 : Describe enhanced JSON array behavior without a fictitious
version
* Issue #8711 : Clarify JSONPath object keys versus array indexes
* Issue #8711 : Preserve JSON null when empty strings differ from null
* Issue #8711 : Validate enhanced JSON group keys during metadata discovery
* Issue #8711 : Report missing enhanced JSON group keys before runtime
lookup
* Issue #8711 : Share enhanced JSON key validation without clearing
metadata on failure
* Issue #8711 : Allow literal JSONPath expressions without changing legacy
resolution
* Issue #8711 : Verify JSONPath mode survives pipeline serialization
* Issue #8711 : Stream enhanced JSON output to the file
* Issue #8711 : Match enhanced JSON output row width to its metadata
* Issue #8711 : Offer unmapped incoming fields as enhanced JSON group keys
* Issue #8711 : Close enhanced JSON generators when disposal interrupts
output
* Issue #8711 : Document the enhanced JSON group key Get Fields button
* Issue #8711 : Record the grouped pretty-print sample from issue 2569
* Issue #8711 : Add compact NDJSON output
* Issue #8711 : Split NDJSON on complete records
* Issue #8711 : Wrap each NDJSON group independently
* Issue #8711 : Reject incompatible NDJSON settings
* Issue #8711 : Validate NDJSON append boundaries
* Issue #8711 : Document newline-delimited enhanced JSON output
* Ignore graphify-out folders
* Issue #8711 : Finish parked JSON output, resolve the block name, and
surface NDJSON append errors
---------
Co-authored-by: Bart Maertens <[email protected]>
---
.gitignore | 1 +
.../hop/core/row/value/ValueMetaTimestampTest.java | 9 +
.../value/timestamp/SimpleTimestampFormatTest.java | 8 +
.../pipeline/transforms/enhancedjsonoutput.adoc | 28 +-
.../ROOT/pages/pipeline/transforms/jsoninput.adoc | 18 +-
.../json/0018-enhanced-json-output-streaming.hpl | 5 +-
.../pipeline/transforms/jsoninput/JsonInput.java | 4 +-
.../transforms/jsoninput/JsonInputDialog.java | 27 ++
.../transforms/jsoninput/JsonInputMeta.java | 24 +
.../jsoninput/reader/RowOutputConverter.java | 6 +-
.../transforms/jsonoutputenhanced/JsonEOutput.java | 191 ++++++--
.../jsonoutputenhanced/JsonEOutputDialog.java | 139 +++++-
.../jsonoutputenhanced/JsonEOutputMeta.java | 70 ++-
.../jsoninput/messages/messages_en_US.properties | 4 +
.../messages/messages_en_US.properties | 8 +
.../messages/messages_it_IT.properties | 4 +-
.../transforms/jsoninput/JsonInputMetaTest.java | 11 +
.../jsoninput/JsonInputPathOptionsDialogTest.java | 162 +++++++
.../transforms/jsoninput/JsonInputTest.java | 93 ++++
.../reader/RowOutputConverterNullTest.java | 67 +++
.../JsonEOutputBehaviorTest.java | 536 +++++++++++++++++++++
.../jsonoutputenhanced/JsonEOutputDialogTest.java | 351 ++++++++++++++
.../jsonoutputenhanced/JsonEOutputMetaTest.java | 49 ++
23 files changed, 1723 insertions(+), 92 deletions(-)
diff --git a/.gitignore b/.gitignore
index 7e674ee2ea..27b47efd29 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,6 +7,7 @@ build
target
release.properties
pom.xml.releaseBackup
+**/graphify-out
*.iml
.idea
.junie
diff --git
a/core/src/test/java/org/apache/hop/core/row/value/ValueMetaTimestampTest.java
b/core/src/test/java/org/apache/hop/core/row/value/ValueMetaTimestampTest.java
index f36de4c9c7..f7fa589115 100644
---
a/core/src/test/java/org/apache/hop/core/row/value/ValueMetaTimestampTest.java
+++
b/core/src/test/java/org/apache/hop/core/row/value/ValueMetaTimestampTest.java
@@ -243,4 +243,13 @@ class ValueMetaTimestampTest {
assertEquals(convertedTimestamp.getTime(), timestamp.getTime());
assertEquals(convertedTimestamp.getNanos(), timestamp.getNanos());
}
+
+ @Test
+ void parsesTheIssue4907TimestampWithAMatchingMask() throws Exception {
+ ValueMetaTimestamp meta = new ValueMetaTimestamp("created");
+ meta.setConversionMask("yyyy-MM-dd'T'HH:mm:ss.SSSSSS");
+ assertEquals(
+ Timestamp.valueOf("2025-02-13 02:10:14.746582"),
+ meta.convertStringToTimestamp("2025-02-13T02:10:14.746582"));
+ }
}
diff --git
a/core/src/test/java/org/apache/hop/core/row/value/timestamp/SimpleTimestampFormatTest.java
b/core/src/test/java/org/apache/hop/core/row/value/timestamp/SimpleTimestampFormatTest.java
index dcc7a09048..2647a45cc5 100644
---
a/core/src/test/java/org/apache/hop/core/row/value/timestamp/SimpleTimestampFormatTest.java
+++
b/core/src/test/java/org/apache/hop/core/row/value/timestamp/SimpleTimestampFormatTest.java
@@ -18,6 +18,7 @@ package org.apache.hop.core.row.value.timestamp;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import java.sql.Timestamp;
import java.text.ParseException;
@@ -247,4 +248,11 @@ class SimpleTimestampFormatTest {
// Should return null (and set error index) instead of throwing NPE
assertNull(result);
}
+
+ @Test
+ void missingRequiredOffsetIsAParseFailureNotANullPointer() {
+ SimpleTimestampFormat format =
+ new SimpleTimestampFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX", Locale.US);
+ assertThrows(ParseException.class, () ->
format.parse("2025-02-13T02:10:14.746582"));
+ }
}
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/enhancedjsonoutput.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/enhancedjsonoutput.adoc
index 38aa8b3e2a..80a1110739 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/enhancedjsonoutput.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/enhancedjsonoutput.adoc
@@ -71,8 +71,16 @@ If file with specified name exists already, it will be
replaced by a new one.
If checked - new JSON output will be appended to the end of existing file.
Or if file does not exist, it will be created as in previous case.
-TIP: If you want to create a ND-JSON (Newline-Delimited JSON) file, you may
get better results by outputting the JSON rows and then printing them with a
xref:pipeline/transforms/textfileoutput.adoc[Text file output] transform
-
+NOTE: Without newline-delimited JSON, Append concatenates complete JSON
documents without adding separators.
+It does not merge arrays or produce one valid JSON document.
+
+|Newline-delimited JSON (NDJSON)|Disabled by default for compatibility.
+When enabled for file output, each input-row item or completed group is
written as one compact JSON value followed by LF.
+A JSON block name wraps each line separately.
+There is no outer document array; Force arrays applies to the value inside a
grouped item.
+UTF-8 is required and Pretty Print JSON must be disabled.
+Multiline NDJSON records are not supported; newline characters inside string
values are escaped.
+Appending requires an empty file or an existing UTF-8 NDJSON file ending in LF.
|Split JSON after n rows|If this number N is larger than zero, split the
resulting JSON file into multiple parts of N items: N rows, or N groups when
group keys are used.
|Create Parent folder|Check this option to create the folders structure, if
some of them are missing in the provided path.
If this option is not checked and the full path cannot be found, the transform
will fail.
@@ -91,11 +99,14 @@ Same as for 'Include date in filename' option
=== Group Key Tab
-This tab is used to define the key fields used for grouping the rows in JSON
fragments.
-
-Rows with the same values in the key fields allow you to generate JSON
fragments from the row data, and group them in a single JSON array. The key
fields defined here will also be forwarded to the next transform as they are.
+This tab defines the input fields used to group consecutive rows into JSON
fragments.
+The input stream must be sorted by all group key fields.
+The key fields are forwarded under their original field names when generated
JSON is output to the next transform.
+In Write to file mode, the original input rows are forwarded unchanged.
-If no group field is defined, all the rows will be grouped in a JSON array and
the transform output will be a single row and a single column.
+The Get Fields button adds available input fields that are not already listed
in Fields or Group Key.
+Existing key rows and their element names are retained.
+Fields can still be added manually to both tabs when a group key must also
appear in the generated fragment.
In the output file, every group is one item: the key fields plus the JSON of
the group, under the name of the output value.
For example, with key field `group` and output value `rows`:
@@ -111,8 +122,9 @@ A JSON block name wraps the whole file, not every group:
`{"data":[{"group":"a",
|===
|Option|Description
|Fieldname|Input transform field name that will contribute to define the input
transform fields key.
-|Element name|JSON element name.
-For example "A":"B" - A is a element name, B is actual input value mapped for
this Element name.
+|Element name|The property name for this group key in each grouped file-output
item.
+It does not rename the downstream Hop field and does not insert the key into
the generated output-value fragment.
+To include a key in that fragment, also select it on the Fields tab.
|===
=== Fields Tab
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/jsoninput.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/jsoninput.adoc
index 59a4c9ab44..27f2529aa0 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/jsoninput.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/jsoninput.adoc
@@ -40,7 +40,7 @@ Use JSON Input when each column should come from its *own
JsonPath* (or you are
== Examples
* For example, connect a REST client to a JSON input transform to read JSON
into data rows.
-* To read nested JSON elements, use a sequence of JSON Input transforms, where
the first JSON Input transform reads the nested element as a String field. In
the second JSON Input transform, use the `Source is from a previous transform`
option to select the String field you used in the first JSON Input transform to
read the information from the nested element.
+* To read a nested JSON value, use either a String field containing serialized
JSON or a JSON-typed field containing a Jackson JSON value. In the next JSON
Input transform, enable *Source is from a previous transform* and select that
field. Leave *Use field as file names* and *Read source as URL* cleared for
direct JSON content. A filename or URL source remains a String field; an
arbitrary Java Object is not a supported JSON source.
Check the `json-input-nested-elements.hpl` sample in the Samples project for
an example.
@@ -116,6 +116,10 @@ Results are not limited when set to zero.
|Add filenames to result|Select to add processed files to the result file list.
|===
+=== JSONPath tab
+
+*Resolve variables in JSONPath* is enabled by default, including for existing
pipelines. Clear it to use every extraction Path literally. This is required
for root-array paths such as `$[0].similarity`, whose `$[0]` otherwise has a
meaning in Hop variable resolution. Clearing it also disables `${VARIABLE}`
substitution inside extraction paths. File and URL settings continue to resolve
variables normally.
+
=== Fields Tab
The Fields tab displays field definitions to extract values from the JSON
structure.
@@ -156,11 +160,9 @@ Click Select fields from snippet button and paste the
entire JSON text and it wi
For Hop types of Date and Timestamp, use the Format column to specify the
incoming format. E.g. for Date: yyyy-MM-dd. You can either select a format from
the dropdown or enter format text directly.
See xref:pipeline/formatting-values.adoc[Formatting numbers and dates].
-*Examples of Timestamp formats:*
-
-* Format: yyyy-MM-dd'T'HH:mm:ss.SSSZ and Result: 2021-10-26T20:51:43.795+0000
+For Timestamp values, match the fraction width in the Format column. For
example, `2025-02-13T02:10:14.746582` uses `yyyy-MM-dd'T'HH:mm:ss.SSSSSS`.
Timestamp preserves the six fractional digits; Date has millisecond precision.
-* Format: 2024-04-22T00:00:00.000Z and Result: 2024-04-22T00:00:00.000Z
+`XXX` requires an offset such as `Z` or `+02:00` in the input. Do not use
`XXX` for a value that contains no offset. A mismatched format must report a
conversion error, not silently assume an offset or discard precision.
*Regex*
@@ -177,6 +179,12 @@ The aggregation functions provided by JayWay can be used
at the end of a Path ex
A function is applied to the value the Path points to. When the Path points to
a single value, the function produces a single value and therefore a single
output row, even when that value is itself a list, as with `keys()`.
+*Object keys and array indexes*
+
+A key containing a dot is one key, not a nested path. Use `$['doc.tipo']` to
retrieve its value. The existing `$.keys()` function returns the object's keys
as one JSON value; it does not define portable object-member ordering or return
a key/value pair by ordinal.
+
+Array indexes address array elements. To select the first key/value pair by
position, make the order explicit in the source, for example
`[{"name":"doc.tipo","value":"XG"}]`, and use `$[0].name` and `$[0].value` with
JsonPath variable resolution disabled. The expressions `$.[0].name()` and
`$.[0]` do not define positional access to object members.
+
When the Path contains a wildcard before the function, the function is applied
to every match and produces one row per match. For example,
`$.orders[*].lines.length()` returns the number of lines of each order. To
aggregate over all matches instead, use a deep scan or pass the Path as a
parameter to the function: `$..price.sum()` or `$.sum($.orders[*].price)`. Note
that `$.orders[*].price.sum()` does not work: the sum is applied to each single
price.
Combining a single-value function field with a field whose Path matches
several elements is not supported: the transform reports a structure error
because the two Paths do not yield the same number of rows. Read such a value
in a dedicated JSON Input transform when the other fields return multiple rows.
diff --git a/integration-tests/json/0018-enhanced-json-output-streaming.hpl
b/integration-tests/json/0018-enhanced-json-output-streaming.hpl
index 22ac42c597..c6709759af 100644
--- a/integration-tests/json/0018-enhanced-json-output-streaming.hpl
+++ b/integration-tests/json/0018-enhanced-json-output-streaming.hpl
@@ -63,8 +63,9 @@ limitations under the License.
<height>60</height>
<xloc>64</xloc>
<yloc>144</yloc>
- <note>Run by main-0018 in a separate hop-run with a small heap. The JSON
file is several
-times larger than that heap, so it only gets written when the transform
streams it.</note>
+ <note>Run by main-0018 in a separate hop-run with -Xmx128m. The JSON
file is about 127MB,
+smaller than that heap, so it catches a retained document rather than a file
larger than the
+entire heap. One unlimited group is still held until the group ends.</note>
<width>560</width>
</notepad>
</notepads>
diff --git
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInput.java
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInput.java
index bc05c4c31c..3ee496c9e9 100644
---
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInput.java
+++
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInput.java
@@ -630,7 +630,9 @@ public class JsonInput extends
BaseFileInputTransform<JsonInputMeta, JsonInputDa
JsonInputField[] inputFields = new JsonInputField[data.nrInputFields];
for (int i = 0; i < data.nrInputFields; i++) {
JsonInputField field = new JsonInputField(meta.getInputFields().get(i));
- field.setPath(resolve(field.getPath()));
+ if (meta.isResolveJsonPaths()) {
+ field.setPath(resolve(field.getPath()));
+ }
inputFields[i] = field;
}
// Instead of putting in the meta.inputFields, we put in our json path
resolved input fields
diff --git
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputDialog.java
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputDialog.java
index c6bd71c94f..03bfc3ab45 100644
---
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputDialog.java
+++
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputDialog.java
@@ -54,6 +54,8 @@ import org.apache.hop.ui.core.dialog.EnterTextDialog;
import org.apache.hop.ui.core.dialog.ErrorDialog;
import org.apache.hop.ui.core.dialog.MessageBox;
import org.apache.hop.ui.core.dialog.PreviewRowsDialog;
+import org.apache.hop.ui.core.gui.GuiCompositeWidgets;
+import org.apache.hop.ui.core.gui.GuiCompositeWidgetsAdapter;
import org.apache.hop.ui.core.gui.GuiResource;
import org.apache.hop.ui.core.widget.ColumnInfo;
import org.apache.hop.ui.core.widget.NamingSchemeTypes;
@@ -158,6 +160,8 @@ public class JsonInputDialog extends BaseTransformDialog {
private final JsonInputMeta input;
+ private GuiCompositeWidgets pathWidgets;
+
private ModifyListener lsMod;
public JsonInputDialog(
@@ -182,6 +186,8 @@ public class JsonInputDialog extends BaseTransformDialog {
addContentTab();
+ addPathOptionsTab();
+
addFieldsTab();
addAdditionalFieldsTab();
@@ -457,6 +463,26 @@ public class JsonInputDialog extends BaseTransformDialog {
}
}
+ private void addPathOptionsTab() {
+ CTabItem tab = new CTabItem(wTabFolder, SWT.NONE);
+ tab.setText(BaseMessages.getString(PKG,
"JsonInputDialog.PathOptions.TabTitle"));
+ Composite parent = new Composite(wTabFolder, SWT.NONE);
+ PropsUi.setLook(parent);
+ parent.setLayout(new FormLayout());
+ pathWidgets =
+ GuiCompositeWidgets.addScrolledComposite(
+ parent, variables, null, null,
JsonInputMeta.GUI_PLUGIN_ELEMENT_PARENT_ID, input);
+ pathWidgets.setWidgetsListener(
+ new GuiCompositeWidgetsAdapter() {
+ @Override
+ public void widgetModified(
+ GuiCompositeWidgets widgets, Control control, String widgetId) {
+ input.setChanged();
+ }
+ });
+ tab.setControl(parent);
+ }
+
private void addContentTab() {
// ////////////////////////
// START OF CONTENT TAB///
@@ -1295,6 +1321,7 @@ public class JsonInputDialog extends BaseTransformDialog {
/** dialog -> meta */
private void getInfo(JsonInputMeta in) {
+ pathWidgets.getWidgetsContents(in,
JsonInputMeta.GUI_PLUGIN_ELEMENT_PARENT_ID);
transformName = wTransformName.getText(); // return value
in.setRowLimit(Const.toLongExpanded(wLimit.getText(), 0L));
diff --git
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMeta.java
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMeta.java
index c81632afe0..6ea05987a1 100644
---
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMeta.java
+++
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMeta.java
@@ -31,6 +31,10 @@ import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopTransformException;
import org.apache.hop.core.fileinput.FileInputList;
import org.apache.hop.core.fileinput.InputFile;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.gui.plugin.GuiWidgetGroupType;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.value.ValueMetaInteger;
@@ -58,11 +62,14 @@ import org.w3c.dom.Node;
description = "i18n::JsonInput.description",
keywords = "i18n::JsonInputMeta.keyword",
categoryDescription = "i18n::JsonInput.category")
+@GuiPlugin
@Getter
@Setter
public class JsonInputMeta extends BaseFileInputMeta<JsonInput, JsonInputData,
BaseFileInput> {
private static final Class<?> PKG = JsonInputMeta.class;
+ public static final String GUI_PLUGIN_ELEMENT_PARENT_ID =
"JsonInput.JsonPathOptions";
+
protected static final String[] RequiredFilesDesc =
new String[] {
BaseMessages.getString(PKG, "System.Combo.No"),
@@ -176,6 +183,23 @@ public class JsonInputMeta extends
BaseFileInputMeta<JsonInput, JsonInputData, B
injectionKeyDescription =
"JsonInput.Injection.DEFAULT_PATH_LEAF_TO_NULL")
private boolean defaultPathLeafToNull;
+ @HopMetadataProperty(
+ key = "resolveJsonPaths",
+ defaultBoolean = true,
+ injectionKey = "RESOLVE_JSON_PATHS",
+ injectionKeyDescription = "JsonInput.Injection.RESOLVE_JSON_PATHS")
+ @GuiWidgetElement(
+ id = "resolveJsonPaths",
+ order = "0100",
+ type = GuiElementType.CHECKBOX,
+ label = "i18n::JsonInputMeta.ResolveJsonPaths.Label",
+ toolTip = "i18n::JsonInputMeta.ResolveJsonPaths.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = "JSONPath",
+ groupOrder = "0100")
+ private boolean resolveJsonPaths = true;
+
@HopMetadataProperty(inline = true)
protected BaseFileInputAdditionalFields additionalOutputFields;
diff --git
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/reader/RowOutputConverter.java
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/reader/RowOutputConverter.java
index d10b266987..4869feef34 100644
---
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/reader/RowOutputConverter.java
+++
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/reader/RowOutputConverter.java
@@ -51,11 +51,9 @@ public class RowOutputConverter {
}
// convert from string
String strValue = getStringValue(value);
+ // A JSON null or missing node is already a Hop null. Do not let the
empty-string
+ // convention turn that null into "".
if (strValue == null) {
- // JSON null and a missing path are null. convertDataFromString turns a
null string into ""
- // when HOP_EMPTY_STRING_DIFFERS_FROM_NULL is Y, which hides the
difference between "" and
- // null. An empty JSON string ("") is not null and still goes through
conversion below.
- // See Apache Hop #4373.
return null;
}
return targetMeta.convertDataFromString(
diff --git
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutput.java
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutput.java
index fb9035e157..282f780002 100644
---
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutput.java
+++
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutput.java
@@ -29,8 +29,10 @@ import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
+import java.io.Serial;
import java.math.BigDecimal;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import org.apache.commons.vfs2.FileObject;
@@ -97,9 +99,28 @@ public class JsonEOutput extends
BaseTransform<JsonEOutputMeta, JsonEOutputData>
data.isWriteToFile && meta.getKeyFields().isEmpty() &&
!meta.isUseSingleItemPerGroup();
data.collectGroupItems = data.isOutputValue || (data.isWriteToFile &&
!data.streamFileRows);
- if (data.isWriteToFile) {
- if (!meta.getFileSettings().isDoNotOpenNewFileInit() &&
!openNewFile()) {
- logError(BaseMessages.getString(PKG, "JsonOutput.Error.OpenNewFile",
buildFilename()));
+ if (meta.isNewlineDelimited()
+ && meta.getOperationType() !=
JsonEOutputMeta.OperationType.OUTPUT_VALUE
+ && (meta.isJsonPrettified()
+ || !Const.UTF_8.equalsIgnoreCase(resolve(meta.getEncoding())))) {
+ logError(BaseMessages.getString(PKG,
"JsonEOutput.Error.NdJsonSettings"));
+ setErrors(1);
+ stopAll();
+ return false;
+ }
+
+ if (data.isWriteToFile &&
!meta.getFileSettings().isDoNotOpenNewFileInit()) {
+ try {
+ if (!openNewFile()) {
+ logError(BaseMessages.getString(PKG,
"JsonOutput.Error.OpenNewFile", buildFilename()));
+ stopAll();
+ setErrors(1);
+ return false;
+ }
+ } catch (HopTransformException e) {
+ // The NDJSON append check names the boundary. Do not replace it
with a generic open
+ // failure.
+ logError(e.getSuperMessage());
stopAll();
setErrors(1);
return false;
@@ -339,16 +360,16 @@ public class JsonEOutput extends
BaseTransform<JsonEOutputMeta, JsonEOutputData>
Object[] keyRow = getKeyValues(rowData);
- Object[] additionalRowFields = new Object[2];
-
- additionalRowFields[0] = data.jsonSerialized;
-
- // Fill accessory fields
- if (!Utils.isEmpty(meta.getJsonSizeFieldName())) {
- additionalRowFields[1] = data.jsonLength;
- }
+ Object[] additionalRowFields =
+ Utils.isEmpty(meta.getJsonSizeFieldName())
+ ? new Object[] {data.jsonSerialized}
+ : new Object[] {data.jsonSerialized, data.jsonLength};
Object[] outputRowData = RowDataUtil.addRowData(keyRow, keyRow.length,
additionalRowFields);
+ // addRowData over-allocates. The published row matches the metadata width.
+ if (outputRowData.length != data.outputRowMeta.size()) {
+ outputRowData = Arrays.copyOf(outputRowData, data.outputRowMeta.size());
+ }
incrementLinesOutput();
putRow(data.outputRowMeta, outputRowData);
@@ -419,6 +440,10 @@ public class JsonEOutput extends
BaseTransform<JsonEOutputMeta, JsonEOutputData>
* forced.
*/
private void writeFileItem(JsonNode item) throws HopException {
+ if (meta.isNewlineDelimited()) {
+ writeNdJsonItem(item);
+ return;
+ }
try {
if (data.fileItemCount == 0) {
data.pendingFileItem = item;
@@ -455,20 +480,60 @@ public class JsonEOutput extends
BaseTransform<JsonEOutputMeta, JsonEOutputData>
if (meta.isJsonPrettified()) {
data.fileGenerator.setPrettyPrinter(new DefaultPrettyPrinter());
}
- if (!Utils.isEmpty(meta.getJsonBloc())) {
+ if (!Utils.isEmpty(data.realBlocName)) {
data.fileGenerator.writeStartObject();
- data.fileGenerator.writeFieldName(meta.getJsonBloc());
+ data.fileGenerator.writeFieldName(data.realBlocName);
}
if (array) {
data.fileGenerator.writeStartArray();
}
}
+ /** One compact JSON value followed by LF. There is no outer document array.
*/
+ private void writeNdJsonItem(JsonNode item) throws HopException {
+ try {
+ if (!openNewFile()) {
+ throw new HopTransformException(
+ BaseMessages.getString(PKG, "JsonEOutput.Error.OpenNewFile",
buildFilename()));
+ }
+ if (data.fileGenerator == null) {
+ data.fileGenerator =
fileMapper.getFactory().createGenerator(data.writer);
+ // The file is closed separately, with its lineage.
+ data.fileGenerator.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET);
+ data.fileGenerator.setPrettyPrinter(null);
+ data.fileGenerator.setRootValueSeparator(null);
+ }
+ if (!Utils.isEmpty(data.realBlocName)) {
+ data.fileGenerator.writeStartObject();
+ data.fileGenerator.writeFieldName(data.realBlocName);
+ }
+ data.fileGenerator.writeTree(item);
+ if (!Utils.isEmpty(data.realBlocName)) {
+ data.fileGenerator.writeEndObject();
+ }
+ data.fileGenerator.writeRaw('\n');
+ } catch (IOException e) {
+ throw new HopTransformException(BaseMessages.getString(PKG,
"JsonEOutput.Error.Writing"), e);
+ }
+ data.fileItemCount++;
+ if (!data.isOutputValue) {
+ incrementLinesOutput();
+ }
+ int split = meta.getFileSettings().getSplitOutputAfter();
+ if (split > 0 && data.fileItemCount >= split) {
+ finishFile();
+ }
+ }
+
/** Close the JSON document and the file, if any item was written to it. */
private void finishFile() throws HopTransformException {
if (data.fileItemCount == 0) {
return;
}
+ if (meta.isNewlineDelimited()) {
+ closeNdJsonFile();
+ return;
+ }
try {
if (data.fileItemCount == 1) {
startFileDocument(meta.isUseArrayWithSingleInstance());
@@ -477,7 +542,7 @@ public class JsonEOutput extends
BaseTransform<JsonEOutputMeta, JsonEOutputData>
if (data.fileItemCount > 1 || meta.isUseArrayWithSingleInstance()) {
data.fileGenerator.writeEndArray();
}
- if (!Utils.isEmpty(meta.getJsonBloc())) {
+ if (!Utils.isEmpty(data.realBlocName)) {
data.fileGenerator.writeEndObject();
}
data.fileGenerator.close();
@@ -490,14 +555,33 @@ public class JsonEOutput extends
BaseTransform<JsonEOutputMeta, JsonEOutputData>
closeFile();
}
+ /** Close the NDJSON generator, then the writer. Do not finish an outer
array or object. */
+ private void closeNdJsonFile() throws HopTransformException {
+ try {
+ if (data.fileGenerator != null) {
+ data.fileGenerator.close();
+ }
+ } catch (IOException e) {
+ throw new HopTransformException(BaseMessages.getString(PKG,
"JsonEOutput.Error.Writing"), e);
+ }
+ String filename = data.openedFilename;
+ data.fileGenerator = null;
+ data.pendingFileItem = null;
+ data.fileItemCount = 0;
+ if (!closeFile()) {
+ throw new HopTransformException(
+ BaseMessages.getString(PKG, "JsonEOutput.Error.ClosingFile",
filename));
+ }
+ }
+
private void serializeJson(List<ObjectNode> jsonItemsList) throws
HopException {
ObjectNode theNode = new ObjectNode(nc);
Object listValue = meta.isUseArrayWithSingleInstance() ? jsonItemsList :
jsonItemsList.get(0);
try {
- if (!Utils.isEmpty(meta.getJsonBloc())) {
+ if (!Utils.isEmpty(data.realBlocName)) {
// TBD Try to understand if this can have a performance impact and do
it better...
theNode.set(
- meta.getJsonBloc(),
+ data.realBlocName,
mapper.readTree(
mapper.writeValueAsString(jsonItemsList.size() > 1 ?
jsonItemsList : listValue)));
if (meta.isJsonPrettified()) {
@@ -534,18 +618,16 @@ public class JsonEOutput extends
BaseTransform<JsonEOutputMeta, JsonEOutputData>
first = false;
data.inputRowMeta = getInputRowMeta();
data.inputRowMetaSize = data.inputRowMeta.size();
+ data.keysGroupIndexes = meta.resolveKeyFieldIndexes(data.inputRowMeta);
// Init previous row copy to this first row
prevRow = data.inputRowMeta.cloneRow(r); // copy the row to previous
// Create new structure for output fields
data.outputRowMeta = new RowMeta();
- List<JsonEOutputKeyField> keyFields = meta.getKeyFields();
for (int i = 0; i < meta.getKeyFields().size(); i++) {
- IValueMeta vmi =
- data.inputRowMeta.getValueMeta(
- data.inputRowMeta.indexOfValue(keyFields.get(i).getFieldName()));
- data.outputRowMeta.addValueMeta(i, vmi);
+ data.outputRowMeta.addValueMeta(
+ data.inputRowMeta.getValueMeta(data.keysGroupIndexes[i]).clone());
}
// This is JSON block's column
@@ -560,7 +642,6 @@ public class JsonEOutput extends
BaseTransform<JsonEOutputMeta, JsonEOutputData>
initDataFieldsPositionsArray();
- if (initKeyFieldsPositionArray(r)) return true;
return false;
}
@@ -591,21 +672,6 @@ public class JsonEOutput extends
BaseTransform<JsonEOutputMeta, JsonEOutputData>
}
}
- private boolean initKeyFieldsPositionArray(Object[] r) {
- data.keysGroupIndexes = new int[meta.getKeyFields().size()];
-
- for (int i = 0; i < meta.getKeyFields().size(); i++) {
- data.keysGroupIndexes[i] =
-
data.inputRowMeta.indexOfValue(meta.getKeyFields().get(i).getFieldName());
- if ((r != null) && (data.keysGroupIndexes[i] < 0)) {
- setErrors(1);
- stopAll();
- return true;
- }
- }
- return false;
- }
-
@Override
public void dispose() {
@@ -613,17 +679,15 @@ public class JsonEOutput extends
BaseTransform<JsonEOutputMeta, JsonEOutputData>
data.jsonKeyGroupItems = null;
}
- // The file was not finished, for example after an error: flush what was
written and close any
- // open array or object, so the partial file is at least well-formed.
- if (data.fileGenerator != null) {
+ // The parked item is a finished row or group. Write it, then close the
file. Do not finish
+ // the group that is still being built. End-of-input has already set the
count to 0.
+ if (data.fileItemCount > 0) {
try {
- data.fileGenerator.close();
- } catch (IOException e) {
- logError(BaseMessages.getString(PKG, "JsonOutput.Error.ClosingFile",
e.toString()));
+ finishFile();
+ } catch (HopTransformException e) {
+ logError(BaseMessages.getString(PKG, "JsonEOutput.Error.ClosingFile",
e.toString()));
setErrors(1);
}
- data.fileGenerator = null;
- data.pendingFileItem = null;
}
closeFile();
@@ -679,7 +743,7 @@ public class JsonEOutput extends
BaseTransform<JsonEOutputMeta, JsonEOutputData>
}
}
- public boolean openNewFile() {
+ public boolean openNewFile() throws HopTransformException {
if (data.writer != null) return true;
boolean retval = false;
@@ -687,6 +751,7 @@ public class JsonEOutput extends
BaseTransform<JsonEOutputMeta, JsonEOutputData>
String filename = buildFilename();
createParentFolder(filename);
+ validateNdJsonAppend(filename);
if (meta.isAddingToResult()) {
// Add this to the result file names...
ResultFile resultFile =
@@ -721,6 +786,8 @@ public class JsonEOutput extends
BaseTransform<JsonEOutputMeta, JsonEOutputData>
retval = true;
+ } catch (NdJsonAppendBoundaryException e) {
+ throw new HopTransformException(e.getMessage(), e);
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "JsonOutput.Error.OpeningFile",
e.toString()));
}
@@ -728,6 +795,29 @@ public class JsonEOutput extends
BaseTransform<JsonEOutputMeta, JsonEOutputData>
return retval;
}
+ /** An appended NDJSON file must already end in LF. The check does not read
the earlier lines. */
+ private void validateNdJsonAppend(String filename) throws Exception {
+ if (!meta.isNewlineDelimited() ||
!meta.getFileSettings().isFileAppended()) {
+ return;
+ }
+ try (FileObject file = HopVfs.getFileObject(filename, variables)) {
+ if (!file.exists()) {
+ return;
+ }
+ long size = file.getContent().getSize();
+ if (size == 0) {
+ return;
+ }
+ try (var stream = HopVfs.getInputStream(file)) {
+ stream.skipNBytes(size - 1);
+ if (stream.read() != '\n') {
+ throw new NdJsonAppendBoundaryException(
+ BaseMessages.getString(PKG,
"JsonEOutput.Error.NdJsonAppendBoundary", filename));
+ }
+ }
+ }
+ }
+
public String buildFilename() {
return meta.getFileSettings()
.buildFilename(variables, getCopy() + "", getPartitionId(),
data.splitnr + "", false);
@@ -765,4 +855,13 @@ public class JsonEOutput extends
BaseTransform<JsonEOutputMeta, JsonEOutputData>
return retval;
}
+
+ /** Append was refused before the output stream existed, so the existing
file is unchanged. */
+ private static final class NdJsonAppendBoundaryException extends IOException
{
+ @Serial private static final long serialVersionUID = 1L;
+
+ private NdJsonAppendBoundaryException(String message) {
+ super(message);
+ }
+ }
}
diff --git
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputDialog.java
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputDialog.java
index 85eedd17d6..03ba9a723d 100644
---
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputDialog.java
+++
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputDialog.java
@@ -18,7 +18,9 @@
package org.apache.hop.pipeline.transforms.jsonoutputenhanced;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import org.apache.hop.core.Const;
import org.apache.hop.core.Props;
import org.apache.hop.core.exception.HopException;
@@ -35,6 +37,8 @@ import org.apache.hop.ui.core.dialog.EnterSelectionDialog;
import org.apache.hop.ui.core.dialog.ErrorDialog;
import org.apache.hop.ui.core.dialog.MessageBox;
import org.apache.hop.ui.core.dialog.MessageDialogWithToggle;
+import org.apache.hop.ui.core.gui.GuiCompositeWidgets;
+import org.apache.hop.ui.core.gui.GuiCompositeWidgetsAdapter;
import org.apache.hop.ui.core.gui.GuiResource;
import org.apache.hop.ui.core.widget.ColumnInfo;
import org.apache.hop.ui.core.widget.ComboVar;
@@ -65,6 +69,10 @@ import org.eclipse.swt.widgets.Shell;
import org.eclipse.swt.widgets.TableItem;
public class JsonEOutputDialog extends BaseTransformDialog {
+ public static final String GUI_PLUGIN_ELEMENT_PARENT_ID =
JsonEOutputMeta.FORMAT_GUI_PARENT;
+
+ private GuiCompositeWidgets keyWidgets;
+ private GuiCompositeWidgets formatWidgets;
private static final Class<?> PKG = JsonEOutputMeta.class; // needed by
Translator!!
public static final String STRING_SORT_WARNING_PARAMETER = "JSONSortWarning";
@@ -232,8 +240,6 @@ public class JsonEOutputDialog extends BaseTransformDialog {
Composite wKeyConfigComp = new Composite(wTabFolder, SWT.NONE);
PropsUi.setLook(wKeyConfigComp);
- final int keyFieldsRows = input.getKeyFields().size();
-
keyColInf =
new ColumnInfo[] {
new ColumnInfo(
@@ -247,24 +253,24 @@ public class JsonEOutputDialog extends
BaseTransformDialog {
false)
};
keyColInf[1].setUsingVariables(true);
- wKeyFields =
- new TableView(
- variables,
- wKeyConfigComp,
- SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI,
- keyColInf,
- keyFieldsRows,
- lsMod,
- props);
-
- FormData fdKeyFields = new FormData();
- fdKeyFields.left = new FormAttachment(0, 0);
- fdKeyFields.top = new FormAttachment(0, 0);
- fdKeyFields.right = new FormAttachment(100, 0);
- fdKeyFields.bottom = new FormAttachment(100, 0);
- wKeyFields.setLayoutData(fdKeyFields);
-
wKeyConfigComp.setLayout(keyConfigLayout);
+ keyWidgets =
+ GuiCompositeWidgets.addScrolledComposite(
+ wKeyConfigComp,
+ variables,
+ null,
+ null,
+ JsonEOutputMeta.KEY_GUI_PARENT,
+ input,
+ widgets -> {
+ keyWidgets = widgets;
+ widgets.registerExtraGroup(
+ BaseMessages.getString(PKG,
"JsonEOutputDialog.KeyConfigTab.TabTitle"),
+ "0100",
+ null,
+ this::createKeyFieldsTable);
+ });
+ keyWidgets.setCompositeButtonsListener(source ->
getKeyFieldsFromPrevious());
wKeyConfigComp.layout();
wKeyConfigTab.setControl(wKeyConfigComp);
@@ -424,6 +430,24 @@ public class JsonEOutputDialog extends BaseTransformDialog
{
wFieldsComp.layout();
wFieldsTab.setControl(wFieldsComp);
+ CTabItem formatTab = new CTabItem(wTabFolder, SWT.NONE);
+ formatTab.setText(BaseMessages.getString(PKG,
"JsonEOutputDialog.FileFormat.TabTitle"));
+ Composite formatComp = new Composite(wTabFolder, SWT.NONE);
+ PropsUi.setLook(formatComp);
+ formatComp.setLayout(new FormLayout());
+ formatWidgets =
+ GuiCompositeWidgets.addScrolledComposite(
+ formatComp, variables, null, null, GUI_PLUGIN_ELEMENT_PARENT_ID,
input);
+ formatWidgets.setWidgetsListener(
+ new GuiCompositeWidgetsAdapter() {
+ @Override
+ public void widgetModified(
+ GuiCompositeWidgets widgets, Control control, String widgetId) {
+ input.setChanged();
+ }
+ });
+ formatTab.setControl(formatComp);
+
FormData fdTabFolder = new FormData();
fdTabFolder.left = new FormAttachment(0, 0);
fdTabFolder.top = new FormAttachment(wSpacer, margin);
@@ -1033,6 +1057,7 @@ public class JsonEOutputDialog extends
BaseTransformDialog {
}
private void getInfo(JsonEOutputMeta jsometa) {
+ formatWidgets.getWidgetsContents(jsometa, GUI_PLUGIN_ELEMENT_PARENT_ID);
jsometa.setJsonBloc(wBlocName.getText());
jsometa.setEncoding(wEncoding.getText());
@@ -1186,5 +1211,81 @@ public class JsonEOutputDialog extends
BaseTransformDialog {
wlAddToResult.setEnabled(activeFile);
wAddToResult.setEnabled(activeFile);
wbShowFiles.setEnabled(activeFile);
+ if (formatWidgets != null) {
+ for (Control control : formatWidgets.getWidgetsMap().values()) {
+ control.setEnabled(activeFile);
+ }
+ for (Control control : formatWidgets.getLabelsMap().values()) {
+ control.setEnabled(activeFile);
+ }
+ }
+ }
+
+ private void createKeyFieldsTable(Composite parent) {
+ wKeyFields =
+ new TableView(
+ variables,
+ parent,
+ SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI,
+ keyColInf,
+ input.getKeyFields().size(),
+ event -> input.setChanged(),
+ props);
+ Control button =
keyWidgets.getWidgetsMap().get(JsonEOutputMeta.WIDGET_GET_KEYS);
+ FormData fd = new FormData();
+ fd.left = new FormAttachment(0, 0);
+ fd.right = new FormAttachment(100, 0);
+ fd.top = new FormAttachment(button, margin);
+ fd.bottom = new FormAttachment(100, 0);
+ wKeyFields.setLayoutData(fd);
+ }
+
+ static List<String> suggestKeyFieldNames(
+ IRowMeta previous, List<String> outputNames, List<String> existingKeys) {
+ Set<String> excluded = new HashSet<>(outputNames);
+ excluded.addAll(existingKeys);
+ List<String> names = new ArrayList<>();
+ for (int i = 0; i < previous.size(); i++) {
+ String name = previous.getValueMeta(i).getName();
+ if (excluded.add(name)) {
+ names.add(name);
+ }
+ }
+ return names;
+ }
+
+ private void getKeyFieldsFromPrevious() {
+ try {
+ IRowMeta previous = pipelineMeta.getPrevTransformFields(variables,
transformName);
+ if (previous == null) {
+ return;
+ }
+ List<String> outputNames = new ArrayList<>();
+ for (TableItem item : wFields.getNonEmptyItems()) {
+ outputNames.add(item.getText(1));
+ }
+ List<String> existingKeys = new ArrayList<>();
+ for (TableItem item : wKeyFields.getNonEmptyItems()) {
+ existingKeys.add(item.getText(1));
+ }
+ List<String> additions = suggestKeyFieldNames(previous, outputNames,
existingKeys);
+ for (String name : additions) {
+ TableItem item = new TableItem(wKeyFields.table, SWT.NONE);
+ item.setText(1, name);
+ item.setText(2, name);
+ }
+ if (!additions.isEmpty()) {
+ wKeyFields.removeEmptyRows();
+ wKeyFields.setRowNums();
+ wKeyFields.optWidth(true);
+ input.setChanged();
+ }
+ } catch (HopException e) {
+ new ErrorDialog(
+ shell,
+ BaseMessages.getString(PKG, "System.Dialog.GetFieldsFailed.Title"),
+ BaseMessages.getString(PKG, "System.Dialog.GetFieldsFailed.Message"),
+ e);
+ }
}
}
diff --git
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputMeta.java
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputMeta.java
index bff6217327..86f3d670bc 100644
---
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputMeta.java
+++
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputMeta.java
@@ -27,9 +27,12 @@ import org.apache.hop.core.ICheckResult;
import org.apache.hop.core.annotations.Transform;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.gui.plugin.GuiWidgetGroupType;
import org.apache.hop.core.injection.Injection;
import org.apache.hop.core.row.IRowMeta;
-import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.value.ValueMetaInteger;
import org.apache.hop.core.row.value.ValueMetaString;
import org.apache.hop.core.util.Utils;
@@ -52,16 +55,51 @@ import org.w3c.dom.Node;
categoryDescription =
"i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Output",
keywords = "i18n::JsonEOutputMeta.keyword",
documentationUrl = "/pipeline/transforms/enhancedjsonoutput.html")
+@GuiPlugin
@Getter
@Setter
public class JsonEOutputMeta extends BaseTransformMeta<JsonEOutput,
JsonEOutputData> {
private static final Class<?> PKG = JsonEOutputMeta.class;
+
+ public static final String KEY_GUI_PARENT = "JsonEOutput.GroupKeys";
+ public static final String FORMAT_GUI_PARENT = "JsonEOutput.FileFormat";
+ public static final String WIDGET_GET_KEYS = "getKeyFields";
public static final String CONST_SPACES_LONG = " ";
public static final String CONST_SPACES = " ";
public static final String CONST_OUTPUT_VALUE = "outputValue";
public static final String CONST_KEY_FIELD = "key_field";
public static final String CONST_FIELD = "field";
+ @GuiWidgetElement(
+ id = WIDGET_GET_KEYS,
+ order = "0100",
+ type = GuiElementType.BUTTON,
+ label = "i18n::JsonEOutputDialog.Get.Button",
+ toolTip = "i18n::JsonEOutputDialog.GetKeys.Tooltip",
+ parentId = KEY_GUI_PARENT,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = "i18n::JsonEOutputDialog.KeyConfigTab.TabTitle",
+ groupOrder = "0100")
+ public void getKeyFieldsFromPrevious(Object source) {
+ // The dialog listener fills the live Group Key table. The button invoker
passes the meta.
+ }
+
+ @HopMetadataProperty(
+ key = "newline_delimited",
+ injectionKey = "NEWLINE_DELIMITED",
+ injectionKeyDescription = "JsonEOutput.Injection.NEWLINE_DELIMITED")
+ @GuiWidgetElement(
+ id = "newlineDelimited",
+ order = "0100",
+ type = GuiElementType.CHECKBOX,
+ label = "i18n::JsonEOutputDialog.NdJson.Label",
+ toolTip = "i18n::JsonEOutputDialog.NdJson.Tooltip",
+ parentId = FORMAT_GUI_PARENT,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = "i18n::JsonEOutputDialog.FileFormat.TabTitle",
+ groupOrder = "0100")
+ private boolean newlineDelimited;
+
@Getter
public enum OperationType implements IEnumHasCodeAndDescription {
OUTPUT_VALUE(
@@ -184,6 +222,23 @@ public class JsonEOutputMeta extends
BaseTransformMeta<JsonEOutput, JsonEOutputD
jsonBloc = "";
}
+ /**
+ * Locate every group key in {@code row}. The row is not modified. A missing
key is reported
+ * before any caller clears or rewrites its metadata.
+ */
+ public int[] resolveKeyFieldIndexes(IRowMeta row) throws
HopTransformException {
+ int[] indexes = new int[keyFields.size()];
+ for (int i = 0; i < keyFields.size(); i++) {
+ String name = keyFields.get(i).getFieldName();
+ indexes[i] = row.indexOfValue(name);
+ if (indexes[i] < 0) {
+ throw new HopTransformException(
+ BaseMessages.getString(PKG,
"JsonEOutput.Error.GroupFieldNotFound", name));
+ }
+ }
+ return indexes;
+ }
+
@Override
public void getFields(
IRowMeta row,
@@ -196,12 +251,10 @@ public class JsonEOutputMeta extends
BaseTransformMeta<JsonEOutput, JsonEOutputD
if (getOperationType() != OperationType.WRITE_TO_FILE) {
IRowMeta rowMeta = row.clone();
+ int[] keyIndexes = resolveKeyFieldIndexes(rowMeta);
row.clear();
-
- for (int i = 0; i < this.getKeyFields().size(); i++) {
- JsonEOutputKeyField keyField = this.getKeyFields().get(i);
- IValueMeta vmi =
rowMeta.getValueMeta(rowMeta.indexOfValue(keyField.getFieldName()));
- row.addValueMeta(i, vmi);
+ for (int keyIndex : keyIndexes) {
+ row.addValueMeta(rowMeta.getValueMeta(keyIndex).clone());
}
ValueMetaString vm = new ValueMetaString(this.getOutputValue());
@@ -247,6 +300,11 @@ public class JsonEOutputMeta extends
BaseTransformMeta<JsonEOutput, JsonEOutputD
}
// Check output fields
if (prev != null && !prev.isEmpty()) {
+ try {
+ resolveKeyFieldIndexes(prev);
+ } catch (HopTransformException e) {
+ remarks.add(new CheckResult(ICheckResult.TYPE_RESULT_ERROR,
e.getMessage(), transformMeta));
+ }
cr =
new CheckResult(
ICheckResult.TYPE_RESULT_OK,
diff --git
a/plugins/transforms/json/src/main/resources/org/apache/hop/pipeline/transforms/jsoninput/messages/messages_en_US.properties
b/plugins/transforms/json/src/main/resources/org/apache/hop/pipeline/transforms/jsoninput/messages/messages_en_US.properties
index 3fe6627064..368d894085 100644
---
a/plugins/transforms/json/src/main/resources/org/apache/hop/pipeline/transforms/jsoninput/messages/messages_en_US.properties
+++
b/plugins/transforms/json/src/main/resources/org/apache/hop/pipeline/transforms/jsoninput/messages/messages_en_US.properties
@@ -58,6 +58,10 @@ TextFileInput.Injection.INCLUDE_SUBFOLDERS=Set this flag to
include subfolders w
JsonInput.Injection.READ_SOURCE_AS_URL=Set this flag to treat the source as a
URL instead of a filename.
JsonInput.Injection.REMOVE_SOURCE_FIELDS=Set this flag to remove the source
fields from the output of the transform.
JsonInput.Injection.DEFAULT_PATH_LEAF_TO_NULL=Default path leaf to null
+JsonInput.Injection.RESOLVE_JSON_PATHS=Resolve variables in JSONPath
expressions? (Y/N)
+JsonInputMeta.ResolveJsonPaths.Label=Resolve variables in JSONPath
+JsonInputMeta.ResolveJsonPaths.Tooltip=Clear this option to use extraction
paths literally, including root array indexes such as '$[0]'. When selected,
existing variable expressions such as '${JSON_PATH}' are resolved.
+JsonInputDialog.PathOptions.TabTitle=JSONPath
JsonInput.Injection.ROW_LIMIT=The maximum number of lines to read.
JsonInput.Injection.ROW_NUMBER_FIELDNAME=The fieldname to use to output the
row number.
JsonInput.Injection.ROW_NUMBER_OUTPUT=Set this flag to include the row number
in the output.
diff --git
a/plugins/transforms/json/src/main/resources/org/apache/hop/pipeline/transforms/jsonoutputenhanced/messages/messages_en_US.properties
b/plugins/transforms/json/src/main/resources/org/apache/hop/pipeline/transforms/jsonoutputenhanced/messages/messages_en_US.properties
index 74e00f0234..9b2234adc3 100644
---
a/plugins/transforms/json/src/main/resources/org/apache/hop/pipeline/transforms/jsonoutputenhanced/messages/messages_en_US.properties
+++
b/plugins/transforms/json/src/main/resources/org/apache/hop/pipeline/transforms/jsonoutputenhanced/messages/messages_en_US.properties
@@ -22,8 +22,11 @@ EnhancedJsonOutput.name=Enhanced JSON output
JsonEOutput.Error.Casting=Error casting String to JSON Fragment
JsonEOutput.Error.ClosingFile=Exception trying to close file: {0}
JsonEOutput.Error.ErrorCreatingParentFolder=Could not create parent folder
[{0}]!
+JsonEOutput.Error.GroupFieldNotFound=Group key field ''{0}'' was not found in
the incoming row.
JsonEOutput.Error.MissingOutputFieldName=Output fieldname that will contain
value is empty\!
JsonEOutput.Error.MissingTargetFilename=Target filename is missing\!
+JsonEOutput.Error.NdJsonAppendBoundary=Cannot append NDJSON to [{0}]: the
nonempty existing file must end with LF.
+JsonEOutput.Error.NdJsonSettings=NDJSON file output requires UTF-8 encoding
and Pretty Print JSON disabled.
JsonEOutput.Error.OpeningFile=Error opening new file : {0}
JsonEOutput.Error.OpenNewFile=Could not open file [{0}]!
JsonEOutput.Error.ParentFolderNotExist=Folder parent [{0}] does not exist!
@@ -82,6 +85,11 @@ JsonEOutputDialog.FieldsTab.TabTitle=Fields
JsonEOutputDialog.Filename.Label=Filename
JsonEOutputDialog.GeneralTab.TabTitle=General
JsonEOutputDialog.Get.Button=\ &Get Fields
+JsonEOutputDialog.GetKeys.Tooltip=Add input fields not already listed in
Fields or Group Key. Existing keys and element names are retained.
+JsonEOutputDialog.FileFormat.TabTitle=File format
+JsonEOutputDialog.NdJson.Label=Newline-delimited JSON (NDJSON)
+JsonEOutputDialog.NdJson.Tooltip=Write one compact JSON item per line.
Requires UTF-8 and Pretty Print JSON disabled. Applies only to file output.
+JsonEOutput.Injection.NEWLINE_DELIMITED=Write newline-delimited JSON to the
output file? (Y/N)
JsonEOutputDialog.Get.Tooltip=Get the fields as defined in previous transforms.
JsonEOutputDialog.Group.File.Label=Output File
JsonEOutputDialog.Group.Settings.Label=Settings
diff --git
a/plugins/transforms/json/src/main/resources/org/apache/hop/pipeline/transforms/jsonoutputenhanced/messages/messages_it_IT.properties
b/plugins/transforms/json/src/main/resources/org/apache/hop/pipeline/transforms/jsonoutputenhanced/messages/messages_it_IT.properties
index cef455f541..3a9cd577af 100644
---
a/plugins/transforms/json/src/main/resources/org/apache/hop/pipeline/transforms/jsonoutputenhanced/messages/messages_it_IT.properties
+++
b/plugins/transforms/json/src/main/resources/org/apache/hop/pipeline/transforms/jsonoutputenhanced/messages/messages_it_IT.properties
@@ -37,7 +37,9 @@ JsonOutputDialog.AddTime.Label=Includere l''orario nel nome
del file?
JsonOutputDialog.Append.Label=Accoda
JsonOutputDialog.Append.Tooltip=Accoda il file (se esiste) altrimenti
verr\u00E0 sovrascritto
JsonOutputDialog.BlocName.Label=Nome blocco Json
-JsonOutputDialog.CompatibilityMode.Tooltip=Selezionare quest''opzione per
generare strutture JSON come se fossero state create con precedenti versioni
(prima di Apache Hop 4.3.0)
+JsonOutputDialog.CompatibilityMode.Tooltip=Selezionare quest''opzione per
generare array JSON anche quando la lista contiene un solo elemento
+JsonEOutputDialog.CompatibilityMode.Tooltip=Selezionare quest''opzione per
generare array JSON anche quando la lista contiene un solo elemento
+JsonEOutputDialog.UseArrayWihSingleInstanceMode.Tooltip=Selezionare
quest''opzione per generare array JSON anche quando la lista contiene un solo
elemento
JsonOutputDialog.CreateParentFolder.Label=Crea cartella padre
JsonOutputDialog.CreateParentFolder.Tooltip=Selezionare qua per creare la
cartella padre quando necessario. \nAltrimenti Apache Hop lancer\u00E0
un''eccezione quando non esiste la cartella padre.
JsonOutputDialog.DialogTitle=Output JSON
diff --git
a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMetaTest.java
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMetaTest.java
index b731a86c96..a1b5ca1819 100644
---
a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMetaTest.java
+++
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMetaTest.java
@@ -108,6 +108,17 @@ class JsonInputMetaTest {
"valueField must sync acceptingField after XML load");
}
+ @Test
+ void missingResolveJsonPathsPropertyDefaultsToTrue() throws Exception {
+ JsonInputMeta loaded = new JsonInputMeta();
+ XmlMetadataUtil.deSerializeFromXml(
+ XmlHandler.loadXmlString("<transform><fields/></transform>",
TransformMeta.XML_TAG),
+ JsonInputMeta.class,
+ loaded,
+ new MemoryMetadataProvider());
+ assertTrue(loaded.isResolveJsonPaths());
+ }
+
private static void validate(JsonInputMeta meta) {
assertNotNull(meta.getInputFields());
assertNotNull(meta.getFileInput().getInputFiles());
diff --git
a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputPathOptionsDialogTest.java
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputPathOptionsDialogTest.java
new file mode 100644
index 0000000000..53045d811c
--- /dev/null
+++
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputPathOptionsDialogTest.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.jsoninput;
+
+import static
org.eclipse.swtbot.swt.finder.matchers.WidgetMatcherFactory.widgetOfType;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.gui.plugin.GuiRegistry;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.ui.testing.SwtBotTestBase;
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.custom.CTabFolder;
+import org.eclipse.swt.custom.CTabItem;
+import org.eclipse.swt.widgets.Button;
+import org.eclipse.swt.widgets.Control;
+import org.eclipse.swt.widgets.Label;
+import org.eclipse.swtbot.swt.finder.SWTBot;
+import org.eclipse.swtbot.swt.finder.utils.SWTBotPreferences;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+
+@Tag("uitest")
+class JsonInputPathOptionsDialogTest extends SwtBotTestBase {
+ /**
+ * JSON Input builds several tables before its event loop starts. SWTBot's 5
s default expires
+ * while that construction is still on the UI thread, so the JSONPath tab is
missed.
+ */
+ private static final long DIALOG_TIMEOUT_MS = 30_000L;
+
+ private static long defaultTimeout;
+
+ @BeforeAll
+ static void slowDownSwtBot() {
+ defaultTimeout = SWTBotPreferences.TIMEOUT;
+ SWTBotPreferences.TIMEOUT = DIALOG_TIMEOUT_MS;
+ }
+
+ @AfterAll
+ static void restoreSwtBotTimeout() {
+ SWTBotPreferences.TIMEOUT = defaultTimeout;
+ }
+
+ @BeforeAll
+ static void registerWidgets() {
+ GuiRegistry registry = GuiRegistry.getInstance();
+ for (Field field : JsonInputMeta.class.getDeclaredFields()) {
+ GuiWidgetElement element = field.getAnnotation(GuiWidgetElement.class);
+ if (element != null) {
+ registry.addGuiWidgetElement(JsonInputMeta.class.getName(), element,
field);
+ }
+ }
+ }
+
+ private PipelineMeta pipeline(JsonInputMeta meta) {
+ PipelineMeta pipeline = new PipelineMeta();
+ pipeline.setMetadataProvider(new MemoryMetadataProvider());
+ pipeline.addTransform(new TransformMeta("JsonInput", "json", meta));
+ return pipeline;
+ }
+
+ @Test
+ void okPersistsLiteralPathMode() {
+ JsonInputMeta meta = new JsonInputMeta();
+ PipelineMeta pipeline = pipeline(meta);
+ withDialog(
+ parent -> new JsonInputDialog(parent, new Variables(), meta,
pipeline).open(),
+ bot -> {
+ SWTBot dialog =
+ bot.shell(BaseMessages.getString(JsonInputMeta.class,
"JsonInputDialog.DialogTitle"))
+ .activate()
+ .bot();
+ activateTab(dialog, "JSONPath");
+ setResolveJsonPaths(dialog, false);
+ dialog.button(buttonLabel("System.Button.OK")).click();
+ });
+ assertFalse(meta.isResolveJsonPaths());
+ }
+
+ @Test
+ void cancelDoesNotPersistLiteralPathMode() {
+ JsonInputMeta meta = new JsonInputMeta();
+ PipelineMeta pipeline = pipeline(meta);
+ withDialog(
+ parent -> new JsonInputDialog(parent, new Variables(), meta,
pipeline).open(),
+ bot -> {
+ SWTBot dialog =
+ bot.shell(BaseMessages.getString(JsonInputMeta.class,
"JsonInputDialog.DialogTitle"))
+ .activate()
+ .bot();
+ activateTab(dialog, "JSONPath");
+ setResolveJsonPaths(dialog, false);
+ dialog.button(buttonLabel("System.Button.Cancel")).click();
+ });
+ assertTrue(meta.isResolveJsonPaths());
+ }
+
+ /**
+ * SWTBot's finder walks controls and does not see the items of a {@code
CTabFolder}. Select the
+ * tab on the folder, the same way the other transform dialog tests do.
+ */
+ private void activateTab(SWTBot dialog, String title) {
+ CTabFolder tabFolder = (CTabFolder)
dialog.widget(widgetOfType(CTabFolder.class));
+ display.syncExec(
+ () -> {
+ List<String> titles = new ArrayList<>();
+ for (CTabItem item : tabFolder.getItems()) {
+ String itemTitle = item.getText() == null ? "" :
item.getText().trim();
+ titles.add(itemTitle);
+ if (title.equals(itemTitle)) {
+ tabFolder.setSelection(item);
+ tabFolder.layout();
+ return;
+ }
+ }
+ throw new AssertionError("no tab titled " + title + " in " + titles);
+ });
+ }
+
+ /**
+ * The annotated checkbox keeps its text on a separate label. The button
itself has no title, so a
+ * {@code checkBox("Resolve variables in JSONPath")} lookup does not see it.
+ */
+ private void setResolveJsonPaths(SWTBot dialog, boolean selected) {
+ Label label = dialog.label("Resolve variables in JSONPath").widget;
+ display.syncExec(
+ () -> {
+ for (Control child : label.getParent().getChildren()) {
+ if (child instanceof Button button && (button.getStyle() &
SWT.CHECK) != 0) {
+ button.setSelection(selected);
+ return;
+ }
+ }
+ throw new AssertionError("no checkbox next to Resolve variables in
JSONPath");
+ });
+ }
+}
diff --git
a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputTest.java
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputTest.java
index da7199790c..ca49eb6752 100644
---
a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputTest.java
+++
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputTest.java
@@ -751,6 +751,99 @@ class JsonInputTest {
assertEquals(1, jsonInput.getLinesWritten(), "rows written");
}
+ @Test
+ void jsonTypedSourceFieldExtractsAValue() throws Exception {
+ JsonInputField field = new JsonInputField("value");
+ field.setPath("$.value");
+ field.setType(IValueMeta.TYPE_STRING);
+ JsonInputMeta meta = createSimpleMeta("json", field);
+ meta.setRemoveSourceField(true);
+ JsonNode input = HopJson.newMapper().readTree("{\"value\":\"ok\"}");
+ JsonInput transform = createJsonInputWithJsonNode("json", meta, new
Object[] {input});
+ List<Object[]> rows = new ArrayList<>();
+ transform.addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) {
+ rows.add(row.clone());
+ }
+ });
+ try {
+ processRows(transform, 3);
+ assertEquals(0, transform.getErrors());
+ assertEquals(1, rows.size());
+ assertEquals("ok", rows.getFirst()[0]);
+ } finally {
+ transform.dispose();
+ }
+ }
+
+ @Test
+ void literalRootArrayPathExtractsTheFirstValue() throws Exception {
+ String path = "$[0].similarity";
+ String input = "[{\"Id\":3902113,\"similarity\":0.95}]";
+ JsonInputField field = new JsonInputField("similarity");
+ field.setPath(path);
+ field.setType(IValueMeta.TYPE_NUMBER);
+ JsonInputMeta meta = createSimpleMeta("json", field);
+ meta.setResolveJsonPaths(false);
+ meta.setRemoveSourceField(true);
+ JsonInput transform = createJsonInput("json", meta, new Object[] {input});
+ List<Object[]> rows = new ArrayList<>();
+ transform.addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) {
+ rows.add(row.clone());
+ }
+ });
+ try {
+ assertEquals(0, transform.getErrors(), "literal JSONPath must compile");
+ processRows(transform, 3);
+ assertEquals(0, transform.getErrors());
+ assertEquals(1, rows.size());
+ assertEquals(0.95d, rows.getFirst()[0]);
+ assertEquals(path, meta.getInputFields().getFirst().getPath());
+ } finally {
+ transform.dispose();
+ }
+ }
+
+ @Test
+ void serializedLiteralPathModeStillExtractsRootArray() throws Exception {
+ JsonInputMeta original = new JsonInputMeta();
+ original.setResolveJsonPaths(false);
+ String xml =
+ "<transform>"
+ +
org.apache.hop.metadata.serializer.xml.XmlMetadataUtil.serializeObjectToXml(original)
+ + "</transform>";
+ JsonInputMeta loaded = new JsonInputMeta();
+ org.apache.hop.metadata.serializer.xml.XmlMetadataUtil.deSerializeFromXml(
+ org.apache.hop.core.xml.XmlHandler.loadXmlString(
+ xml, org.apache.hop.pipeline.transform.TransformMeta.XML_TAG),
+ JsonInputMeta.class,
+ loaded,
+ new
org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider());
+ assertFalse(loaded.isResolveJsonPaths());
+ JsonInputField field = new JsonInputField("value");
+ field.setPath("$[0].value");
+ field.setType(IValueMeta.TYPE_STRING);
+ loaded.setInputFields(new ArrayList<>(List.of(field)));
+ loaded.setInFields(true);
+ loaded.setFieldValue("json");
+ loaded.setRemoveSourceField(true);
+ JsonInput transform = createJsonInput("json", loaded, new Object[]
{"[{\"value\":\"ok\"}]"});
+ RowComparatorListener rows = new RowComparatorListener(new Object[]
{"ok"});
+ transform.addRowListener(rows);
+ try {
+ processRows(transform, 3);
+ assertEquals(0, transform.getErrors());
+ assertEquals(1, rows.rowNbr);
+ } finally {
+ transform.dispose();
+ }
+ }
+
@Test
void testArrayOut() throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
diff --git
a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/reader/RowOutputConverterNullTest.java
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/reader/RowOutputConverterNullTest.java
new file mode 100644
index 0000000000..378f71dd0d
--- /dev/null
+++
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/reader/RowOutputConverterNullTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.jsoninput.reader;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+
+import com.fasterxml.jackson.databind.node.NullNode;
+import com.fasterxml.jackson.databind.node.TextNode;
+import java.util.BitSet;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.pipeline.transforms.jsoninput.JsonInputData;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+class RowOutputConverterNullTest {
+ @ParameterizedTest
+ @ValueSource(strings = {"Y", "N"})
+ void jsonNullIsNotConvertedIntoAnEmptyString(String distinguish) throws
Exception {
+ String before =
System.getProperty(Const.HOP_EMPTY_STRING_DIFFERS_FROM_NULL);
+ try {
+ System.setProperty(Const.HOP_EMPTY_STRING_DIFFERS_FROM_NULL,
distinguish);
+ JsonInputData data = new JsonInputData();
+ data.outputRowMeta = new RowMeta();
+ data.outputRowMeta.addValueMeta(new ValueMetaString("nullValue"));
+ data.outputRowMeta.addValueMeta(new ValueMetaString("emptyValue"));
+ data.convertRowMeta =
data.outputRowMeta.cloneToType(ValueMetaString.TYPE_STRING);
+ data.totalpreviousfields = 0;
+ data.repeatedFields = new BitSet();
+ RowOutputConverter converter = new
RowOutputConverter(mock(ILogChannel.class));
+ Object[] row =
+ converter.getRow(
+ new Object[0], new Object[] {NullNode.getInstance(),
TextNode.valueOf("")}, data);
+ assertNull(row[0]);
+ assertEquals("", row[1]);
+ if ("Y".equals(distinguish)) {
+ assertTrue(!data.outputRowMeta.getValueMeta(1).isNull(row[1]));
+ }
+ } finally {
+ if (before == null) {
+ System.clearProperty(Const.HOP_EMPTY_STRING_DIFFERS_FROM_NULL);
+ } else {
+ System.setProperty(Const.HOP_EMPTY_STRING_DIFFERS_FROM_NULL, before);
+ }
+ }
+ }
+}
diff --git
a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputBehaviorTest.java
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputBehaviorTest.java
new file mode 100644
index 0000000000..c16b0d00d0
--- /dev/null
+++
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputBehaviorTest.java
@@ -0,0 +1,536 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.jsonoutputenhanced;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.hop.core.HopClientEnvironment;
+import org.apache.hop.core.RowMetaAndData;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.json.HopJson;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+class JsonEOutputBehaviorTest {
+ private final String base = "ram:///enhanced-json-" + UUID.randomUUID();
+
+ @BeforeAll
+ static void initialize() throws HopException {
+ HopClientEnvironment.init();
+ }
+
+ @AfterEach
+ void removeFiles() throws Exception {
+ try (FileObject file = HopVfs.getFileObject(base)) {
+ if (file.exists()) {
+ file.deleteAll();
+ }
+ }
+ }
+
+ private JsonEOutputMeta meta(JsonEOutputMeta.OperationType operation) {
+ JsonEOutputMeta meta = new JsonEOutputMeta();
+ meta.setOperationType(operation);
+ meta.setOutputValue("rows");
+ meta.setJsonBloc("");
+ meta.setEncoding("UTF-8");
+ JsonEOutputField field = new JsonEOutputField();
+ field.setFieldName("payload");
+ field.setElementName("payload");
+ meta.getOutputFields().add(field);
+ meta.getFileSettings().setFileName(base + "/out");
+ meta.getFileSettings().setExtension("json");
+ meta.getFileSettings().setCreateParentFolder(true);
+ meta.getFileSettings().setDoNotOpenNewFileInit(true);
+ return meta;
+ }
+
+ private static String read(String path) throws Exception {
+ try (FileObject file = HopVfs.getFileObject(path);
+ var input = HopVfs.getInputStream(file)) {
+ return new String(input.readAllBytes(), StandardCharsets.UTF_8);
+ }
+ }
+
+ private static JsonNode parse(String json) throws Exception {
+ return HopJson.newMapper()
+ .enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
+ .readTree(json);
+ }
+
+ private static String payload(String value) {
+ return HopJson.newMapper().createObjectNode().put("payload",
value).toString();
+ }
+
+ private static IRowMeta rowMeta(String... names) {
+ IRowMeta row = new RowMeta();
+ for (String name : names) {
+ row.addValueMeta(new ValueMetaString(name));
+ }
+ return row;
+ }
+
+ private static final class Harness implements AutoCloseable {
+ final TransformMockHelper<JsonEOutputMeta, JsonEOutputData> helper;
+ final List<RowMetaAndData> written = new ArrayList<>();
+ final JsonEOutputData data;
+ final JsonEOutput transform;
+
+ Harness(JsonEOutputMeta meta, IRowMeta inputMeta, Object[]... rows) {
+ helper =
+ new TransformMockHelper<>("enhanced JSON", JsonEOutputMeta.class,
JsonEOutputData.class);
+ when(helper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+ .thenReturn(helper.iLogChannel);
+ when(helper.pipeline.isRunning()).thenReturn(true);
+ when(helper.transformMeta.getTransform()).thenReturn(meta);
+ var iterator = Arrays.asList(rows).iterator();
+ data = new JsonEOutputData();
+ transform =
+ new JsonEOutput(
+ helper.transformMeta, meta, data, 0, helper.pipelineMeta,
helper.pipeline) {
+ @Override
+ public Object[] getRow() {
+ return iterator.hasNext() ? iterator.next() : null;
+ }
+
+ @Override
+ public void putRow(IRowMeta rowMeta, Object[] row) {
+ written.add(new RowMetaAndData(rowMeta.clone(), row.clone()));
+ }
+ };
+ transform.setInputRowMeta(inputMeta);
+ }
+
+ void run() throws HopException {
+ assertTrue(transform.init());
+ while (transform.processRow()) {
+ // Drain the real transform through EOF.
+ }
+ }
+
+ @Override
+ public void close() {
+ try {
+ transform.dispose();
+ } finally {
+ helper.cleanUp();
+ }
+ }
+ }
+
+ @Test
+ void missingRuntimeGroupKeyIsAnActionableError() throws Exception {
+ JsonEOutputMeta meta = meta(JsonEOutputMeta.OperationType.OUTPUT_VALUE);
+ meta.getKeyFields().add(new JsonEOutputKeyField("gone"));
+ try (Harness h = new Harness(meta, rowMeta("payload"), new Object[]
{"x"})) {
+ assertTrue(h.transform.init());
+ HopException error = assertThrows(HopException.class,
h.transform::processRow);
+ assertTrue(error.getMessage().contains("gone"), error.getMessage());
+ }
+ }
+
+ @Test
+ void missingKeyIsNamedDuringSchemaDiscovery() {
+ JsonEOutputMeta meta = meta(JsonEOutputMeta.OperationType.OUTPUT_VALUE);
+ meta.getKeyFields().add(new JsonEOutputKeyField("gone"));
+ IRowMeta input = rowMeta("payload");
+ HopTransformException error =
+ assertThrows(
+ HopTransformException.class,
+ () -> meta.getFields(input, "json", null, null, new Variables(),
null));
+ assertTrue(error.getMessage().contains("gone"), error.getMessage());
+ assertEquals("payload", input.getValueMeta(0).getName());
+ }
+
+ @Test
+ void bothKeepsRowShapeAndWritesEveryGroup() throws Exception {
+ JsonEOutputMeta meta = meta(JsonEOutputMeta.OperationType.BOTH);
+ JsonEOutputKeyField key = new JsonEOutputKeyField("grp");
+ key.setElementName("group");
+ meta.getKeyFields().add(key);
+ try (Harness h =
+ new Harness(
+ meta,
+ rowMeta("grp", "payload"),
+ new Object[] {"a", "x"},
+ new Object[] {"a", "y"},
+ new Object[] {"b", "z"})) {
+ h.run();
+ assertEquals(2, h.written.size());
+ for (RowMetaAndData row : h.written) {
+ assertEquals(row.getRowMeta().size(), row.getData().length);
+ assertTrue(row.getRowMeta().indexOfValue("grp") >= 0);
+ assertEquals(-1, row.getRowMeta().indexOfValue("group"));
+ }
+ }
+ JsonNode file = parse(read(base + "/out.json"));
+ assertEquals(2, file.size());
+ assertEquals("a", file.get(0).get("group").asText());
+ assertEquals(2, file.get(0).get("rows").size());
+ assertEquals("z", file.get(1).get("rows").get("payload").asText());
+ }
+
+ @Test
+ void keySuggestionsExcludeLiveOutputAndExistingKeys() {
+ assertEquals(
+ List.of("grp"),
+ JsonEOutputDialog.suggestKeyFieldNames(
+ rowMeta("payload", "grp", "already"), List.of("payload"),
List.of("already")));
+ assertEquals(
+ List.of(),
+ JsonEOutputDialog.suggestKeyFieldNames(
+ rowMeta("payload", "grp"), List.of("payload"), List.of("grp")));
+ }
+
+ @Test
+ void disposalClosesAnActiveFileGenerator() throws Exception {
+ JsonEOutputMeta meta = meta(JsonEOutputMeta.OperationType.WRITE_TO_FILE);
+ try (Harness h =
+ new Harness(
+ meta, rowMeta("payload"), new Object[] {"a"}, new Object[] {"b"},
new Object[] {"c"})) {
+ assertTrue(h.transform.init());
+ assertTrue(h.transform.processRow());
+ assertTrue(h.transform.processRow());
+ var generator = h.data.fileGenerator;
+ assertNotNull(generator);
+ h.transform.dispose();
+ assertTrue(generator.isClosed());
+ assertNull(h.data.fileGenerator);
+ assertNull(h.data.writer);
+ }
+ }
+
+ /**
+ * The pipeline attached to issue #2569: pretty-printed file output, one
group key with an element
+ * name, a JSON block around the file, and a mix of one-row and many-row
groups. Parsed structure
+ * is what matters; pretty-print whitespace is not.
+ */
+ @Test
+ void issue2569SampleGroupsPrettyFileByKeyAlias() throws Exception {
+ JsonEOutputMeta meta = meta(JsonEOutputMeta.OperationType.WRITE_TO_FILE);
+ meta.setOutputValue("lvl1Details");
+ meta.setJsonBloc("result");
+ meta.setJsonPrettified(true);
+ meta.getOutputFields().clear();
+ JsonEOutputField field2 = new JsonEOutputField();
+ field2.setFieldName("Field2");
+ field2.setElementName("campo2");
+ JsonEOutputField field3 = new JsonEOutputField();
+ field3.setFieldName("Field3");
+ field3.setElementName("campo3");
+ meta.getOutputFields().add(field2);
+ meta.getOutputFields().add(field3);
+ JsonEOutputKeyField key = new JsonEOutputKeyField("Field1");
+ key.setElementName("recordKey");
+ meta.getKeyFields().add(key);
+
+ RowMeta input = new RowMeta();
+ input.addValueMeta(new ValueMetaString("Field1"));
+ input.addValueMeta(new ValueMetaString("Field2"));
+ input.addValueMeta(new ValueMetaInteger("Field3"));
+ try (Harness h =
+ new Harness(
+ meta,
+ input,
+ new Object[] {"A", "B", 2L},
+ new Object[] {"B", "C", 1L},
+ new Object[] {"B", "C", 2L},
+ new Object[] {"B", "D", 4L},
+ new Object[] {"C", "F", 5L},
+ new Object[] {"C", "F", 6L},
+ new Object[] {"C", "V", 6L},
+ new Object[] {"C", "B", 7L})) {
+ h.run();
+ }
+ String content = read(base + "/out.json");
+ assertTrue(content.contains("\n"), content);
+ JsonNode file = parse(content);
+ JsonNode result = file.get("result");
+ assertEquals(3, result.size());
+ assertEquals("A", result.get(0).get("recordKey").asText());
+ assertEquals("B", result.get(0).get("lvl1Details").get("campo2").asText());
+ assertEquals(2, result.get(0).get("lvl1Details").get("campo3").asInt());
+ assertFalse(result.get(0).get("lvl1Details").isArray());
+ assertEquals(3, result.get(1).get("lvl1Details").size());
+ assertEquals(4,
result.get(1).get("lvl1Details").get(2).get("campo3").asInt());
+ assertEquals(4, result.get(2).get("lvl1Details").size());
+ assertEquals("B",
result.get(2).get("lvl1Details").get(3).get("campo2").asText());
+ assertEquals(7,
result.get(2).get("lvl1Details").get(3).get("campo3").asInt());
+ }
+
+ @Test
+ void ndjsonAppendEscapesEmbeddedNewlines() throws Exception {
+ JsonEOutputMeta meta = meta(JsonEOutputMeta.OperationType.WRITE_TO_FILE);
+ meta.setNewlineDelimited(true);
+ meta.getFileSettings().setFileAppended(true);
+ try (Harness h = new Harness(meta, rowMeta("payload"), new Object[]
{"a\nb"})) {
+ h.run();
+ }
+ try (Harness h = new Harness(meta, rowMeta("payload"), new Object[]
{"c\rd"})) {
+ h.run();
+ }
+ String content = read(base + "/out.json");
+ assertEquals(payload("a\nb") + "\n" + payload("c\rd") + "\n", content);
+ List<String> lines = content.lines().toList();
+ assertEquals(2, lines.size());
+ assertEquals("a\nb", parse(lines.get(0)).get("payload").asText());
+ assertEquals("c\rd", parse(lines.get(1)).get("payload").asText());
+ }
+
+ @Test
+ void ndjsonSplitsRecordsWithoutOuterArrays() throws Exception {
+ JsonEOutputMeta meta = meta(JsonEOutputMeta.OperationType.WRITE_TO_FILE);
+ meta.setNewlineDelimited(true);
+ meta.setUseArrayWithSingleInstance(true);
+ meta.getFileSettings().setSplitOutputAfter(2);
+ try (Harness h =
+ new Harness(
+ meta, rowMeta("payload"), new Object[] {"x"}, new Object[] {"y"},
new Object[] {"z"})) {
+ h.run();
+ assertEquals(3, h.written.size());
+ }
+ assertEquals(payload("x") + "\n" + payload("y") + "\n", read(base +
"/out_0.json"));
+ assertEquals(payload("z") + "\n", read(base + "/out_1.json"));
+ try (FileObject extra = HopVfs.getFileObject(base + "/out_2.json")) {
+ assertFalse(extra.exists());
+ }
+ }
+
+ @Test
+ void ndjsonWrapsEachCompleteGroup() throws Exception {
+ JsonEOutputMeta meta = meta(JsonEOutputMeta.OperationType.BOTH);
+ meta.setNewlineDelimited(true);
+ meta.setJsonBloc("data");
+ JsonEOutputKeyField key = new JsonEOutputKeyField("grp");
+ key.setElementName("group");
+ meta.getKeyFields().add(key);
+ try (Harness h =
+ new Harness(
+ meta,
+ rowMeta("grp", "payload"),
+ new Object[] {"a", "x"},
+ new Object[] {"a", "y"},
+ new Object[] {"b", "z"})) {
+ h.run();
+ assertEquals(2, h.written.size());
+ }
+ List<String> lines = read(base + "/out.json").lines().toList();
+ assertEquals(2, lines.size());
+ JsonNode first = parse(lines.get(0)).get("data");
+ JsonNode last = parse(lines.get(1)).get("data");
+ assertEquals("a", first.get("group").asText());
+ assertEquals(2, first.get("rows").size());
+ assertEquals("b", last.get("group").asText());
+ assertEquals("z", last.get("rows").get("payload").asText());
+ }
+
+ @Test
+ void ndjsonRejectsPrettyPrintingBeforeOpeningFile() throws Exception {
+ JsonEOutputMeta meta = meta(JsonEOutputMeta.OperationType.WRITE_TO_FILE);
+ meta.setNewlineDelimited(true);
+ meta.setJsonPrettified(true);
+ meta.getFileSettings().setDoNotOpenNewFileInit(false);
+ try (Harness h = new Harness(meta, rowMeta("payload"), new Object[]
{"x"})) {
+ assertFalse(h.transform.init());
+ }
+ try (FileObject file = HopVfs.getFileObject(base + "/out.json")) {
+ assertFalse(file.exists());
+ }
+ }
+
+ @Test
+ void ndjsonRejectsNonUtf8BeforeOpeningFile() throws Exception {
+ JsonEOutputMeta meta = meta(JsonEOutputMeta.OperationType.WRITE_TO_FILE);
+ meta.setNewlineDelimited(true);
+ meta.setEncoding("UTF-16");
+ meta.getFileSettings().setDoNotOpenNewFileInit(false);
+ try (Harness h = new Harness(meta, rowMeta("payload"), new Object[]
{"x"})) {
+ assertFalse(h.transform.init());
+ }
+ try (FileObject file = HopVfs.getFileObject(base + "/out.json")) {
+ assertFalse(file.exists());
+ }
+ }
+
+ @Test
+ void ndjsonRejectsAppendWithoutFinalLf() throws Exception {
+ String existing = payload("old");
+ try (FileObject folder = HopVfs.getFileObject(base)) {
+ folder.createFolder();
+ }
+ try (var out = HopVfs.getOutputStream(base + "/out.json", false)) {
+ out.write(existing.getBytes(StandardCharsets.UTF_8));
+ }
+ JsonEOutputMeta meta = meta(JsonEOutputMeta.OperationType.WRITE_TO_FILE);
+ meta.setNewlineDelimited(true);
+ meta.getFileSettings().setFileAppended(true);
+ try (Harness h = new Harness(meta, rowMeta("payload"), new Object[]
{"new"})) {
+ assertTrue(h.transform.init());
+ HopException error = assertThrows(HopException.class,
h.transform::processRow);
+ assertTrue(error.getMessage().contains("must end with LF"),
error.getMessage());
+ assertFalse(error.getMessage().contains("Could not open file"),
error.getMessage());
+ assertFalse(error.getMessage().contains("Error opening new file"),
error.getMessage());
+ assertFalse(error.getMessage().contains("Error writing"),
error.getMessage());
+ verify(h.helper.iLogChannel, never())
+ .logError(argThat((String message) -> message.contains("Could not
open file")));
+ }
+ assertEquals(existing, read(base + "/out.json"));
+ }
+
+ @Test
+ void ndjsonAppendBoundaryIsReportedWhenTheFileIsOpenedAtInit() throws
Exception {
+ String existing = payload("old");
+ try (FileObject folder = HopVfs.getFileObject(base)) {
+ folder.createFolder();
+ }
+ try (var out = HopVfs.getOutputStream(base + "/out.json", false)) {
+ out.write(existing.getBytes(StandardCharsets.UTF_8));
+ }
+ JsonEOutputMeta meta = meta(JsonEOutputMeta.OperationType.WRITE_TO_FILE);
+ meta.setNewlineDelimited(true);
+ meta.getFileSettings().setFileAppended(true);
+ meta.getFileSettings().setDoNotOpenNewFileInit(false);
+ try (Harness h = new Harness(meta, rowMeta("payload"), new Object[]
{"new"})) {
+ assertFalse(h.transform.init());
+ verify(h.helper.iLogChannel)
+ .logError(
+ argThat((String message) -> message != null &&
message.contains("must end with LF")));
+ verify(h.helper.iLogChannel, never())
+ .logError(argThat((String message) -> message.contains("Could not
open file")));
+ verify(h.helper.iLogChannel, never())
+ .logError(argThat((String message) -> message.contains("Error
opening new file")));
+ }
+ assertEquals(existing, read(base + "/out.json"));
+ }
+
+ @Test
+ void disposeWritesParkedItemAndSkipsTheOpenGroup() throws Exception {
+ JsonEOutputMeta meta = meta(JsonEOutputMeta.OperationType.WRITE_TO_FILE);
+ meta.getFileSettings().setDoNotOpenNewFileInit(false);
+ JsonEOutputKeyField key = new JsonEOutputKeyField("grp");
+ key.setElementName("group");
+ meta.getKeyFields().add(key);
+ try (Harness h =
+ new Harness(
+ meta, rowMeta("grp", "payload"), new Object[] {"a", "x"}, new
Object[] {"b", "y"})) {
+ assertTrue(h.transform.init());
+ assertNotNull(h.data.writer);
+ assertTrue(h.transform.processRow());
+ assertEquals(0, h.data.fileItemCount);
+ assertTrue(h.transform.processRow());
+ assertEquals(1, h.data.fileItemCount);
+ assertNotNull(h.data.pendingFileItem);
+ assertNull(h.data.fileGenerator);
+ h.transform.dispose();
+ assertNull(h.data.writer);
+ assertNull(h.data.pendingFileItem);
+ assertEquals(0, h.data.fileItemCount);
+ assertEquals(0L, h.transform.getErrors());
+ }
+ JsonNode file = parse(read(base + "/out.json"));
+ assertEquals("a", file.get("group").asText());
+ assertEquals("x", file.get("rows").get("payload").asText());
+ assertFalse(file.toString().contains("\"b\""), file.toString());
+ assertFalse(file.toString().contains("\"y\""), file.toString());
+ }
+
+ @Test
+ void disposeClosesAFileOpenedAtInitWithNoItems() throws Exception {
+ JsonEOutputMeta meta = meta(JsonEOutputMeta.OperationType.WRITE_TO_FILE);
+ meta.getFileSettings().setDoNotOpenNewFileInit(false);
+ try (Harness h = new Harness(meta, rowMeta("payload"))) {
+ assertTrue(h.transform.init());
+ assertNotNull(h.data.writer);
+ assertEquals(0, h.data.fileItemCount);
+ h.transform.dispose();
+ assertNull(h.data.writer);
+ assertEquals(0L, h.transform.getErrors());
+ }
+ assertEquals("", read(base + "/out.json"));
+ }
+
+ @Test
+ void resolvedBlockNameMatchesTheFileAndTheOutputField() throws Exception {
+ JsonEOutputMeta regular = meta(JsonEOutputMeta.OperationType.BOTH);
+ regular.setJsonBloc("${BLOC}");
+ regular.getFileSettings().setFileName(base + "/regular");
+ try (Harness h = new Harness(regular, rowMeta("payload"), new Object[]
{"x"})) {
+ h.transform.setVariable("BLOC", "wrapper");
+ h.run();
+ assertEquals("wrapper", h.data.realBlocName);
+ JsonNode field = parse(h.written.get(0).getString("rows", null));
+ JsonNode file = parse(read(base + "/regular.json"));
+ assertEquals(field, file);
+ assertEquals("x", field.get("wrapper").get("payload").asText());
+ assertNull(field.get("${BLOC}"));
+ }
+
+ JsonEOutputMeta ndjson = meta(JsonEOutputMeta.OperationType.BOTH);
+ ndjson.setJsonBloc("${BLOC}");
+ ndjson.setNewlineDelimited(true);
+ ndjson.getFileSettings().setFileName(base + "/lines");
+ try (Harness h = new Harness(ndjson, rowMeta("payload"), new Object[]
{"y"})) {
+ h.transform.setVariable("BLOC", "wrapper");
+ h.run();
+ JsonNode field = parse(h.written.get(0).getString("rows", null));
+ List<String> lines = read(base + "/lines.json").lines().toList();
+ assertEquals(1, lines.size());
+ assertEquals(field, parse(lines.get(0)));
+ assertEquals("y", field.get("wrapper").get("payload").asText());
+ assertNull(field.get("${BLOC}"));
+ }
+
+ JsonEOutputMeta literal = meta(JsonEOutputMeta.OperationType.BOTH);
+ literal.setJsonBloc("plain");
+ literal.getFileSettings().setFileName(base + "/literal");
+ try (Harness h = new Harness(literal, rowMeta("payload"), new Object[]
{"z"})) {
+ h.run();
+ JsonNode field = parse(h.written.get(0).getString("rows", null));
+ assertEquals(field, parse(read(base + "/literal.json")));
+ assertEquals("z", field.get("plain").get("payload").asText());
+ }
+ }
+}
diff --git
a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputDialogTest.java
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputDialogTest.java
new file mode 100644
index 0000000000..00f4b143dc
--- /dev/null
+++
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputDialogTest.java
@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.jsonoutputenhanced;
+
+import static
org.eclipse.swtbot.swt.finder.matchers.WidgetMatcherFactory.widgetOfType;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.hop.core.gui.plugin.GuiRegistry;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.ui.core.PropsUi;
+import org.apache.hop.ui.core.widget.TableView;
+import org.apache.hop.ui.testing.SwtBotTestBase;
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.custom.CTabFolder;
+import org.eclipse.swt.custom.CTabItem;
+import org.eclipse.swt.graphics.Point;
+import org.eclipse.swt.graphics.Rectangle;
+import org.eclipse.swt.widgets.Button;
+import org.eclipse.swt.widgets.Composite;
+import org.eclipse.swt.widgets.Control;
+import org.eclipse.swt.widgets.Label;
+import org.eclipse.swt.widgets.TableItem;
+import org.eclipse.swtbot.swt.finder.SWTBot;
+import org.eclipse.swtbot.swt.finder.utils.SWTBotPreferences;
+import org.eclipse.swtbot.swt.finder.widgets.SWTBotButton;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+
+@Tag("uitest")
+class JsonEOutputDialogTest extends SwtBotTestBase {
+ /**
+ * Enhanced JSON output builds several tables before its event loop starts.
SWTBot's 5 s default
+ * expires while that construction is still on the UI thread.
+ */
+ private static final long DIALOG_TIMEOUT_MS = 30_000L;
+
+ private static long defaultTimeout;
+
+ @BeforeAll
+ static void slowDownSwtBot() {
+ defaultTimeout = SWTBotPreferences.TIMEOUT;
+ SWTBotPreferences.TIMEOUT = DIALOG_TIMEOUT_MS;
+ }
+
+ @AfterAll
+ static void restoreSwtBotTimeout() {
+ SWTBotPreferences.TIMEOUT = defaultTimeout;
+ }
+
+ @BeforeAll
+ static void registerWidgets() {
+ GuiRegistry registry = GuiRegistry.getInstance();
+ Class<JsonEOutputMeta> type = JsonEOutputMeta.class;
+ for (Field field : type.getDeclaredFields()) {
+ GuiWidgetElement element = field.getAnnotation(GuiWidgetElement.class);
+ if (element != null) {
+ registry.addGuiWidgetElement(type.getName(), element, field);
+ }
+ }
+ for (Method method : type.getDeclaredMethods()) {
+ GuiWidgetElement element = method.getAnnotation(GuiWidgetElement.class);
+ if (element != null) {
+ registry.addGuiWidgetElement(element, method, type.getName(),
type.getClassLoader());
+ }
+ }
+ }
+
+ private static TableView table(JsonEOutputDialog dialog, String name) {
+ try {
+ Field field = JsonEOutputDialog.class.getDeclaredField(name);
+ field.setAccessible(true);
+ return (TableView) field.get(dialog);
+ } catch (ReflectiveOperationException e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ private static Rectangle onDisplay(Control control) {
+ Point point = control.toDisplay(0, 0);
+ Rectangle bounds = control.getBounds();
+ return new Rectangle(point.x, point.y, bounds.width, bounds.height);
+ }
+
+ /**
+ * Upstream fields come from the dialog's {@code
getPrevTransformFields(variables, transformName)}
+ * call. The string overload is not final, so the test supplies the live row
without the field
+ * helper from the unmerged dialog-retention pull request.
+ */
+ private PipelineMeta pipeline(JsonEOutputMeta meta, String... fieldNames) {
+ PipelineMeta pipeline =
+ new PipelineMeta() {
+ @Override
+ public IRowMeta getPrevTransformFields(IVariables variables, String
transformName) {
+ RowMeta row = new RowMeta();
+ for (String fieldName : fieldNames) {
+ row.addValueMeta(new ValueMetaString(fieldName));
+ }
+ return row;
+ }
+ };
+ pipeline.setMetadataProvider(new MemoryMetadataProvider());
+ pipeline.addTransform(new TransformMeta("EnhancedJsonOutput", "json",
meta));
+ return pipeline;
+ }
+
+ @Test
+ void getKeysUsesLiveSelectionsAndCancelKeepsMetadata() {
+ JsonEOutputMeta meta = new JsonEOutputMeta();
+ meta.setOperationType(JsonEOutputMeta.OperationType.OUTPUT_VALUE);
+ JsonEOutputField payload = new JsonEOutputField();
+ payload.setFieldName("payload");
+ payload.setElementName("payload");
+ meta.getOutputFields().add(payload);
+ JsonEOutputKeyField existing = new JsonEOutputKeyField("already");
+ existing.setElementName("custom_alias");
+ meta.getKeyFields().add(existing);
+ PipelineMeta pipeline = pipeline(meta, "payload", "grp", "already",
"unsaved");
+ AtomicReference<JsonEOutputDialog> opened = new AtomicReference<>();
+ AtomicReference<List<String>> names = new AtomicReference<>();
+ AtomicReference<List<String>> aliases = new AtomicReference<>();
+ AtomicReference<Rectangle> tableBounds = new AtomicReference<>();
+ AtomicReference<Rectangle> buttonBounds = new AtomicReference<>();
+ String title = BaseMessages.getString(JsonEOutputMeta.class,
"JsonEOutputDialog.DialogTitle");
+ String keyTab =
+ BaseMessages.getString(JsonEOutputMeta.class,
"JsonEOutputDialog.KeyConfigTab.TabTitle");
+ String getButton =
+ BaseMessages.getString(JsonEOutputMeta.class,
"JsonEOutputDialog.Get.Button");
+ withDialog(
+ parent -> {
+ opened.set(new JsonEOutputDialog(parent, new Variables(), meta,
pipeline));
+ opened.get().open();
+ },
+ bot -> {
+ SWTBot dialog = bot.shell(title).activate().bot();
+ display.syncExec(
+ () -> {
+ TableItem item = new TableItem(table(opened.get(),
"wFields").table, SWT.NONE);
+ item.setText(1, "unsaved");
+ item.setText(2, "unsaved_alias");
+ });
+ Composite keyTabControl = activateTab(dialog, keyTab);
+ Button getControl = findPushButton(keyTabControl, getButton);
+ new SWTBotButton(getControl).click();
+ new SWTBotButton(getControl).click();
+ var shell = bot.shell(title);
+ display.syncExec(
+ () -> {
+ TableView keys = table(opened.get(), "wKeyFields");
+ names.set(keys.getNonEmptyItems().stream().map(item ->
item.getText(1)).toList());
+ aliases.set(keys.getNonEmptyItems().stream().map(item ->
item.getText(2)).toList());
+ shell.widget.setSize(600, 400);
+ shell.widget.layout(true, true);
+ tableBounds.set(onDisplay(keys));
+ buttonBounds.set(onDisplay(getControl));
+ });
+ dialog.button(buttonLabel("System.Button.Cancel")).click();
+ });
+ assertEquals(List.of("already", "grp"), names.get());
+ assertEquals(List.of("custom_alias", "grp"), aliases.get());
+ assertEquals(1, meta.getKeyFields().size(), "Cancel does not persist table
additions");
+ assertTrue(tableBounds.get().width > 0 && tableBounds.get().height > 0);
+ assertFalse(
+ tableBounds.get().intersects(buttonBounds.get()),
+ tableBounds.get() + " overlaps " + buttonBounds.get());
+ }
+
+ @Test
+ void ndjsonOptionIsSavedFromTheGroupedTab() {
+ JsonEOutputMeta meta = new JsonEOutputMeta();
+ meta.setOperationType(JsonEOutputMeta.OperationType.WRITE_TO_FILE);
+ PipelineMeta pipeline = pipeline(meta, "payload");
+ String title = BaseMessages.getString(JsonEOutputMeta.class,
"JsonEOutputDialog.DialogTitle");
+ String formatTab =
+ BaseMessages.getString(JsonEOutputMeta.class,
"JsonEOutputDialog.FileFormat.TabTitle");
+ String ndjson = BaseMessages.getString(JsonEOutputMeta.class,
"JsonEOutputDialog.NdJson.Label");
+ withDialog(
+ parent -> {
+ silenceSortWarning();
+ new JsonEOutputDialog(parent, new Variables(), meta,
pipeline).open();
+ },
+ bot -> {
+ SWTBot dialog = bot.shell(title).activate().bot();
+ activateTab(dialog, formatTab);
+ Button checkbox = checkBoxNextTo(dialog, ndjson);
+ display.syncExec(() -> checkbox.setSelection(true));
+ dialog.button(buttonLabel("System.Button.OK")).click();
+ });
+ assertTrue(meta.isNewlineDelimited());
+ }
+
+ @Test
+ void ndjsonOptionCancelKeepsMetadata() {
+ JsonEOutputMeta meta = new JsonEOutputMeta();
+ meta.setOperationType(JsonEOutputMeta.OperationType.WRITE_TO_FILE);
+ PipelineMeta pipeline = pipeline(meta, "payload");
+ String title = BaseMessages.getString(JsonEOutputMeta.class,
"JsonEOutputDialog.DialogTitle");
+ String formatTab =
+ BaseMessages.getString(JsonEOutputMeta.class,
"JsonEOutputDialog.FileFormat.TabTitle");
+ String ndjson = BaseMessages.getString(JsonEOutputMeta.class,
"JsonEOutputDialog.NdJson.Label");
+ withDialog(
+ parent -> new JsonEOutputDialog(parent, new Variables(), meta,
pipeline).open(),
+ bot -> {
+ SWTBot dialog = bot.shell(title).activate().bot();
+ activateTab(dialog, formatTab);
+ Button checkbox = checkBoxNextTo(dialog, ndjson);
+ display.syncExec(() -> checkbox.setSelection(true));
+ dialog.button(buttonLabel("System.Button.Cancel")).click();
+ });
+ assertFalse(meta.isNewlineDelimited());
+ }
+
+ @Test
+ void ndjsonOptionIsDisabledForOutputValue() {
+ JsonEOutputMeta meta = new JsonEOutputMeta();
+ meta.setOperationType(JsonEOutputMeta.OperationType.OUTPUT_VALUE);
+ PipelineMeta pipeline = pipeline(meta, "payload");
+ String title = BaseMessages.getString(JsonEOutputMeta.class,
"JsonEOutputDialog.DialogTitle");
+ String formatTab =
+ BaseMessages.getString(JsonEOutputMeta.class,
"JsonEOutputDialog.FileFormat.TabTitle");
+ String ndjson = BaseMessages.getString(JsonEOutputMeta.class,
"JsonEOutputDialog.NdJson.Label");
+ AtomicBoolean enabled = new AtomicBoolean(true);
+ withDialog(
+ parent -> new JsonEOutputDialog(parent, new Variables(), meta,
pipeline).open(),
+ bot -> {
+ SWTBot dialog = bot.shell(title).activate().bot();
+ activateTab(dialog, formatTab);
+ Button checkbox = checkBoxNextTo(dialog, ndjson);
+ display.syncExec(() -> enabled.set(checkbox.getEnabled()));
+ dialog.button(buttonLabel("System.Button.Cancel")).click();
+ });
+ assertFalse(enabled.get());
+ }
+
+ private static void silenceSortWarning() {
+
PropsUi.getInstance().setCustomParameter(JsonEOutputDialog.STRING_SORT_WARNING_PARAMETER,
"N");
+ }
+
+ /** Annotated checkboxes keep their title on a separate label. */
+ private Button checkBoxNextTo(SWTBot dialog, String labelText) {
+ Label label = dialog.label(labelText).widget;
+ AtomicReference<Button> found = new AtomicReference<>();
+ display.syncExec(
+ () -> {
+ for (Control child : label.getParent().getChildren()) {
+ if (child instanceof Button button && (button.getStyle() &
SWT.CHECK) != 0) {
+ found.set(button);
+ return;
+ }
+ }
+ throw new AssertionError("no checkbox next to " + labelText);
+ });
+ if (found.get() == null) {
+ throw new AssertionError("no checkbox next to " + labelText);
+ }
+ return found.get();
+ }
+
+ /**
+ * SWTBot's finder walks controls and does not see the items of a {@code
CTabFolder}. Select the
+ * tab on the folder. The Fields tab has another button with the same Get
Fields label, so the
+ * click stays inside this tab.
+ */
+ private Composite activateTab(SWTBot dialog, String title) {
+ CTabFolder tabFolder = (CTabFolder)
dialog.widget(widgetOfType(CTabFolder.class));
+ AtomicReference<Composite> selected = new AtomicReference<>();
+ display.syncExec(
+ () -> {
+ List<String> titles = new ArrayList<>();
+ for (CTabItem item : tabFolder.getItems()) {
+ String itemTitle = item.getText() == null ? "" :
item.getText().trim();
+ titles.add(itemTitle);
+ if (title.equals(itemTitle)) {
+ tabFolder.setSelection(item);
+ tabFolder.layout(true, true);
+ selected.set((Composite) item.getControl());
+ return;
+ }
+ }
+ throw new AssertionError("no tab titled " + title + " in " + titles);
+ });
+ return selected.get();
+ }
+
+ private Button findPushButton(Composite root, String message) {
+ AtomicReference<Button> found = new AtomicReference<>();
+ List<String> seen = new ArrayList<>();
+ display.syncExec(() -> found.set(searchPushButton(root, message, seen)));
+ if (found.get() == null) {
+ throw new AssertionError("no button [" + message + "] among " + seen);
+ }
+ return found.get();
+ }
+
+ private static Button searchPushButton(Control control, String message,
List<String> seen) {
+ if (control instanceof Button button && (button.getStyle() & SWT.PUSH) !=
0) {
+ seen.add(button.getText());
+ if (sameButtonLabel(button.getText(), message)) {
+ return button;
+ }
+ }
+ if (control instanceof Composite composite) {
+ for (Control child : composite.getChildren()) {
+ Button match = searchPushButton(child, message, seen);
+ if (match != null) {
+ return match;
+ }
+ }
+ }
+ return null;
+ }
+
+ private static boolean sameButtonLabel(String widgetText, String message) {
+ String widget = widgetText == null ? "" : widgetText.replace("&", "");
+ String expected = message == null ? "" : message.replace("&", "");
+ return widget.equals(expected);
+ }
+}
diff --git
a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputMetaTest.java
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputMetaTest.java
index 9f0391fd2d..4c0e316ed1 100644
---
a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputMetaTest.java
+++
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputMetaTest.java
@@ -22,13 +22,21 @@ import static org.apache.hop.core.util.Utils.isEmpty;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Objects;
import org.apache.hop.core.Const;
+import org.apache.hop.core.ICheckResult;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.variables.Variables;
import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
import org.apache.hop.metadata.serializer.xml.XmlMetadataUtil;
@@ -150,4 +158,45 @@ class JsonEOutputMetaTest {
assertEquals("f3", f3.getFieldName());
assertEquals("element3", f3.getElementName());
}
+
+ @Test
+ void missingGroupKeyNamesTheMissingField() {
+ JsonEOutputMeta meta = new JsonEOutputMeta();
+ meta.setOperationType(JsonEOutputMeta.OperationType.OUTPUT_VALUE);
+ meta.setKeyFields(List.of(new JsonEOutputKeyField("Field1")));
+ RowMeta row = new RowMeta();
+ row.addValueMeta(new ValueMetaString("Field2"));
+ HopTransformException error =
+ assertThrows(
+ HopTransformException.class,
+ () ->
+ meta.getFields(
+ row, "json", null, null, new Variables(), new
MemoryMetadataProvider()));
+ assertTrue(error.getMessage().contains("Field1"));
+ }
+
+ @Test
+ void validationReportsAMissingGroupKey() {
+ JsonEOutputMeta meta = new JsonEOutputMeta();
+ meta.setKeyFields(List.of(new JsonEOutputKeyField("gone")));
+ RowMeta previous = new RowMeta();
+ previous.addValueMeta(new ValueMetaString("payload"));
+ List<ICheckResult> remarks = new ArrayList<>();
+ meta.check(
+ remarks,
+ null,
+ new TransformMeta("EnhancedJsonOutput", "json", meta),
+ previous,
+ new String[] {"upstream"},
+ new String[0],
+ null,
+ new Variables(),
+ new MemoryMetadataProvider());
+ assertTrue(
+ remarks.stream()
+ .anyMatch(
+ result ->
+ result.getType() == ICheckResult.TYPE_RESULT_ERROR
+ && result.getText().contains("gone")));
+ }
}
