This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new bf7518dd0b Issue apache#2362 Evaluate JsonPath functions in the JSON
Input transform (#8198)
bf7518dd0b is described below
commit bf7518dd0b249053d13ee66f68d202d0ee746d59
Author: Sergio Ramazzina <[email protected]>
AuthorDate: Fri Sep 25 11:21:30 2026 +0200
Issue apache#2362 Evaluate JsonPath functions in the JSON Input transform
(#8198)
* Issue apache#2362 Evaluate JsonPath functions in the JSON Input transform
* Issue apache#2362 Honour SUPPRESS_EXCEPTIONS for failing JsonPath
functions
* Issue apache#2362 Add integration test for JsonPath functions in JSON
Input
* minor cleanup and extra tests
---------
Co-authored-by: Hans Van Akelyen <[email protected]>
---
.../ROOT/pages/pipeline/transforms/jsoninput.adoc | 14 +
.../json/0015-json-input-jsonpath-functions.hpl | 210 +++++++++++++++
.../0016-json-input-jsonpath-functions-rows.hpl | 293 +++++++++++++++++++++
.../0017-json-input-jsonpath-functions-missing.hpl | 207 +++++++++++++++
.../golden-json-input-jsonpath-functions-lines.csv | 4 +
...olden-json-input-jsonpath-functions-missing.csv | 4 +
...golden-json-input-jsonpath-functions-totals.csv | 2 +
.../golden-json-input-jsonpath-functions.csv | 4 +
.../main-0015-json-input-jsonpath-functions.hwf | 80 ++++++
...ain-0016-json-input-jsonpath-functions-rows.hwf | 80 ++++++
...-0017-json-input-jsonpath-functions-missing.hwf | 80 ++++++
...golden-json-input-jsonpath-functions-lines.json | 25 ++
...lden-json-input-jsonpath-functions-missing.json | 41 +++
...olden-json-input-jsonpath-functions-totals.json | 33 +++
.../golden-json-input-jsonpath-functions.json | 41 +++
.../0015-json-input-jsonpath-functions UNIT.json | 43 +++
...16-json-input-jsonpath-functions-rows UNIT.json | 56 ++++
...json-input-jsonpath-functions-missing UNIT.json | 43 +++
.../jsoninput/reader/FastJsonReader.java | 85 +++++-
.../jsoninput/messages/messages_en_US.properties | 1 +
.../jsoninput/reader/FastJsonReaderTest.java | 219 +++++++++++++++
21 files changed, 1560 insertions(+), 5 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/jsoninput.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/jsoninput.adoc
index ab51eb67f3..59a4c9ab44 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/jsoninput.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/jsoninput.adoc
@@ -169,6 +169,20 @@ You can use Regex in the Path expression.
*Example Path to filter on those that start with the letter 'a':*
$.data[?(@=~/a.*/i)]
+*JsonPath functions*
+
+The aggregation functions provided by JayWay can be used at the end of a Path
expression, e.g. `length()` (or its alias `size()`), `sum()`, `avg()`, `min()`,
`max()`, `stddev()`, `keys()`, `first()`, `last()` and `index()`.
+
+*Example Path to retrieve the number of elements of an array:*
$.someArray.length()
+
+A function is applied to the value the Path points to. When the Path points to
a single value, the function produces a single value and therefore a single
output row, even when that value is itself a list, as with `keys()`.
+
+When the Path contains a wildcard before the function, the function is applied
to every match and produces one row per match. For example,
`$.orders[*].lines.length()` returns the number of lines of each order. To
aggregate over all matches instead, use a deep scan or pass the Path as a
parameter to the function: `$..price.sum()` or `$.sum($.orders[*].price)`. Note
that `$.orders[*].price.sum()` does not work: the sum is applied to each single
price.
+
+Combining a single-value function field with a field whose Path matches
several elements is not supported: the transform reports a structure error
because the two Paths do not yield the same number of rows. Read such a value
in a dedicated JSON Input transform when the other fields return multiple rows.
+
+When a function cannot be evaluated, for example `sum()` or `first()` on an
empty array, the value is treated as missing: it is empty when *Ignore missing
path* is enabled, otherwise the transform fails with the reason.
+
=== Additional output fields tab
diff --git a/integration-tests/json/0015-json-input-jsonpath-functions.hpl
b/integration-tests/json/0015-json-input-jsonpath-functions.hpl
new file mode 100644
index 0000000000..e87504dc6f
--- /dev/null
+++ b/integration-tests/json/0015-json-input-jsonpath-functions.hpl
@@ -0,0 +1,210 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0015-json-input-jsonpath-functions</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Verifies that JsonPath functions (length(), sum(), max()) are
evaluated by the JSON Input transform</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/09/01 09:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/01 09:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Dummy JSON</from>
+ <to>Get Attributes</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Get Attributes</from>
+ <to>Validate</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Dummy JSON</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>{"name": "Goku", "moves": [{"moveName": "Kamehameha"},
{"moveName": "Spirit Bomb"}], "powers": [9500, 12000]}</item>
+ </line>
+ <line>
+ <item>{"name": "Vegeta", "moves": [{"moveName": "Galick Gun"}],
"powers": [9000]}</item>
+ </line>
+ <line>
+ <item>{"name": "Gohan", "moves": [{"moveName": "Masenko"},
{"moveName": "Father-Son Kamehameha"}, {"moveName": "Ki Blast"}], "powers":
[8500, 11000, 3000]}</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>z_fighter</name>
+ <type>JSON</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>192</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Get Attributes</name>
+ <type>JsonInput</type>
+ <description/>
+ <distribute>N</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <include>N</include>
+ <include_field/>
+ <rownum>N</rownum>
+ <addresultfile>N</addresultfile>
+ <readurl>N</readurl>
+ <removeSourceField>Y</removeSourceField>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <ignoreMissingPath>N</ignoreMissingPath>
+ <defaultPathLeafToNull>Y</defaultPathLeafToNull>
+ <rownum_field/>
+ <file>
+ <name/>
+ <filemask/>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
+ </file>
+ <fields>
+ <field>
+ <name>name</name>
+ <path>$.name</path>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>moveCount</name>
+ <path>$.moves.length()</path>
+ <type>Integer</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>totalPower</name>
+ <path>$.powers.sum()</path>
+ <type>Number</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>maxPower</name>
+ <path>$.powers.max()</path>
+ <type>Number</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ </fields>
+ <limit>0</limit>
+ <IsInFields>Y</IsInFields>
+ <IsAFile>N</IsAFile>
+ <valueField>z_fighter</valueField>
+ <shortFileFieldName/>
+ <pathFieldName/>
+ <hiddenFieldName/>
+ <lastModificationTimeFieldName/>
+ <uriNameFieldName/>
+ <rootUriNameFieldName/>
+ <extensionFieldName/>
+ <sizeFieldName/>
+ <attributes/>
+ <GUI>
+ <xloc>352</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Validate</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>512</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/json/0016-json-input-jsonpath-functions-rows.hpl
b/integration-tests/json/0016-json-input-jsonpath-functions-rows.hpl
new file mode 100644
index 0000000000..1eab101c96
--- /dev/null
+++ b/integration-tests/json/0016-json-input-jsonpath-functions-rows.hpl
@@ -0,0 +1,293 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0016-json-input-jsonpath-functions-rows</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Verifies JsonPath functions behind a wildcard (one row per
match), aggregation across matches and a list-valued function</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/09/25 09:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/25 09:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Orders</from>
+ <to>Order lines</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Order lines</from>
+ <to>Validate lines</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Orders</from>
+ <to>Order totals</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Order totals</from>
+ <to>Validate totals</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Orders</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>N</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>{"customer": "Acme", "orders": [{"id": "A1", "lines": [1, 2],
"price": 10}, {"id": "A2", "lines": [1, 2, 3], "price": 20}, {"id": "A3",
"lines": [], "price": 5}]}</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>doc</name>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>128</xloc>
+ <yloc>176</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Order lines</name>
+ <type>JsonInput</type>
+ <description/>
+ <distribute>N</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <include>N</include>
+ <include_field/>
+ <rownum>N</rownum>
+ <addresultfile>N</addresultfile>
+ <readurl>N</readurl>
+ <removeSourceField>Y</removeSourceField>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <ignoreMissingPath>N</ignoreMissingPath>
+ <defaultPathLeafToNull>Y</defaultPathLeafToNull>
+ <rownum_field/>
+ <file>
+ <name/>
+ <filemask/>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
+ </file>
+ <fields>
+ <field>
+ <name>orderId</name>
+ <path>$.orders[*].id</path>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>lineCount</name>
+ <path>$.orders[*].lines.length()</path>
+ <type>Integer</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ </fields>
+ <limit>0</limit>
+ <IsInFields>Y</IsInFields>
+ <IsAFile>N</IsAFile>
+ <valueField>doc</valueField>
+ <shortFileFieldName/>
+ <pathFieldName/>
+ <hiddenFieldName/>
+ <lastModificationTimeFieldName/>
+ <uriNameFieldName/>
+ <rootUriNameFieldName/>
+ <extensionFieldName/>
+ <sizeFieldName/>
+ <attributes/>
+ <GUI>
+ <xloc>320</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Validate lines</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>512</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Order totals</name>
+ <type>JsonInput</type>
+ <description/>
+ <distribute>N</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <include>N</include>
+ <include_field/>
+ <rownum>N</rownum>
+ <addresultfile>N</addresultfile>
+ <readurl>N</readurl>
+ <removeSourceField>Y</removeSourceField>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <ignoreMissingPath>N</ignoreMissingPath>
+ <defaultPathLeafToNull>Y</defaultPathLeafToNull>
+ <rownum_field/>
+ <file>
+ <name/>
+ <filemask/>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
+ </file>
+ <fields>
+ <field>
+ <name>totalPrice</name>
+ <path>$..price.sum()</path>
+ <type>Number</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>maxPrice</name>
+ <path>$.max($.orders[*].price)</path>
+ <type>Number</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>orderKeys</name>
+ <path>$.orders[0].keys()</path>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ </fields>
+ <limit>0</limit>
+ <IsInFields>Y</IsInFields>
+ <IsAFile>N</IsAFile>
+ <valueField>doc</valueField>
+ <shortFileFieldName/>
+ <pathFieldName/>
+ <hiddenFieldName/>
+ <lastModificationTimeFieldName/>
+ <uriNameFieldName/>
+ <rootUriNameFieldName/>
+ <extensionFieldName/>
+ <sizeFieldName/>
+ <attributes/>
+ <GUI>
+ <xloc>320</xloc>
+ <yloc>240</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Validate totals</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>512</xloc>
+ <yloc>240</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/json/0017-json-input-jsonpath-functions-missing.hpl
b/integration-tests/json/0017-json-input-jsonpath-functions-missing.hpl
new file mode 100644
index 0000000000..295f85b1dc
--- /dev/null
+++ b/integration-tests/json/0017-json-input-jsonpath-functions-missing.hpl
@@ -0,0 +1,207 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0017-json-input-jsonpath-functions-missing</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Verifies that a JsonPath function that cannot be evaluated
gives an empty value when Ignore missing path is enabled</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/09/25 09:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/25 09:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Documents</from>
+ <to>Get Values</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Get Values</from>
+ <to>Validate</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Documents</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>N</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>{"items": [], "tags": ["x", "y"]}</item>
+ </line>
+ <line>
+ <item>{"items": [3, 4], "tags": ["z"]}</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>doc</name>
+ <type>JSON</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>192</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Get Values</name>
+ <type>JsonInput</type>
+ <description/>
+ <distribute>N</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <include>N</include>
+ <include_field/>
+ <rownum>N</rownum>
+ <addresultfile>N</addresultfile>
+ <readurl>N</readurl>
+ <removeSourceField>Y</removeSourceField>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <ignoreMissingPath>Y</ignoreMissingPath>
+ <defaultPathLeafToNull>Y</defaultPathLeafToNull>
+ <rownum_field/>
+ <file>
+ <name/>
+ <filemask/>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
+ </file>
+ <fields>
+ <field>
+ <name>tag</name>
+ <path>$.tags[*]</path>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>firstItem</name>
+ <path>$.items.first()</path>
+ <type>Integer</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>itemTotal</name>
+ <path>$.items.sum()</path>
+ <type>Number</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>absentCount</name>
+ <path>$.absent.length()</path>
+ <type>Integer</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ </fields>
+ <limit>0</limit>
+ <IsInFields>Y</IsInFields>
+ <IsAFile>N</IsAFile>
+ <valueField>doc</valueField>
+ <shortFileFieldName/>
+ <pathFieldName/>
+ <hiddenFieldName/>
+ <lastModificationTimeFieldName/>
+ <uriNameFieldName/>
+ <rootUriNameFieldName/>
+ <extensionFieldName/>
+ <sizeFieldName/>
+ <attributes/>
+ <GUI>
+ <xloc>352</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Validate</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>512</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/json/datasets/golden-json-input-jsonpath-functions-lines.csv
b/integration-tests/json/datasets/golden-json-input-jsonpath-functions-lines.csv
new file mode 100644
index 0000000000..33e366e98c
--- /dev/null
+++
b/integration-tests/json/datasets/golden-json-input-jsonpath-functions-lines.csv
@@ -0,0 +1,4 @@
+orderId,lineCount
+A1,2
+A2,3
+A3,0
diff --git
a/integration-tests/json/datasets/golden-json-input-jsonpath-functions-missing.csv
b/integration-tests/json/datasets/golden-json-input-jsonpath-functions-missing.csv
new file mode 100644
index 0000000000..94a935d1fd
--- /dev/null
+++
b/integration-tests/json/datasets/golden-json-input-jsonpath-functions-missing.csv
@@ -0,0 +1,4 @@
+tag,firstItem,itemTotal,absentCount
+x,,,
+y,,,
+z,3,7.0,
diff --git
a/integration-tests/json/datasets/golden-json-input-jsonpath-functions-totals.csv
b/integration-tests/json/datasets/golden-json-input-jsonpath-functions-totals.csv
new file mode 100644
index 0000000000..341086e03d
--- /dev/null
+++
b/integration-tests/json/datasets/golden-json-input-jsonpath-functions-totals.csv
@@ -0,0 +1,2 @@
+totalPrice,maxPrice,orderKeys
+35.0,20.0,"[id, lines, price]"
diff --git
a/integration-tests/json/datasets/golden-json-input-jsonpath-functions.csv
b/integration-tests/json/datasets/golden-json-input-jsonpath-functions.csv
new file mode 100644
index 0000000000..01b57f2762
--- /dev/null
+++ b/integration-tests/json/datasets/golden-json-input-jsonpath-functions.csv
@@ -0,0 +1,4 @@
+name,moveCount,totalPower,maxPower
+Goku,2,21500.0,12000.0
+Vegeta,1,9000.0,9000.0
+Gohan,3,22500.0,11000.0
diff --git a/integration-tests/json/main-0015-json-input-jsonpath-functions.hwf
b/integration-tests/json/main-0015-json-input-jsonpath-functions.hwf
new file mode 100644
index 0000000000..ebfcedfe0d
--- /dev/null
+++ b/integration-tests/json/main-0015-json-input-jsonpath-functions.hwf
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0015-json-input-jsonpath-functions</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/09/01 09:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/01 09:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>50</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Run Pipeline Unit Tests</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0015-json-input-jsonpath-functions UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>256</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Run Pipeline Unit Tests</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/json/main-0016-json-input-jsonpath-functions-rows.hwf
b/integration-tests/json/main-0016-json-input-jsonpath-functions-rows.hwf
new file mode 100644
index 0000000000..4b92240476
--- /dev/null
+++ b/integration-tests/json/main-0016-json-input-jsonpath-functions-rows.hwf
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0016-json-input-jsonpath-functions-rows</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/09/25 09:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/25 09:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>50</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Run Pipeline Unit Tests</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0016-json-input-jsonpath-functions-rows UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>256</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Run Pipeline Unit Tests</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/json/main-0017-json-input-jsonpath-functions-missing.hwf
b/integration-tests/json/main-0017-json-input-jsonpath-functions-missing.hwf
new file mode 100644
index 0000000000..f30233f843
--- /dev/null
+++ b/integration-tests/json/main-0017-json-input-jsonpath-functions-missing.hwf
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0017-json-input-jsonpath-functions-missing</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/09/25 09:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/09/25 09:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>50</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Run Pipeline Unit Tests</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0017-json-input-jsonpath-functions-missing UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>256</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Run Pipeline Unit Tests</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/json/metadata/dataset/golden-json-input-jsonpath-functions-lines.json
b/integration-tests/json/metadata/dataset/golden-json-input-jsonpath-functions-lines.json
new file mode 100644
index 0000000000..9db5c9caec
--- /dev/null
+++
b/integration-tests/json/metadata/dataset/golden-json-input-jsonpath-functions-lines.json
@@ -0,0 +1,25 @@
+{
+ "base_filename": "golden-json-input-jsonpath-functions-lines.csv",
+ "virtualPath": "",
+ "name": "golden-json-input-jsonpath-functions-lines",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "orderId",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": -1,
+ "field_name": "lineCount",
+ "field_format": ""
+ }
+ ],
+ "folder_name": "${HOP_DATASETS_FOLDER}"
+}
diff --git
a/integration-tests/json/metadata/dataset/golden-json-input-jsonpath-functions-missing.json
b/integration-tests/json/metadata/dataset/golden-json-input-jsonpath-functions-missing.json
new file mode 100644
index 0000000000..653e3dff85
--- /dev/null
+++
b/integration-tests/json/metadata/dataset/golden-json-input-jsonpath-functions-missing.json
@@ -0,0 +1,41 @@
+{
+ "base_filename": "golden-json-input-jsonpath-functions-missing.csv",
+ "virtualPath": "",
+ "name": "golden-json-input-jsonpath-functions-missing",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "tag",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": -1,
+ "field_name": "firstItem",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 1,
+ "field_precision": -1,
+ "field_name": "itemTotal",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": -1,
+ "field_name": "absentCount",
+ "field_format": ""
+ }
+ ],
+ "folder_name": "${HOP_DATASETS_FOLDER}"
+}
diff --git
a/integration-tests/json/metadata/dataset/golden-json-input-jsonpath-functions-totals.json
b/integration-tests/json/metadata/dataset/golden-json-input-jsonpath-functions-totals.json
new file mode 100644
index 0000000000..cdeca9f4fe
--- /dev/null
+++
b/integration-tests/json/metadata/dataset/golden-json-input-jsonpath-functions-totals.json
@@ -0,0 +1,33 @@
+{
+ "base_filename": "golden-json-input-jsonpath-functions-totals.csv",
+ "virtualPath": "",
+ "name": "golden-json-input-jsonpath-functions-totals",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 1,
+ "field_precision": -1,
+ "field_name": "totalPrice",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 1,
+ "field_precision": -1,
+ "field_name": "maxPrice",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "orderKeys",
+ "field_format": ""
+ }
+ ],
+ "folder_name": "${HOP_DATASETS_FOLDER}"
+}
diff --git
a/integration-tests/json/metadata/dataset/golden-json-input-jsonpath-functions.json
b/integration-tests/json/metadata/dataset/golden-json-input-jsonpath-functions.json
new file mode 100644
index 0000000000..fb86b3c8bc
--- /dev/null
+++
b/integration-tests/json/metadata/dataset/golden-json-input-jsonpath-functions.json
@@ -0,0 +1,41 @@
+{
+ "base_filename": "golden-json-input-jsonpath-functions.csv",
+ "virtualPath": "",
+ "name": "golden-json-input-jsonpath-functions",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "name",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": -1,
+ "field_name": "moveCount",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 1,
+ "field_precision": -1,
+ "field_name": "totalPower",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 1,
+ "field_precision": -1,
+ "field_name": "maxPower",
+ "field_format": ""
+ }
+ ],
+ "folder_name": "${HOP_DATASETS_FOLDER}"
+}
diff --git
a/integration-tests/json/metadata/unit-test/0015-json-input-jsonpath-functions
UNIT.json
b/integration-tests/json/metadata/unit-test/0015-json-input-jsonpath-functions
UNIT.json
new file mode 100644
index 0000000000..04adb8cc7d
--- /dev/null
+++
b/integration-tests/json/metadata/unit-test/0015-json-input-jsonpath-functions
UNIT.json
@@ -0,0 +1,43 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "name",
+ "data_set_field": "name"
+ },
+ {
+ "transform_field": "moveCount",
+ "data_set_field": "moveCount"
+ },
+ {
+ "transform_field": "totalPower",
+ "data_set_field": "totalPower"
+ },
+ {
+ "transform_field": "maxPower",
+ "data_set_field": "maxPower"
+ }
+ ],
+ "field_order": [
+ "name",
+ "moveCount",
+ "totalPower",
+ "maxPower"
+ ],
+ "data_set_name": "golden-json-input-jsonpath-functions",
+ "transform_name": "Validate"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0015-json-input-jsonpath-functions UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0015-json-input-jsonpath-functions.hpl"
+}
diff --git
a/integration-tests/json/metadata/unit-test/0016-json-input-jsonpath-functions-rows
UNIT.json
b/integration-tests/json/metadata/unit-test/0016-json-input-jsonpath-functions-rows
UNIT.json
new file mode 100644
index 0000000000..dd64826e7f
--- /dev/null
+++
b/integration-tests/json/metadata/unit-test/0016-json-input-jsonpath-functions-rows
UNIT.json
@@ -0,0 +1,56 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "orderId",
+ "data_set_field": "orderId"
+ },
+ {
+ "transform_field": "lineCount",
+ "data_set_field": "lineCount"
+ }
+ ],
+ "field_order": [
+ "orderId",
+ "lineCount"
+ ],
+ "data_set_name": "golden-json-input-jsonpath-functions-lines",
+ "transform_name": "Validate lines"
+ },
+ {
+ "field_mappings": [
+ {
+ "transform_field": "totalPrice",
+ "data_set_field": "totalPrice"
+ },
+ {
+ "transform_field": "maxPrice",
+ "data_set_field": "maxPrice"
+ },
+ {
+ "transform_field": "orderKeys",
+ "data_set_field": "orderKeys"
+ }
+ ],
+ "field_order": [
+ "totalPrice",
+ "maxPrice",
+ "orderKeys"
+ ],
+ "data_set_name": "golden-json-input-jsonpath-functions-totals",
+ "transform_name": "Validate totals"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0016-json-input-jsonpath-functions-rows UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0016-json-input-jsonpath-functions-rows.hpl"
+}
diff --git
a/integration-tests/json/metadata/unit-test/0017-json-input-jsonpath-functions-missing
UNIT.json
b/integration-tests/json/metadata/unit-test/0017-json-input-jsonpath-functions-missing
UNIT.json
new file mode 100644
index 0000000000..1e7720b74b
--- /dev/null
+++
b/integration-tests/json/metadata/unit-test/0017-json-input-jsonpath-functions-missing
UNIT.json
@@ -0,0 +1,43 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "tag",
+ "data_set_field": "tag"
+ },
+ {
+ "transform_field": "firstItem",
+ "data_set_field": "firstItem"
+ },
+ {
+ "transform_field": "itemTotal",
+ "data_set_field": "itemTotal"
+ },
+ {
+ "transform_field": "absentCount",
+ "data_set_field": "absentCount"
+ }
+ ],
+ "field_order": [
+ "tag",
+ "firstItem",
+ "itemTotal",
+ "absentCount"
+ ],
+ "data_set_name": "golden-json-input-jsonpath-functions-missing",
+ "transform_name": "Validate"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0017-json-input-jsonpath-functions-missing UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0017-json-input-jsonpath-functions-missing.hpl"
+}
diff --git
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/reader/FastJsonReader.java
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/reader/FastJsonReader.java
index cc4cd5acf8..fe8e3c6539 100644
---
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/reader/FastJsonReader.java
+++
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/reader/FastJsonReader.java
@@ -24,11 +24,13 @@ import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.Option;
import com.jayway.jsonpath.ParseContext;
import com.jayway.jsonpath.ReadContext;
+import com.jayway.jsonpath.internal.path.PathCompiler;
import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider;
import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider;
import java.io.InputStream;
import java.util.AbstractList;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.RandomAccess;
@@ -54,11 +56,29 @@ public class FastJsonReader implements IJsonReader {
/** used if the incoming value is a JsonNode */
private final Configuration jsonNodeConfiguration;
+ /**
+ * Jayway refuses path functions under ALWAYS_RETURN_LIST, so function paths
are read with these
+ * copies of the configurations above that leave the option out.
+ */
+ private Configuration jsonFunctionConfiguration;
+
+ private final Configuration jsonNodeFunctionConfiguration;
+
+ /** The function configuration matching the current read context. */
+ private Configuration functionConfiguration;
+
private boolean ignoreMissingPath;
@Getter private boolean defaultPathLeafToNull;
private JsonInputField[] fields;
private JsonPath[] paths = null;
+
+ /** Per path: it ends in a function such as length(). */
+ private boolean[] functionPaths = new boolean[0];
+
+ /** Per path: it matches at most one element, so a function on it yields one
value. */
+ private boolean[] definitePaths = new boolean[0];
+
private final ILogChannel log;
private static final Option[] DEFAULT_OPTIONS = {
@@ -70,6 +90,8 @@ public class FastJsonReader implements IJsonReader {
this.defaultPathLeafToNull = true;
this.jsonConfiguration =
Configuration.defaultConfiguration().addOptions(DEFAULT_OPTIONS);
this.jsonNodeConfiguration = getJacksonNodeJsonPathConfig();
+ this.jsonFunctionConfiguration =
withoutAlwaysReturnList(this.jsonConfiguration);
+ this.jsonNodeFunctionConfiguration =
withoutAlwaysReturnList(this.jsonNodeConfiguration);
this.log = log;
}
@@ -90,6 +112,7 @@ public class FastJsonReader implements IJsonReader {
if (!this.defaultPathLeafToNull) {
this.jsonConfiguration =
deleteOptionFromConfiguration(this.jsonConfiguration,
Option.DEFAULT_PATH_LEAF_TO_NULL);
+ this.jsonFunctionConfiguration =
withoutAlwaysReturnList(this.jsonConfiguration);
}
}
}
@@ -102,6 +125,13 @@ public class FastJsonReader implements IJsonReader {
.build();
}
+ private static Configuration withoutAlwaysReturnList(Configuration config) {
+ EnumSet<Option> options = EnumSet.noneOf(Option.class);
+ options.addAll(config.getOptions());
+ options.remove(Option.ALWAYS_RETURN_LIST);
+ return config.setOptions(options.toArray(new Option[0]));
+ }
+
@SuppressWarnings("javabugs:S2259") // the configuration is created before
it is logged
private Configuration deleteOptionFromConfiguration(Configuration config,
Option option) {
Configuration currentConf = config;
@@ -161,6 +191,7 @@ public class FastJsonReader implements IJsonReader {
protected void readInput(InputStream is) throws HopException {
jsonReadContext = getParseContext().parse(is, Const.UTF_8);
+ functionConfiguration = jsonFunctionConfiguration;
if (jsonReadContext == null) {
throw new HopException(BaseMessages.getString(PKG,
"JsonReader.Error.ReadUrl.Null"));
}
@@ -168,6 +199,7 @@ public class FastJsonReader implements IJsonReader {
protected void readInput(JsonNode node) throws HopException {
jsonReadContext = getJsonNodeParseContext().parse(node);
+ functionConfiguration = jsonNodeFunctionConfiguration;
if (jsonReadContext == null) {
throw new HopException(BaseMessages.getString(PKG,
"JsonReader.Error.ReadUrl.Null"));
}
@@ -182,6 +214,13 @@ public class FastJsonReader implements IJsonReader {
public void setFields(JsonInputField[] fields) throws HopException {
this.fields = fields;
this.paths = compilePaths(fields);
+ this.functionPaths = new boolean[paths.length];
+ this.definitePaths = new boolean[paths.length];
+ for (int i = 0; i < paths.length; i++) {
+ // JsonPath does not expose whether a path ends in a function; its
compiler does.
+ functionPaths[i] =
PathCompiler.compile(fields[i].getPath()).isFunctionPath();
+ definitePaths[i] = paths[i].isDefinite();
+ }
}
@Override
@@ -296,10 +335,11 @@ public class FastJsonReader implements IJsonReader {
int lastSize = -1;
String prevPath = null;
List<List<?>> results = new ArrayList<>(paths.length);
- int i = 0;
- for (JsonPath path : paths) {
- Object raw = getReadContext().read(path);
- List<Object> result = normalizeJsonPathResult(raw);
+ for (int i = 0; i < paths.length; i++) {
+ List<Object> result =
+ functionPaths[i]
+ ? readFunctionPath(i)
+ : normalizeJsonPathResult(getReadContext().read(paths[i]));
if (result.size() != lastSize && lastSize > 0 && !result.isEmpty()) {
throw new JsonInputException(
BaseMessages.getString(
@@ -317,11 +357,46 @@ public class FastJsonReader implements IJsonReader {
results.add(result);
lastSize = result.size();
prevPath = fields[i].getPath();
- i++;
}
return results;
}
+ /**
+ * Evaluate a path that ends in a function. A definite path gives one value,
even when that value
+ * is itself an array (keys(), first() of a nested array). A function behind
a wildcard, such as
+ * {@code $.items[*].name.length()}, is applied to each match and gives one
value per match. A
+ * missing value is an empty result, like a regular path that matches
nothing.
+ */
+ private List<Object> readFunctionPath(int index) throws JsonInputException {
+ Object document = getReadContext().json();
+ Object value;
+ try {
+ value = paths[index].read(document, functionConfiguration);
+ } catch (RuntimeException e) {
+ // SUPPRESS_EXCEPTIONS does not cover errors raised by the function
itself, such as sum() or
+ // first() on an empty array. Treat them as a missing value, or report
the actual cause.
+ if (!isIgnoreMissingPath()) {
+ throw new JsonInputException(
+ BaseMessages.getString(
+ PKG, "JsonReader.Error.PathFunction", fields[index].getPath(),
e.getMessage()),
+ e);
+ }
+ if (log.isDebug()) {
+ log.logDebug(
+ BaseMessages.getString(
+ PKG, "JsonReader.Error.PathFunction", fields[index].getPath(),
e.getMessage()));
+ }
+ return Collections.emptyList();
+ }
+ if (value == null) {
+ return Collections.emptyList();
+ }
+ if (definitePaths[index]) {
+ return Collections.singletonList(value);
+ }
+ return normalizeJsonPathResult(value);
+ }
+
public static boolean isAllNull(Iterable<?> list) {
for (Object obj : list) {
if (obj != null) {
diff --git
a/plugins/transforms/json/src/main/resources/org/apache/hop/pipeline/transforms/jsoninput/messages/messages_en_US.properties
b/plugins/transforms/json/src/main/resources/org/apache/hop/pipeline/transforms/jsoninput/messages/messages_en_US.properties
index b99ea336a2..3fe6627064 100644
---
a/plugins/transforms/json/src/main/resources/org/apache/hop/pipeline/transforms/jsoninput/messages/messages_en_US.properties
+++
b/plugins/transforms/json/src/main/resources/org/apache/hop/pipeline/transforms/jsoninput/messages/messages_en_US.properties
@@ -182,6 +182,7 @@ JsonParser.JsonPath.Compile.Error=Error compiling JSONPath
expression: {0}
JsonReader.Debug.Configuration.Option.Delete={0} option has been deleted from
JSON configuration.
JsonReader.Debug.Configuration.Options=JSON configuration options: {0}.
JsonReader.Error.CanNotFindPath=We can not find any data with path [{0}]\!
+JsonReader.Error.PathFunction=Could not evaluate the function in path [{0}]:
{1}
JsonReader.Error.ParsingFile=Error parsing file [{0}]\!
JsonReader.Error.ParsingString=Error parsing string [{0}]\!
JsonReader.Error.ReadUrl.Null=JSON parsing failed. Parser returned null.
diff --git
a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/reader/FastJsonReaderTest.java
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/reader/FastJsonReaderTest.java
index 7f44079b78..f9a749614a 100644
---
a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/reader/FastJsonReaderTest.java
+++
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/reader/FastJsonReaderTest.java
@@ -20,18 +20,26 @@ package org.apache.hop.pipeline.transforms.jsoninput.reader;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.Option;
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
+import org.apache.hop.core.IRowSet;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.logging.ILogChannel;
import org.apache.hop.pipeline.transforms.jsoninput.JsonInputField;
+import
org.apache.hop.pipeline.transforms.jsoninput.exception.JsonInputException;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -110,4 +118,215 @@ class FastJsonReaderTest {
mainList.add(l3);
assertEquals(3,
FastJsonReader.getMaxRowSize(Collections.singletonList(mainList)));
}
+
+ private static final String ARRAY_JSON =
"{\"someArray\":[{\"x\":1},{\"x\":2}]}";
+
+ private static IRowSet readString(String json, String path) throws
HopException {
+ JsonInputField field = new JsonInputField("value");
+ field.setPath(path);
+ FastJsonReader reader =
+ new FastJsonReader(new JsonInputField[] {field},
mock(ILogChannel.class));
+ return reader.parseStringValue(new
ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8)));
+ }
+
+ @Test
+ void testLengthFunctionPathReturnsNumberOfArrayElements() throws Exception {
+ IRowSet rowSet = readString(ARRAY_JSON, "$.someArray.length()");
+ Object[] row = rowSet.getRow();
+ assertNotNull(row);
+ assertEquals(2, ((Number) row[0]).intValue());
+ }
+
+ @Test
+ void testSumFunctionPathIsEvaluated() throws Exception {
+ IRowSet rowSet = readString("{\"values\":[1,2,4]}", "$.values.sum()");
+ Object[] row = rowSet.getRow();
+ assertNotNull(row);
+ assertEquals(7, ((Number) row[0]).intValue());
+ }
+
+ @Test
+ void testFunctionPathReturnsSingleRow() throws Exception {
+ IRowSet rowSet = readString(ARRAY_JSON, "$.someArray.length()");
+ assertNotNull(rowSet.getRow());
+ assertNull(rowSet.getRow());
+ }
+
+ @Test
+ void testLengthFunctionPathOnJsonNodeInput() throws Exception {
+ JsonInputField field = new JsonInputField("value");
+ field.setPath("$.someArray.length()");
+ FastJsonReader reader =
+ new FastJsonReader(new JsonInputField[] {field},
mock(ILogChannel.class));
+ IRowSet rowSet = reader.parseJsonNodeValue(new
ObjectMapper().readTree(ARRAY_JSON));
+ Object[] row = rowSet.getRow();
+ assertNotNull(row);
+ assertEquals(2, ((Number) row[0]).intValue());
+ }
+
+ @Test
+ void testRegularPathStillReturnsOneRowPerMatch() throws Exception {
+ IRowSet rowSet = readString(ARRAY_JSON, "$.someArray[*].x");
+ assertNotNull(rowSet.getRow());
+ assertNotNull(rowSet.getRow());
+ assertNull(rowSet.getRow());
+ }
+
+ @Test
+ void testFailingFunctionPathIsSuppressedAsNullValue() throws Exception {
+ JsonInputField field = new JsonInputField("value");
+ field.setPath("$.emptyList.sum()");
+ FastJsonReader reader =
+ new FastJsonReader(new JsonInputField[] {field},
mock(ILogChannel.class));
+ reader.setIgnoreMissingPath(true);
+ IRowSet rowSet =
+ reader.parseStringValue(
+ new
ByteArrayInputStream("{\"emptyList\":[]}".getBytes(StandardCharsets.UTF_8)));
+ Object[] row = rowSet.getRow();
+ assertNotNull(row);
+ assertNull(row[0]);
+ }
+
+ private static final String ITEMS_JSON =
+ "{\"items\":[{\"name\":\"ab\",\"p\":1,\"tags\":[1,2]},"
+ + "{\"name\":\"abc\",\"p\":2,\"tags\":[1,2,3]},"
+ +
"{\"name\":\"abcd\",\"p\":4,\"tags\":[]}],\"empty\":[],\"matrix\":[[1,2],[3]],"
+ + "\"obj\":{\"a\":1,\"b\":2}}";
+
+ private static FastJsonReader reader(boolean ignoreMissingPath, String...
paths)
+ throws HopException {
+ JsonInputField[] fields = new JsonInputField[paths.length];
+ for (int i = 0; i < paths.length; i++) {
+ fields[i] = new JsonInputField("value" + i);
+ fields[i].setPath(paths[i]);
+ }
+ FastJsonReader reader = new FastJsonReader(fields,
mock(ILogChannel.class));
+ reader.setIgnoreMissingPath(ignoreMissingPath);
+ return reader;
+ }
+
+ private static List<Object[]> rows(IRowSet rowSet) {
+ List<Object[]> rows = new ArrayList<>();
+ Object[] row;
+ while ((row = rowSet.getRow()) != null) {
+ rows.add(row);
+ }
+ return rows;
+ }
+
+ private static List<Object[]> readRows(String json, boolean
ignoreMissingPath, String... paths)
+ throws HopException {
+ return rows(
+ reader(ignoreMissingPath, paths)
+ .parseStringValue(new
ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8))));
+ }
+
+ private static List<Object[]> readNodeRows(
+ String json, boolean ignoreMissingPath, String... paths) throws
Exception {
+ return rows(
+ reader(ignoreMissingPath, paths).parseJsonNodeValue(new
ObjectMapper().readTree(json)));
+ }
+
+ @Test
+ void testFunctionBehindWildcardGivesOneValuePerMatch() throws Exception {
+ List<Object[]> rows = readRows(ITEMS_JSON, false,
"$.items[*].tags.length()");
+ assertEquals(3, rows.size());
+ assertEquals(2, ((Number) rows.get(0)[0]).intValue());
+ assertEquals(3, ((Number) rows.get(1)[0]).intValue());
+ assertEquals(0, ((Number) rows.get(2)[0]).intValue());
+ }
+
+ @Test
+ void testFunctionBehindWildcardOnJsonNodeInput() throws Exception {
+ List<Object[]> rows = readNodeRows(ITEMS_JSON, false,
"$.items[*].tags.length()");
+ assertEquals(3, rows.size());
+ assertEquals(0, ((JsonNode) rows.get(2)[0]).intValue());
+ }
+
+ @Test
+ void testFunctionBehindWildcardCombinesWithRegularField() throws Exception {
+ List<Object[]> rows =
+ readRows(ITEMS_JSON, false, "$.items[*].name",
"$.items[*].tags.length()");
+ assertEquals(3, rows.size());
+ assertEquals("abc", rows.get(1)[0]);
+ assertEquals(3, ((Number) rows.get(1)[1]).intValue());
+ }
+
+ @Test
+ void testAggregationOverDeepScan() throws Exception {
+ List<Object[]> rows = readRows(ITEMS_JSON, false, "$..p.sum()");
+ assertEquals(1, rows.size());
+ assertEquals(7, ((Number) rows.get(0)[0]).intValue());
+ }
+
+ @Test
+ void testAggregationWithPathParameter() throws Exception {
+ List<Object[]> rows = readRows(ITEMS_JSON, false, "$.max($.items[*].p)");
+ assertEquals(1, rows.size());
+ assertEquals(4, ((Number) rows.get(0)[0]).intValue());
+ }
+
+ @Test
+ void testFunctionReturningArrayGivesOneRow() throws Exception {
+ List<Object[]> rows = readRows(ITEMS_JSON, false, "$.matrix.first()");
+ assertEquals(1, rows.size());
+ assertEquals(List.of(1, 2), rows.get(0)[0]);
+ }
+
+ @Test
+ void testKeysGivesOneRowForStringAndJsonNodeInput() throws Exception {
+ List<Object[]> stringRows = readRows(ITEMS_JSON, false, "$.obj.keys()");
+ List<Object[]> nodeRows = readNodeRows(ITEMS_JSON, false, "$.obj.keys()");
+ assertEquals(1, stringRows.size());
+ assertEquals(1, nodeRows.size());
+ }
+
+ @Test
+ void testFirstOnEmptyArrayIsMissingValueWhenIgnoringMissingPath() throws
Exception {
+ List<Object[]> rows = readRows(ITEMS_JSON, true, "$.empty.first()");
+ assertEquals(1, rows.size());
+ assertNull(rows.get(0)[0]);
+ }
+
+ @Test
+ void testIndexOutOfRangeIsMissingValueWhenIgnoringMissingPath() throws
Exception {
+ List<Object[]> rows = readRows(ITEMS_JSON, true, "$.matrix.index(5)");
+ assertEquals(1, rows.size());
+ assertNull(rows.get(0)[0]);
+ }
+
+ @Test
+ void testFailingFunctionReportsCauseWhenNotIgnoringMissingPath() {
+ JsonInputException e =
+ assertThrows(JsonInputException.class, () -> readRows(ITEMS_JSON,
false, "$.empty.sum()"));
+ assertTrue(e.getMessage().contains("$.empty.sum()"), e.getMessage());
+ assertTrue(e.getMessage().contains("empty array"), e.getMessage());
+ }
+
+ @Test
+ void testFirstOnEmptyArrayFailsWhenNotIgnoringMissingPath() {
+ assertThrows(JsonInputException.class, () -> readRows(ITEMS_JSON, false,
"$.empty.first()"));
+ }
+
+ @Test
+ void testMissingFunctionValueNextToMultiRowField() throws Exception {
+ List<Object[]> rows = readRows(ITEMS_JSON, true, "$.items[*].name",
"$.absent.length()");
+ assertEquals(3, rows.size());
+ assertEquals("abcd", rows.get(2)[0]);
+ assertNull(rows.get(2)[1]);
+ }
+
+ @Test
+ void testFunctionPathWithoutDefaultPathLeafToNull() throws Exception {
+ JsonInputField field = new JsonInputField("value");
+ field.setPath("$.items.length()");
+ FastJsonReader reader =
+ new FastJsonReader(new JsonInputField[] {field}, false,
mock(ILogChannel.class));
+ List<Object[]> rows =
+ rows(
+ reader.parseStringValue(
+ new
ByteArrayInputStream(ITEMS_JSON.getBytes(StandardCharsets.UTF_8))));
+ assertEquals(1, rows.size());
+ assertEquals(3, ((Number) rows.get(0)[0]).intValue());
+ }
}
