This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 64b28593b0 [IT] fix broken certificate and tests (#9568)
64b28593b0 is described below
commit 64b28593b0799b4f2294b907bbbe7f1dac599bd0
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Wed Nov 18 15:27:37 2026 +0100
[IT] fix broken certificate and tests (#9568)
---
.../resource/http/mtls/regenerate-certs.sh | 106 +++++++++++
.../resource/http/mtls/test-certs/ca.crt | 31 ++--
.../resource/http/mtls/test-certs/server.crt | 60 +++---
integration-tests/hop_run/hop-import-exit-code.sh | 0
.../http/test-certs/client-expired.p12 | Bin 5956 -> 5932 bytes
integration-tests/http/test-certs/client-valid.crt | 60 +++---
integration-tests/http/test-certs/client-valid.jks | Bin 5417 -> 5378 bytes
integration-tests/http/test-certs/client-valid.p12 | Bin 5934 -> 5910 bytes
integration-tests/http/test-certs/client-wrong.crt | 56 +++---
integration-tests/http/test-certs/client-wrong.p12 | Bin 5720 -> 5696 bytes
integration-tests/http/test-certs/truststore.jks | Bin 1830 -> 1488 bytes
integration-tests/http/test-certs/truststore.p12 | Bin 1830 -> 1798 bytes
.../json/0018-enhanced-json-output-streaming.hpl | 202 ---------------------
.../main-0018-enhanced-json-output-streaming.hwf | 188 -------------------
.../JsonEOutputBehaviorTest.java | 38 ++++
15 files changed, 247 insertions(+), 494 deletions(-)
diff --git a/docker/integration-tests/resource/http/mtls/regenerate-certs.sh
b/docker/integration-tests/resource/http/mtls/regenerate-certs.sh
new file mode 100755
index 0000000000..44ee7c5d89
--- /dev/null
+++ b/docker/integration-tests/resource/http/mtls/regenerate-certs.sh
@@ -0,0 +1,106 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Re-issues the certificates used by the http mTLS integration tests
(0018-mts-*).
+#
+# The existing private keys are reused, only the certificates and the key and
trust stores built
+# from them are written again. The certificates are valid for DAYS days, so
this only needs to run
+# again when they expire.
+#
+# client-expired.crt is not touched: it expired on purpose and is still signed
by the same CA key.
+#
+# Requires openssl and keytool (JDK).
+
+set -euo pipefail
+
+DAYS=7300 # 20 years
+PASS=hop123
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+SERVER_DIR="${HERE}/test-certs"
+CLIENT_DIR="${HERE}/../../../../../integration-tests/http/test-certs"
+CLIENT_DIR="$(cd "${CLIENT_DIR}" && pwd)"
+
+WORK="$(mktemp -d)"
+trap 'rm -rf "${WORK}"' EXIT
+
+cat > "${WORK}/leaf.ext" <<EOF
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid
+EOF
+
+# The CA that the mTLS server trusts
+openssl req -x509 -new -sha256 -days "${DAYS}" \
+ -key "${SERVER_DIR}/ca.key" \
+ -subj "/C=BE/ST=Flanders/L=Brussels/O=Hop Test CA/CN=Hop Test Root CA" \
+ -addext "basicConstraints=critical,CA:TRUE" \
+ -out "${SERVER_DIR}/ca.crt"
+
+# A CA the server does not trust
+openssl req -x509 -new -sha256 -days "${DAYS}" \
+ -key "${SERVER_DIR}/wrong-ca.key" \
+ -subj "/C=US/O=Wrong CA/CN=Evil CA" \
+ -addext "basicConstraints=critical,CA:TRUE" \
+ -out "${WORK}/wrong-ca.crt"
+
+# sign <key> <subject> <ca.crt> <ca.key> <out.crt>
+sign() {
+ openssl req -new -sha256 -key "$1" -subj "$2" -out "${WORK}/req.csr"
+ openssl x509 -req -sha256 -days "${DAYS}" -in "${WORK}/req.csr" \
+ -CA "$3" -CAkey "$4" -set_serial "0x$(openssl rand -hex 16)" \
+ -extfile "${WORK}/leaf.ext" -out "$5"
+}
+
+sign "${SERVER_DIR}/server.key" "/C=BE/ST=Flanders/L=Brussels/O=Hop
Test/CN=localhost" \
+ "${SERVER_DIR}/ca.crt" "${SERVER_DIR}/ca.key" "${SERVER_DIR}/server.crt"
+
+sign "${CLIENT_DIR}/client-valid.key" \
+ "/C=BE/ST=Flanders/L=Brussels/O=Hop Development/CN=Hop Test User" \
+ "${SERVER_DIR}/ca.crt" "${SERVER_DIR}/ca.key"
"${CLIENT_DIR}/client-valid.crt"
+
+sign "${CLIENT_DIR}/client-wrong.key" "/C=US/O=Evil Corp/CN=Wrong User" \
+ "${WORK}/wrong-ca.crt" "${SERVER_DIR}/wrong-ca.key"
"${CLIENT_DIR}/client-wrong.crt"
+
+# Client key stores
+openssl pkcs12 -export -name "Hop Test User" -passout "pass:${PASS}" \
+ -inkey "${CLIENT_DIR}/client-valid.key" -in "${CLIENT_DIR}/client-valid.crt"
\
+ -certfile "${SERVER_DIR}/ca.crt" -out "${CLIENT_DIR}/client-valid.p12"
+
+openssl pkcs12 -export -name "Wrong User" -passout "pass:${PASS}" \
+ -inkey "${CLIENT_DIR}/client-wrong.key" -in "${CLIENT_DIR}/client-wrong.crt"
\
+ -certfile "${WORK}/wrong-ca.crt" -out "${CLIENT_DIR}/client-wrong.p12"
+
+openssl pkcs12 -export -name "Hop Expired User" -passout "pass:${PASS}" \
+ -inkey "${CLIENT_DIR}/client-expired.key" -in
"${CLIENT_DIR}/client-expired.crt" \
+ -certfile "${SERVER_DIR}/ca.crt" -out "${CLIENT_DIR}/client-expired.p12"
+
+rm -f "${CLIENT_DIR}/client-valid.jks"
+keytool -importkeystore -noprompt \
+ -srckeystore "${CLIENT_DIR}/client-valid.p12" -srcstoretype PKCS12
-srcstorepass "${PASS}" \
+ -destkeystore "${CLIENT_DIR}/client-valid.jks" -deststoretype JKS
-deststorepass "${PASS}" \
+ -destkeypass "${PASS}"
+
+# Trust stores with the test CA
+for store in truststore.jks:JKS truststore.p12:PKCS12; do
+ file="${CLIENT_DIR}/${store%%:*}"
+ rm -f "${file}"
+ keytool -importcert -noprompt -alias hop-test-ca -file
"${SERVER_DIR}/ca.crt" \
+ -keystore "${file}" -storetype "${store##*:}" -storepass "${PASS}"
+done
+
+echo "Certificates valid until $(openssl x509 -in "${SERVER_DIR}/ca.crt"
-noout -enddate)"
diff --git a/docker/integration-tests/resource/http/mtls/test-certs/ca.crt
b/docker/integration-tests/resource/http/mtls/test-certs/ca.crt
index 607d5a5d1f..6486ba979c 100644
--- a/docker/integration-tests/resource/http/mtls/test-certs/ca.crt
+++ b/docker/integration-tests/resource/http/mtls/test-certs/ca.crt
@@ -1,8 +1,8 @@
-----BEGIN CERTIFICATE-----
-MIIFqTCCA5GgAwIBAgIUH9VHq+hA0ge99DEnhJ5tUUNvJwowDQYJKoZIhvcNAQEL
+MIIFiDCCA3CgAwIBAgIUDcpwpc9j935tqTsKA3fslXoh0U8wDQYJKoZIhvcNAQEL
BQAwZDELMAkGA1UEBhMCQkUxETAPBgNVBAgMCEZsYW5kZXJzMREwDwYDVQQHDAhC
cnVzc2VsczEUMBIGA1UECgwLSG9wIFRlc3QgQ0ExGTAXBgNVBAMMEEhvcCBUZXN0
-IFJvb3QgQ0EwHhcNMjUxMTEwMTQyNTIzWhcNMjYxMTEwMTQyNTIzWjBkMQswCQYD
+IFJvb3QgQ0EwHhcNMjYxMTE4MTA1NDI1WhcNNDYxMTEzMTA1NDI1WjBkMQswCQYD
VQQGEwJCRTERMA8GA1UECAwIRmxhbmRlcnMxETAPBgNVBAcMCEJydXNzZWxzMRQw
EgYDVQQKDAtIb3AgVGVzdCBDQTEZMBcGA1UEAwwQSG9wIFRlc3QgUm9vdCBDQTCC
AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK1lhw/eCNJXu680iHRJcjph
@@ -16,18 +16,17 @@
PVUd20BFNqAY1kYMB+jPTDs3IqMspANB7H58PK404Mzjk1aZaejPK6bhOeMSjmgF
ueuNczdAOiD818OvQ9APGmobMvpzkWyDuFK4WBZVGRV84RRD3MBwc5OqTS4nqB5G
y2tNOIipQduR7/7OGA6gLuaV7NH6qf/nEka1N++WCQDeZBfwGx4pURJ3JU6Ch1ly
vv+oA3bE22I2dZBiMLu8pGLfawULzg1MVLQuyewgheRe32DVC+wVB9aC4xc7QuhH
-eDsx3Uz37f/JAqi8rwfpAgMBAAGjUzBRMB0GA1UdDgQWBBQkNo3JV8pl4DMeYgpL
-fcmKaQaenjAfBgNVHSMEGDAWgBQkNo3JV8pl4DMeYgpLfcmKaQaenjAPBgNVHRMB
-Af8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQBWkC/46D8MUS3ZRYoqilb9PIYy
-edtdS4te1nrJHB8Y/J7fO06WZCKuTnBSBiT3ItQwGa2KL9/uGDoOljcxGc2x9YO0
-s/yh0sOPVEhT72b9wHbcRO83l02nOBfyK3iRDBYvdK0F/SwjElIxFpjJIuFsDbos
-9CmBTYiCkiiEUC+nF08HmpKQWps8jgh+JB0EfZKa47UqCYKimn60NShtIgmoFB6y
-fYfYZ0+KG7VH0ke5PIK2Os4qNz9RhF/g+Sj0eoE/+hFLSLWtkDvFhuAp+TXvs6zr
-7GtBWk2NaFCW3+890XI+0lp7GjdEyUO0vP4ggLiPEUboqGNQAiSn8BdiNAwr7M94
-ksusqLDPedPQatVop+USDRalZ8+GjNW9xA6m7TOt7U/z9W3/KoVc+NzvleEjBb9Y
-HXT5A3yCYbAgbAwnkqAbkQ2N38SnSH40FM2IEvLUl1wx4GXlvpjSTCnWISBBvrmv
-NK9M3RQ5XNONe9mfRZSxei2e/qtsbS4bUCg4rcqJ2Ic6bUWW15Ioe+Ae1vPJVWmK
-tj6WjHwjhWXYzYZO9ifQF+nsDn8VIJv0UtZrMzBIErkOUGGH1U15mrwfRlEPeQ/H
-ppuV9zWhkghWLbWETN+p+vopMYcgMHh3PSmWOJL68MIfmx2N7dS4CEgOKFf5gKw3
-IRzv0zraM7kCSJOipw==
+eDsx3Uz37f/JAqi8rwfpAgMBAAGjMjAwMB0GA1UdDgQWBBQkNo3JV8pl4DMeYgpL
+fcmKaQaenjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQAa6XNC
+wwQyW06dWGKC0beYfkHYSWX+/6pSddN7XJ9H+TpbCbRezdl3Q1Hu2pwdfGDNZVur
+TY2UMeST19qyao3+C+P78Iu9zYvwVJwrPencNg0AhuURSh35slZczHJ3lRZ0Vu4a
+uGpBBVJO8oixr5xkoj1+V/xWsm51iryNe2wMkiB65AVzjTcVH1JitmdmTKMklnQl
+a1H/N9rIU6b4Jlx6HVI0p+5Naey6wIaVrUaBmJcAlBohpNdtucV66vmf4aSUrJsq
+zT37irzMzNU8RrfY21kQWFn4PufULcYf9Sn8TRrgPsOHl2GIjHOBcNwA/P13lPQl
+uxGIsz9WLE3WKsYPN7imHeWbP0Qecups1jFOc5McWtht+0A6Q89IfpRlaQHySkgf
+QW+WDZ3HL6OyTiuaFJuY411E8ywXuR3Vqc7ncgJKFfLhKjqv5yn6IVrwUFQqptKl
+Hf+kH6wEdP/r6zk7dzBbkblFn0JSUQzps6bTuBaDVlbhllAc4z0RyH8BAgvEN04d
+vLUIagJmprDZqD7gG1ZJeTALtnBsgSDUE2FQa97Qi4ID6hnwLrZiCAFZYnFP1Ewa
+yq48+9V/WsustiYYcbgXovoB0pdmk5xwm9ezv7Zb3y90NTe8vz9AZYf0I3EvA0hl
+KBMHs825O2V4AH3USOrV7CBPoewRiQop9Io9og==
-----END CERTIFICATE-----
diff --git a/docker/integration-tests/resource/http/mtls/test-certs/server.crt
b/docker/integration-tests/resource/http/mtls/test-certs/server.crt
index 7315c3d6b0..be5ccfd12f 100644
--- a/docker/integration-tests/resource/http/mtls/test-certs/server.crt
+++ b/docker/integration-tests/resource/http/mtls/test-certs/server.crt
@@ -1,32 +1,32 @@
-----BEGIN CERTIFICATE-----
-MIIFjjCCA3agAwIBAgIUBgZ/VnTG4ZThwDKUo9onmn5G190wDQYJKoZIhvcNAQEL
-BQAwZDELMAkGA1UEBhMCQkUxETAPBgNVBAgMCEZsYW5kZXJzMREwDwYDVQQHDAhC
-cnVzc2VsczEUMBIGA1UECgwLSG9wIFRlc3QgQ0ExGTAXBgNVBAMMEEhvcCBUZXN0
-IFJvb3QgQ0EwHhcNMjUxMTEwMTQyNTIzWhcNMjYxMTEwMTQyNTIzWjBaMQswCQYD
-VQQGEwJCRTERMA8GA1UECAwIRmxhbmRlcnMxETAPBgNVBAcMCEJydXNzZWxzMREw
-DwYDVQQKDAhIb3AgVGVzdDESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG
-9w0BAQEFAAOCAg8AMIICCgKCAgEAtdvY2jdVw9bdGXLCOQkgLHdIQsU7r1XG7D3v
-5nNNdN2QlZ6PBOqn8QveSVd9H82HZkIDZXhUw2GVc+xoJSCGnTH1y3KndWRWDb4e
-SUaWrSsAtn2HTT2544oQsQZWqFJDbMSkDg48rCa5ybRtvBKsHEXhl79AXNt1n6D5
-kLBMaK6ys7x1uu1vTvDuu1npkNU/VicM1cILAJNBpvGzuU/rqfk7F1+ot6pG/t9z
-Z8Ko5Fbw1zm0gSs7lxQAm5pfMjxT5y3hCMlVf14mWrv0zwqNvOQ/gR7tCLmwwzxn
-GI5ySbAanuafgbJaSJGq7J8riFTr7LG39R0XM3pJ7Cd9tzoidnBjsWI0vvATTZXn
-a8iGYndnax+pRERY3w0YSq5Y37fP8LWtcCEDdUIJR9ksAN4yrhevMLfdqi5TXsPs
-XivhC5tKbJKI/tRr/H6+ACV5APUhcxMo1keBl+Q/HlGQKZgtSC6xSt+2e5H6RDGg
-qfuwP0MwJcM9ZunO0mLAsoTcwgJl7iOKnsFi82VVTxCycNTgH+mLzUdTicrwegmA
-/rUZRjyXX7fVP9HI7WrF0yPmquXojJmYCH0hRDEKb9+uBkl0biXZ5RH9pSxn05SN
-ssQFimOJiFmCzePNenu5EVlmoQJ8V5eFqw9gh/zKGGTclqchmeiiRetS1kbnOy+n
-/SgJUNcCAwEAAaNCMEAwHQYDVR0OBBYEFOnOXT+CiTm5DkI8SDq//ArN/79VMB8G
-A1UdIwQYMBaAFCQ2jclXymXgMx5iCkt9yYppBp6eMA0GCSqGSIb3DQEBCwUAA4IC
-AQBwCFtrlIuGq63Mcs489QVFCPlMNrpF3nmCzqBAEWRi0SoFIA0l2dmxeqCCEpL6
-xsjx6Kr+ic6PBgdQxm+k3NodSv+T+z9t4mqKWGpxWMpN9k7HROCJCyw114rKNcz4
-TffmGlG1m8p0Qw5v5+vNmUsMoAiDxt6PTj2IdwRFme/VbgoFihhqGuq5Zt1xC7dB
-gikwO0U10PX0aTcKPW96PuyMoYkCH1+Wo+4j0pWCv66Pjit5adc8sIODS3UYW0ab
-hZJUd20wM1jmjWP8+dYZGoNli67KdrpYNcVtHux8d7fK92lDkBJIC0gPxY5rFf28
-sy94IGIZLCGPUgiBaaS+tKfK5HYSi1l3v55j4qyWv36kMImAI+uoOqGvaLLuKFRw
-6Hvr+r3SrF+vZEqzrnOUBqRkASUorL7f4UKtiCIGd6Xo1tEjunyTiq0ega7xGBf2
-SYFGLWpAwiW0Qm6lxP0v/1WU2Q4UcLUS/ot50dPOCt2oZupqXgWK19CcHyklUnTg
-HHGn40GM9YYO8IZubFzGSlpm+CfC3h/Skh9ndP5ktFzqmUe0u/LL26WrnAfgT08c
-dn+48IqPw68CPEGenhtAIvyDyS2KUjcolhawxkpXf33QmRgS7Dcnyn3kp96Q4Geh
-E8IIARk3CBtKYkBFS2IcXcP6jdiZi/swz8/bD7+BsWWcJQ==
+MIIFizCCA3OgAwIBAgIRAJ7t+E1q00aYNq4tK9tTPIgwDQYJKoZIhvcNAQELBQAw
+ZDELMAkGA1UEBhMCQkUxETAPBgNVBAgMCEZsYW5kZXJzMREwDwYDVQQHDAhCcnVz
+c2VsczEUMBIGA1UECgwLSG9wIFRlc3QgQ0ExGTAXBgNVBAMMEEhvcCBUZXN0IFJv
+b3QgQ0EwHhcNMjYxMTE4MTA1NDI1WhcNNDYxMTEzMTA1NDI1WjBaMQswCQYDVQQG
+EwJCRTERMA8GA1UECAwIRmxhbmRlcnMxETAPBgNVBAcMCEJydXNzZWxzMREwDwYD
+VQQKDAhIb3AgVGVzdDESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0B
+AQEFAAOCAg8AMIICCgKCAgEAtdvY2jdVw9bdGXLCOQkgLHdIQsU7r1XG7D3v5nNN
+dN2QlZ6PBOqn8QveSVd9H82HZkIDZXhUw2GVc+xoJSCGnTH1y3KndWRWDb4eSUaW
+rSsAtn2HTT2544oQsQZWqFJDbMSkDg48rCa5ybRtvBKsHEXhl79AXNt1n6D5kLBM
+aK6ys7x1uu1vTvDuu1npkNU/VicM1cILAJNBpvGzuU/rqfk7F1+ot6pG/t9zZ8Ko
+5Fbw1zm0gSs7lxQAm5pfMjxT5y3hCMlVf14mWrv0zwqNvOQ/gR7tCLmwwzxnGI5y
+SbAanuafgbJaSJGq7J8riFTr7LG39R0XM3pJ7Cd9tzoidnBjsWI0vvATTZXna8iG
+Yndnax+pRERY3w0YSq5Y37fP8LWtcCEDdUIJR9ksAN4yrhevMLfdqi5TXsPsXivh
+C5tKbJKI/tRr/H6+ACV5APUhcxMo1keBl+Q/HlGQKZgtSC6xSt+2e5H6RDGgqfuw
+P0MwJcM9ZunO0mLAsoTcwgJl7iOKnsFi82VVTxCycNTgH+mLzUdTicrwegmA/rUZ
+RjyXX7fVP9HI7WrF0yPmquXojJmYCH0hRDEKb9+uBkl0biXZ5RH9pSxn05SNssQF
+imOJiFmCzePNenu5EVlmoQJ8V5eFqw9gh/zKGGTclqchmeiiRetS1kbnOy+n/SgJ
+UNcCAwEAAaNCMEAwHQYDVR0OBBYEFOnOXT+CiTm5DkI8SDq//ArN/79VMB8GA1Ud
+IwQYMBaAFCQ2jclXymXgMx5iCkt9yYppBp6eMA0GCSqGSIb3DQEBCwUAA4ICAQAR
+9ETMHU1f/NDMMlnR3ZsS2/s0q+Nvrnwf99d10kPpCMsUVSq6JWQ1GA7sSvINmQfV
+JTsj0W3Rl3MT/PZOA9pYt+D8NWXTTDG8/TvFnu/MYkw4RTEmeiH56yJGYgkE5QEN
+xo9Hx7OlYnEomN86WYNZCEndlTwDL+LcuC6OO0m8OpQSdq3IOpco9ghvxbBTmBwv
+ZK5WvtTUm6OOa2d5hJPFQwsjeUBU7L1B/JNLp0Y89WbWoFmlUrahN+V5PxP08D6M
+aWwFDGbSpTlFe4x0kXQV4AYuOD82XLi80HG8X2Es4f9Eyc4u2r4rPnQcZ+lWd9X3
+ezvTDirjxMe9e0csks3lC2zwuJBgj8t5EXoc0VK2dtTtonBsKodiCLoI9m0fwDii
+ES6qUWB2vNSmOrBk7FxR0KVriYn42dMQwtxh648/3y54+WVQHhUepiaTu85u1uG3
+cwi8ld+SBGcjLq3aOw9cKdldUZb/oDlF0Uns+TYjYhEwpgYxd8eJ6kdusZ+BuQyg
+OrDyNY0JZhGRslpDV2n3lk2Uku9MXmBS+Lc0zd63LheydqEh184/BtMPz+wCypCs
+xhUGnXFUqkIjLy2tYAcMAGpXemiJbm6E/UDtIMdBATF/+yGCYEl+JvjL1q3nk5nD
+e1WxOIUzzMryMsZs8SGhVLi6SEqadhV6CdOQMHvadw==
-----END CERTIFICATE-----
diff --git a/integration-tests/hop_run/hop-import-exit-code.sh
b/integration-tests/hop_run/hop-import-exit-code.sh
old mode 100644
new mode 100755
diff --git a/integration-tests/http/test-certs/client-expired.p12
b/integration-tests/http/test-certs/client-expired.p12
index c6c8b0b019..fe13823bb5 100644
Binary files a/integration-tests/http/test-certs/client-expired.p12 and
b/integration-tests/http/test-certs/client-expired.p12 differ
diff --git a/integration-tests/http/test-certs/client-valid.crt
b/integration-tests/http/test-certs/client-valid.crt
index baaf4019c6..80dabc4067 100644
--- a/integration-tests/http/test-certs/client-valid.crt
+++ b/integration-tests/http/test-certs/client-valid.crt
@@ -1,32 +1,32 @@
-----BEGIN CERTIFICATE-----
-MIIFmTCCA4GgAwIBAgIUBgZ/VnTG4ZThwDKUo9onmn5G194wDQYJKoZIhvcNAQEL
-BQAwZDELMAkGA1UEBhMCQkUxETAPBgNVBAgMCEZsYW5kZXJzMREwDwYDVQQHDAhC
-cnVzc2VsczEUMBIGA1UECgwLSG9wIFRlc3QgQ0ExGTAXBgNVBAMMEEhvcCBUZXN0
-IFJvb3QgQ0EwHhcNMjUxMTEwMTQyNTIzWhcNMjYxMTEwMTQyNTIzWjBlMQswCQYD
-VQQGEwJCRTERMA8GA1UECAwIRmxhbmRlcnMxETAPBgNVBAcMCEJydXNzZWxzMRgw
-FgYDVQQKDA9Ib3AgRGV2ZWxvcG1lbnQxFjAUBgNVBAMMDUhvcCBUZXN0IFVzZXIw
-ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCnGE0o9JhjsKSvT0T3/s3w
-Cu3dkS4hrBsNCcL6QzN/Tyk+djqUZlRAGL+ScStbyurKeQTu88K+AlEpBoxgR/8i
-PWJKhyhKPLW/shI99b6hsCe5A0JQseOA8CERJxFswA33OtOzyvojosK4yQegFTsT
-80aHkTf7gdb1SSSakZ56Ji4YVVEJfw2p+ohmSHQHPiVEjihGo4VSRkmvzRNIlNZq
-EFPzyB1WwTb1zrs2QbBEYvLhwgF1fG09cehp8UfPq3PlkMM1Z4zWnlagC7q/6uvP
-OnGSvIwKcsglOUBN8w7DKzX2gkfenX/TaRJW39qDzOym4VzroNXJQWtVrOvGntWu
-m+Qytd1743FH6ij8gNLDXjXwTqaMTCzx6EbxyFOGT3gFcRie4KpyxIGG2Tbo2RRp
-RkBpsWCMSIJXqzSVVoO9rQz0+mzTJtuggyGVhPrK01jylC/G42Ur+Szex5igpohr
-QNylZDhij1+r/c+LnClnDGWVPwH5SXGAfPIG3dscbiJ1v3Ef8WDn5w7l3Hw5AjtS
-GkcOwkK8c6GTSlUB9sWYjhiy4bThecEX+IEUJA59czi93CxrRPJc4de+qBD2Q0ee
-cKAATPZQckbdnegS8LmnOUhMdIrhGO90+wdG5TEZI52/3KvtCV7UpSUo2+7Q9vtT
-l2DcVJacNxXP1Po96aL7BQIDAQABo0IwQDAdBgNVHQ4EFgQUs8ZH4wOdqkRkZg28
-0GhafJXlL6gwHwYDVR0jBBgwFoAUJDaNyVfKZeAzHmIKS33JimkGnp4wDQYJKoZI
-hvcNAQELBQADggIBAH5S9Xrb/aYhLqnJkI+RbbEn7pB3kG5yAfYZYJUAgoLCtkVl
-nFRGIiyFC2adFsPrDpNisLhlfehInCdobNXtiXJh7OEnx55GGf4kjkTpb77fFFqU
-Mo5b1n/NXClgZIOb+6f3K7VLSGaurmAQ0xl0lc08LDuuxBCAIEW8/iLTex7MsZA+
-4a+/Xz+dwawUeBN5VLZNXQJEtoApPGjExcYr4gFSWgz8L0FjXM041Thw4go2mtuW
-NA5bi1Yo/FFJ2bh+w+xdTXvy0k6P2fUWv7e3tJhEAV7k52y6PKwo5YQW7dCK1aJb
-YAvxIwJHPyNqp5rw9iMcYHrNkybULgipLTtDWH0jkvQ4zakx8Uxs7zFz5ZJijegb
-eQH4jiCD0PxsTBu92Uo0fA3Ib99/k5Xk3Ep+Fo5iW3GbENAckLa7rZoPmMlHtoQo
-ZRY688zMvWaDUMZ8+nAHsgY6h7fltQEKaRonnqT/CltUA5dmeRQUbvOpFsoFy7F5
-F2CmkOp5kr6Id7aqyyWgThj3cYdH1cae15MmX6tJQGd6DLQ1IVoKUtw5vrhZfQva
-sv4f60dhDLAuiSX/mrMaYmCVsOx85Ouf+vcHREN8FYerD7SVlJcVm+eWdOD/Z1wR
-9tV3/PJs67S5A9a3714NAymiBW1fvUS19i4ip7ZPzfwolnZ4nzJk2PfRau2J
+MIIFlTCCA32gAwIBAgIQPpeAY6c1rS3TURCXsLAVRzANBgkqhkiG9w0BAQsFADBk
+MQswCQYDVQQGEwJCRTERMA8GA1UECAwIRmxhbmRlcnMxETAPBgNVBAcMCEJydXNz
+ZWxzMRQwEgYDVQQKDAtIb3AgVGVzdCBDQTEZMBcGA1UEAwwQSG9wIFRlc3QgUm9v
+dCBDQTAeFw0yNjExMTgxMDU0MjVaFw00NjExMTMxMDU0MjVaMGUxCzAJBgNVBAYT
+AkJFMREwDwYDVQQIDAhGbGFuZGVyczERMA8GA1UEBwwIQnJ1c3NlbHMxGDAWBgNV
+BAoMD0hvcCBEZXZlbG9wbWVudDEWMBQGA1UEAwwNSG9wIFRlc3QgVXNlcjCCAiIw
+DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAKcYTSj0mGOwpK9PRPf+zfAK7d2R
+LiGsGw0JwvpDM39PKT52OpRmVEAYv5JxK1vK6sp5BO7zwr4CUSkGjGBH/yI9YkqH
+KEo8tb+yEj31vqGwJ7kDQlCx44DwIREnEWzADfc607PK+iOiwrjJB6AVOxPzRoeR
+N/uB1vVJJJqRnnomLhhVUQl/Dan6iGZIdAc+JUSOKEajhVJGSa/NE0iU1moQU/PI
+HVbBNvXOuzZBsERi8uHCAXV8bT1x6GnxR8+rc+WQwzVnjNaeVqALur/q6886cZK8
+jApyyCU5QE3zDsMrNfaCR96df9NpElbf2oPM7KbhXOug1clBa1Ws68ae1a6b5DK1
+3XvjcUfqKPyA0sNeNfBOpoxMLPHoRvHIU4ZPeAVxGJ7gqnLEgYbZNujZFGlGQGmx
+YIxIglerNJVWg72tDPT6bNMm26CDIZWE+srTWPKUL8bjZSv5LN7HmKCmiGtA3KVk
+OGKPX6v9z4ucKWcMZZU/AflJcYB88gbd2xxuInW/cR/xYOfnDuXcfDkCO1IaRw7C
+QrxzoZNKVQH2xZiOGLLhtOF5wRf4gRQkDn1zOL3cLGtE8lzh176oEPZDR55woABM
+9lByRt2d6BLwuac5SEx0iuEY73T7B0blMRkjnb/cq+0JXtSlJSjb7tD2+1OXYNxU
+lpw3Fc/U+j3povsFAgMBAAGjQjBAMB0GA1UdDgQWBBSzxkfjA52qRGRmDbzQaFp8
+leUvqDAfBgNVHSMEGDAWgBQkNo3JV8pl4DMeYgpLfcmKaQaenjANBgkqhkiG9w0B
+AQsFAAOCAgEAOZYWn4i8xLVqs0r5IaJAOsZd2KNsW905w22sKhWP+I3pt9C/zPkY
+s4q33OSfPClWMPdmmk5di84tACYezujFVlY8CmEx+BqXgEpPyys1A8ucbYtu9qa5
+eAJnOyUHwnegZC4WFSxTo9IBcCVqU8Abj6Dww1G56T/t0uRcBEG9RPNrdnAPJXIz
+Y/A11e3DUFqIHZUN6M1exisC1LTzy3JMVSC/zzm0dUUaAqwuohRabMlBCB6XCrc4
+7dltuA7WQL3gdSoCF2lau3LBx31w7lidfRvO4nRlLC3Cxlum9UMb8bKbY7M114o1
+FKcrwPLz1Ny+CCfUwGEeIBVcpLF59XhJPduHcDFXU45xpkBvjHW26i4DazUJtERl
+fsW7of4XpxTKHRjRrEYjAiD8a0YZgRijmk+AmtTj4IINnhBj1aTqpk7tg1Ps46iD
+tNr7kr/qlgL391LTAb2yv1qiF8vtTCi1QG3QLFwejbNA/ncw9Nk9eaRVWUl2Cqe6
+5zmM22ipH/GRDjYUYIqPKIfck1Jc/Nc6z55kM+lMzMb8LR/w3Y/pWQQ2Lr1Ampv/
+vx14ywIO7xCYe2QSI62HIG/TGav+4hdtRBvOh4VILCIaMJnQUJM2kJzvfjgW/wMB
+PyXurJ/GRF4u/q45KWpCD59xAG389uCw0VGaemcbzPnm3xJ30ACCb68=
-----END CERTIFICATE-----
diff --git a/integration-tests/http/test-certs/client-valid.jks
b/integration-tests/http/test-certs/client-valid.jks
index cafff2f60d..da54503933 100644
Binary files a/integration-tests/http/test-certs/client-valid.jks and
b/integration-tests/http/test-certs/client-valid.jks differ
diff --git a/integration-tests/http/test-certs/client-valid.p12
b/integration-tests/http/test-certs/client-valid.p12
index a365f3605a..0832f00d21 100644
Binary files a/integration-tests/http/test-certs/client-valid.p12 and
b/integration-tests/http/test-certs/client-valid.p12 differ
diff --git a/integration-tests/http/test-certs/client-wrong.crt
b/integration-tests/http/test-certs/client-wrong.crt
index 6d05812c87..103e83c306 100644
--- a/integration-tests/http/test-certs/client-wrong.crt
+++ b/integration-tests/http/test-certs/client-wrong.crt
@@ -1,30 +1,30 @@
-----BEGIN CERTIFICATE-----
-MIIFODCCAyCgAwIBAgIUHeKLE6HOwYyxYckTs1UoDfENZLcwDQYJKoZIhvcNAQEL
-BQAwMjELMAkGA1UEBhMCVVMxETAPBgNVBAoMCFdyb25nIENBMRAwDgYDVQQDDAdF
-dmlsIENBMB4XDTI1MTExMDE0MjUyNFoXDTI2MTExMDE0MjUyNFowNjELMAkGA1UE
-BhMCVVMxEjAQBgNVBAoMCUV2aWwgQ29ycDETMBEGA1UEAwwKV3JvbmcgVXNlcjCC
-AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANwPU0Itlj43jVcPTIwXvgsW
-ZdLXODqboiRZV9xLlh1AIMU5uMRdTNs9BGxnsH8xImfGlbt4Q4slivHsZhKfXeia
-FntoiSoM4NG7mEv7n893h76fzc0KOTmcj77fI9Wv5i0gwpPpD6rAOKiWeAI3Tq2i
-YOC9NaeJypnwhOg0PKiTtQ0kDaTlNvV19CKNP+5Op1ch9iV67PBqWTshvw96UmPx
-VVOAFIz0J1FRIPuk1x4QbgeRBPDvYxDuzPBg9qGONSLtmxFFa65lPb5qzt/HDYce
-CxTj0chI5z/myV/PaTgnvNZ7vtt7l0HhvDWsba2QiHVDWkXIAz6xgwKM04+YA/d4
-yPTUAP+2/LskS/3mw8YLQSzqPw/lQ9CVN1mgR26BWOOrHJwdIzVXGdAcqk4zgLzh
-WeKOJfwHryUQdPnChnlpzIMu2XqoZ+CVnxVXUS6gi1LyVAA59kGlrdv3mnWcKnZ1
-FvmOrVgtBxnJY8rbOSRzdOWu53pXvB85gE8SNorOhoRL9HHGl5i33FDqjb0SV78U
-xWdpcPeHvvLeXKyQjGcVlcv3n47uwjAuC9bdb0A6o4gLfmPMUkkWNFIQ6D+kAgQl
-uD6lwABkmGt5l63D0/85+mJiMA1Fbz2Xh1EhDNqZbarD/FW6NS5evvmnsxPmkVD1
-C/CAMzsejgaiK1rBpdEVAgMBAAGjQjBAMB0GA1UdDgQWBBTkNB9TY8b6T0bn/bzT
-X2wWNm0OlzAfBgNVHSMEGDAWgBSQZuVC2ER7eM/AEs7JjFWU5U35aDANBgkqhkiG
-9w0BAQsFAAOCAgEAAs79SlGyAP1o+cSvDAyDmhwjj8vkupA/ASbO7uFmq3yEkcH2
-BhR8LzFLwK0MkZIQWnINswhWnsg2fMX3JVAhrW3bt5BAXSOJIccbPoTfqrw5GoIU
-F6DsDzQr0Pp64aH/hJXDd59//GVm8F2ss0oIzysG80bCEUD3JKXUjQbFlCNYUUQg
-9HPVcyEJ1yQtzCYZbuub23DyZDU+Fpv7zi7BcDnKJiM0W6IWY5Rr/Rwulm42DsyA
-VIXw9hDqsOFh4Aaw3NfyHgcSZXxNfcfDV+Yne8D1X7FYEnWs7SCbZHA93Ym//QCv
-dhxO1AQqAKKTL8F8z2fZUpICBgrcgoY/kpLBI69laFmAvlAoppWxHJ4l+GMlWir8
-Vcs8UpbFgh+8TxWgh//hFdyAKDKr98eE5pFYNk2k45tS88HWmB4OvgWQ88AmPS4P
-wEkQdtLRb1Qteg2A2L2tHNGuV8AKpHCfkBRRhpsD6VoLuMb8VCoO9qf7PF8DWoE4
-/RteM6MpYqtlllnZlWQkOvxHI9TbQK433aVOq7ViCGUyXqRnkc0aq0WfQnpTbiso
-+/q/Cl0zuLLe3Qz72Enz82DLDYgsAWE/zCwTSaxokxs9pNHv7nfvsc0C565fyw2B
-YWQn992U7S9Y5ceCXys2kiWqASDXbvp7loMAubn7DGyLc0r+q1Q9joY6LGE=
+MIIFNTCCAx2gAwIBAgIRAJIA2DlDk7y/B00ZYFq5OVswDQYJKoZIhvcNAQELBQAw
+MjELMAkGA1UEBhMCVVMxETAPBgNVBAoMCFdyb25nIENBMRAwDgYDVQQDDAdFdmls
+IENBMB4XDTI2MTExODEwNTQyNVoXDTQ2MTExMzEwNTQyNVowNjELMAkGA1UEBhMC
+VVMxEjAQBgNVBAoMCUV2aWwgQ29ycDETMBEGA1UEAwwKV3JvbmcgVXNlcjCCAiIw
+DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANwPU0Itlj43jVcPTIwXvgsWZdLX
+ODqboiRZV9xLlh1AIMU5uMRdTNs9BGxnsH8xImfGlbt4Q4slivHsZhKfXeiaFnto
+iSoM4NG7mEv7n893h76fzc0KOTmcj77fI9Wv5i0gwpPpD6rAOKiWeAI3Tq2iYOC9
+NaeJypnwhOg0PKiTtQ0kDaTlNvV19CKNP+5Op1ch9iV67PBqWTshvw96UmPxVVOA
+FIz0J1FRIPuk1x4QbgeRBPDvYxDuzPBg9qGONSLtmxFFa65lPb5qzt/HDYceCxTj
+0chI5z/myV/PaTgnvNZ7vtt7l0HhvDWsba2QiHVDWkXIAz6xgwKM04+YA/d4yPTU
+AP+2/LskS/3mw8YLQSzqPw/lQ9CVN1mgR26BWOOrHJwdIzVXGdAcqk4zgLzhWeKO
+JfwHryUQdPnChnlpzIMu2XqoZ+CVnxVXUS6gi1LyVAA59kGlrdv3mnWcKnZ1FvmO
+rVgtBxnJY8rbOSRzdOWu53pXvB85gE8SNorOhoRL9HHGl5i33FDqjb0SV78UxWdp
+cPeHvvLeXKyQjGcVlcv3n47uwjAuC9bdb0A6o4gLfmPMUkkWNFIQ6D+kAgQluD6l
+wABkmGt5l63D0/85+mJiMA1Fbz2Xh1EhDNqZbarD/FW6NS5evvmnsxPmkVD1C/CA
+MzsejgaiK1rBpdEVAgMBAAGjQjBAMB0GA1UdDgQWBBTkNB9TY8b6T0bn/bzTX2wW
+Nm0OlzAfBgNVHSMEGDAWgBSQZuVC2ER7eM/AEs7JjFWU5U35aDANBgkqhkiG9w0B
+AQsFAAOCAgEAJhMWoDN0GxfZ7oJdBFlM7s/5+xhH6n7x40GP40ORUPdJenw4OcsD
+hiluDxedIxyOGqndqL8qsIMLG83GbSNgzedNKDPu5UKFzSP0eGYIIhd6NcxO07zy
+mU6u+nfRjzczn2AMiCN8LiX9eLLuIW08JFO7wo4y82Xz28+ZBLa+o3ZR20pldpiN
+1C8RGsCZRbrEpgP9kygJ+/Qawdmkk/7Knd+G2YlezgEgyMraXjsRpZFHZ+majgm2
+hI9lPQ5QlLs6JScz4Rz0GOO/SDYswglroNo3aNn08oZwhUh+LjRO8JKQW+LDUMYJ
+IZ8kAjf0b67zravl7w6X6G3hdiHPdhqtDCeADe+JEAMktkO6+j7rzur2TrtTwmr8
+g1a0EWVpEYAcGisWKDItoPg6nX1kI874aT7hikDHykIgSDltDATtJUkPKQjtzp/w
+aDjRyn6au9z4X5JOXBBmsuxXmzgLnc2D/mLVdspMoAebFOKYtRSvTvQXoRE2TafH
+mM6tBz+pobKDZSbQVN284ac4IH6TjFkTacdl8pup6TXeY/aSxQXm6dWE0R1Sz80z
+A2Ob47xiDdWnG9ZdE3WnQNzCZP9GG5ods2ZA+S5TL6BEkDuloyYTJV4tXbiLwpri
+B3esjEv436zN0gTVEvcaArL+8agvau/CJBe/hKPhTNXFlHQN5V1lMoE=
-----END CERTIFICATE-----
diff --git a/integration-tests/http/test-certs/client-wrong.p12
b/integration-tests/http/test-certs/client-wrong.p12
index 1974f1dff2..e0a11e3ef3 100644
Binary files a/integration-tests/http/test-certs/client-wrong.p12 and
b/integration-tests/http/test-certs/client-wrong.p12 differ
diff --git a/integration-tests/http/test-certs/truststore.jks
b/integration-tests/http/test-certs/truststore.jks
index 4d1e4e8906..c267a072cf 100644
Binary files a/integration-tests/http/test-certs/truststore.jks and
b/integration-tests/http/test-certs/truststore.jks differ
diff --git a/integration-tests/http/test-certs/truststore.p12
b/integration-tests/http/test-certs/truststore.p12
index 7d9235acdd..bbcd2e89ac 100644
Binary files a/integration-tests/http/test-certs/truststore.p12 and
b/integration-tests/http/test-certs/truststore.p12 differ
diff --git a/integration-tests/json/0018-enhanced-json-output-streaming.hpl
b/integration-tests/json/0018-enhanced-json-output-streaming.hpl
deleted file mode 100644
index c6709759af..0000000000
--- a/integration-tests/json/0018-enhanced-json-output-streaming.hpl
+++ /dev/null
@@ -1,202 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
--->
-<pipeline>
- <info>
- <name>0018-enhanced-json-output-streaming</name>
- <name_sync_with_filename>Y</name_sync_with_filename>
- <description/>
- <extended_description/>
- <pipeline_version/>
- <pipeline_type>Normal</pipeline_type>
- <parameters>
- <parameter>
- <name>OUTPUT_FILE</name>
- <default_value/>
- <description>The JSON file to write, without extension</description>
- </parameter>
- <parameter>
- <name>ROWS</name>
- <default_value>1000</default_value>
- <description>The number of rows to write</description>
- </parameter>
- </parameters>
- <capture_transform_performance>N</capture_transform_performance>
-
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
-
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
- <created_user>-</created_user>
- <created_date>2026/09/30 16:00:00.000</created_date>
- <modified_user>-</modified_user>
- <modified_date>2026/09/30 16:00:00.000</modified_date>
- </info>
- <notepads>
- <notepad>
- <backgroundcolorblue>251</backgroundcolorblue>
- <backgroundcolorgreen>232</backgroundcolorgreen>
- <backgroundcolorred>201</backgroundcolorred>
- <bordercolorblue>90</bordercolorblue>
- <bordercolorgreen>58</bordercolorgreen>
- <bordercolorred>14</bordercolorred>
- <fontbold>N</fontbold>
- <fontcolorblue>90</fontcolorblue>
- <fontcolorgreen>58</fontcolorgreen>
- <fontcolorred>14</fontcolorred>
- <fontitalic>N</fontitalic>
- <fontname>Noto Sans</fontname>
- <fontsize>10</fontsize>
- <height>60</height>
- <xloc>64</xloc>
- <yloc>144</yloc>
- <note>Run by main-0018 in a separate hop-run with -Xmx128m. The JSON
file is about 127MB,
-smaller than that heap, so it catches a retained document rather than a file
larger than the
-entire heap. One unlimited group is still held until the group ends.</note>
- <width>560</width>
- </notepad>
- </notepads>
- <order>
- <hop>
- <from>generate rows</from>
- <to>add id</to>
- <enabled>Y</enabled>
- </hop>
- <hop>
- <from>add id</from>
- <to>Enhanced JSON Output</to>
- <enabled>Y</enabled>
- </hop>
- </order>
- <transform>
- <name>generate rows</name>
- <type>RowGenerator</type>
- <description/>
- <distribute>Y</distribute>
- <custom_distribution/>
- <copies>1</copies>
- <partitioning>
- <method>none</method>
- <schema_name/>
- </partitioning>
- <fields>
- <field>
- <currency/>
- <decimal/>
- <format/>
- <group/>
- <length>-1</length>
- <name>payload</name>
- <precision>-1</precision>
- <set_empty_string>N</set_empty_string>
- <type>String</type>
-
<nullif>xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx</nullif>
- </field>
- </fields>
- <interval_in_ms>5000</interval_in_ms>
- <last_time_field>FiveSecondsAgo</last_time_field>
- <limit>${ROWS}</limit>
- <never_ending>N</never_ending>
- <row_time_field>now</row_time_field>
- <attributes/>
- <GUI>
- <xloc>64</xloc>
- <yloc>64</yloc>
- </GUI>
- </transform>
- <transform>
- <name>add id</name>
- <type>Sequence</type>
- <description/>
- <distribute>Y</distribute>
- <custom_distribution/>
- <copies>1</copies>
- <partitioning>
- <method>none</method>
- <schema_name/>
- </partitioning>
- <counter_name/>
- <use_counter>Y</use_counter>
- <connection/>
- <use_database>N</use_database>
- <increment_by>1</increment_by>
- <max_value>999999999</max_value>
- <schema/>
- <seqname>SEQ_</seqname>
- <start_at>1</start_at>
- <valuename>id</valuename>
- <attributes/>
- <GUI>
- <xloc>240</xloc>
- <yloc>64</yloc>
- </GUI>
- </transform>
- <transform>
- <name>Enhanced JSON Output</name>
- <type>EnhancedJsonOutput</type>
- <description/>
- <distribute>Y</distribute>
- <custom_distribution/>
- <copies>1</copies>
- <partitioning>
- <method>none</method>
- <schema_name/>
- </partitioning>
- <outputValue>outputValue</outputValue>
- <jsonBloc>data</jsonBloc>
- <operation_type>writetofile</operation_type>
- <use_arrays_with_single_instance>N</use_arrays_with_single_instance>
- <use_single_item_per_group>N</use_single_item_per_group>
- <json_prittified>N</json_prittified>
- <encoding>UTF-8</encoding>
- <addtoresult>N</addtoresult>
- <file>
- <name>${OUTPUT_FILE}</name>
- <split_output_after>0</split_output_after>
- <extention>json</extention>
- <append>N</append>
- <create_parent_folder>Y</create_parent_folder>
- <doNotOpenNewFileInit>N</doNotOpenNewFileInit>
- </file>
- <json_size_field/>
- <key_fields>
- </key_fields>
- <fields>
- <field>
- <name>id</name>
- <element>id</element>
- <json_fragment>N</json_fragment>
- <is_without_enclosing>N</is_without_enclosing>
- <remove_if_blank>N</remove_if_blank>
- </field>
- <field>
- <name>payload</name>
- <element>payload</element>
- <json_fragment>N</json_fragment>
- <is_without_enclosing>N</is_without_enclosing>
- <remove_if_blank>N</remove_if_blank>
- </field>
- </fields>
- <attributes/>
- <GUI>
- <xloc>416</xloc>
- <yloc>64</yloc>
- </GUI>
- </transform>
- <transform_error_handling>
- </transform_error_handling>
- <attributes/>
-</pipeline>
diff --git
a/integration-tests/json/main-0018-enhanced-json-output-streaming.hwf
b/integration-tests/json/main-0018-enhanced-json-output-streaming.hwf
deleted file mode 100644
index c4aa06758e..0000000000
--- a/integration-tests/json/main-0018-enhanced-json-output-streaming.hwf
+++ /dev/null
@@ -1,188 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
--->
-<workflow>
- <name>main-0018-enhanced-json-output-streaming</name>
- <name_sync_with_filename>Y</name_sync_with_filename>
- <description>Enhanced JSON Output writes a file that is larger than the
heap</description>
- <extended_description/>
- <workflow_version/>
- <created_user>-</created_user>
- <created_date>2026/09/30 16:00:00.000</created_date>
- <modified_user>-</modified_user>
- <modified_date>2026/09/30 16:00:00.000</modified_date>
- <parameters>
- </parameters>
- <actions>
- <action>
- <name>Start</name>
- <description/>
- <type>SPECIAL</type>
- <attributes/>
- <DayOfMonth>1</DayOfMonth>
- <hour>12</hour>
- <intervalMinutes>60</intervalMinutes>
- <intervalSeconds>0</intervalSeconds>
- <minutes>0</minutes>
- <repeat>N</repeat>
- <schedulerType>0</schedulerType>
- <weekDay>1</weekDay>
- <parallel>N</parallel>
- <xloc>112</xloc>
- <yloc>96</yloc>
- <attributes_hac/>
- </action>
- <action>
- <name>Write a JSON file larger than the heap</name>
- <description/>
- <type>SHELL</type>
- <attributes/>
- <filename/>
- <work_directory/>
- <arg_from_previous>N</arg_from_previous>
- <exec_per_row>N</exec_per_row>
- <set_logfile>N</set_logfile>
- <logfile/>
- <set_append_logfile>N</set_append_logfile>
- <logext/>
- <add_date>N</add_date>
- <add_time>N</add_time>
- <insertScript>Y</insertScript>
- <script>OUT="${PROJECT_HOME}/output/0018-enhanced-json-output-streaming"
-LOG="${OUT}.log"
-FILE="${OUT}/streamed.json"
-
-rm -rf "$OUT" "$LOG"
-mkdir -p "$OUT"
-
-# One million rows make a JSON file of about 127MB. The child hop-run gets a
128MB heap, a good
-# part of which Hop needs for itself. Building the file in memory runs out of
heap, streaming it
-# does not. Without ExitOnOutOfMemoryError the child keeps hanging after
running out of memory.
-if ! HOP_OPTIONS="-Xmx128m -XX:+ExitOnOutOfMemoryError" timeout 600 sh
hop-run.sh -r 'local' \
- -f "${PROJECT_HOME}/0018-enhanced-json-output-streaming.hpl" \
- -p ROWS=1000000 -p OUTPUT_FILE="${OUT}/streamed" -l Basic > "$LOG"
2>&1; then
- echo "Writing the JSON file with a 128MB heap failed:"
- tail -n 50 "$LOG"
- exit 1
-fi
-
-PAYLOAD="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-FIRST="{\"data\":[{\"id\":1,\"payload\":\"${PAYLOAD}\"},"
-LAST="{\"id\":1000000,\"payload\":\"${PAYLOAD}\"}]}"
-
-failures=0
-fail() {
- echo "FAIL: $1"
- failures=$((failures + 1))
-}
-
-if [ ! -f "$FILE" ]; then
- fail "${FILE} was not written"
-else
- [ "$(head -c ${#FIRST} "$FILE")" = "$FIRST" ] || fail "the file does not
start with ${FIRST}"
- [ "$(tail -c ${#LAST} "$FILE")" = "$LAST" ] || fail "the file does not end
with ${LAST}"
- # One object per row plus the one around the JSON block
- objects="$(tr -cd '{' < "$FILE" | wc -c | tr -d ' ')"
- [ "$objects" = "1000001" ] || fail "expected 1000001 JSON objects, found
${objects}"
- size="$(wc -c < "$FILE" | tr -d ' ')"
- [ "$size" = "126888906" ] || fail "expected 126888906 bytes, found ${size}"
-fi
-
-# Do not leave 127MB behind
-rm -f "$FILE"
-
-[ "$failures" -eq 0 ]</script>
- <loglevel>Basic</loglevel>
- <parallel>N</parallel>
- <xloc>320</xloc>
- <yloc>96</yloc>
- <attributes_hac/>
- </action>
- <action>
- <name>Success</name>
- <description/>
- <type>SUCCESS</type>
- <attributes/>
- <parallel>N</parallel>
- <xloc>560</xloc>
- <yloc>96</yloc>
- <attributes_hac/>
- </action>
- <action>
- <name>Failure: the JSON file was not streamed</name>
- <description/>
- <type>ABORT</type>
- <attributes/>
- <always_log_rows>N</always_log_rows>
- <message>Failure: the JSON file was not streamed</message>
- <parallel>N</parallel>
- <xloc>560</xloc>
- <yloc>192</yloc>
- <attributes_hac/>
- </action>
- </actions>
- <hops>
- <hop>
- <from>Start</from>
- <to>Write a JSON file larger than the heap</to>
- <enabled>Y</enabled>
- <evaluation>Y</evaluation>
- <unconditional>Y</unconditional>
- </hop>
- <hop>
- <from>Write a JSON file larger than the heap</from>
- <to>Success</to>
- <enabled>Y</enabled>
- <evaluation>Y</evaluation>
- <unconditional>N</unconditional>
- </hop>
- <hop>
- <from>Write a JSON file larger than the heap</from>
- <to>Failure: the JSON file was not streamed</to>
- <enabled>Y</enabled>
- <evaluation>N</evaluation>
- <unconditional>N</unconditional>
- </hop>
- </hops>
- <notepads>
- <notepad>
- <backgroundcolorblue>251</backgroundcolorblue>
- <backgroundcolorgreen>232</backgroundcolorgreen>
- <backgroundcolorred>201</backgroundcolorred>
- <bordercolorblue>90</bordercolorblue>
- <bordercolorgreen>58</bordercolorgreen>
- <bordercolorred>14</bordercolorred>
- <fontbold>N</fontbold>
- <fontcolorblue>90</fontcolorblue>
- <fontcolorgreen>58</fontcolorgreen>
- <fontcolorred>14</fontcolorred>
- <fontitalic>N</fontitalic>
- <fontname>Noto Sans</fontname>
- <fontsize>10</fontsize>
- <height>80</height>
- <xloc>112</xloc>
- <yloc>256</yloc>
- <note>Issue #2929: Enhanced JSON Output built the whole file in memory
and ran out of heap on
-large files. 0018-enhanced-json-output-streaming.hpl runs in a separate
hop-run with a 128MB heap
-and writes a JSON file of about 127MB. That only works when the rows are
streamed to the file.</note>
- <width>620</width>
- </notepad>
- </notepads>
- <attributes/>
-</workflow>
diff --git
a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputBehaviorTest.java
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputBehaviorTest.java
index 7b87d0cb28..0d5f852562 100644
---
a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputBehaviorTest.java
+++
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutputenhanced/JsonEOutputBehaviorTest.java
@@ -477,6 +477,44 @@ class JsonEOutputBehaviorTest {
assertFalse(file.toString().contains("\"y\""), file.toString());
}
+ /**
+ * Without group keys every row is written to the file as it arrives
(#2929): no row is kept in
+ * memory and the bytes reach the file before the input ends.
+ */
+ @Test
+ void rowsWithoutGroupKeysAreStreamedToTheFile() throws Exception {
+ final int rows = 1000;
+ final String value = "x".repeat(100);
+ JsonEOutputMeta meta = meta(JsonEOutputMeta.OperationType.WRITE_TO_FILE);
+ meta.setJsonBloc("data");
+ Object[][] input = new Object[rows][];
+ for (int i = 0; i < rows; i++) {
+ input[i] = new Object[] {value};
+ }
+ try (Harness h = new Harness(meta, rowMeta("payload"), input)) {
+ assertTrue(h.transform.init());
+ assertTrue(h.data.streamFileRows);
+ assertFalse(h.data.collectGroupItems);
+ for (int i = 1; i <= rows; i++) {
+ assertTrue(h.transform.processRow());
+ assertTrue(h.data.jsonKeyGroupItems.isEmpty(), "row " + i + " was kept
in memory");
+ assertEquals(i, h.data.fileItemCount);
+ if (i > 1) {
+ assertNull(h.data.pendingFileItem, "row " + i + " was parked instead
of written");
+ }
+ }
+ // About 115KB of rows, only the stream buffers may still hold some of
it.
+ long writtenBeforeEnd = h.data.countingStream.getCount();
+ assertTrue(writtenBeforeEnd > 50_000, writtenBeforeEnd + " bytes written
before the end");
+ assertFalse(h.transform.processRow());
+ assertEquals(0L, h.transform.getErrors());
+ }
+ JsonNode data = parse(read(base + "/out.json")).get("data");
+ assertEquals(rows, data.size());
+ assertEquals(value, data.get(0).get("payload").asText());
+ assertEquals(value, data.get(rows - 1).get("payload").asText());
+ }
+
@Test
void disposeClosesAFileOpenedAtInitWithNoItems() throws Exception {
JsonEOutputMeta meta = meta(JsonEOutputMeta.OperationType.WRITE_TO_FILE);