This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 4279406ea6 Issue #3745 : Raise the JSON string limit used to open
execution data (#8683)
4279406ea6 is described below
commit 4279406ea6be5fd1a1b096963d517ab3e745465c
Author: Matt Casters <[email protected]>
AuthorDate: Thu Oct 1 09:11:59 2026 +0200
Issue #3745 : Raise the JSON string limit used to open execution data
(#8683)
---
.../java/org/apache/hop/core/json/HopJson.java | 28 ++++++++++++--
.../java/org/apache/hop/core/json/HopJsonTest.java | 44 ++++++++++++++++++++++
.../caching/CachingFileExecutionInfoLocation.java | 3 +-
.../CachingFileExecutionInfoLocationTest.java | 32 ++++++++++++++++
.../elastic/ElasticExecutionInfoLocation.java | 3 +-
.../OpenSearchExecutionInfoLocation.java | 3 +-
6 files changed, 107 insertions(+), 6 deletions(-)
diff --git a/core/src/main/java/org/apache/hop/core/json/HopJson.java
b/core/src/main/java/org/apache/hop/core/json/HopJson.java
index dfaf933e69..0da8c9a97a 100644
--- a/core/src/main/java/org/apache/hop/core/json/HopJson.java
+++ b/core/src/main/java/org/apache/hop/core/json/HopJson.java
@@ -18,17 +18,39 @@
package org.apache.hop.core.json;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonFactoryBuilder;
+import com.fasterxml.jackson.core.StreamReadConstraints;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
public class HopJson {
+
+ /**
+ * Jackson's default maximum string length is 20,000,000 characters
(5,000,000 in 2.15.0). Sampled
+ * execution rows are stored as one Base64 string and need a higher limit to
be read back.
+ */
+ public static final int MAX_STRING_LENGTH = 50_000_000;
+
+ private HopJson() {}
+
+ /**
+ * @return a factory that accepts a string of up to {@link
#MAX_STRING_LENGTH} characters
+ */
+ public static JsonFactory newFactory() {
+ return new JsonFactoryBuilder()
+ .streamReadConstraints(
+
StreamReadConstraints.builder().maxStringLength(MAX_STRING_LENGTH).build())
+ .build();
+ }
+
/**
- * @return Create a new ObjectMapper instance with the default options set
for Hop file
- * serialization and de-serialization..
+ * @return a new ObjectMapper with the default options set for Hop file
serialization and
+ * de-serialization
*/
public static final ObjectMapper newMapper() {
- ObjectMapper objectMapper = new ObjectMapper();
+ ObjectMapper objectMapper = new ObjectMapper(newFactory());
objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
objectMapper.disable(SerializationFeature.INDENT_OUTPUT);
return objectMapper;
diff --git a/core/src/test/java/org/apache/hop/core/json/HopJsonTest.java
b/core/src/test/java/org/apache/hop/core/json/HopJsonTest.java
index 25bd1fcd0c..69a84883a3 100644
--- a/core/src/test/java/org/apache/hop/core/json/HopJsonTest.java
+++ b/core/src/test/java/org/apache/hop/core/json/HopJsonTest.java
@@ -19,10 +19,16 @@ package org.apache.hop.core.json;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.StreamReadConstraints;
+import com.fasterxml.jackson.core.exc.StreamConstraintsException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
+import java.util.Map;
import org.junit.jupiter.api.Test;
class HopJsonTest {
@@ -54,6 +60,44 @@ class HopJsonTest {
assertEquals("{\"a\":42}", json);
}
+ @Test
+ void newMapperReadsAStringLongerThanTheJacksonDefault() throws Exception {
+ int length = StreamReadConstraints.DEFAULT_MAX_STRING_LEN + 1;
+ String json = "{\"rowsBinaryGzipBase64Encoded\":\"" + "x".repeat(length) +
"\"}";
+
+ JsonProcessingException rejected =
+ assertThrows(
+ JsonProcessingException.class, () -> new
ObjectMapper().readValue(json, Map.class));
+ assertTrue(causedBy(rejected, StreamConstraintsException.class),
rejected.toString());
+
+ ObjectMapper mapper = HopJson.newMapper();
+ StreamReadConstraints constraints =
mapper.getFactory().streamReadConstraints();
+ assertEquals(HopJson.MAX_STRING_LENGTH, constraints.getMaxStringLength());
+ assertEquals(
+ StreamReadConstraints.defaults().getMaxNestingDepth(),
constraints.getMaxNestingDepth());
+ assertEquals(
+ StreamReadConstraints.defaults().getMaxNameLength(),
constraints.getMaxNameLength());
+
+ Map<?, ?> back = mapper.readValue(json, Map.class);
+ assertEquals(length,
String.valueOf(back.get("rowsBinaryGzipBase64Encoded")).length());
+
+ ObjectMapper strict = new ObjectMapper(HopJson.newFactory());
+ assertEquals(
+ length,
+ String.valueOf(strict.readValue(json,
Map.class).get("rowsBinaryGzipBase64Encoded"))
+ .length());
+ }
+
+ private static boolean causedBy(Throwable throwable, Class<? extends
Throwable> type) {
+ while (throwable != null) {
+ if (type.isInstance(throwable)) {
+ return true;
+ }
+ throwable = throwable.getCause();
+ }
+ return false;
+ }
+
public static class Simple {
public int a;
}
diff --git
a/engine/src/main/java/org/apache/hop/execution/caching/CachingFileExecutionInfoLocation.java
b/engine/src/main/java/org/apache/hop/execution/caching/CachingFileExecutionInfoLocation.java
index bf26cef081..c08653f99f 100644
---
a/engine/src/main/java/org/apache/hop/execution/caching/CachingFileExecutionInfoLocation.java
+++
b/engine/src/main/java/org/apache/hop/execution/caching/CachingFileExecutionInfoLocation.java
@@ -38,6 +38,7 @@ import org.apache.hop.core.exception.HopFileException;
import org.apache.hop.core.gui.plugin.GuiElementType;
import org.apache.hop.core.gui.plugin.GuiPlugin;
import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.json.HopJson;
import org.apache.hop.core.logging.LogChannel;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.vfs.HopVfs;
@@ -210,7 +211,7 @@ public class CachingFileExecutionInfoLocation extends
BaseCachingExecutionInfoLo
if (!HopVfs.fileExists(filename, variables)) {
return null;
}
- ObjectMapper objectMapper = new ObjectMapper();
+ ObjectMapper objectMapper = new ObjectMapper(HopJson.newFactory());
return objectMapper.readValue(HopVfs.getInputStream(filename,
variables), CacheEntry.class);
} catch (Exception e) {
throw new HopException(
diff --git
a/engine/src/test/java/org/apache/hop/execution/caching/CachingFileExecutionInfoLocationTest.java
b/engine/src/test/java/org/apache/hop/execution/caching/CachingFileExecutionInfoLocationTest.java
index 4bd6f68be6..69f8b15a90 100644
---
a/engine/src/test/java/org/apache/hop/execution/caching/CachingFileExecutionInfoLocationTest.java
+++
b/engine/src/test/java/org/apache/hop/execution/caching/CachingFileExecutionInfoLocationTest.java
@@ -24,6 +24,9 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import com.fasterxml.jackson.core.StreamReadConstraints;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
@@ -36,6 +39,7 @@ import org.apache.hop.core.logging.HopLogStore;
import org.apache.hop.core.row.RowBuffer;
import org.apache.hop.core.row.RowMetaBuilder;
import org.apache.hop.core.variables.Variables;
+import org.apache.hop.core.vfs.HopVfs;
import org.apache.hop.execution.Execution;
import org.apache.hop.execution.ExecutionData;
import org.apache.hop.execution.ExecutionDataBuilder;
@@ -393,4 +397,32 @@ class CachingFileExecutionInfoLocationTest {
reader.close();
}
}
+
+ @Test
+ void loadCacheEntryReadsAStringLongerThanTheJacksonDefault() throws
Exception {
+ Path root = tempDir.resolve("large-string");
+ String id = "large-exec";
+ int length = StreamReadConstraints.DEFAULT_MAX_STRING_LEN + 1;
+ String name = "n".repeat(length);
+
+ CachingFileExecutionInfoLocation location = new
CachingFileExecutionInfoLocation();
+ location.setRootFolder(root.toAbsolutePath().toString());
+ location.initialize(new Variables(), null);
+ try {
+ CacheEntry marker = new CacheEntry();
+ marker.setId(id);
+ String filename = marker.calculateFilename(location.actualRootFolder);
+ String json = "{\"id\":\"" + id + "\",\"name\":\"" + name + "\"}";
+ try (OutputStream out = HopVfs.getOutputStream(filename, false, new
Variables())) {
+ out.write(json.getBytes(StandardCharsets.UTF_8));
+ }
+
+ CacheEntry loaded = location.loadCacheEntry(id);
+ assertNotNull(loaded);
+ assertEquals(id, loaded.getId());
+ assertEquals(length, loaded.getName().length());
+ } finally {
+ location.close();
+ }
+ }
}
diff --git
a/plugins/tech/elastic/src/main/java/org/apache/hop/execution/elastic/ElasticExecutionInfoLocation.java
b/plugins/tech/elastic/src/main/java/org/apache/hop/execution/elastic/ElasticExecutionInfoLocation.java
index 523ff36736..26199d8794 100644
---
a/plugins/tech/elastic/src/main/java/org/apache/hop/execution/elastic/ElasticExecutionInfoLocation.java
+++
b/plugins/tech/elastic/src/main/java/org/apache/hop/execution/elastic/ElasticExecutionInfoLocation.java
@@ -34,6 +34,7 @@ import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.gui.plugin.GuiElementType;
import org.apache.hop.core.gui.plugin.GuiPlugin;
import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.json.HopJson;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.execution.ExecutionInfoLocation;
@@ -331,7 +332,7 @@ public class ElasticExecutionInfoLocation extends
BaseCachingExecutionInfoLocati
// Convert this source object to CacheEntry
//
String json = jSource.toJSONString();
- ObjectMapper objectMapper = new ObjectMapper();
+ ObjectMapper objectMapper = new ObjectMapper(HopJson.newFactory());
return objectMapper.readValue(json, CacheEntry.class);
} catch (Exception e) {
throw new HopException(
diff --git
a/plugins/tech/opensearch/src/main/java/org/apache/hop/execution/opensearch/OpenSearchExecutionInfoLocation.java
b/plugins/tech/opensearch/src/main/java/org/apache/hop/execution/opensearch/OpenSearchExecutionInfoLocation.java
index bce37555c3..083e241a07 100644
---
a/plugins/tech/opensearch/src/main/java/org/apache/hop/execution/opensearch/OpenSearchExecutionInfoLocation.java
+++
b/plugins/tech/opensearch/src/main/java/org/apache/hop/execution/opensearch/OpenSearchExecutionInfoLocation.java
@@ -36,6 +36,7 @@ import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.gui.plugin.GuiElementType;
import org.apache.hop.core.gui.plugin.GuiPlugin;
import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.json.HopJson;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.execution.ExecutionInfoLocation;
@@ -400,7 +401,7 @@ public class OpenSearchExecutionInfoLocation extends
BaseCachingExecutionInfoLoc
// Convert this source object to CacheEntry
//
String json = jSource.toJSONString();
- ObjectMapper objectMapper = new ObjectMapper();
+ ObjectMapper objectMapper = new ObjectMapper(HopJson.newFactory());
return objectMapper.readValue(json, CacheEntry.class);
} catch (Exception e) {
throw new HopException(
