This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new f69c2e1790 Issue #8617 : Limit large String, JSON, Binary, and Avro
values in data profiles (#8632)
f69c2e1790 is described below
commit f69c2e1790ab5d74ef4e5a421e81a8b3ac5c25a3
Author: Matt Casters <[email protected]>
AuthorDate: Mon Sep 28 08:20:51 2026 +0200
Issue #8617 : Limit large String, JSON, Binary, and Avro values in data
profiles (#8632)
* Issue #8617 : Limit large String, JSON, Binary, and Avro values in data
profiles
An execution data profile can leave oversized String, JSON, Binary, and Avro
values out of the sample rows it stores. An empty limit keeps today's
behavior.
Zero stores none of that type. A longer String or JSON cell is replaced
with a
short marker, and Binary or Avro cells over the limit are left empty.
fixes #8617
* Issue #8617 : Store sample markers in the field storage type
Replacing a lazy-conversion value with a Java String made execution data
registration fail for the whole tick. Markers now follow binary-string and
indexed storage. Kept cells are cloned the same way as a normal row copy,
a profiled row is measured once, and a negative or non-numeric limit is
logged.
---
.../hop/core/row/value/ValueMetaAvroRecord.java | 46 +++
.../metadata-types/execution-data-profile.png | Bin 0 -> 111201 bytes
.../metadata-types/execution-data-profile.adoc | 8 +
.../execution/profiling/ExecutionDataProfile.java | 117 ++++--
.../execution/sampler/IExecutionDataSampler.java | 16 +
.../hop/execution/sampler/SampledValueLimits.java | 353 ++++++++++++++++++
.../sampler/plugins/ExecutionDataSamplerBase.java | 18 +
.../dataprof/BasicDataProfilingDataSampler.java | 68 +++-
.../first/FirstRowsExecutionDataSampler.java | 2 +-
.../plugins/last/LastRowsExecutionDataSampler.java | 2 +-
.../random/RandomRowsExecutionDataSampler.java | 4 +-
.../engines/local/LocalPipelineEngine.java | 1 +
.../profiling/messages/messages_en_US.properties | 11 +-
.../execution/sampler/SampledValueLimitsTest.java | 415 +++++++++++++++++++++
.../hop/beam/core/transform/TransformBaseFn.java | 6 +-
.../execution/SparkTransformExecutionSampling.java | 8 +-
.../profiling/ExecutionDataProfileEditor.java | 38 ++
17 files changed, 1053 insertions(+), 60 deletions(-)
diff --git
a/core/src/main/java/org/apache/hop/core/row/value/ValueMetaAvroRecord.java
b/core/src/main/java/org/apache/hop/core/row/value/ValueMetaAvroRecord.java
index 95ebf68f09..39d66ef5d5 100644
--- a/core/src/main/java/org/apache/hop/core/row/value/ValueMetaAvroRecord.java
+++ b/core/src/main/java/org/apache/hop/core/row/value/ValueMetaAvroRecord.java
@@ -21,7 +21,9 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.IOException;
+import java.io.OutputStream;
import java.net.SocketTimeoutException;
+import java.nio.charset.StandardCharsets;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumReader;
@@ -456,6 +458,33 @@ public class ValueMetaAvroRecord extends ValueMetaBase {
}
}
+ /**
+ * Bytes of the compact schema JSON plus the binary datum. Sample storage
uses this as the size of
+ * an Avro value: the schema is part of the payload and {@link #writeData}
writes the datum.
+ *
+ * @param object An Avro {@link GenericRecord}
+ * @return The schema and datum size in bytes
+ * @throws HopValueException When the value is not a record or cannot be
encoded
+ */
+ public static int storedPayloadBytes(Object object) throws HopValueException
{
+ if (!(object instanceof GenericRecord genericRecord) ||
genericRecord.getSchema() == null) {
+ throw new HopValueException(
+ "An Avro GenericRecord with a schema is needed to measure its size");
+ }
+ Schema recordSchema = genericRecord.getSchema();
+ int schemaBytes =
recordSchema.toString(false).getBytes(StandardCharsets.UTF_8).length;
+ CountingOutputStream counting = new CountingOutputStream();
+ try {
+ BinaryEncoder encoder =
EncoderFactory.get().directBinaryEncoder(counting, null);
+ new GenericDatumWriter<GenericRecord>(recordSchema).write(genericRecord,
encoder);
+ encoder.flush();
+ } catch (IOException e) {
+ throw new HopValueException("Unable to measure the size of an Avro
record", e);
+ }
+ long total = (long) schemaBytes + counting.count;
+ return total > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) total;
+ }
+
public Schema getSchema() {
return schema;
}
@@ -484,4 +513,21 @@ public class ValueMetaAvroRecord extends ValueMetaBase {
return super.getInteger(object);
}
+
+ /** Counts bytes written by an Avro encoder without keeping them. */
+ private static final class CountingOutputStream extends OutputStream {
+ private long count;
+
+ @Override
+ public void write(int b) {
+ count++;
+ }
+
+ @Override
+ public void write(byte[] bytes, int offset, int length) {
+ if (length > 0) {
+ count += length;
+ }
+ }
+ }
}
diff --git
a/docs/hop-user-manual/modules/ROOT/assets/images/metadata-types/execution-data-profile.png
b/docs/hop-user-manual/modules/ROOT/assets/images/metadata-types/execution-data-profile.png
new file mode 100644
index 0000000000..762de0af5d
Binary files /dev/null and
b/docs/hop-user-manual/modules/ROOT/assets/images/metadata-types/execution-data-profile.png
differ
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/execution-data-profile.adoc
b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/execution-data-profile.adoc
index 27123d05aa..bac4c439e0 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/execution-data-profile.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/execution-data-profile.adoc
@@ -26,14 +26,22 @@ An Apache Hop Execution Data Profile builds data profiles
as data flow through p
== Options
+image::metadata-types/execution-data-profile.png[Execution data profile
editor, with the large value limits,width="90%"]
+
[options="header"]
|===
|Option |Description
|Name|The name to be used for this Execution Data Profile
|Description|A description to be used for this Execution Data Profile
+|Maximum string length|Maximum number of characters to keep for a String
field. A JSON document kept in a String field uses this limit. Empty means no
limit. `0` stores none. A longer value is replaced with `<not stored, N
characters>`.
+|Maximum JSON length|Maximum number of characters of the compact JSON text to
keep for a JSON field. Empty means no limit. `0` stores none. A longer value is
replaced with a JSON text `<not stored, N characters>`.
+|Maximum binary size|Maximum number of bytes to keep for a Binary field. Empty
means no limit. `0` stores none. A larger value is left empty in the sample.
+|Maximum Avro size|Maximum number of bytes of schema JSON plus Avro datum to
keep for an Avro Record field. Empty means no limit. `0` stores none. A larger
value is left empty in the sample.
|Data Samplers to use|One or more data samplers to use with this Execution
Data Profile. See details below.
|===
+The four size limits apply to every sampler in the profile. Leave a field
empty to keep today's behavior for that type. Minimum and maximum values are
calculated only from values that were stored. Null counts and string lengths
are still recorded for values that are left out. Minimum and maximum length
example rows keep the other columns and replace the large value.
+
== Data Samplers
[options="header"]
diff --git
a/engine/src/main/java/org/apache/hop/execution/profiling/ExecutionDataProfile.java
b/engine/src/main/java/org/apache/hop/execution/profiling/ExecutionDataProfile.java
index 9d36867d5a..f5828f4e15 100644
---
a/engine/src/main/java/org/apache/hop/execution/profiling/ExecutionDataProfile.java
+++
b/engine/src/main/java/org/apache/hop/execution/profiling/ExecutionDataProfile.java
@@ -20,7 +20,15 @@ package org.apache.hop.execution.profiling;
import java.util.ArrayList;
import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.gui.plugin.GuiWidgetGroupType;
+import org.apache.hop.core.variables.IVariables;
import org.apache.hop.execution.sampler.IExecutionDataSampler;
+import org.apache.hop.execution.sampler.SampledValueLimits;
import org.apache.hop.metadata.api.HopMetadata;
import org.apache.hop.metadata.api.HopMetadataBase;
import org.apache.hop.metadata.api.HopMetadataCategory;
@@ -32,6 +40,9 @@ import org.apache.hop.metadata.api.IHopMetadata;
* This data profile makes it easy to specify what kind of information you
want to capture about the
* output of a transform during its execution.
*/
+@Getter
+@Setter
+@GuiPlugin(description = "Execution data profile widgets")
@HopMetadata(
key = "execution-data-profile",
name = "i18n::ExecutionDataProfile.name",
@@ -44,8 +55,60 @@ import org.apache.hop.metadata.api.IHopMetadata;
public class ExecutionDataProfile extends HopMetadataBase implements
IHopMetadata, Cloneable {
public static final String GUI_PLUGIN_ELEMENT_PARENT_ID =
"ExecutionDataSamplerParent";
+ public static final String GUI_PLUGIN_LIMITS_PARENT_ID =
"ExecutionDataProfileLimits";
+
+ private static final String LARGE_VALUES_GROUP =
"i18n::ExecutionDataProfile.Group.LargeValues";
+
@HopMetadataProperty private String description;
+ @GuiWidgetElement(
+ id = "stringValueLimit",
+ order = "100",
+ type = GuiElementType.TEXT,
+ parentId = GUI_PLUGIN_LIMITS_PARENT_ID,
+ label = "i18n::ExecutionDataProfile.Label.StringValueLimit",
+ toolTip = "i18n::ExecutionDataProfile.Tooltip.StringValueLimit",
+ groupType = GuiWidgetGroupType.BOXES,
+ group = LARGE_VALUES_GROUP)
+ @HopMetadataProperty
+ private String stringValueLimit;
+
+ @GuiWidgetElement(
+ id = "jsonValueLimit",
+ order = "110",
+ type = GuiElementType.TEXT,
+ parentId = GUI_PLUGIN_LIMITS_PARENT_ID,
+ label = "i18n::ExecutionDataProfile.Label.JsonValueLimit",
+ toolTip = "i18n::ExecutionDataProfile.Tooltip.JsonValueLimit",
+ groupType = GuiWidgetGroupType.BOXES,
+ group = LARGE_VALUES_GROUP)
+ @HopMetadataProperty
+ private String jsonValueLimit;
+
+ @GuiWidgetElement(
+ id = "binaryValueLimit",
+ order = "120",
+ type = GuiElementType.TEXT,
+ parentId = GUI_PLUGIN_LIMITS_PARENT_ID,
+ label = "i18n::ExecutionDataProfile.Label.BinaryValueLimit",
+ toolTip = "i18n::ExecutionDataProfile.Tooltip.BinaryValueLimit",
+ groupType = GuiWidgetGroupType.BOXES,
+ group = LARGE_VALUES_GROUP)
+ @HopMetadataProperty
+ private String binaryValueLimit;
+
+ @GuiWidgetElement(
+ id = "avroValueLimit",
+ order = "130",
+ type = GuiElementType.TEXT,
+ parentId = GUI_PLUGIN_LIMITS_PARENT_ID,
+ label = "i18n::ExecutionDataProfile.Label.AvroValueLimit",
+ toolTip = "i18n::ExecutionDataProfile.Tooltip.AvroValueLimit",
+ groupType = GuiWidgetGroupType.BOXES,
+ group = LARGE_VALUES_GROUP)
+ @HopMetadataProperty
+ private String avroValueLimit;
+
@HopMetadataProperty(groupKey = "samplers", key = "sampler")
private List<IExecutionDataSampler> samplers;
@@ -68,49 +131,35 @@ public class ExecutionDataProfile extends HopMetadataBase
implements IHopMetadat
public ExecutionDataProfile(ExecutionDataProfile profile) {
this(profile.name);
this.description = profile.description;
+ this.stringValueLimit = profile.stringValueLimit;
+ this.jsonValueLimit = profile.jsonValueLimit;
+ this.binaryValueLimit = profile.binaryValueLimit;
+ this.avroValueLimit = profile.avroValueLimit;
for (IExecutionDataSampler<?> sampler : profile.samplers) {
this.samplers.add(sampler.clone());
}
}
- @Override
- protected ExecutionDataProfile clone() {
- return new ExecutionDataProfile(this);
- }
-
- /**
- * Gets description
- *
- * @return value of description
- */
- public String getDescription() {
- return description;
- }
-
/**
- * Sets description
+ * Resolve this profile's value limits and attach them to every sampler that
will store rows.
*
- * @param description value of description
+ * @param samplers Profile samplers and any extra samplers added by the
engine
+ * @param variables Pipeline variables used to resolve the limit fields
*/
- public void setDescription(String description) {
- this.description = description;
- }
-
- /**
- * Gets samplers
- *
- * @return value of samplers
- */
- public List<IExecutionDataSampler> getSamplers() {
- return samplers;
+ public void applyLimits(List<IExecutionDataSampler> samplers, IVariables
variables) {
+ if (samplers == null) {
+ return;
+ }
+ SampledValueLimits limits = SampledValueLimits.from(this, variables);
+ for (IExecutionDataSampler sampler : samplers) {
+ if (sampler != null) {
+ sampler.setSampledValueLimits(limits);
+ }
+ }
}
- /**
- * Sets samplers
- *
- * @param samplers value of samplers
- */
- public void setSamplers(List<IExecutionDataSampler> samplers) {
- this.samplers = samplers;
+ @Override
+ protected ExecutionDataProfile clone() {
+ return new ExecutionDataProfile(this);
}
}
diff --git
a/engine/src/main/java/org/apache/hop/execution/sampler/IExecutionDataSampler.java
b/engine/src/main/java/org/apache/hop/execution/sampler/IExecutionDataSampler.java
index 748526350b..28ed68c938 100644
---
a/engine/src/main/java/org/apache/hop/execution/sampler/IExecutionDataSampler.java
+++
b/engine/src/main/java/org/apache/hop/execution/sampler/IExecutionDataSampler.java
@@ -18,6 +18,7 @@
package org.apache.hop.execution.sampler;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.plugins.IPlugin;
import org.apache.hop.core.plugins.PluginRegistry;
@@ -60,6 +61,21 @@ public interface IExecutionDataSampler<Store extends
IExecutionDataSamplerStore>
void sampleRow(Store samplerStore, IStream.StreamType streamType, IRowMeta
rowMeta, Object[] row)
throws HopException;
+ /**
+ * Limits resolved from the execution data profile before sampling starts.
Not metadata: Beam and
+ * Spark must not write this into the sampler JSON.
+ */
+ @JsonIgnore
+ default void setSampledValueLimits(SampledValueLimits limits) {}
+
+ /**
+ * @return The limits for this sampler, or no limits when the profile did
not set any.
+ */
+ @JsonIgnore
+ default SampledValueLimits getSampledValueLimits() {
+ return SampledValueLimits.unlimited();
+ }
+
/**
* This object factory is needed to instantiate the correct plugin class
based on the value of the
* Object ID which is simply the object ID.
diff --git
a/engine/src/main/java/org/apache/hop/execution/sampler/SampledValueLimits.java
b/engine/src/main/java/org/apache/hop/execution/sampler/SampledValueLimits.java
new file mode 100644
index 0000000000..51240bac8e
--- /dev/null
+++
b/engine/src/main/java/org/apache/hop/execution/sampler/SampledValueLimits.java
@@ -0,0 +1,353 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.execution.sampler;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.TextNode;
+import java.nio.charset.StandardCharsets;
+import lombok.Getter;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.exception.HopValueException;
+import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.value.ValueMetaAvroRecord;
+import org.apache.hop.core.row.value.ValueMetaJson;
+import org.apache.hop.core.util.JsonUtil;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.execution.profiling.ExecutionDataProfile;
+
+/**
+ * Resolved limits for String, JSON, Binary, and Avro values kept by an
execution data profile. A
+ * blank setting means no limit. A negative or non-numeric setting is logged
and then treated as no
+ * limit. Zero stores none of that type.
+ */
+@Getter
+public final class SampledValueLimits {
+ public static final String NOT_STORED = "<not stored>";
+
+ private static final SampledValueLimits UNLIMITED =
+ new SampledValueLimits(null, null, null, null);
+ private static final String CHARACTERS = "characters";
+
+ private final Integer stringLimit;
+ private final Integer jsonLimit;
+ private final Integer binaryLimit;
+ private final Integer avroLimit;
+
+ private SampledValueLimits(
+ Integer stringLimit, Integer jsonLimit, Integer binaryLimit, Integer
avroLimit) {
+ this.stringLimit = stringLimit;
+ this.jsonLimit = jsonLimit;
+ this.binaryLimit = binaryLimit;
+ this.avroLimit = avroLimit;
+ }
+
+ public static SampledValueLimits unlimited() {
+ return UNLIMITED;
+ }
+
+ /**
+ * Marker stored in place of a String or JSON value that is over its limit.
+ *
+ * @param size The measured size that was left out
+ * @param unit {@code characters} for String and JSON
+ * @return The short replacement text
+ */
+ public static String notStored(int size, String unit) {
+ return "<not stored, " + size + " " + unit + ">";
+ }
+
+ /**
+ * Resolve the profile fields with {@code variables}. A blank value is
unlimited. A negative or
+ * non-numeric value is unlimited and logged, so a typo does not turn the
limit off quietly. Zero
+ * stores nothing of that type.
+ */
+ public static SampledValueLimits from(ExecutionDataProfile profile,
IVariables variables) {
+ if (profile == null) {
+ return unlimited();
+ }
+ return new SampledValueLimits(
+ parseLimit(resolve(variables, profile.getStringValueLimit())),
+ parseLimit(resolve(variables, profile.getJsonValueLimit())),
+ parseLimit(resolve(variables, profile.getBinaryValueLimit())),
+ parseLimit(resolve(variables, profile.getAvroValueLimit())));
+ }
+
+ public boolean hasLimits() {
+ return stringLimit != null || jsonLimit != null || binaryLimit != null ||
avroLimit != null;
+ }
+
+ /**
+ * @return true when this value must not be kept as itself. Nulls are kept
as null.
+ */
+ public boolean omit(IValueMeta valueMeta, Object value) throws
HopValueException {
+ return decide(valueMeta, value).omit;
+ }
+
+ /**
+ * Measure every field once. Profiling calls {@link #copyRow(IRowMeta,
Object[], Decision[])}
+ * several times for the same row, and measuring JSON or Avro serialises the
value.
+ */
+ public Decision[] decisionsFor(IRowMeta rowMeta, Object[] row) throws
HopValueException {
+ if (row == null || rowMeta == null) {
+ return new Decision[0];
+ }
+ int fields = Math.min(rowMeta.size(), row.length);
+ Decision[] decisions = new Decision[fields];
+ for (int i = 0; i < fields; i++) {
+ decisions[i] = decide(rowMeta.getValueMeta(i), row[i]);
+ }
+ return decisions;
+ }
+
+ /**
+ * A new row array for storage. Cells over a limit are replaced. Cells that
stay are cloned. The
+ * pipeline row is left unchanged, and a value that is dropped is not copied
first.
+ */
+ public Object[] copyRow(IRowMeta rowMeta, Object[] row) throws
HopValueException {
+ return copyRow(rowMeta, row, null);
+ }
+
+ /**
+ * Same as {@link #copyRow(IRowMeta, Object[])}, using decisions already
computed for this row.
+ */
+ public Object[] copyRow(IRowMeta rowMeta, Object[] row, Decision[] decisions)
+ throws HopValueException {
+ if (row == null) {
+ return null;
+ }
+ if (rowMeta == null || !hasLimits()) {
+ return rowMeta == null ? row.clone() : rowMeta.cloneRow(row);
+ }
+
+ int fields = Math.min(rowMeta.size(), row.length);
+ if (decisions == null) {
+ decisions = decisionsFor(rowMeta, row);
+ } else {
+ fields = Math.min(fields, decisions.length);
+ }
+ boolean anyOmitted = false;
+ for (int i = 0; i < fields; i++) {
+ anyOmitted = anyOmitted || decisions[i].omit();
+ }
+ if (!anyOmitted) {
+ return rowMeta.cloneRow(row);
+ }
+
+ Object[] copy = row.clone();
+ for (int i = 0; i < fields; i++) {
+ if (decisions[i].omit()) {
+ copy[i] = decisions[i].replacement();
+ } else {
+ copy[i] = copyKept(rowMeta.getValueMeta(i), row[i]);
+ }
+ }
+ return copy;
+ }
+
+ private static Object copyKept(IValueMeta valueMeta, Object value) throws
HopValueException {
+ if (value == null || valueMeta == null) {
+ return value;
+ }
+ return valueMeta.cloneValueData(value);
+ }
+
+ private Decision decide(IValueMeta valueMeta, Object value) throws
HopValueException {
+ if (valueMeta == null || value == null || valueMeta.isNull(value)) {
+ return Decision.keep();
+ }
+ return switch (valueMeta.getType()) {
+ case IValueMeta.TYPE_STRING -> decideString(valueMeta, value);
+ case IValueMeta.TYPE_JSON -> decideJson(valueMeta, value);
+ case IValueMeta.TYPE_BINARY -> decideBinary(valueMeta, value);
+ case IValueMeta.TYPE_AVRO -> decideAvro(value);
+ default -> Decision.keep();
+ };
+ }
+
+ private Decision decideString(IValueMeta valueMeta, Object value) throws
HopValueException {
+ if (stringLimit == null) {
+ return Decision.keep();
+ }
+ if (stringLimit == 0) {
+ return Decision.drop(marker(valueMeta, NOT_STORED));
+ }
+ int length = stringLength(valueMeta, value);
+ if (length > stringLimit) {
+ return Decision.drop(marker(valueMeta, notStored(length, CHARACTERS)));
+ }
+ return Decision.keep();
+ }
+
+ private Decision decideJson(IValueMeta valueMeta, Object value) throws
HopValueException {
+ if (jsonLimit == null) {
+ return Decision.keep();
+ }
+ if (jsonLimit == 0) {
+ return Decision.drop(marker(valueMeta, NOT_STORED));
+ }
+ int length;
+ try {
+ length = jsonLength(valueMeta, value);
+ } catch (Exception e) {
+ return Decision.drop(marker(valueMeta, NOT_STORED));
+ }
+ if (length > jsonLimit) {
+ return Decision.drop(marker(valueMeta, notStored(length, CHARACTERS)));
+ }
+ return Decision.keep();
+ }
+
+ private Decision decideBinary(IValueMeta valueMeta, Object value) throws
HopValueException {
+ if (binaryLimit == null) {
+ return Decision.keep();
+ }
+ if (binaryLimit == 0) {
+ return Decision.drop(null);
+ }
+ int size;
+ try {
+ size = binarySize(valueMeta, value);
+ } catch (Exception e) {
+ return Decision.drop(null);
+ }
+ if (size > binaryLimit) {
+ return Decision.drop(null);
+ }
+ return Decision.keep();
+ }
+
+ private Decision decideAvro(Object value) {
+ if (avroLimit == null) {
+ return Decision.keep();
+ }
+ if (avroLimit == 0) {
+ return Decision.drop(null);
+ }
+ int size;
+ try {
+ size = ValueMetaAvroRecord.storedPayloadBytes(value);
+ } catch (Exception e) {
+ return Decision.drop(null);
+ }
+ if (size > avroLimit) {
+ return Decision.drop(null);
+ }
+ return Decision.keep();
+ }
+
+ private static int stringLength(IValueMeta valueMeta, Object value) throws
HopValueException {
+ if (valueMeta.getStorageType() == IValueMeta.STORAGE_TYPE_BINARY_STRING
+ && value instanceof byte[] bytes) {
+ String text = valueMeta.getString(value);
+ return text == null ? bytes.length : text.length();
+ }
+ if (value instanceof String text) {
+ return text.length();
+ }
+ String text = valueMeta.getString(value);
+ return text == null ? 0 : text.length();
+ }
+
+ private static int jsonLength(IValueMeta valueMeta, Object value) throws
Exception {
+ if (valueMeta.getStorageType() == IValueMeta.STORAGE_TYPE_BINARY_STRING
+ && value instanceof byte[] bytes) {
+ return new String(bytes, StandardCharsets.UTF_8).length();
+ }
+ JsonNode node;
+ if (value instanceof JsonNode jsonNode) {
+ node = jsonNode;
+ } else if (valueMeta instanceof ValueMetaJson jsonMeta) {
+ node = jsonMeta.getJson(value);
+ } else {
+ node = valueMeta.getJson(value);
+ }
+ if (node == null) {
+ return 0;
+ }
+ String text = JsonUtil.mapJsonToString(node, false);
+ return text == null ? 0 : text.length();
+ }
+
+ private static int binarySize(IValueMeta valueMeta, Object value) throws
HopValueException {
+ if (value instanceof byte[] bytes) {
+ return bytes.length;
+ }
+ byte[] bytes = valueMeta.getBinary(value);
+ return bytes == null ? 0 : bytes.length;
+ }
+
+ private static String resolve(IVariables variables, String value) {
+ if (value == null || variables == null) {
+ return value;
+ }
+ return variables.resolve(value);
+ }
+
+ /**
+ * A replacement that {@link
org.apache.hop.core.row.value.ValueMetaBase#writeData} can store.
+ * Lazy conversion keeps a {@code byte[]} and an indexed field keeps an
index, so a Java String
+ * marker would fail execution-data registration for the whole tick.
+ */
+ private static Object marker(IValueMeta valueMeta, String text) {
+ if (valueMeta == null) {
+ return text;
+ }
+ return switch (valueMeta.getStorageType()) {
+ case IValueMeta.STORAGE_TYPE_BINARY_STRING ->
text.getBytes(StandardCharsets.UTF_8);
+ case IValueMeta.STORAGE_TYPE_INDEXED -> null;
+ default -> valueMeta.getType() == IValueMeta.TYPE_JSON ?
TextNode.valueOf(text) : text;
+ };
+ }
+
+ private static Integer parseLimit(String raw) {
+ if (StringUtils.isBlank(raw)) {
+ return null;
+ }
+ String trimmed = raw.trim();
+ try {
+ int parsed = Integer.parseInt(trimmed);
+ if (parsed < 0) {
+ LogChannel.GENERAL.logError(
+ "Execution data profile value limit '"
+ + trimmed
+ + "' is negative and is ignored. The value is stored without a
limit.");
+ return null;
+ }
+ return parsed;
+ } catch (NumberFormatException e) {
+ LogChannel.GENERAL.logError(
+ "Execution data profile value limit '"
+ + trimmed
+ + "' is not a number and is ignored. The value is stored without
a limit.");
+ return null;
+ }
+ }
+
+ /** Whether one cell is kept, and the storage-typed marker used when it is
not. */
+ public record Decision(boolean omit, Object replacement) {
+ private static Decision keep() {
+ return new Decision(false, null);
+ }
+
+ private static Decision drop(Object replacement) {
+ return new Decision(true, replacement);
+ }
+ }
+}
diff --git
a/engine/src/main/java/org/apache/hop/execution/sampler/plugins/ExecutionDataSamplerBase.java
b/engine/src/main/java/org/apache/hop/execution/sampler/plugins/ExecutionDataSamplerBase.java
index ba36dd0172..c774a4b500 100644
---
a/engine/src/main/java/org/apache/hop/execution/sampler/plugins/ExecutionDataSamplerBase.java
+++
b/engine/src/main/java/org/apache/hop/execution/sampler/plugins/ExecutionDataSamplerBase.java
@@ -18,6 +18,7 @@
package org.apache.hop.execution.sampler.plugins;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import java.util.Objects;
import org.apache.hop.core.gui.plugin.GuiElementType;
import org.apache.hop.core.gui.plugin.GuiWidgetElement;
@@ -25,6 +26,7 @@ import
org.apache.hop.execution.profiling.ExecutionDataProfile;
import org.apache.hop.execution.sampler.ExecutionDataSamplerMeta;
import org.apache.hop.execution.sampler.IExecutionDataSampler;
import org.apache.hop.execution.sampler.IExecutionDataSamplerStore;
+import org.apache.hop.execution.sampler.SampledValueLimits;
import org.apache.hop.metadata.api.HopMetadataProperty;
public abstract class ExecutionDataSamplerBase<Store extends
IExecutionDataSamplerStore>
@@ -39,6 +41,9 @@ public abstract class ExecutionDataSamplerBase<Store extends
IExecutionDataSampl
@HopMetadataProperty
protected String sampleSize;
+ /** Resolved from the profile at runtime. Not part of the sampler metadata.
*/
+ @JsonIgnore protected SampledValueLimits sampledValueLimits;
+
protected String pluginId;
protected String pluginName;
@@ -57,6 +62,7 @@ public abstract class ExecutionDataSamplerBase<Store extends
IExecutionDataSampl
this.sampleSize = base.sampleSize;
this.pluginId = base.pluginId;
this.pluginName = base.pluginName;
+ this.sampledValueLimits = base.sampledValueLimits;
}
@Override
@@ -96,6 +102,18 @@ public abstract class ExecutionDataSamplerBase<Store
extends IExecutionDataSampl
this.sampleSize = sampleSize;
}
+ @Override
+ @JsonIgnore
+ public void setSampledValueLimits(SampledValueLimits sampledValueLimits) {
+ this.sampledValueLimits = sampledValueLimits;
+ }
+
+ @Override
+ @JsonIgnore
+ public SampledValueLimits getSampledValueLimits() {
+ return sampledValueLimits == null ? SampledValueLimits.unlimited() :
sampledValueLimits;
+ }
+
/**
* Gets pluginId
*
diff --git
a/engine/src/main/java/org/apache/hop/execution/sampler/plugins/dataprof/BasicDataProfilingDataSampler.java
b/engine/src/main/java/org/apache/hop/execution/sampler/plugins/dataprof/BasicDataProfilingDataSampler.java
index 568886d095..7ad676ab9a 100644
---
a/engine/src/main/java/org/apache/hop/execution/sampler/plugins/dataprof/BasicDataProfilingDataSampler.java
+++
b/engine/src/main/java/org/apache/hop/execution/sampler/plugins/dataprof/BasicDataProfilingDataSampler.java
@@ -18,9 +18,11 @@
package org.apache.hop.execution.sampler.plugins.dataprof;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import java.util.HashMap;
import java.util.Map;
import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.exception.HopValueException;
import org.apache.hop.core.gui.plugin.GuiElementType;
import org.apache.hop.core.gui.plugin.GuiPlugin;
import org.apache.hop.core.gui.plugin.GuiWidgetElement;
@@ -31,6 +33,7 @@ import
org.apache.hop.execution.profiling.ExecutionDataProfile;
import org.apache.hop.execution.sampler.ExecutionDataSamplerMeta;
import org.apache.hop.execution.sampler.ExecutionDataSamplerPlugin;
import org.apache.hop.execution.sampler.IExecutionDataSampler;
+import org.apache.hop.execution.sampler.SampledValueLimits;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.HopMetadataProperty;
import org.apache.hop.pipeline.transform.stream.IStream;
@@ -72,6 +75,9 @@ public class BasicDataProfilingDataSampler
private String pluginId;
private String pluginName;
+ /** Resolved from the profile at runtime. Not part of the sampler metadata.
*/
+ @JsonIgnore private SampledValueLimits sampledValueLimits;
+
// The metadata fields
//
@@ -163,6 +169,7 @@ public class BasicDataProfilingDataSampler
this.profilingNrNonNull = sampler.profilingNrNonNull;
this.profilingMinLength = sampler.profilingMinLength;
this.profilingMaxLength = sampler.profilingMaxLength;
+ this.sampledValueLimits = sampler.sampledValueLimits;
}
public BasicDataProfilingDataSampler clone() {
@@ -195,6 +202,12 @@ public class BasicDataProfilingDataSampler
try {
+ // Measure each field once. Min, max, length and null samples all store
a copy of this row.
+ //
+ SampledValueLimits limits = getSampledValueLimits();
+ SampledValueLimits.Decision[] decisions =
+ limits.hasLimits() ? limits.decisionsFor(rowMeta, row) : null;
+
// Profile all columns
//
for (int i = 0; i < rowMeta.size(); i++) {
@@ -206,66 +219,71 @@ public class BasicDataProfilingDataSampler
if (profilingNrNull) {
long counter = store.getNullCounters().getOrDefault(name, 0L);
store.getNullCounters().put(name, ++counter);
- addSampleRow(store, name, ProfilingType.NrNulls, rowMeta, row);
+ addSampleRow(store, name, ProfilingType.NrNulls, rowMeta, row,
decisions);
}
} else {
if (profilingNrNonNull) {
long counter = store.getNonNullCounters().getOrDefault(name, 0L);
store.getNonNullCounters().put(name, ++counter);
- addSampleRow(store, name, ProfilingType.NrNonNulls, rowMeta, row);
+ addSampleRow(store, name, ProfilingType.NrNonNulls, rowMeta, row,
decisions);
}
}
+ // A value over its type limit is still counted above. It is not kept
as a minimum or
+ // maximum, because that would retain the large value.
+ //
+ boolean storeValue = decisions == null || i >= decisions.length ||
!decisions[i].omit();
+
// Minimum
//
- if (profilingMinimum) {
+ if (profilingMinimum && storeValue) {
Object oldMin = store.getMinValues().get(name);
if (oldMin == null) {
- store.getMinValues().put(name, valueData);
+ store.getMinValues().put(name,
valueMeta.cloneValueData(valueData));
store.getMinMeta().put(name, valueMeta);
} else {
int compare = valueMeta.compare(valueData, oldMin);
if (compare < 0) {
// We have a new minimum
//
- store.getMinValues().put(name, valueData);
+ store.getMinValues().put(name,
valueMeta.cloneValueData(valueData));
store.getMinMeta().put(name, valueMeta);
clearSampleRows(store, name, ProfilingType.MinValue);
// Also save the row of data as a sample
//
- addSampleRow(store, name, ProfilingType.MinValue, rowMeta, row);
+ addSampleRow(store, name, ProfilingType.MinValue, rowMeta, row,
decisions);
} else if (compare == 0) {
// We found another value at the current minimum
- addSampleRow(store, name, ProfilingType.MinValue, rowMeta, row);
+ addSampleRow(store, name, ProfilingType.MinValue, rowMeta, row,
decisions);
}
}
}
// Maximum
//
- if (profilingMaximum) {
+ if (profilingMaximum && storeValue) {
Object oldMax = store.getMaxValues().get(name);
if (oldMax == null) {
- store.getMaxValues().put(name, valueData);
+ store.getMaxValues().put(name,
valueMeta.cloneValueData(valueData));
store.getMaxMeta().put(name, valueMeta);
} else {
int compare = valueMeta.compare(valueData, oldMax);
if (compare > 0) {
// We have a new maximum
//
- store.getMaxValues().put(name, valueData);
+ store.getMaxValues().put(name,
valueMeta.cloneValueData(valueData));
store.getMaxMeta().put(name, valueMeta);
clearSampleRows(store, name, ProfilingType.MaxValue);
// Also save the row of data as a sample
//
- addSampleRow(store, name, ProfilingType.MaxValue, rowMeta, row);
+ addSampleRow(store, name, ProfilingType.MaxValue, rowMeta, row,
decisions);
} else if (compare == 0) {
// We found another value at the current maximum
- addSampleRow(store, name, ProfilingType.MaxValue, rowMeta, row);
+ addSampleRow(store, name, ProfilingType.MaxValue, rowMeta, row,
decisions);
}
}
}
@@ -292,10 +310,10 @@ public class BasicDataProfilingDataSampler
// Also save the row of data as a sample
//
- addSampleRow(store, name, ProfilingType.MinLength, rowMeta,
row);
+ addSampleRow(store, name, ProfilingType.MinLength, rowMeta,
row, decisions);
} else if (length == oldMin) {
// We found another value at the current minimum length
- addSampleRow(store, name, ProfilingType.MinLength, rowMeta,
row);
+ addSampleRow(store, name, ProfilingType.MinLength, rowMeta,
row, decisions);
}
}
}
@@ -319,10 +337,10 @@ public class BasicDataProfilingDataSampler
// Also save the row of data as a sample
//
- addSampleRow(store, name, ProfilingType.MaxLength, rowMeta,
row);
+ addSampleRow(store, name, ProfilingType.MaxLength, rowMeta,
row, decisions);
} else if (length == oldMax) {
// We found another value at the current maximum length
- addSampleRow(store, name, ProfilingType.MaxLength, rowMeta,
row);
+ addSampleRow(store, name, ProfilingType.MaxLength, rowMeta,
row, decisions);
}
}
}
@@ -346,7 +364,9 @@ public class BasicDataProfilingDataSampler
String name,
ProfilingType profilingType,
IRowMeta rowMeta,
- Object[] row) {
+ Object[] row,
+ SampledValueLimits.Decision[] decisions)
+ throws HopValueException {
synchronized (store.getProfileSamples()) {
Map<ProfilingType, RowBuffer> typeBufferMap =
store.getProfileSamples().computeIfAbsent(name, k -> new
HashMap<>());
@@ -356,7 +376,7 @@ public class BasicDataProfilingDataSampler
// Keep the memory consumption sane
//
if (rowBuffer.size() < store.getMaxRows()) {
- rowBuffer.addRow(row);
+ rowBuffer.addRow(getSampledValueLimits().copyRow(rowMeta, row,
decisions));
}
}
}
@@ -544,4 +564,16 @@ public class BasicDataProfilingDataSampler
public void setProfilingMaxLength(boolean profilingMaxLength) {
this.profilingMaxLength = profilingMaxLength;
}
+
+ @Override
+ @JsonIgnore
+ public void setSampledValueLimits(SampledValueLimits sampledValueLimits) {
+ this.sampledValueLimits = sampledValueLimits;
+ }
+
+ @Override
+ @JsonIgnore
+ public SampledValueLimits getSampledValueLimits() {
+ return sampledValueLimits == null ? SampledValueLimits.unlimited() :
sampledValueLimits;
+ }
}
diff --git
a/engine/src/main/java/org/apache/hop/execution/sampler/plugins/first/FirstRowsExecutionDataSampler.java
b/engine/src/main/java/org/apache/hop/execution/sampler/plugins/first/FirstRowsExecutionDataSampler.java
index 27239e77a8..6cfbe00470 100644
---
a/engine/src/main/java/org/apache/hop/execution/sampler/plugins/first/FirstRowsExecutionDataSampler.java
+++
b/engine/src/main/java/org/apache/hop/execution/sampler/plugins/first/FirstRowsExecutionDataSampler.java
@@ -79,7 +79,7 @@ public class FirstRowsExecutionDataSampler
if (rows.isEmpty()) {
samplerStore.setRowMeta(rowMeta);
}
- rows.add(rowMeta.cloneRow(row));
+ rows.add(getSampledValueLimits().copyRow(rowMeta, row));
}
}
}
diff --git
a/engine/src/main/java/org/apache/hop/execution/sampler/plugins/last/LastRowsExecutionDataSampler.java
b/engine/src/main/java/org/apache/hop/execution/sampler/plugins/last/LastRowsExecutionDataSampler.java
index f7d1a707fa..c3ff990ef1 100644
---
a/engine/src/main/java/org/apache/hop/execution/sampler/plugins/last/LastRowsExecutionDataSampler.java
+++
b/engine/src/main/java/org/apache/hop/execution/sampler/plugins/last/LastRowsExecutionDataSampler.java
@@ -82,7 +82,7 @@ public class LastRowsExecutionDataSampler
// Add to the front, remove the last on overflow
// This is why we use a linked list
//
- rows.add(0, rowMeta.cloneRow(row));
+ rows.add(0, getSampledValueLimits().copyRow(rowMeta, row));
if (rows.size() > samplerStore.getMaxRows()) {
rows.remove(rows.size() - 1);
}
diff --git
a/engine/src/main/java/org/apache/hop/execution/sampler/plugins/random/RandomRowsExecutionDataSampler.java
b/engine/src/main/java/org/apache/hop/execution/sampler/plugins/random/RandomRowsExecutionDataSampler.java
index 6f5511551c..0d947422a1 100644
---
a/engine/src/main/java/org/apache/hop/execution/sampler/plugins/random/RandomRowsExecutionDataSampler.java
+++
b/engine/src/main/java/org/apache/hop/execution/sampler/plugins/random/RandomRowsExecutionDataSampler.java
@@ -83,11 +83,11 @@ public class RandomRowsExecutionDataSampler
if (rows.isEmpty()) {
samplerStore.setRowMeta(rowMeta);
}
- rows.add(row);
+ rows.add(getSampledValueLimits().copyRow(rowMeta, row));
} else {
int randomIndex = random.nextInt(samplerStore.getMaxRows());
if (randomIndex < samplerStore.getMaxRows()) {
- rows.set(randomIndex, rowMeta.cloneRow(row));
+ rows.set(randomIndex, getSampledValueLimits().copyRow(rowMeta, row));
}
}
}
diff --git
a/engine/src/main/java/org/apache/hop/pipeline/engines/local/LocalPipelineEngine.java
b/engine/src/main/java/org/apache/hop/pipeline/engines/local/LocalPipelineEngine.java
index 8359f8c884..7d77daea84 100644
---
a/engine/src/main/java/org/apache/hop/pipeline/engines/local/LocalPipelineEngine.java
+++
b/engine/src/main/java/org/apache/hop/pipeline/engines/local/LocalPipelineEngine.java
@@ -398,6 +398,7 @@ public class LocalPipelineEngine extends Pipeline
implements IPipelineEngine<Pip
if (samplers.isEmpty()) {
return;
}
+ profile.applyLimits(samplers, this);
samplerStoresMap = new HashMap<>();
diff --git
a/engine/src/main/resources/org/apache/hop/execution/profiling/messages/messages_en_US.properties
b/engine/src/main/resources/org/apache/hop/execution/profiling/messages/messages_en_US.properties
index 04a62301be..a65615d560 100644
---
a/engine/src/main/resources/org/apache/hop/execution/profiling/messages/messages_en_US.properties
+++
b/engine/src/main/resources/org/apache/hop/execution/profiling/messages/messages_en_US.properties
@@ -17,4 +17,13 @@
#
ExecutionDataProfile.name=Execution Data Profile
-ExecutionDataProfile.description=Describes how transform output data is
profiled
\ No newline at end of file
+ExecutionDataProfile.description=Describes how transform output data is
profiled
+ExecutionDataProfile.Group.LargeValues=Large values
+ExecutionDataProfile.Label.StringValueLimit=Maximum string length
+ExecutionDataProfile.Tooltip.StringValueLimit=Maximum number of characters of
a String field to keep in sample rows and in minimum and maximum values. A REST
or JSON body stored in a String field uses this limit. Leave empty for no
limit. 0 stores none of these values. A longer value is replaced with '<not
stored, N characters>'. You can use a variable, for example
'${SAMPLE_STRING_LIMIT}'. Minimum and maximum values are calculated only from
values that are stored.
+ExecutionDataProfile.Label.JsonValueLimit=Maximum JSON length
+ExecutionDataProfile.Tooltip.JsonValueLimit=Maximum number of characters of
the compact JSON text to keep for a JSON field. Leave empty for no limit. 0
stores none of these values. A longer value is replaced with a JSON text '<not
stored, N characters>'. You can use a variable, for example
'${SAMPLE_JSON_LIMIT}'.
+ExecutionDataProfile.Label.BinaryValueLimit=Maximum binary size (bytes)
+ExecutionDataProfile.Tooltip.BinaryValueLimit=Maximum number of bytes to keep
for a Binary field. Leave empty for no limit. 0 stores none of these values. A
larger value is left empty in the sample. You can use a variable, for example
'${SAMPLE_BINARY_LIMIT}'.
+ExecutionDataProfile.Label.AvroValueLimit=Maximum Avro size (bytes)
+ExecutionDataProfile.Tooltip.AvroValueLimit=Maximum number of bytes of schema
JSON plus Avro datum to keep for an Avro Record field. Leave empty for no
limit. 0 stores none of these values. A larger value is left empty in the
sample. You can use a variable, for example '${SAMPLE_AVRO_LIMIT}'.
\ No newline at end of file
diff --git
a/engine/src/test/java/org/apache/hop/execution/sampler/SampledValueLimitsTest.java
b/engine/src/test/java/org/apache/hop/execution/sampler/SampledValueLimitsTest.java
new file mode 100644
index 0000000000..8f6ba079bf
--- /dev/null
+++
b/engine/src/test/java/org/apache/hop/execution/sampler/SampledValueLimitsTest.java
@@ -0,0 +1,415 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.execution.sampler;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.lang.reflect.Field;
+import java.nio.charset.StandardCharsets;
+import java.sql.Timestamp;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hop.core.HopClientEnvironment;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.gui.plugin.GuiWidgetGroupType;
+import org.apache.hop.core.json.HopJson;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.RowBuffer;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaAvroRecord;
+import org.apache.hop.core.row.value.ValueMetaBinary;
+import org.apache.hop.core.row.value.ValueMetaJson;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.row.value.ValueMetaTimestamp;
+import org.apache.hop.core.util.JsonUtil;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.execution.ExecutionData;
+import org.apache.hop.execution.profiling.ExecutionDataProfile;
+import
org.apache.hop.execution.sampler.plugins.dataprof.BasicDataProfilingDataSampler;
+import
org.apache.hop.execution.sampler.plugins.dataprof.BasicDataProfilingDataSamplerStore;
+import
org.apache.hop.execution.sampler.plugins.first.FirstRowsExecutionDataSampler;
+import
org.apache.hop.execution.sampler.plugins.first.FirstRowsExecutionDataSamplerStore;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.pipeline.transform.stream.IStream;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+class SampledValueLimitsTest {
+
+ @BeforeAll
+ static void initHop() throws Exception {
+ HopClientEnvironment.init();
+ }
+
+ @Test
+ void unlimitedCopyLeavesEveryValueInANewRow() throws Exception {
+ IRowMeta rowMeta = sampleRowMeta();
+ byte[] binary = new byte[] {1, 2, 3, 4};
+ JsonNode json = JsonUtil.jsonMapper().readTree("{\"ok\":true}");
+ GenericRecord avro = avroRecord("hi");
+ Object[] row = new Object[] {"text", json, binary, avro};
+
+ Object[] copy = SampledValueLimits.unlimited().copyRow(rowMeta, row);
+
+ assertNotSame(row, copy);
+ assertEquals("text", copy[0]);
+ assertEquals(json, copy[1]);
+ assertArrayEquals(binary, (byte[]) copy[2]);
+ assertNotSame(binary, copy[2]);
+ assertEquals("hi", ((GenericRecord) copy[3]).get("a").toString());
+ assertSame("text", row[0]);
+ assertSame(json, row[1]);
+ assertSame(binary, row[2]);
+ assertSame(avro, row[3]);
+ }
+
+ @Test
+ void zeroOnOneTypeReplacesOnlyThatType() throws Exception {
+ IRowMeta rowMeta = sampleRowMeta();
+ Object[] row =
+ new Object[] {
+ "text", JsonUtil.jsonMapper().readTree("{\"ok\":true}"), new byte[]
{9}, avroRecord("hi")
+ };
+
+ Object[] copy = limits(null, null, "0", null).copyRow(rowMeta, row);
+
+ assertEquals("text", copy[0]);
+ assertEquals(row[1], copy[1]);
+ assertNull(copy[2]);
+ assertEquals("hi", ((GenericRecord) copy[3]).get("a").toString());
+ assertArrayEquals(new byte[] {9}, (byte[]) row[2]);
+ }
+
+ @Test
+ void positiveStringLimitKeepsShortValuesAndReplacesLongOnes() throws
Exception {
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("body"));
+ String longText = "this is longer than five";
+ Object[] shortRow = new Object[] {"short"};
+ Object[] longRow = new Object[] {longText};
+ SampledValueLimits valueLimits = limits("5", null, null, null);
+
+ Object[] shortCopy = valueLimits.copyRow(rowMeta, shortRow);
+ Object[] longCopy = valueLimits.copyRow(rowMeta, longRow);
+
+ assertEquals("short", shortCopy[0]);
+ assertEquals(SampledValueLimits.notStored(longText.length(),
"characters"), longCopy[0]);
+ assertSame(longText, longRow[0]);
+ assertNotSame(longRow, longCopy);
+ }
+
+ @Test
+ void binaryAndAvroOverTheLimitAreNullAndSchemaBytesCount() throws Exception {
+ IRowMeta rowMeta = sampleRowMeta();
+ byte[] binary = new byte[] {1, 2, 3, 4, 5};
+ GenericRecord small = avroRecord("hi");
+ int smallSize = ValueMetaAvroRecord.storedPayloadBytes(small);
+ GenericRecord wideSchema = wideSchemaRecord();
+ int wideSize = ValueMetaAvroRecord.storedPayloadBytes(wideSchema);
+ assertTrue(wideSize > smallSize);
+
+ Object[] overBinary =
+ limits(null, null, "4", null)
+ .copyRow(rowMeta, new Object[] {"text", json("{\"a\":1}"), binary,
small});
+ assertNull(overBinary[2]);
+ assertNotNull(overBinary[3]);
+
+ Object[] overSchema =
+ limits(null, null, null, Integer.toString(smallSize))
+ .copyRow(rowMeta, new Object[] {"text", json("{\"a\":1}"), new
byte[] {1}, wideSchema});
+ assertNull(overSchema[3]);
+
+ Object[] underSchema =
+ limits(null, null, "0", Integer.toString(smallSize))
+ .copyRow(rowMeta, new Object[] {"text", json("{\"a\":1}"), new
byte[] {1}, small});
+ assertNull(underSchema[2]);
+ assertEquals("hi", ((GenericRecord) underSchema[3]).get("a").toString());
+ assertNotSame(small, underSchema[3]);
+ }
+
+ @Test
+ void binaryStringAndIndexedMarkersMatchTheFieldStorage() throws Exception {
+ String longText = "this is longer than five";
+ SampledValueLimits valueLimits = limits("5", "5", null, null);
+
+ ValueMetaString lazy = new ValueMetaString("body");
+ lazy.setStorageType(IValueMeta.STORAGE_TYPE_BINARY_STRING);
+ lazy.setStorageMetadata(new ValueMetaString("body"));
+ IRowMeta lazyMeta = new RowMeta();
+ lazyMeta.addValueMeta(lazy);
+ Object[] lazyCopy =
+ valueLimits.copyRow(lazyMeta, new Object[]
{longText.getBytes(StandardCharsets.UTF_8)});
+ assertInstanceOf(byte[].class, lazyCopy[0]);
+ assertEquals(
+ SampledValueLimits.notStored(longText.length(), "characters"),
+ new String((byte[]) lazyCopy[0], StandardCharsets.UTF_8));
+
+ RowBuffer buffer = new RowBuffer(lazyMeta);
+ buffer.addRow(lazyCopy);
+ ExecutionData executionData = new ExecutionData();
+ executionData.getDataSets().put("rows", buffer);
+ assertNotNull(executionData.getRowsBinaryGzipBase64Encoded());
+
+ ValueMetaJson lazyJson = new ValueMetaJson("payload");
+ lazyJson.setStorageType(IValueMeta.STORAGE_TYPE_BINARY_STRING);
+ lazyJson.setStorageMetadata(new ValueMetaString("payload"));
+ IRowMeta jsonMeta = new RowMeta();
+ jsonMeta.addValueMeta(lazyJson);
+ byte[] jsonBytes =
"{\"a\":\"0123456789\"}".getBytes(StandardCharsets.UTF_8);
+ Object[] jsonCopy = valueLimits.copyRow(jsonMeta, new Object[]
{jsonBytes});
+ assertInstanceOf(byte[].class, jsonCopy[0]);
+
+ ValueMetaString indexed = new ValueMetaString("body");
+ indexed.setStorageType(IValueMeta.STORAGE_TYPE_INDEXED);
+ indexed.setIndex(new Object[] {"short", longText});
+ indexed.setStorageMetadata(new ValueMetaString("body"));
+ IRowMeta indexedMeta = new RowMeta();
+ indexedMeta.addValueMeta(indexed);
+ Object[] indexedRow = new Object[] {Integer.valueOf(1)};
+ Object[] indexedCopy = valueLimits.copyRow(indexedMeta, indexedRow);
+ assertNull(indexedCopy[0]);
+ assertEquals(Integer.valueOf(1), indexedRow[0]);
+ }
+
+ @Test
+ void aKeptTimestampIsClonedWhenAnotherCellIsOmitted() throws Exception {
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("body"));
+ rowMeta.addValueMeta(new ValueMetaTimestamp("when"));
+ Timestamp when = new Timestamp(1_700_000_000_000L);
+ when.setNanos(123_456);
+ Object[] row = new Object[] {"this is longer than five", when};
+
+ Object[] copy = limits("5", null, null, null).copyRow(rowMeta, row);
+
+ assertNotSame(when, copy[1]);
+ assertEquals(when, copy[1]);
+ assertEquals(when.getNanos(), ((Timestamp) copy[1]).getNanos());
+ }
+
+ @Test
+ void nonNumericLimitIsUnlimited() throws Exception {
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("body"));
+ String longText = "still stored";
+ Object[] row = new Object[] {longText};
+
+ Object[] copy = limits("many", null, null, null).copyRow(rowMeta, row);
+ Object[] negative = limits("-3", null, null, null).copyRow(rowMeta, row);
+
+ assertEquals(longText, copy[0]);
+ assertEquals(longText, negative[0]);
+ }
+
+ @Test
+ void variableLimitIsResolvedOnce() throws Exception {
+ ExecutionDataProfile profile = new ExecutionDataProfile("profile");
+ profile.setStringValueLimit("${MAX}");
+ Variables variables = new Variables();
+ variables.setVariable("MAX", "0");
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("body"));
+
+ Object[] copy =
+ SampledValueLimits.from(profile, variables).copyRow(rowMeta, new
Object[] {"kept?"});
+
+ assertEquals(SampledValueLimits.NOT_STORED, copy[0]);
+ }
+
+ @Test
+ void profileCopyKeepsTheLimitFields() {
+ ExecutionDataProfile profile = new ExecutionDataProfile("profile");
+ profile.setStringValueLimit("10");
+ profile.setJsonValueLimit("0");
+ profile.setBinaryValueLimit("20");
+ profile.setAvroValueLimit("30");
+
+ ExecutionDataProfile copy = new ExecutionDataProfile(profile);
+
+ assertEquals("10", copy.getStringValueLimit());
+ assertEquals("0", copy.getJsonValueLimit());
+ assertEquals("20", copy.getBinaryValueLimit());
+ assertEquals("30", copy.getAvroValueLimit());
+ }
+
+ @Test
+ void limitsRoundTripThroughMetadata() throws Exception {
+ ExecutionDataProfile profile = new ExecutionDataProfile("limits");
+ profile.setStringValueLimit("10");
+ profile.setJsonValueLimit("0");
+ profile.setBinaryValueLimit("20");
+ profile.setAvroValueLimit("${SAMPLE_AVRO_LIMIT}");
+ IHopMetadataProvider provider = new MemoryMetadataProvider();
+ provider.getSerializer(ExecutionDataProfile.class).save(profile);
+
+ ExecutionDataProfile loaded =
provider.getSerializer(ExecutionDataProfile.class).load("limits");
+
+ assertEquals("10", loaded.getStringValueLimit());
+ assertEquals("0", loaded.getJsonValueLimit());
+ assertEquals("20", loaded.getBinaryValueLimit());
+ assertEquals("${SAMPLE_AVRO_LIMIT}", loaded.getAvroValueLimit());
+ }
+
+ @Test
+ void profileDeclaresTheLimitWidgets() throws Exception {
+ assertNotNull(ExecutionDataProfile.class.getAnnotation(GuiPlugin.class));
+ assertLimitWidget("stringValueLimit");
+ assertLimitWidget("jsonValueLimit");
+ assertLimitWidget("binaryValueLimit");
+ assertLimitWidget("avroValueLimit");
+ }
+
+ private static void assertLimitWidget(String fieldName) throws Exception {
+ Field field = ExecutionDataProfile.class.getDeclaredField(fieldName);
+ GuiWidgetElement element = field.getAnnotation(GuiWidgetElement.class);
+ assertNotNull(element, fieldName);
+ assertEquals(fieldName, element.id());
+ assertEquals(GuiElementType.TEXT, element.type());
+ assertEquals(ExecutionDataProfile.GUI_PLUGIN_LIMITS_PARENT_ID,
element.parentId());
+ assertEquals(GuiWidgetGroupType.BOXES, element.groupType());
+ assertEquals("i18n::ExecutionDataProfile.Group.LargeValues",
element.group());
+ assertTrue(element.label().startsWith("i18n::"));
+ assertTrue(element.toolTip().startsWith("i18n::"));
+ }
+
+ @Test
+ void jacksonDoesNotWriteTheRuntimeLimits() throws Exception {
+ FirstRowsExecutionDataSampler sampler = new
FirstRowsExecutionDataSampler("10");
+ sampler.setSampledValueLimits(limits("0", "0", "0", "0"));
+
+ String json = HopJson.newMapper().writeValueAsString(sampler);
+
+ assertFalse(json.contains("sampledValueLimits"));
+ assertFalse(json.contains("stringLimit"));
+ }
+
+ @Test
+ void profilingKeepsLengthStatsWithoutTheLongString() throws Exception {
+ BasicDataProfilingDataSampler sampler = new
BasicDataProfilingDataSampler();
+ sampler.setSampledValueLimits(limits("5", null, null, null));
+ ExecutionDataSamplerMeta samplerMeta =
+ new ExecutionDataSamplerMeta("transform", "0", "log", false, true);
+ BasicDataProfilingDataSamplerStore store =
sampler.createSamplerStore(samplerMeta);
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("name"));
+ rowMeta.addValueMeta(new ValueMetaString("body"));
+ store.init(new Variables(), rowMeta, rowMeta);
+
+ String longText = "this is long";
+ sampler.sampleRow(store, IStream.StreamType.OUTPUT, rowMeta, new Object[]
{"a", "short"});
+ sampler.sampleRow(store, IStream.StreamType.OUTPUT, rowMeta, new Object[]
{"b", longText});
+
+ assertEquals(2L, store.getNonNullCounters().get("body"));
+ assertEquals(longText.length(), store.getMaxLengths().get("body"));
+ assertEquals("short", store.getMaxValues().get("body"));
+ assertFalse(store.getMaxValues().containsValue(longText));
+
+ RowBuffer lengthRows =
+ store
+ .getProfileSamples()
+ .get("body")
+ .get(BasicDataProfilingDataSampler.ProfilingType.MaxLength);
+ assertEquals(1, lengthRows.size());
+ Object[] example = lengthRows.getBuffer().get(0);
+ assertEquals("b", example[0]);
+ assertEquals(SampledValueLimits.notStored(longText.length(),
"characters"), example[1]);
+ }
+
+ @Test
+ void firstRowsReplaceJsonAndKeepTheOtherColumns() throws Exception {
+ FirstRowsExecutionDataSampler sampler = new
FirstRowsExecutionDataSampler("10");
+ sampler.setSampledValueLimits(limits(null, "0", null, null));
+ ExecutionDataSamplerMeta samplerMeta =
+ new ExecutionDataSamplerMeta("transform", "0", "log", false, true);
+ FirstRowsExecutionDataSamplerStore store =
sampler.createSamplerStore(samplerMeta);
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("id"));
+ rowMeta.addValueMeta(new ValueMetaJson("payload"));
+ store.init(new Variables(), rowMeta, rowMeta);
+ JsonNode payload = json("{\"n\":1}");
+ Object[] row = new Object[] {"row-1", payload};
+
+ sampler.sampleRow(store, IStream.StreamType.OUTPUT, rowMeta, row);
+
+ Object[] stored = store.getRows().get(0);
+ assertNotSame(row, stored);
+ assertEquals("row-1", stored[0]);
+ JsonNode marker = assertInstanceOf(JsonNode.class, stored[1]);
+ assertEquals(SampledValueLimits.NOT_STORED, marker.asText());
+ assertSame(payload, row[1]);
+ }
+
+ private static SampledValueLimits limits(
+ String stringLimit, String jsonLimit, String binaryLimit, String
avroLimit) {
+ ExecutionDataProfile profile = new ExecutionDataProfile("profile");
+ profile.setStringValueLimit(stringLimit);
+ profile.setJsonValueLimit(jsonLimit);
+ profile.setBinaryValueLimit(binaryLimit);
+ profile.setAvroValueLimit(avroLimit);
+ return SampledValueLimits.from(profile, new Variables());
+ }
+
+ private static IRowMeta sampleRowMeta() {
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("text"));
+ rowMeta.addValueMeta(new ValueMetaJson("json"));
+ rowMeta.addValueMeta(new ValueMetaBinary("binary"));
+ rowMeta.addValueMeta(new ValueMetaAvroRecord("avro"));
+ return rowMeta;
+ }
+
+ private static JsonNode json(String text) throws Exception {
+ return JsonUtil.jsonMapper().readTree(text);
+ }
+
+ private static GenericRecord avroRecord(String value) {
+ Schema schema =
+ new Schema.Parser()
+ .parse(
+
"{\"type\":\"record\",\"name\":\"R\",\"fields\":[{\"name\":\"a\",\"type\":\"string\"}]}");
+ GenericRecord record = new GenericData.Record(schema);
+ record.put("a", value);
+ return record;
+ }
+
+ private static GenericRecord wideSchemaRecord() {
+ Schema schema =
+ new Schema.Parser()
+ .parse(
+ "{\"type\":\"record\",\"name\":\"Wide\",\"doc\":\""
+ + "x".repeat(400)
+ + "\",\"fields\":[{\"name\":\"a\",\"type\":\"string\"}]}");
+ GenericRecord record = new GenericData.Record(schema);
+ record.put("a", "hi");
+ return record;
+ }
+}
diff --git
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/core/transform/TransformBaseFn.java
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/core/transform/TransformBaseFn.java
index a7990fdc3f..2f49235912 100644
---
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/core/transform/TransformBaseFn.java
+++
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/core/transform/TransformBaseFn.java
@@ -166,8 +166,9 @@ public abstract class TransformBaseFn extends DoFn<HopRow,
HopRow> {
// If not there's nothing we have to do in this transform really
//
String profileName = runConf.getExecutionDataProfileName();
+ ExecutionDataProfile dataProfile = null;
if (StringUtils.isNotEmpty(profileName)) {
- ExecutionDataProfile dataProfile =
+ dataProfile =
metadataProvider.getSerializer(ExecutionDataProfile.class).load(profileName);
if (dataProfile != null) {
dataSamplers.addAll(dataProfile.getSamplers());
@@ -180,6 +181,9 @@ public abstract class TransformBaseFn extends DoFn<HopRow,
HopRow> {
HopJson.newMapper().readValue(dataSamplersJson,
IExecutionDataSampler[].class);
dataSamplers.addAll(Arrays.asList(extraSamplers));
}
+ if (dataProfile != null) {
+ dataProfile.applyLimits(dataSamplers, variables);
+ }
executionInfoLocation = location;
diff --git
a/plugins/engines/spark/src/main/java/org/apache/hop/spark/execution/SparkTransformExecutionSampling.java
b/plugins/engines/spark/src/main/java/org/apache/hop/spark/execution/SparkTransformExecutionSampling.java
index 3c6a2c717c..f9064adebd 100644
---
a/plugins/engines/spark/src/main/java/org/apache/hop/spark/execution/SparkTransformExecutionSampling.java
+++
b/plugins/engines/spark/src/main/java/org/apache/hop/spark/execution/SparkTransformExecutionSampling.java
@@ -168,9 +168,9 @@ public class SparkTransformExecutionSampling {
}
String profileName = runConf.getExecutionDataProfileName();
+ ExecutionDataProfile dataProfile = null;
if (StringUtils.isNotEmpty(profileName)) {
- ExecutionDataProfile dataProfile =
-
metadataProvider.getSerializer(ExecutionDataProfile.class).load(profileName);
+ dataProfile =
metadataProvider.getSerializer(ExecutionDataProfile.class).load(profileName);
if (dataProfile != null && dataProfile.getSamplers() != null) {
dataSamplers.addAll(dataProfile.getSamplers());
} else {
@@ -201,6 +201,10 @@ public class SparkTransformExecutionSampling {
}
}
+ if (dataProfile != null) {
+ dataProfile.applyLimits(dataSamplers, variables);
+ }
+
if (dataSamplers.isEmpty()) {
LogChannel.GENERAL.logBasic(
"Spark sampling inactive for '"
diff --git
a/ui/src/main/java/org/apache/hop/ui/execution/profiling/ExecutionDataProfileEditor.java
b/ui/src/main/java/org/apache/hop/ui/execution/profiling/ExecutionDataProfileEditor.java
index b1782a6fdd..2ac60109e2 100644
---
a/ui/src/main/java/org/apache/hop/ui/execution/profiling/ExecutionDataProfileEditor.java
+++
b/ui/src/main/java/org/apache/hop/ui/execution/profiling/ExecutionDataProfileEditor.java
@@ -44,6 +44,7 @@ import org.eclipse.swt.layout.FormData;
import org.eclipse.swt.layout.FormLayout;
import org.eclipse.swt.widgets.Composite;
import org.eclipse.swt.widgets.Control;
+import org.eclipse.swt.widgets.Group;
import org.eclipse.swt.widgets.Label;
import org.eclipse.swt.widgets.Listener;
import org.eclipse.swt.widgets.Text;
@@ -66,8 +67,10 @@ public class ExecutionDataProfileEditor extends
MetadataEditor<ExecutionDataProf
private Text wDescription;
private org.eclipse.swt.widgets.List wSamplers;
+ private Composite editorComposite;
private Composite wPluginSpecificComp;
private GuiCompositeWidgets guiCompositeWidgets;
+ private GuiCompositeWidgets limitsWidgets;
private Map<String, IExecutionDataSampler> metaMap;
@@ -86,6 +89,7 @@ public class ExecutionDataProfileEditor extends
MetadataEditor<ExecutionDataProf
@Override
public void createControl(Composite parent) {
+ this.editorComposite = parent;
PropsUi props = PropsUi.getInstance();
// Create a tabbed interface instead of the confusing left-hand side
options
@@ -135,6 +139,18 @@ public class ExecutionDataProfileEditor extends
MetadataEditor<ExecutionDataProf
wDescription.setLayoutData(fdDescription);
lastControl = wDescription;
+ limitsWidgets = new GuiCompositeWidgets(manager.getVariables());
+ limitsWidgets.createCompositeWidgets(
+ workingProfile,
+ null,
+ parent,
+ ExecutionDataProfile.GUI_PLUGIN_LIMITS_PARENT_ID,
+ wDescription);
+ Control limitsGroup = findGroup(parent);
+ if (limitsGroup != null) {
+ lastControl = limitsGroup;
+ }
+
Label wlSamplers = new Label(parent, SWT.LEFT);
PropsUi.setLook(wlSamplers);
wlSamplers.setText(BaseMessages.getString(PKG,
"ExecutionDataProfileEditor.label.Samplers"));
@@ -202,6 +218,25 @@ public class ExecutionDataProfileEditor extends
MetadataEditor<ExecutionDataProf
wName.addListener(SWT.Modify, modifyListener);
wDescription.addListener(SWT.Modify, modifyListener);
wSamplers.addListener(SWT.Selection, e -> addSamplerPluginWidgets());
+ limitsWidgets.setWidgetsListener(
+ new GuiCompositeWidgetsAdapter() {
+ @Override
+ public void widgetModified(
+ GuiCompositeWidgets compositeWidgets, Control changedWidget,
String widgetId) {
+ setChanged();
+ }
+ });
+ }
+
+ /** The limits group is created on the editor composite, so the sampler list
can sit under it. */
+ private static Control findGroup(Composite parent) {
+ Control group = null;
+ for (Control child : parent.getChildren()) {
+ if (child instanceof Group) {
+ group = child;
+ }
+ }
+ return group;
}
private IExecutionDataSampler previousSampler;
@@ -271,6 +306,8 @@ public class ExecutionDataProfileEditor extends
MetadataEditor<ExecutionDataProf
wName.setText(Const.NVL(workingProfile.getName(), ""));
wDescription.setText(Const.NVL(workingProfile.getDescription(), ""));
+ limitsWidgets.setWidgetsContents(
+ workingProfile, editorComposite,
ExecutionDataProfile.GUI_PLUGIN_LIMITS_PARENT_ID);
// Add all the sampler plugins in the profile
//
@@ -294,6 +331,7 @@ public class ExecutionDataProfileEditor extends
MetadataEditor<ExecutionDataProf
profile.setName(wName.getText());
profile.setDescription(wDescription.getText());
+ limitsWidgets.getWidgetsContents(profile,
ExecutionDataProfile.GUI_PLUGIN_LIMITS_PARENT_ID);
savePreviousSampler();
