This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 37040b2890 Script transform: resolve output field types once, and run
Groovy scr… (#9442)
37040b2890 is described below
commit 37040b289034bcf2345d4008b9988116484f1ea9
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Sat Nov 14 12:49:40 2026 +0100
Script transform: resolve output field types once, and run Groovy scr…
(#9442)
* Script transform: resolve output field types once, and run Groovy scripts
without re-creating them for every row, fixes #9439
* fix remarks
---
.../transforms/script/GroovyRowScript.java | 331 ++++++++++++
.../hop/pipeline/transforms/script/Script.java | 47 +-
.../hop/pipeline/transforms/script/ScriptData.java | 7 +
.../script/ScriptGroovyDirectRunTest.java | 562 +++++++++++++++++++++
4 files changed, 945 insertions(+), 2 deletions(-)
diff --git
a/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/GroovyRowScript.java
b/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/GroovyRowScript.java
new file mode 100644
index 0000000000..7a50b8e53e
--- /dev/null
+++
b/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/GroovyRowScript.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.script;
+
+import groovy.lang.Binding;
+import groovy.lang.Closure;
+import groovy.lang.DelegatingMetaClass;
+import groovy.lang.GroovyRuntimeException;
+import groovy.lang.MetaClass;
+import groovy.lang.MissingMethodException;
+import groovy.lang.MissingPropertyException;
+import groovy.lang.Script;
+import groovy.lang.Tuple;
+import java.io.PrintWriter;
+import java.io.Writer;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import javax.script.ScriptContext;
+import javax.script.ScriptEngine;
+import javax.script.ScriptException;
+import org.codehaus.groovy.jsr223.GroovyScriptEngineImpl;
+import org.codehaus.groovy.runtime.InvokerHelper;
+import org.codehaus.groovy.util.ManagedConcurrentValueMap;
+
+/**
+ * Runs the row script of a Groovy Script transform without going through the
JSR-223 engine for
+ * every row.
+ *
+ * <p>The engine's {@code CompiledScript.eval()} builds, for every call, a new
script object, a
+ * closure for each of its methods and a metaclass for that one object. The
new metaclass makes
+ * Groovy link every call in the script again, which is most of the cost of a
short script. Here the
+ * script class is compiled once and each row gets a new instance that is run
directly, with the
+ * same metaclass object as the row before.
+ *
+ * <p>What a script can see and do is kept the same as with the engine:
+ *
+ * <ul>
+ * <li>a new script object per row, so {@code @Field} variables start fresh
for every row;
+ * <li>variables are read from and written to the script context, with the
engine's own rules for
+ * {@code out} and {@code context};
+ * <li>a method the script does not have is looked up among the functions of
the scripts the
+ * engine evaluated before, such as the start script, and then among the
closures in the
+ * context, as the engine does;
+ * <li>the functions of the row script are known to the engine, for the
start and end scripts;
+ * <li>an exception leaves as a {@link ScriptException} around the original
one.
+ * </ul>
+ */
+final class GroovyRowScript {
+
+ private final Class<? extends Script> scriptClass;
+ private final Constructor<? extends Script> constructor;
+ private final Binding binding;
+ private final ManagedConcurrentValueMap<String, Closure<?>> globalClosures;
+ private final ScriptContext context;
+
+ /** The metaclass of the script objects, see {@link #newScript()}. */
+ private GlobalFunctionsMetaClass metaClass;
+
+ /** The script object of the current row. */
+ private Script lastScript;
+
+ private GroovyRowScript(
+ Class<? extends Script> scriptClass,
+ Constructor<? extends Script> constructor,
+ ScriptContext context,
+ ManagedConcurrentValueMap<String, Closure<?>> globalClosures) {
+ this.scriptClass = scriptClass;
+ this.constructor = constructor;
+ this.binding = new ContextBinding(context);
+ this.globalClosures = globalClosures;
+ this.context = context;
+ }
+
+ /**
+ * @return the row script, or null when this is not Groovy's JSR-223 engine,
the source is a class
+ * and not a script, or the engine's functions cannot be reached: the
caller then keeps using
+ * the engine
+ */
+ static GroovyRowScript create(ScriptEngine engine, String source,
ScriptContext context) {
+ if (!(engine instanceof GroovyScriptEngineImpl groovyEngine) || context ==
null) {
+ return null;
+ }
+ // The engine's own class loader: classes that the start script defined
are visible.
+ Class<?> parsed = groovyEngine.getClassLoader().parseClass(source);
+ if (!Script.class.isAssignableFrom(parsed)) {
+ return null;
+ }
+ Class<? extends Script> scriptClass = parsed.asSubclass(Script.class);
+ ManagedConcurrentValueMap<String, Closure<?>> globalClosures =
globalClosures(groovyEngine);
+ if (globalClosures == null) {
+ return null;
+ }
+ Constructor<? extends Script> constructor;
+ try {
+ constructor = scriptClass.getConstructor(Binding.class);
+ } catch (NoSuchMethodException e) {
+ return null;
+ }
+ GroovyRowScript rowScript =
+ new GroovyRowScript(scriptClass, constructor, context, globalClosures);
+ rowScript.registerFunctions();
+ return rowScript;
+ }
+
+ /**
+ * The functions of every script the engine evaluated, by name. The engine
keeps them in a private
+ * map, so this needs reflection.
+ *
+ * @return the map, or null when it cannot be reached
+ */
+ @SuppressWarnings("unchecked")
+ private static ManagedConcurrentValueMap<String, Closure<?>> globalClosures(
+ GroovyScriptEngineImpl engine) {
+ try {
+ Field field =
GroovyScriptEngineImpl.class.getDeclaredField("globalClosures");
+ field.setAccessible(true);
+ return (ManagedConcurrentValueMap<String, Closure<?>>) field.get(engine);
+ } catch (ReflectiveOperationException | RuntimeException e) {
+ return null;
+ }
+ }
+
+ /** Runs the script for one row and returns its result. */
+ Object run() throws ScriptException {
+ try {
+ Script script = newScript();
+ lastScript = script;
+ return script.run();
+ } catch (Exception e) {
+ throw new ScriptException(e);
+ }
+ }
+
+ /**
+ * A new script object for the row. Like the engine, its metaclass wraps the
one the object got
+ * from its class. Unlike the engine, the wrapper is the same object from
row to row as long as
+ * the class keeps its metaclass: Groovy's call sites check the metaclass of
the object they call,
+ * and stay linked as long as it does not change.
+ *
+ * <p>The wrapper is set on the object, not registered for the class.
Registering a metaclass
+ * makes Groovy relink the calls of every Groovy script in the JVM, and a
script that changes the
+ * metaclass of its own class would remove it.
+ */
+ private Script newScript() {
+ Script script;
+ try {
+ script = constructor.newInstance(binding);
+ } catch (ReflectiveOperationException e) {
+ // As InvokerHelper.createScript, which the engine uses, reports it
+ throw new GroovyRuntimeException(
+ "Failed to create Script instance for class: " + scriptClass + ".
Reason: " + e, e);
+ }
+ MetaClass classMetaClass = script.getMetaClass();
+ if (metaClass == null || metaClass.getAdaptee() != classMetaClass) {
+ metaClass = new GlobalFunctionsMetaClass(classMetaClass, globalClosures,
context);
+ }
+ script.setMetaClass(metaClass);
+ return script;
+ }
+
+ /**
+ * Tells the engine about the functions of the row script, as it does itself
for every script it
+ * runs: a function of the start script that calls one of the row script,
and the end script, find
+ * them there.
+ */
+ private void registerFunctions() {
+ for (Method method : scriptClass.getMethods()) {
+ String name = method.getName();
+ globalClosures.put(name, new CurrentScriptFunction(name));
+ }
+ }
+
+ /** A function of the row script, called on the script object of the current
row. */
+ private final class CurrentScriptFunction extends Closure<Object> {
+ private static final long serialVersionUID = 1L;
+ private final String name;
+
+ CurrentScriptFunction(String name) {
+ super(GroovyRowScript.class);
+ this.name = name;
+ }
+
+ @Override
+ public Object call(Object... args) {
+ Script script = lastScript;
+ if (script == null) {
+ throw new MissingMethodException(name, scriptClass, args);
+ }
+ return InvokerHelper.invokeMethod(script, name, args);
+ }
+ }
+
+ /** Variable lookup in the script context, as the engine's own binding does
it. */
+ private static final class ContextBinding extends Binding {
+ private final transient ScriptContext context;
+
+ ContextBinding(ScriptContext context) {
+ super(context.getBindings(ScriptContext.ENGINE_SCOPE));
+ this.context = context;
+ }
+
+ @Override
+ public Object getVariable(String name) {
+ synchronized (context) {
+ int scope = context.getAttributesScope(name);
+ if (scope != -1) {
+ return context.getAttribute(name, scope);
+ }
+ // Script output goes to the context writer when no out variable is
provided
+ if ("out".equals(name)) {
+ Writer writer = context.getWriter();
+ if (writer != null) {
+ return writer instanceof PrintWriter printWriter
+ ? printWriter
+ : new PrintWriter(writer, true);
+ }
+ }
+ if ("context".equals(name)) {
+ return context;
+ }
+ }
+ throw new MissingPropertyException(name, getClass());
+ }
+
+ @Override
+ public void setVariable(String name, Object value) {
+ synchronized (context) {
+ int scope = context.getAttributesScope(name);
+ if (scope == -1) {
+ scope = ScriptContext.ENGINE_SCOPE;
+ }
+ context.setAttribute(name, value, scope);
+ }
+ }
+ }
+
+ /**
+ * Passes a method the script does not have on to the functions of the
scripts the engine
+ * evaluated, the start script's for one, and then to the closures in the
context. This is the
+ * engine's own lookup: what such a function throws reaches the script as it
is.
+ */
+ private static final class GlobalFunctionsMetaClass extends
DelegatingMetaClass {
+ private final ManagedConcurrentValueMap<String, Closure<?>> globalClosures;
+ private final ScriptContext context;
+
+ GlobalFunctionsMetaClass(
+ MetaClass delegate,
+ ManagedConcurrentValueMap<String, Closure<?>> globalClosures,
+ ScriptContext context) {
+ super(delegate);
+ this.globalClosures = globalClosures;
+ this.context = context;
+ }
+
+ @Override
+ public Object invokeMethod(Object object, String name, Object args) {
+ if (args == null) {
+ return invokeMethod(object, name, new Object[0]);
+ }
+ if (args instanceof Tuple<?> tuple) {
+ return invokeMethod(object, name, tuple.toArray());
+ }
+ if (args instanceof Object[] array) {
+ return invokeMethod(object, name, array);
+ }
+ return invokeMethod(object, name, new Object[] {args});
+ }
+
+ @Override
+ public Object invokeMethod(Object object, String name, Object[] args) {
+ try {
+ return super.invokeMethod(object, name, args);
+ } catch (MissingMethodException e) {
+ if (!isAboutThisCall(e, object, name)) {
+ throw e;
+ }
+ return callGlobal(name, args);
+ }
+ }
+
+ @Override
+ public Object invokeStaticMethod(Object object, String name, Object[]
args) {
+ try {
+ return super.invokeStaticMethod(object, name, args);
+ } catch (MissingMethodException e) {
+ if (!isAboutThisCall(e, object, name)) {
+ throw e;
+ }
+ return callGlobal(name, args);
+ }
+ }
+
+ /**
+ * Whether the method that is missing is the one called here, the check
Groovy's own call sites
+ * make. A method that is missing further down, inside the method that was
found, must reach the
+ * script as it is: looking it up among the functions would run the method
that was found again.
+ */
+ private static boolean isAboutThisCall(MissingMethodException e, Object
object, String name) {
+ Class<?> type = object instanceof Class<?> c ? c : object.getClass();
+ return name.equals(e.getMethod()) && e.getType() == type;
+ }
+
+ private Object callGlobal(String name, Object[] args) {
+ Closure<?> closure = globalClosures.get(name);
+ if (closure != null) {
+ return closure.call(args);
+ }
+ Object value = context.getAttribute(name);
+ if (value instanceof Closure<?> variable) {
+ return variable.call(args);
+ }
+ throw new MissingMethodException(name, GroovyScriptEngineImpl.class,
args);
+ }
+ }
+}
diff --git
a/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/Script.java
b/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/Script.java
index e0254860e0..79e7d97c0a 100644
---
a/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/Script.java
+++
b/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/Script.java
@@ -92,6 +92,12 @@ public class Script extends BaseTransform<ScriptMeta,
ScriptData> implements ITr
private Object[] previousRow = null;
+ /**
+ * Run a Groovy row script directly instead of through the engine's eval,
see {@link
+ * GroovyRowScript}. Tests switch it off to compare both ways.
+ */
+ boolean directGroovyRowScript = true;
+
/**
* This is the base transform that forms that basis for all transforms. You
can derive from this
* class to implement your own transforms.
@@ -201,6 +207,12 @@ public class Script extends BaseTransform<ScriptMeta,
ScriptData> implements ITr
}
}
+ // The type of each output field, looked up once: the lookup searches
the plugin registry.
+ data.fieldTypes = new int[meta.getFields().size()];
+ for (int i = 0; i < data.fieldTypes.length; i++) {
+ data.fieldTypes[i] = meta.getFields().get(i).getHopType();
+ }
+
data.context = data.engine.getContext();
if (data.context == null) {
data.context = new SimpleScriptContext();
@@ -260,6 +272,7 @@ public class Script extends BaseTransform<ScriptMeta,
ScriptData> implements ITr
} else {
data.compiledScript = null;
}
+ data.groovyRowScript = createGroovyRowScript();
} catch (Exception e) {
throw new HopValueException(
BaseMessages.getString(PKG, "Script.Log.CouldNotCompileScript"),
e);
@@ -420,7 +433,28 @@ public class Script extends BaseTransform<ScriptMeta,
ScriptData> implements ITr
}
}
+ /**
+ * @return the direct runner for a Groovy row script, or null to evaluate
through the engine
+ */
+ private GroovyRowScript createGroovyRowScript() {
+ if (!directGroovyRowScript || data.compiledScript == null) {
+ return null;
+ }
+ try {
+ return GroovyRowScript.create(data.engine, strTransformScript,
data.context);
+ } catch (Exception | LinkageError e) {
+ // The engine compiled the script, so it can run it: fall back to that
+ if (isDetailed()) {
+ logDetailed("The Groovy script is run through the script engine: " +
e);
+ }
+ return null;
+ }
+ }
+
protected Object evalScript() throws ScriptException {
+ if (data.groovyRowScript != null) {
+ return data.groovyRowScript.run();
+ }
if (data.compiledScript != null) {
try {
return data.compiledScript.eval(data.context);
@@ -449,7 +483,7 @@ public class Script extends BaseTransform<ScriptMeta,
ScriptData> implements ITr
try {
if (result != null) {
String classType = result.getClass().getName();
- switch (field.getHopType()) {
+ switch (hopType(field, i)) {
case IValueMeta.TYPE_NUMBER:
if
(classType.equalsIgnoreCase(CONST_ORG_MOZILLA_JAVASCRIPT_UNDEFINED)) {
return null;
@@ -603,7 +637,7 @@ public class Script extends BaseTransform<ScriptMeta,
ScriptData> implements ITr
{
throw new HopRuntimeException(
"JavaScript conversion not implemented for type "
- + field.getHopType()
+ + hopType(field, i)
+ " ("
+ field.getType()
+ ")");
@@ -620,6 +654,15 @@ public class Script extends BaseTransform<ScriptMeta,
ScriptData> implements ITr
}
}
+ /**
+ * The Hop type of output field i, as resolved on the first row. Without a
first row, as when this
+ * class is called on its own, it is looked up.
+ */
+ private int hopType(ScriptMeta.SField field, int i) {
+ int[] fieldTypes = data.fieldTypes;
+ return fieldTypes != null ? fieldTypes[i] : field.getHopType();
+ }
+
/**
* A floating point script result as a BigNumber, converted the way any
Number field is (no
* scientific notation for whole numbers from 1e7 upwards). NaN and
Infinity, which no BigDecimal
diff --git
a/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/ScriptData.java
b/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/ScriptData.java
index cca9d06fa5..91f3587335 100644
---
a/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/ScriptData.java
+++
b/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/ScriptData.java
@@ -38,6 +38,13 @@ public class ScriptData extends BaseTransformData implements
ITransformData {
public IRowMeta outputRowMeta;
public int[] replaceIndex;
+
+ /** The Hop type of each output field, resolved on the first row. */
+ public int[] fieldTypes;
+
+ /** Runs a Groovy row script directly; null for other languages, which use
the engine. */
+ GroovyRowScript groovyRowScript;
+
public boolean hasPreviousTransforms;
/** */
diff --git
a/plugins/transforms/script/src/test/java/org/apache/hop/pipeline/transforms/script/ScriptGroovyDirectRunTest.java
b/plugins/transforms/script/src/test/java/org/apache/hop/pipeline/transforms/script/ScriptGroovyDirectRunTest.java
new file mode 100644
index 0000000000..6ab66ae733
--- /dev/null
+++
b/plugins/transforms/script/src/test/java/org/apache/hop/pipeline/transforms/script/ScriptGroovyDirectRunTest.java
@@ -0,0 +1,562 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.script;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+import groovy.lang.GroovySystem;
+import groovy.lang.MetaClassRegistryChangeEventListener;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import org.apache.hop.core.BlockingRowSet;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaNumber;
+import org.apache.hop.core.row.value.ValueMetaPluginType;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/**
+ * A Groovy row script is run directly, not through the script engine's eval.
Every test here runs
+ * the same transform both ways and wants the same rows, errors and side
effects, next to the values
+ * it expects.
+ */
+class ScriptGroovyDirectRunTest {
+
+ @RegisterExtension
+ static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
+
+ private TransformMockHelper<ScriptMeta, ScriptData> mockHelper;
+
+ @BeforeAll
+ static void initPlugins() throws HopException {
+ ValueMetaPluginType.getInstance().searchPlugins();
+ }
+
+ @BeforeEach
+ void setUp() {
+ mockHelper = new TransformMockHelper<>("Script", ScriptMeta.class,
ScriptData.class);
+ when(mockHelper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+ .thenReturn(mockHelper.iLogChannel);
+ when(mockHelper.pipeline.isRunning()).thenReturn(true);
+ }
+
+ @AfterEach
+ void tearDown() {
+ mockHelper.cleanUp();
+ }
+
+ @Test
+ void outputFieldsAreSetFromInputFields() throws Exception {
+ Outcome outcome =
+ bothWays(
+ meta(
+ "gross = price * qty\nstatus_lower = status.toLowerCase()",
+ field("gross", "Number"),
+ field("status_lower", "String")));
+
+ assertEquals(3, outcome.rows.size());
+ assertEquals(List.of(1L, 2.5d, 4L, "NEW", 10.0d, "new"), outcome.row(0));
+ assertEquals(List.of(3L, 1.5d, 2L, "DONE", 3.0d, "done"), outcome.row(2));
+ }
+
+ @Test
+ void aFunctionOfTheStartScriptCanBeCalled() throws Exception {
+ ScriptMeta meta = meta("doubled = twice(qty)", field("doubled",
"Integer"));
+ meta.getScripts().add(start("def twice(x) { x * 2 }"));
+
+ Outcome outcome = bothWays(meta);
+
+ assertEquals(List.of(8L, 6L, 4L), outcome.column(4));
+ }
+
+ @Test
+ void aVariableOfTheStartScriptCarriesOverFromRowToRow() throws Exception {
+ ScriptMeta meta = meta("counter = counter + qty\ntotal = counter",
field("total", "Integer"));
+ meta.getScripts().add(start("counter = 100"));
+
+ Outcome outcome = bothWays(meta);
+
+ assertEquals(List.of(104L, 107L, 109L), outcome.column(4));
+ }
+
+ @Test
+ void theScriptCanDefineAndCallItsOwnFunction() throws Exception {
+ Outcome outcome =
+ bothWays(meta("def inc(x) { x + 1 }\nnext = inc(id)", field("next",
"Integer")));
+
+ assertEquals(List.of(2L, 3L, 4L), outcome.column(4));
+ }
+
+ /** One script object for all rows would count 1, 2, 3 here. */
+ @Test
+ void aFieldAnnotatedVariableStartsFreshForEveryRow() throws Exception {
+ Outcome outcome =
+ bothWays(
+ meta(
+ "import groovy.transform.Field\n@Field int n = 0\nn++\ncount =
n",
+ field("count", "Integer")));
+
+ assertEquals(List.of(1L, 1L, 1L), outcome.column(4));
+ }
+
+ @Test
+ void theRowVariablesOfTheTransformAreThere() throws Exception {
+ Outcome outcome =
+ bothWays(
+ meta(
+ "nr = rowNumber\n"
+ + "prev = previousRow == null ? -1 : previousRow[0]\n"
+ + "fields = rowMeta.size()\n"
+ + "name = transformName + '/' + (context != null) + '/' +
(row[3] == status)",
+ field("nr", "Integer"),
+ field("prev", "Integer"),
+ field("fields", "Integer"),
+ field("name", "String")));
+
+ assertEquals(List.of(1L, 2L, 3L), outcome.column(4));
+ assertEquals(List.of(-1L, 1L, 2L), outcome.column(5));
+ assertEquals(List.of(4L, 4L, 4L), outcome.column(6));
+ assertEquals("Script/true/true", outcome.rows.get(0)[7]);
+ }
+
+ @Test
+ void aClosureKeptInAVariableWorksOnLaterRows() throws Exception {
+ Outcome outcome =
+ bothWays(
+ meta(
+ "if (!binding.hasVariable('addTen')) { addTen = { it + 10 }
}\nplus = addTen(id)",
+ field("plus", "Integer")));
+
+ assertEquals(List.of(11L, 12L, 13L), outcome.column(4));
+ }
+
+ @Test
+ void theScriptResultCanBeAnOutputField() throws Exception {
+ ScriptMeta.SField result = field("gross", "Number");
+ result.setScriptResult(true);
+
+ Outcome outcome = bothWays(meta("price * qty", result));
+
+ assertEquals(List.of(10.0d, 9.0d, 3.0d), outcome.column(4));
+ }
+
+ @Test
+ void aRowCanBeSkippedWithThePipelineStatus() throws Exception {
+ Outcome outcome =
+ bothWays(
+ meta(
+ "pipeline_status = id == 2 ? SKIP_PIPELINE :
CONTINUE_PIPELINE\nseen = id",
+ field("seen", "Integer")));
+
+ assertEquals(List.of(1L, 3L), outcome.column(4));
+ assertEquals(0, outcome.errors);
+ }
+
+ @Test
+ void aFailingRowStopsTheTransformWithAnError() throws Exception {
+ Outcome outcome = bothWays(meta("ratio = 10 / (id - 2)", field("ratio",
"Integer")));
+
+ // the first row passes, the second divides by zero
+ assertEquals(1, outcome.rows.size());
+ assertEquals(1, outcome.errors);
+ }
+
+ @Test
+ void aCallToAFunctionThatDoesNotExistFails() throws Exception {
+ Outcome outcome = bothWays(meta("x = noSuchFunction(id)", field("x",
"Integer")));
+
+ assertEquals(0, outcome.rows.size());
+ assertEquals(1, outcome.errors);
+ }
+
+ @Test
+ void aScriptThatDoesNotCompileFails() throws Exception {
+ Outcome outcome = bothWays(meta("x = (", field("x", "Integer")));
+
+ assertEquals(0, outcome.rows.size());
+ assertEquals(1, outcome.errors);
+ }
+
+ @Test
+ void theEndScriptCanCallAFunctionOfTheRowScript() throws Exception {
+ ScriptMeta meta = meta("def inc(x) { x + 1 }\nnext = inc(id)",
field("next", "Integer"));
+ meta.getScripts()
+ .add(
+ new ScriptMeta.SScript(
+ ScriptMeta.ScriptType.END_SCRIPT,
+ "end",
+ "transform.setVariable('END_RESULT',
String.valueOf(inc(41)))"));
+
+ Outcome outcome = bothWays(meta);
+
+ assertEquals("42", outcome.endResult);
+ assertEquals(0, outcome.errors);
+ }
+
+ @Test
+ void aFunctionOfTheStartScriptCanCallOneOfTheRowScript() throws Exception {
+ ScriptMeta meta =
+ meta("def timesTen(x) { x * 10 }\nresult = viaStart(id)",
field("result", "Integer"));
+ meta.getScripts().add(start("def viaStart(x) { timesTen(x) + 1 }"));
+
+ Outcome outcome = bothWays(meta);
+
+ assertEquals(List.of(11L, 21L, 31L), outcome.column(4));
+ }
+
+ /** The functions of the row script do not show up as variables. */
+ @Test
+ void theScriptSeesTheSameVariables() throws Exception {
+ Outcome outcome =
+ bothWays(
+ meta(
+ "def inc(x) { x + 1 }\nnames =
binding.variables.keySet().sort().join(',')",
+ field("names", "String")));
+
+ assertEquals(3, outcome.rows.size());
+ }
+
+ /**
+ * Several copies of the transform run the same script at the same time,
each on its own rows:
+ * every copy must get its own results, with a start-script function, a
function of its own, a
+ * variable that carries over and a variable that starts fresh for every row.
+ */
+ @Test
+ void copiesRunningAtTheSameTimeDoNotMixTheirRows() throws Exception {
+ ScriptMeta meta =
+ meta(
+ "import groovy.transform.Field\n"
+ + "@Field int fresh = 0\n"
+ + "def plusOne(x) { x + 1 }\n"
+ + "fresh++\n"
+ + "seen = seen + 1\n"
+ + "result = twice(plusOne(id)) + fresh\n"
+ + "count = seen\n"
+ + "label = status + '-' + qty",
+ field("result", "Integer"),
+ field("count", "Integer"),
+ field("label", "String"));
+ meta.getScripts().add(start("def twice(x) { x * 2 }\nseen = 0"));
+
+ int copies = 8;
+ int rowsPerCopy = 500;
+ stubTransformMeta(meta);
+ for (boolean direct : new boolean[] {false, true}) {
+ ExecutorService pool = Executors.newFixedThreadPool(copies);
+ try {
+ CountDownLatch start = new CountDownLatch(1);
+ List<Future<Outcome>> futures = new ArrayList<>();
+ for (int copy = 0; copy < copies; copy++) {
+ Object[][] input = new Object[rowsPerCopy][];
+ for (int r = 0; r < rowsPerCopy; r++) {
+ // ids and quantities differ per copy
+ input[r] = new Object[] {copy * 100_000L + r, 1.0d, (long) copy,
"C" + copy};
+ }
+ futures.add(
+ pool.submit(
+ () -> {
+ start.await();
+ return run(meta, direct, input);
+ }));
+ }
+ start.countDown();
+
+ for (int copy = 0; copy < copies; copy++) {
+ Outcome outcome = futures.get(copy).get(2, TimeUnit.MINUTES);
+ assertEquals(direct, outcome.direct);
+ assertEquals(0, outcome.errors);
+ assertEquals(rowsPerCopy, outcome.rows.size());
+ for (int r = 0; r < rowsPerCopy; r++) {
+ Object[] row = outcome.rows.get(r);
+ long id = copy * 100_000L + r;
+ assertEquals(id, row[0]);
+ assertEquals((id + 1) * 2 + 1, row[4], "copy " + copy + " row " +
r);
+ assertEquals((long) r + 1, row[5], "copy " + copy + " row " + r);
+ assertEquals("C" + copy + "-" + copy, row[6]);
+ }
+ }
+ } finally {
+ pool.shutdownNow();
+ }
+ }
+ }
+
+ /** Other languages keep going through the script engine. */
+ @Test
+ void aJavaScriptRowScriptIsNotRunDirectly() throws Exception {
+ ScriptMeta meta = meta("var doubled = qty * 2;", field("doubled",
"Integer"));
+ meta.setLanguageName("JavaScript");
+
+ Outcome outcome = run(meta, true);
+
+ assertFalse(outcome.direct);
+ assertEquals(List.of(8L, 6L, 4L), outcome.column(4));
+ }
+
+ /**
+ * A start-script function that fails inside reports its own error, not that
the function is
+ * missing: the script sees the method that is really missing.
+ */
+ @Test
+ void aStartScriptFunctionThatFailsInsideShowsItsOwnError() throws Exception {
+ ScriptMeta meta =
+ meta(
+ "try { missing = bad(id) } catch (MissingMethodException e) {
missing = e.method }",
+ field("missing", "String"));
+ meta.getScripts().add(start("def bad(x) { x.substring('wrong') }"));
+
+ Outcome outcome = bothWays(meta);
+
+ assertEquals(List.of("substring", "substring", "substring"),
outcome.column(4));
+ }
+
+ @Test
+ void anExceptionFromAStartScriptFunctionCanBeCaught() throws Exception {
+ ScriptMeta meta =
+ meta(
+ "try { ratio = String.valueOf(divide(id)) } catch
(ArithmeticException e) { ratio = 'caught' }",
+ field("ratio", "String"));
+ meta.getScripts().add(start("def divide(x) { 10.intdiv(x - 2) }"));
+
+ Outcome outcome = bothWays(meta);
+
+ assertEquals(List.of("-10", "caught", "10"), outcome.column(4));
+ }
+
+ @Test
+ void aStartScriptFunctionThatFailsStopsTheTransform() throws Exception {
+ ScriptMeta meta = meta("x = bad(id)", field("x", "String"));
+ meta.getScripts().add(start("def bad(x) { x.substring('wrong') }"));
+
+ Outcome outcome = bothWays(meta);
+
+ assertEquals(0, outcome.rows.size());
+ assertEquals(1, outcome.errors);
+ }
+
+ /**
+ * A method that is missing inside a function of the row script, called from
a start-script
+ * function, is not taken for either function: the row script function runs
once and the script
+ * sees the method that is missing.
+ */
+ @Test
+ void aMissingMethodInsideAFunctionOfTheRowScriptIsNotRetried() throws
Exception {
+ ScriptMeta meta =
+ meta(
+ "def inner(x) { calls = calls + 1; x.nosuch() }\n"
+ + "try { missing = viaStart(id) } catch
(MissingMethodException e) {"
+ + " missing = e.method + '/' + calls }",
+ field("missing", "String"));
+ meta.getScripts().add(start("calls = 0\ndef viaStart(x) { inner(x) }"));
+
+ Outcome outcome = bothWays(meta);
+
+ assertEquals(List.of("nosuch/1", "nosuch/2", "nosuch/3"),
outcome.column(4));
+ }
+
+ /**
+ * A script that changes the metaclass of its own class keeps finding the
start-script functions,
+ * and finds what it added.
+ */
+ @Test
+ void aScriptThatChangesItsClassMetaClassKeepsTheStartScriptFunctions()
throws Exception {
+ ScriptMeta meta =
+ meta(
+ "if (id == 1) { this.class.metaClass.fmt = { -> 'f' } }\n"
+ + "label = twice(id) + '/' + (id > 1 ? fmt() : '-')",
+ field("label", "String"));
+ meta.getScripts().add(start("def twice(x) { x * 2 }"));
+
+ Outcome outcome = bothWays(meta);
+
+ assertEquals(List.of("2/-", "4/f", "6/f"), outcome.column(4));
+ }
+
+ /**
+ * The direct run does not register a metaclass for the script class: Groovy
would relink the
+ * calls of every Groovy script running in the JVM each time a transform
copy starts.
+ */
+ @Test
+ void theDirectRunLeavesGroovysMetaClassRegistryAlone() throws Exception {
+ ScriptMeta meta = meta("doubled = twice(qty)", field("doubled",
"Integer"));
+ meta.getScripts().add(start("def twice(x) { x * 2 }"));
+ List<Class<?>> updated = new CopyOnWriteArrayList<>();
+ MetaClassRegistryChangeEventListener listener = event ->
updated.add(event.getClassToUpdate());
+
+
GroovySystem.getMetaClassRegistry().addMetaClassRegistryChangeEventListener(listener);
+ Outcome outcome;
+ try {
+ outcome = run(meta, true);
+ } finally {
+
GroovySystem.getMetaClassRegistry().removeMetaClassRegistryChangeEventListener(listener);
+ }
+
+ assertTrue(outcome.direct);
+ assertEquals(List.of(8L, 6L, 4L), outcome.column(4));
+ assertTrue(
+ updated.stream().noneMatch(groovy.lang.Script.class::isAssignableFrom),
+ () -> "a metaclass was registered for " + updated);
+ }
+
+ // --- helpers ---
+
+ private static ScriptMeta meta(String rowScript, ScriptMeta.SField...
fields) {
+ ScriptMeta meta = new ScriptMeta();
+ meta.setLanguageName("Groovy");
+ meta.getScripts()
+ .add(new ScriptMeta.SScript(ScriptMeta.ScriptType.TRANSFORM_SCRIPT,
"script", rowScript));
+ meta.getFields().addAll(Arrays.asList(fields));
+ return meta;
+ }
+
+ private static ScriptMeta.SScript start(String script) {
+ return new ScriptMeta.SScript(ScriptMeta.ScriptType.START_SCRIPT, "start",
script);
+ }
+
+ private static ScriptMeta.SField field(String name, String type) {
+ ScriptMeta.SField field = new ScriptMeta.SField();
+ field.setName(name);
+ field.setRename(name);
+ field.setType(type);
+ field.setLength(-1);
+ field.setPrecision(-1);
+ return field;
+ }
+
+ /** Runs the transform through the engine and directly, and wants the same
outcome. */
+ private Outcome bothWays(ScriptMeta meta) throws Exception {
+ Outcome engine = run(meta, false);
+ Outcome direct = run(meta, true);
+
+ assertFalse(engine.direct, "the engine run must not use the direct
runner");
+ assertEquals(engine.rowTexts(), direct.rowTexts());
+ assertEquals(engine.errors, direct.errors);
+ assertEquals(engine.endResult, direct.endResult);
+ if (engine.errors == 0 || !direct.rows.isEmpty()) {
+ // a script that does not compile never gets a runner
+ assertTrue(direct.direct, "the direct run must use the direct runner");
+ }
+ return direct;
+ }
+
+ private Outcome run(ScriptMeta meta, boolean direct) throws Exception {
+ Object[][] input = {
+ {1L, 2.5d, 4L, "NEW"},
+ {2L, 3.0d, 3L, "OPEN"},
+ {3L, 1.5d, 2L, "DONE"}
+ };
+ stubTransformMeta(meta);
+ return run(meta, direct, input);
+ }
+
+ /** Not from several threads at once: a mock cannot be stubbed while others
use it. */
+ private void stubTransformMeta(ScriptMeta meta) {
+ when(mockHelper.transformMeta.getTransform()).thenReturn(meta);
+ when(mockHelper.transformMeta.getName()).thenReturn("Script");
+ }
+
+ /** Call {@link #stubTransformMeta} first. */
+ private Outcome run(ScriptMeta meta, boolean direct, Object[][] input)
throws Exception {
+ RowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaInteger("id"));
+ rowMeta.addValueMeta(new ValueMetaNumber("price"));
+ rowMeta.addValueMeta(new ValueMetaInteger("qty"));
+ rowMeta.addValueMeta(new ValueMetaString("status"));
+
+ BlockingRowSet inputSet = new BlockingRowSet(input.length + 10);
+ inputSet.setThreadNameFromToCopy("previous", 0, "Script", 0);
+ inputSet.setRowMeta(rowMeta);
+ for (Object[] row : input) {
+ inputSet.putRow(rowMeta, row);
+ }
+ inputSet.setDone();
+ BlockingRowSet outputSet = new BlockingRowSet(input.length + 10);
+ outputSet.setThreadNameFromToCopy("Script", 0, "next", 0);
+
+ ScriptData data = new ScriptData();
+ Script transform =
+ new Script(
+ mockHelper.transformMeta, meta, data, 0, mockHelper.pipelineMeta,
mockHelper.pipeline);
+ transform.directGroovyRowScript = direct;
+ transform.addRowSetToInputRowSets(inputSet);
+ transform.addRowSetToOutputRowSets(outputSet);
+
+ assertTrue(transform.init());
+ data.hasPreviousTransforms = true;
+ for (int i = 0; i < input.length + 1 && transform.processRow(); i++) {
+ // next row
+ }
+
+ Outcome outcome = new Outcome();
+ outcome.direct = data.groovyRowScript != null;
+ outcome.errors = transform.getErrors();
+ outcome.endResult = transform.getVariable("END_RESULT");
+ for (Object[] row = outputSet.getRowWait(10, TimeUnit.MILLISECONDS);
+ row != null;
+ row = outputSet.getRowWait(10, TimeUnit.MILLISECONDS)) {
+ outcome.rows.add(row);
+ }
+ if (outcome.rows.isEmpty()) {
+ assertNull(outputSet.getRowWait(10, TimeUnit.MILLISECONDS));
+ } else {
+ assertNotNull(outcome.rows.get(0));
+ }
+ return outcome;
+ }
+
+ private static final class Outcome {
+ final List<Object[]> rows = new ArrayList<>();
+ long errors;
+ String endResult;
+ boolean direct;
+
+ List<Object> row(int index) {
+ return Arrays.asList(rows.get(index)).subList(0, 6);
+ }
+
+ List<Object> column(int index) {
+ return rows.stream().map(row -> row[index]).toList();
+ }
+
+ List<String> rowTexts() {
+ return rows.stream().map(Arrays::deepToString).toList();
+ }
+ }
+}