This is an automated email from the ASF dual-hosted git repository.

bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git


The following commit(s) were added to refs/heads/main by this push:
     new 37040b2890 Script transform: resolve output field types once, and run 
Groovy scr… (#9442)
37040b2890 is described below

commit 37040b289034bcf2345d4008b9988116484f1ea9
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Sat Nov 14 12:49:40 2026 +0100

    Script transform: resolve output field types once, and run Groovy scr… 
(#9442)
    
    * Script transform: resolve output field types once, and run Groovy scripts 
without re-creating them for every row, fixes #9439
    
    * fix remarks
---
 .../transforms/script/GroovyRowScript.java         | 331 ++++++++++++
 .../hop/pipeline/transforms/script/Script.java     |  47 +-
 .../hop/pipeline/transforms/script/ScriptData.java |   7 +
 .../script/ScriptGroovyDirectRunTest.java          | 562 +++++++++++++++++++++
 4 files changed, 945 insertions(+), 2 deletions(-)

diff --git 
a/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/GroovyRowScript.java
 
b/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/GroovyRowScript.java
new file mode 100644
index 0000000000..7a50b8e53e
--- /dev/null
+++ 
b/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/GroovyRowScript.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.script;
+
+import groovy.lang.Binding;
+import groovy.lang.Closure;
+import groovy.lang.DelegatingMetaClass;
+import groovy.lang.GroovyRuntimeException;
+import groovy.lang.MetaClass;
+import groovy.lang.MissingMethodException;
+import groovy.lang.MissingPropertyException;
+import groovy.lang.Script;
+import groovy.lang.Tuple;
+import java.io.PrintWriter;
+import java.io.Writer;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import javax.script.ScriptContext;
+import javax.script.ScriptEngine;
+import javax.script.ScriptException;
+import org.codehaus.groovy.jsr223.GroovyScriptEngineImpl;
+import org.codehaus.groovy.runtime.InvokerHelper;
+import org.codehaus.groovy.util.ManagedConcurrentValueMap;
+
+/**
+ * Runs the row script of a Groovy Script transform without going through the 
JSR-223 engine for
+ * every row.
+ *
+ * <p>The engine's {@code CompiledScript.eval()} builds, for every call, a new 
script object, a
+ * closure for each of its methods and a metaclass for that one object. The 
new metaclass makes
+ * Groovy link every call in the script again, which is most of the cost of a 
short script. Here the
+ * script class is compiled once and each row gets a new instance that is run 
directly, with the
+ * same metaclass object as the row before.
+ *
+ * <p>What a script can see and do is kept the same as with the engine:
+ *
+ * <ul>
+ *   <li>a new script object per row, so {@code @Field} variables start fresh 
for every row;
+ *   <li>variables are read from and written to the script context, with the 
engine's own rules for
+ *       {@code out} and {@code context};
+ *   <li>a method the script does not have is looked up among the functions of 
the scripts the
+ *       engine evaluated before, such as the start script, and then among the 
closures in the
+ *       context, as the engine does;
+ *   <li>the functions of the row script are known to the engine, for the 
start and end scripts;
+ *   <li>an exception leaves as a {@link ScriptException} around the original 
one.
+ * </ul>
+ */
+final class GroovyRowScript {
+
+  private final Class<? extends Script> scriptClass;
+  private final Constructor<? extends Script> constructor;
+  private final Binding binding;
+  private final ManagedConcurrentValueMap<String, Closure<?>> globalClosures;
+  private final ScriptContext context;
+
+  /** The metaclass of the script objects, see {@link #newScript()}. */
+  private GlobalFunctionsMetaClass metaClass;
+
+  /** The script object of the current row. */
+  private Script lastScript;
+
+  private GroovyRowScript(
+      Class<? extends Script> scriptClass,
+      Constructor<? extends Script> constructor,
+      ScriptContext context,
+      ManagedConcurrentValueMap<String, Closure<?>> globalClosures) {
+    this.scriptClass = scriptClass;
+    this.constructor = constructor;
+    this.binding = new ContextBinding(context);
+    this.globalClosures = globalClosures;
+    this.context = context;
+  }
+
+  /**
+   * @return the row script, or null when this is not Groovy's JSR-223 engine, 
the source is a class
+   *     and not a script, or the engine's functions cannot be reached: the 
caller then keeps using
+   *     the engine
+   */
+  static GroovyRowScript create(ScriptEngine engine, String source, 
ScriptContext context) {
+    if (!(engine instanceof GroovyScriptEngineImpl groovyEngine) || context == 
null) {
+      return null;
+    }
+    // The engine's own class loader: classes that the start script defined 
are visible.
+    Class<?> parsed = groovyEngine.getClassLoader().parseClass(source);
+    if (!Script.class.isAssignableFrom(parsed)) {
+      return null;
+    }
+    Class<? extends Script> scriptClass = parsed.asSubclass(Script.class);
+    ManagedConcurrentValueMap<String, Closure<?>> globalClosures = 
globalClosures(groovyEngine);
+    if (globalClosures == null) {
+      return null;
+    }
+    Constructor<? extends Script> constructor;
+    try {
+      constructor = scriptClass.getConstructor(Binding.class);
+    } catch (NoSuchMethodException e) {
+      return null;
+    }
+    GroovyRowScript rowScript =
+        new GroovyRowScript(scriptClass, constructor, context, globalClosures);
+    rowScript.registerFunctions();
+    return rowScript;
+  }
+
+  /**
+   * The functions of every script the engine evaluated, by name. The engine 
keeps them in a private
+   * map, so this needs reflection.
+   *
+   * @return the map, or null when it cannot be reached
+   */
+  @SuppressWarnings("unchecked")
+  private static ManagedConcurrentValueMap<String, Closure<?>> globalClosures(
+      GroovyScriptEngineImpl engine) {
+    try {
+      Field field = 
GroovyScriptEngineImpl.class.getDeclaredField("globalClosures");
+      field.setAccessible(true);
+      return (ManagedConcurrentValueMap<String, Closure<?>>) field.get(engine);
+    } catch (ReflectiveOperationException | RuntimeException e) {
+      return null;
+    }
+  }
+
+  /** Runs the script for one row and returns its result. */
+  Object run() throws ScriptException {
+    try {
+      Script script = newScript();
+      lastScript = script;
+      return script.run();
+    } catch (Exception e) {
+      throw new ScriptException(e);
+    }
+  }
+
+  /**
+   * A new script object for the row. Like the engine, its metaclass wraps the 
one the object got
+   * from its class. Unlike the engine, the wrapper is the same object from 
row to row as long as
+   * the class keeps its metaclass: Groovy's call sites check the metaclass of 
the object they call,
+   * and stay linked as long as it does not change.
+   *
+   * <p>The wrapper is set on the object, not registered for the class. 
Registering a metaclass
+   * makes Groovy relink the calls of every Groovy script in the JVM, and a 
script that changes the
+   * metaclass of its own class would remove it.
+   */
+  private Script newScript() {
+    Script script;
+    try {
+      script = constructor.newInstance(binding);
+    } catch (ReflectiveOperationException e) {
+      // As InvokerHelper.createScript, which the engine uses, reports it
+      throw new GroovyRuntimeException(
+          "Failed to create Script instance for class: " + scriptClass + ". 
Reason: " + e, e);
+    }
+    MetaClass classMetaClass = script.getMetaClass();
+    if (metaClass == null || metaClass.getAdaptee() != classMetaClass) {
+      metaClass = new GlobalFunctionsMetaClass(classMetaClass, globalClosures, 
context);
+    }
+    script.setMetaClass(metaClass);
+    return script;
+  }
+
+  /**
+   * Tells the engine about the functions of the row script, as it does itself 
for every script it
+   * runs: a function of the start script that calls one of the row script, 
and the end script, find
+   * them there.
+   */
+  private void registerFunctions() {
+    for (Method method : scriptClass.getMethods()) {
+      String name = method.getName();
+      globalClosures.put(name, new CurrentScriptFunction(name));
+    }
+  }
+
+  /** A function of the row script, called on the script object of the current 
row. */
+  private final class CurrentScriptFunction extends Closure<Object> {
+    private static final long serialVersionUID = 1L;
+    private final String name;
+
+    CurrentScriptFunction(String name) {
+      super(GroovyRowScript.class);
+      this.name = name;
+    }
+
+    @Override
+    public Object call(Object... args) {
+      Script script = lastScript;
+      if (script == null) {
+        throw new MissingMethodException(name, scriptClass, args);
+      }
+      return InvokerHelper.invokeMethod(script, name, args);
+    }
+  }
+
+  /** Variable lookup in the script context, as the engine's own binding does 
it. */
+  private static final class ContextBinding extends Binding {
+    private final transient ScriptContext context;
+
+    ContextBinding(ScriptContext context) {
+      super(context.getBindings(ScriptContext.ENGINE_SCOPE));
+      this.context = context;
+    }
+
+    @Override
+    public Object getVariable(String name) {
+      synchronized (context) {
+        int scope = context.getAttributesScope(name);
+        if (scope != -1) {
+          return context.getAttribute(name, scope);
+        }
+        // Script output goes to the context writer when no out variable is 
provided
+        if ("out".equals(name)) {
+          Writer writer = context.getWriter();
+          if (writer != null) {
+            return writer instanceof PrintWriter printWriter
+                ? printWriter
+                : new PrintWriter(writer, true);
+          }
+        }
+        if ("context".equals(name)) {
+          return context;
+        }
+      }
+      throw new MissingPropertyException(name, getClass());
+    }
+
+    @Override
+    public void setVariable(String name, Object value) {
+      synchronized (context) {
+        int scope = context.getAttributesScope(name);
+        if (scope == -1) {
+          scope = ScriptContext.ENGINE_SCOPE;
+        }
+        context.setAttribute(name, value, scope);
+      }
+    }
+  }
+
+  /**
+   * Passes a method the script does not have on to the functions of the 
scripts the engine
+   * evaluated, the start script's for one, and then to the closures in the 
context. This is the
+   * engine's own lookup: what such a function throws reaches the script as it 
is.
+   */
+  private static final class GlobalFunctionsMetaClass extends 
DelegatingMetaClass {
+    private final ManagedConcurrentValueMap<String, Closure<?>> globalClosures;
+    private final ScriptContext context;
+
+    GlobalFunctionsMetaClass(
+        MetaClass delegate,
+        ManagedConcurrentValueMap<String, Closure<?>> globalClosures,
+        ScriptContext context) {
+      super(delegate);
+      this.globalClosures = globalClosures;
+      this.context = context;
+    }
+
+    @Override
+    public Object invokeMethod(Object object, String name, Object args) {
+      if (args == null) {
+        return invokeMethod(object, name, new Object[0]);
+      }
+      if (args instanceof Tuple<?> tuple) {
+        return invokeMethod(object, name, tuple.toArray());
+      }
+      if (args instanceof Object[] array) {
+        return invokeMethod(object, name, array);
+      }
+      return invokeMethod(object, name, new Object[] {args});
+    }
+
+    @Override
+    public Object invokeMethod(Object object, String name, Object[] args) {
+      try {
+        return super.invokeMethod(object, name, args);
+      } catch (MissingMethodException e) {
+        if (!isAboutThisCall(e, object, name)) {
+          throw e;
+        }
+        return callGlobal(name, args);
+      }
+    }
+
+    @Override
+    public Object invokeStaticMethod(Object object, String name, Object[] 
args) {
+      try {
+        return super.invokeStaticMethod(object, name, args);
+      } catch (MissingMethodException e) {
+        if (!isAboutThisCall(e, object, name)) {
+          throw e;
+        }
+        return callGlobal(name, args);
+      }
+    }
+
+    /**
+     * Whether the method that is missing is the one called here, the check 
Groovy's own call sites
+     * make. A method that is missing further down, inside the method that was 
found, must reach the
+     * script as it is: looking it up among the functions would run the method 
that was found again.
+     */
+    private static boolean isAboutThisCall(MissingMethodException e, Object 
object, String name) {
+      Class<?> type = object instanceof Class<?> c ? c : object.getClass();
+      return name.equals(e.getMethod()) && e.getType() == type;
+    }
+
+    private Object callGlobal(String name, Object[] args) {
+      Closure<?> closure = globalClosures.get(name);
+      if (closure != null) {
+        return closure.call(args);
+      }
+      Object value = context.getAttribute(name);
+      if (value instanceof Closure<?> variable) {
+        return variable.call(args);
+      }
+      throw new MissingMethodException(name, GroovyScriptEngineImpl.class, 
args);
+    }
+  }
+}
diff --git 
a/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/Script.java
 
b/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/Script.java
index e0254860e0..79e7d97c0a 100644
--- 
a/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/Script.java
+++ 
b/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/Script.java
@@ -92,6 +92,12 @@ public class Script extends BaseTransform<ScriptMeta, 
ScriptData> implements ITr
 
   private Object[] previousRow = null;
 
+  /**
+   * Run a Groovy row script directly instead of through the engine's eval, 
see {@link
+   * GroovyRowScript}. Tests switch it off to compare both ways.
+   */
+  boolean directGroovyRowScript = true;
+
   /**
    * This is the base transform that forms that basis for all transforms. You 
can derive from this
    * class to implement your own transforms.
@@ -201,6 +207,12 @@ public class Script extends BaseTransform<ScriptMeta, 
ScriptData> implements ITr
         }
       }
 
+      // The type of each output field, looked up once: the lookup searches 
the plugin registry.
+      data.fieldTypes = new int[meta.getFields().size()];
+      for (int i = 0; i < data.fieldTypes.length; i++) {
+        data.fieldTypes[i] = meta.getFields().get(i).getHopType();
+      }
+
       data.context = data.engine.getContext();
       if (data.context == null) {
         data.context = new SimpleScriptContext();
@@ -260,6 +272,7 @@ public class Script extends BaseTransform<ScriptMeta, 
ScriptData> implements ITr
         } else {
           data.compiledScript = null;
         }
+        data.groovyRowScript = createGroovyRowScript();
       } catch (Exception e) {
         throw new HopValueException(
             BaseMessages.getString(PKG, "Script.Log.CouldNotCompileScript"), 
e);
@@ -420,7 +433,28 @@ public class Script extends BaseTransform<ScriptMeta, 
ScriptData> implements ITr
     }
   }
 
+  /**
+   * @return the direct runner for a Groovy row script, or null to evaluate 
through the engine
+   */
+  private GroovyRowScript createGroovyRowScript() {
+    if (!directGroovyRowScript || data.compiledScript == null) {
+      return null;
+    }
+    try {
+      return GroovyRowScript.create(data.engine, strTransformScript, 
data.context);
+    } catch (Exception | LinkageError e) {
+      // The engine compiled the script, so it can run it: fall back to that
+      if (isDetailed()) {
+        logDetailed("The Groovy script is run through the script engine: " + 
e);
+      }
+      return null;
+    }
+  }
+
   protected Object evalScript() throws ScriptException {
+    if (data.groovyRowScript != null) {
+      return data.groovyRowScript.run();
+    }
     if (data.compiledScript != null) {
       try {
         return data.compiledScript.eval(data.context);
@@ -449,7 +483,7 @@ public class Script extends BaseTransform<ScriptMeta, 
ScriptData> implements ITr
       try {
         if (result != null) {
           String classType = result.getClass().getName();
-          switch (field.getHopType()) {
+          switch (hopType(field, i)) {
             case IValueMeta.TYPE_NUMBER:
               if 
(classType.equalsIgnoreCase(CONST_ORG_MOZILLA_JAVASCRIPT_UNDEFINED)) {
                 return null;
@@ -603,7 +637,7 @@ public class Script extends BaseTransform<ScriptMeta, 
ScriptData> implements ITr
               {
                 throw new HopRuntimeException(
                     "JavaScript conversion not implemented for type "
-                        + field.getHopType()
+                        + hopType(field, i)
                         + " ("
                         + field.getType()
                         + ")");
@@ -620,6 +654,15 @@ public class Script extends BaseTransform<ScriptMeta, 
ScriptData> implements ITr
     }
   }
 
+  /**
+   * The Hop type of output field i, as resolved on the first row. Without a 
first row, as when this
+   * class is called on its own, it is looked up.
+   */
+  private int hopType(ScriptMeta.SField field, int i) {
+    int[] fieldTypes = data.fieldTypes;
+    return fieldTypes != null ? fieldTypes[i] : field.getHopType();
+  }
+
   /**
    * A floating point script result as a BigNumber, converted the way any 
Number field is (no
    * scientific notation for whole numbers from 1e7 upwards). NaN and 
Infinity, which no BigDecimal
diff --git 
a/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/ScriptData.java
 
b/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/ScriptData.java
index cca9d06fa5..91f3587335 100644
--- 
a/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/ScriptData.java
+++ 
b/plugins/transforms/script/src/main/java/org/apache/hop/pipeline/transforms/script/ScriptData.java
@@ -38,6 +38,13 @@ public class ScriptData extends BaseTransformData implements 
ITransformData {
 
   public IRowMeta outputRowMeta;
   public int[] replaceIndex;
+
+  /** The Hop type of each output field, resolved on the first row. */
+  public int[] fieldTypes;
+
+  /** Runs a Groovy row script directly; null for other languages, which use 
the engine. */
+  GroovyRowScript groovyRowScript;
+
   public boolean hasPreviousTransforms;
 
   /** */
diff --git 
a/plugins/transforms/script/src/test/java/org/apache/hop/pipeline/transforms/script/ScriptGroovyDirectRunTest.java
 
b/plugins/transforms/script/src/test/java/org/apache/hop/pipeline/transforms/script/ScriptGroovyDirectRunTest.java
new file mode 100644
index 0000000000..6ab66ae733
--- /dev/null
+++ 
b/plugins/transforms/script/src/test/java/org/apache/hop/pipeline/transforms/script/ScriptGroovyDirectRunTest.java
@@ -0,0 +1,562 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.script;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+import groovy.lang.GroovySystem;
+import groovy.lang.MetaClassRegistryChangeEventListener;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import org.apache.hop.core.BlockingRowSet;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaNumber;
+import org.apache.hop.core.row.value.ValueMetaPluginType;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/**
+ * A Groovy row script is run directly, not through the script engine's eval. 
Every test here runs
+ * the same transform both ways and wants the same rows, errors and side 
effects, next to the values
+ * it expects.
+ */
+class ScriptGroovyDirectRunTest {
+
+  @RegisterExtension
+  static RestoreHopEngineEnvironmentExtension env = new 
RestoreHopEngineEnvironmentExtension();
+
+  private TransformMockHelper<ScriptMeta, ScriptData> mockHelper;
+
+  @BeforeAll
+  static void initPlugins() throws HopException {
+    ValueMetaPluginType.getInstance().searchPlugins();
+  }
+
+  @BeforeEach
+  void setUp() {
+    mockHelper = new TransformMockHelper<>("Script", ScriptMeta.class, 
ScriptData.class);
+    when(mockHelper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+        .thenReturn(mockHelper.iLogChannel);
+    when(mockHelper.pipeline.isRunning()).thenReturn(true);
+  }
+
+  @AfterEach
+  void tearDown() {
+    mockHelper.cleanUp();
+  }
+
+  @Test
+  void outputFieldsAreSetFromInputFields() throws Exception {
+    Outcome outcome =
+        bothWays(
+            meta(
+                "gross = price * qty\nstatus_lower = status.toLowerCase()",
+                field("gross", "Number"),
+                field("status_lower", "String")));
+
+    assertEquals(3, outcome.rows.size());
+    assertEquals(List.of(1L, 2.5d, 4L, "NEW", 10.0d, "new"), outcome.row(0));
+    assertEquals(List.of(3L, 1.5d, 2L, "DONE", 3.0d, "done"), outcome.row(2));
+  }
+
+  @Test
+  void aFunctionOfTheStartScriptCanBeCalled() throws Exception {
+    ScriptMeta meta = meta("doubled = twice(qty)", field("doubled", 
"Integer"));
+    meta.getScripts().add(start("def twice(x) { x * 2 }"));
+
+    Outcome outcome = bothWays(meta);
+
+    assertEquals(List.of(8L, 6L, 4L), outcome.column(4));
+  }
+
+  @Test
+  void aVariableOfTheStartScriptCarriesOverFromRowToRow() throws Exception {
+    ScriptMeta meta = meta("counter = counter + qty\ntotal = counter", 
field("total", "Integer"));
+    meta.getScripts().add(start("counter = 100"));
+
+    Outcome outcome = bothWays(meta);
+
+    assertEquals(List.of(104L, 107L, 109L), outcome.column(4));
+  }
+
+  @Test
+  void theScriptCanDefineAndCallItsOwnFunction() throws Exception {
+    Outcome outcome =
+        bothWays(meta("def inc(x) { x + 1 }\nnext = inc(id)", field("next", 
"Integer")));
+
+    assertEquals(List.of(2L, 3L, 4L), outcome.column(4));
+  }
+
+  /** One script object for all rows would count 1, 2, 3 here. */
+  @Test
+  void aFieldAnnotatedVariableStartsFreshForEveryRow() throws Exception {
+    Outcome outcome =
+        bothWays(
+            meta(
+                "import groovy.transform.Field\n@Field int n = 0\nn++\ncount = 
n",
+                field("count", "Integer")));
+
+    assertEquals(List.of(1L, 1L, 1L), outcome.column(4));
+  }
+
+  @Test
+  void theRowVariablesOfTheTransformAreThere() throws Exception {
+    Outcome outcome =
+        bothWays(
+            meta(
+                "nr = rowNumber\n"
+                    + "prev = previousRow == null ? -1 : previousRow[0]\n"
+                    + "fields = rowMeta.size()\n"
+                    + "name = transformName + '/' + (context != null) + '/' + 
(row[3] == status)",
+                field("nr", "Integer"),
+                field("prev", "Integer"),
+                field("fields", "Integer"),
+                field("name", "String")));
+
+    assertEquals(List.of(1L, 2L, 3L), outcome.column(4));
+    assertEquals(List.of(-1L, 1L, 2L), outcome.column(5));
+    assertEquals(List.of(4L, 4L, 4L), outcome.column(6));
+    assertEquals("Script/true/true", outcome.rows.get(0)[7]);
+  }
+
+  @Test
+  void aClosureKeptInAVariableWorksOnLaterRows() throws Exception {
+    Outcome outcome =
+        bothWays(
+            meta(
+                "if (!binding.hasVariable('addTen')) { addTen = { it + 10 } 
}\nplus = addTen(id)",
+                field("plus", "Integer")));
+
+    assertEquals(List.of(11L, 12L, 13L), outcome.column(4));
+  }
+
+  @Test
+  void theScriptResultCanBeAnOutputField() throws Exception {
+    ScriptMeta.SField result = field("gross", "Number");
+    result.setScriptResult(true);
+
+    Outcome outcome = bothWays(meta("price * qty", result));
+
+    assertEquals(List.of(10.0d, 9.0d, 3.0d), outcome.column(4));
+  }
+
+  @Test
+  void aRowCanBeSkippedWithThePipelineStatus() throws Exception {
+    Outcome outcome =
+        bothWays(
+            meta(
+                "pipeline_status = id == 2 ? SKIP_PIPELINE : 
CONTINUE_PIPELINE\nseen = id",
+                field("seen", "Integer")));
+
+    assertEquals(List.of(1L, 3L), outcome.column(4));
+    assertEquals(0, outcome.errors);
+  }
+
+  @Test
+  void aFailingRowStopsTheTransformWithAnError() throws Exception {
+    Outcome outcome = bothWays(meta("ratio = 10 / (id - 2)", field("ratio", 
"Integer")));
+
+    // the first row passes, the second divides by zero
+    assertEquals(1, outcome.rows.size());
+    assertEquals(1, outcome.errors);
+  }
+
+  @Test
+  void aCallToAFunctionThatDoesNotExistFails() throws Exception {
+    Outcome outcome = bothWays(meta("x = noSuchFunction(id)", field("x", 
"Integer")));
+
+    assertEquals(0, outcome.rows.size());
+    assertEquals(1, outcome.errors);
+  }
+
+  @Test
+  void aScriptThatDoesNotCompileFails() throws Exception {
+    Outcome outcome = bothWays(meta("x = (", field("x", "Integer")));
+
+    assertEquals(0, outcome.rows.size());
+    assertEquals(1, outcome.errors);
+  }
+
+  @Test
+  void theEndScriptCanCallAFunctionOfTheRowScript() throws Exception {
+    ScriptMeta meta = meta("def inc(x) { x + 1 }\nnext = inc(id)", 
field("next", "Integer"));
+    meta.getScripts()
+        .add(
+            new ScriptMeta.SScript(
+                ScriptMeta.ScriptType.END_SCRIPT,
+                "end",
+                "transform.setVariable('END_RESULT', 
String.valueOf(inc(41)))"));
+
+    Outcome outcome = bothWays(meta);
+
+    assertEquals("42", outcome.endResult);
+    assertEquals(0, outcome.errors);
+  }
+
+  @Test
+  void aFunctionOfTheStartScriptCanCallOneOfTheRowScript() throws Exception {
+    ScriptMeta meta =
+        meta("def timesTen(x) { x * 10 }\nresult = viaStart(id)", 
field("result", "Integer"));
+    meta.getScripts().add(start("def viaStart(x) { timesTen(x) + 1 }"));
+
+    Outcome outcome = bothWays(meta);
+
+    assertEquals(List.of(11L, 21L, 31L), outcome.column(4));
+  }
+
+  /** The functions of the row script do not show up as variables. */
+  @Test
+  void theScriptSeesTheSameVariables() throws Exception {
+    Outcome outcome =
+        bothWays(
+            meta(
+                "def inc(x) { x + 1 }\nnames = 
binding.variables.keySet().sort().join(',')",
+                field("names", "String")));
+
+    assertEquals(3, outcome.rows.size());
+  }
+
+  /**
+   * Several copies of the transform run the same script at the same time, 
each on its own rows:
+   * every copy must get its own results, with a start-script function, a 
function of its own, a
+   * variable that carries over and a variable that starts fresh for every row.
+   */
+  @Test
+  void copiesRunningAtTheSameTimeDoNotMixTheirRows() throws Exception {
+    ScriptMeta meta =
+        meta(
+            "import groovy.transform.Field\n"
+                + "@Field int fresh = 0\n"
+                + "def plusOne(x) { x + 1 }\n"
+                + "fresh++\n"
+                + "seen = seen + 1\n"
+                + "result = twice(plusOne(id)) + fresh\n"
+                + "count = seen\n"
+                + "label = status + '-' + qty",
+            field("result", "Integer"),
+            field("count", "Integer"),
+            field("label", "String"));
+    meta.getScripts().add(start("def twice(x) { x * 2 }\nseen = 0"));
+
+    int copies = 8;
+    int rowsPerCopy = 500;
+    stubTransformMeta(meta);
+    for (boolean direct : new boolean[] {false, true}) {
+      ExecutorService pool = Executors.newFixedThreadPool(copies);
+      try {
+        CountDownLatch start = new CountDownLatch(1);
+        List<Future<Outcome>> futures = new ArrayList<>();
+        for (int copy = 0; copy < copies; copy++) {
+          Object[][] input = new Object[rowsPerCopy][];
+          for (int r = 0; r < rowsPerCopy; r++) {
+            // ids and quantities differ per copy
+            input[r] = new Object[] {copy * 100_000L + r, 1.0d, (long) copy, 
"C" + copy};
+          }
+          futures.add(
+              pool.submit(
+                  () -> {
+                    start.await();
+                    return run(meta, direct, input);
+                  }));
+        }
+        start.countDown();
+
+        for (int copy = 0; copy < copies; copy++) {
+          Outcome outcome = futures.get(copy).get(2, TimeUnit.MINUTES);
+          assertEquals(direct, outcome.direct);
+          assertEquals(0, outcome.errors);
+          assertEquals(rowsPerCopy, outcome.rows.size());
+          for (int r = 0; r < rowsPerCopy; r++) {
+            Object[] row = outcome.rows.get(r);
+            long id = copy * 100_000L + r;
+            assertEquals(id, row[0]);
+            assertEquals((id + 1) * 2 + 1, row[4], "copy " + copy + " row " + 
r);
+            assertEquals((long) r + 1, row[5], "copy " + copy + " row " + r);
+            assertEquals("C" + copy + "-" + copy, row[6]);
+          }
+        }
+      } finally {
+        pool.shutdownNow();
+      }
+    }
+  }
+
+  /** Other languages keep going through the script engine. */
+  @Test
+  void aJavaScriptRowScriptIsNotRunDirectly() throws Exception {
+    ScriptMeta meta = meta("var doubled = qty * 2;", field("doubled", 
"Integer"));
+    meta.setLanguageName("JavaScript");
+
+    Outcome outcome = run(meta, true);
+
+    assertFalse(outcome.direct);
+    assertEquals(List.of(8L, 6L, 4L), outcome.column(4));
+  }
+
+  /**
+   * A start-script function that fails inside reports its own error, not that 
the function is
+   * missing: the script sees the method that is really missing.
+   */
+  @Test
+  void aStartScriptFunctionThatFailsInsideShowsItsOwnError() throws Exception {
+    ScriptMeta meta =
+        meta(
+            "try { missing = bad(id) } catch (MissingMethodException e) { 
missing = e.method }",
+            field("missing", "String"));
+    meta.getScripts().add(start("def bad(x) { x.substring('wrong') }"));
+
+    Outcome outcome = bothWays(meta);
+
+    assertEquals(List.of("substring", "substring", "substring"), 
outcome.column(4));
+  }
+
+  @Test
+  void anExceptionFromAStartScriptFunctionCanBeCaught() throws Exception {
+    ScriptMeta meta =
+        meta(
+            "try { ratio = String.valueOf(divide(id)) } catch 
(ArithmeticException e) { ratio = 'caught' }",
+            field("ratio", "String"));
+    meta.getScripts().add(start("def divide(x) { 10.intdiv(x - 2) }"));
+
+    Outcome outcome = bothWays(meta);
+
+    assertEquals(List.of("-10", "caught", "10"), outcome.column(4));
+  }
+
+  @Test
+  void aStartScriptFunctionThatFailsStopsTheTransform() throws Exception {
+    ScriptMeta meta = meta("x = bad(id)", field("x", "String"));
+    meta.getScripts().add(start("def bad(x) { x.substring('wrong') }"));
+
+    Outcome outcome = bothWays(meta);
+
+    assertEquals(0, outcome.rows.size());
+    assertEquals(1, outcome.errors);
+  }
+
+  /**
+   * A method that is missing inside a function of the row script, called from 
a start-script
+   * function, is not taken for either function: the row script function runs 
once and the script
+   * sees the method that is missing.
+   */
+  @Test
+  void aMissingMethodInsideAFunctionOfTheRowScriptIsNotRetried() throws 
Exception {
+    ScriptMeta meta =
+        meta(
+            "def inner(x) { calls = calls + 1; x.nosuch() }\n"
+                + "try { missing = viaStart(id) } catch 
(MissingMethodException e) {"
+                + " missing = e.method + '/' + calls }",
+            field("missing", "String"));
+    meta.getScripts().add(start("calls = 0\ndef viaStart(x) { inner(x) }"));
+
+    Outcome outcome = bothWays(meta);
+
+    assertEquals(List.of("nosuch/1", "nosuch/2", "nosuch/3"), 
outcome.column(4));
+  }
+
+  /**
+   * A script that changes the metaclass of its own class keeps finding the 
start-script functions,
+   * and finds what it added.
+   */
+  @Test
+  void aScriptThatChangesItsClassMetaClassKeepsTheStartScriptFunctions() 
throws Exception {
+    ScriptMeta meta =
+        meta(
+            "if (id == 1) { this.class.metaClass.fmt = { -> 'f' } }\n"
+                + "label = twice(id) + '/' + (id > 1 ? fmt() : '-')",
+            field("label", "String"));
+    meta.getScripts().add(start("def twice(x) { x * 2 }"));
+
+    Outcome outcome = bothWays(meta);
+
+    assertEquals(List.of("2/-", "4/f", "6/f"), outcome.column(4));
+  }
+
+  /**
+   * The direct run does not register a metaclass for the script class: Groovy 
would relink the
+   * calls of every Groovy script running in the JVM each time a transform 
copy starts.
+   */
+  @Test
+  void theDirectRunLeavesGroovysMetaClassRegistryAlone() throws Exception {
+    ScriptMeta meta = meta("doubled = twice(qty)", field("doubled", 
"Integer"));
+    meta.getScripts().add(start("def twice(x) { x * 2 }"));
+    List<Class<?>> updated = new CopyOnWriteArrayList<>();
+    MetaClassRegistryChangeEventListener listener = event -> 
updated.add(event.getClassToUpdate());
+
+    
GroovySystem.getMetaClassRegistry().addMetaClassRegistryChangeEventListener(listener);
+    Outcome outcome;
+    try {
+      outcome = run(meta, true);
+    } finally {
+      
GroovySystem.getMetaClassRegistry().removeMetaClassRegistryChangeEventListener(listener);
+    }
+
+    assertTrue(outcome.direct);
+    assertEquals(List.of(8L, 6L, 4L), outcome.column(4));
+    assertTrue(
+        updated.stream().noneMatch(groovy.lang.Script.class::isAssignableFrom),
+        () -> "a metaclass was registered for " + updated);
+  }
+
+  // --- helpers ---
+
+  private static ScriptMeta meta(String rowScript, ScriptMeta.SField... 
fields) {
+    ScriptMeta meta = new ScriptMeta();
+    meta.setLanguageName("Groovy");
+    meta.getScripts()
+        .add(new ScriptMeta.SScript(ScriptMeta.ScriptType.TRANSFORM_SCRIPT, 
"script", rowScript));
+    meta.getFields().addAll(Arrays.asList(fields));
+    return meta;
+  }
+
+  private static ScriptMeta.SScript start(String script) {
+    return new ScriptMeta.SScript(ScriptMeta.ScriptType.START_SCRIPT, "start", 
script);
+  }
+
+  private static ScriptMeta.SField field(String name, String type) {
+    ScriptMeta.SField field = new ScriptMeta.SField();
+    field.setName(name);
+    field.setRename(name);
+    field.setType(type);
+    field.setLength(-1);
+    field.setPrecision(-1);
+    return field;
+  }
+
+  /** Runs the transform through the engine and directly, and wants the same 
outcome. */
+  private Outcome bothWays(ScriptMeta meta) throws Exception {
+    Outcome engine = run(meta, false);
+    Outcome direct = run(meta, true);
+
+    assertFalse(engine.direct, "the engine run must not use the direct 
runner");
+    assertEquals(engine.rowTexts(), direct.rowTexts());
+    assertEquals(engine.errors, direct.errors);
+    assertEquals(engine.endResult, direct.endResult);
+    if (engine.errors == 0 || !direct.rows.isEmpty()) {
+      // a script that does not compile never gets a runner
+      assertTrue(direct.direct, "the direct run must use the direct runner");
+    }
+    return direct;
+  }
+
+  private Outcome run(ScriptMeta meta, boolean direct) throws Exception {
+    Object[][] input = {
+      {1L, 2.5d, 4L, "NEW"},
+      {2L, 3.0d, 3L, "OPEN"},
+      {3L, 1.5d, 2L, "DONE"}
+    };
+    stubTransformMeta(meta);
+    return run(meta, direct, input);
+  }
+
+  /** Not from several threads at once: a mock cannot be stubbed while others 
use it. */
+  private void stubTransformMeta(ScriptMeta meta) {
+    when(mockHelper.transformMeta.getTransform()).thenReturn(meta);
+    when(mockHelper.transformMeta.getName()).thenReturn("Script");
+  }
+
+  /** Call {@link #stubTransformMeta} first. */
+  private Outcome run(ScriptMeta meta, boolean direct, Object[][] input) 
throws Exception {
+    RowMeta rowMeta = new RowMeta();
+    rowMeta.addValueMeta(new ValueMetaInteger("id"));
+    rowMeta.addValueMeta(new ValueMetaNumber("price"));
+    rowMeta.addValueMeta(new ValueMetaInteger("qty"));
+    rowMeta.addValueMeta(new ValueMetaString("status"));
+
+    BlockingRowSet inputSet = new BlockingRowSet(input.length + 10);
+    inputSet.setThreadNameFromToCopy("previous", 0, "Script", 0);
+    inputSet.setRowMeta(rowMeta);
+    for (Object[] row : input) {
+      inputSet.putRow(rowMeta, row);
+    }
+    inputSet.setDone();
+    BlockingRowSet outputSet = new BlockingRowSet(input.length + 10);
+    outputSet.setThreadNameFromToCopy("Script", 0, "next", 0);
+
+    ScriptData data = new ScriptData();
+    Script transform =
+        new Script(
+            mockHelper.transformMeta, meta, data, 0, mockHelper.pipelineMeta, 
mockHelper.pipeline);
+    transform.directGroovyRowScript = direct;
+    transform.addRowSetToInputRowSets(inputSet);
+    transform.addRowSetToOutputRowSets(outputSet);
+
+    assertTrue(transform.init());
+    data.hasPreviousTransforms = true;
+    for (int i = 0; i < input.length + 1 && transform.processRow(); i++) {
+      // next row
+    }
+
+    Outcome outcome = new Outcome();
+    outcome.direct = data.groovyRowScript != null;
+    outcome.errors = transform.getErrors();
+    outcome.endResult = transform.getVariable("END_RESULT");
+    for (Object[] row = outputSet.getRowWait(10, TimeUnit.MILLISECONDS);
+        row != null;
+        row = outputSet.getRowWait(10, TimeUnit.MILLISECONDS)) {
+      outcome.rows.add(row);
+    }
+    if (outcome.rows.isEmpty()) {
+      assertNull(outputSet.getRowWait(10, TimeUnit.MILLISECONDS));
+    } else {
+      assertNotNull(outcome.rows.get(0));
+    }
+    return outcome;
+  }
+
+  private static final class Outcome {
+    final List<Object[]> rows = new ArrayList<>();
+    long errors;
+    String endResult;
+    boolean direct;
+
+    List<Object> row(int index) {
+      return Arrays.asList(rows.get(index)).subList(0, 6);
+    }
+
+    List<Object> column(int index) {
+      return rows.stream().map(row -> row[index]).toList();
+    }
+
+    List<String> rowTexts() {
+      return rows.stream().map(Arrays::deepToString).toList();
+    }
+  }
+}

Reply via email to