bamaer commented on code in PR #9598:
URL: https://github.com/apache/hop/pull/9598#discussion_r4643545687


##########
engine/src/main/java/org/apache/hop/pipeline/TransformPrepareOnce.java:
##########
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.pipeline.transform.BaseTransform;
+import org.apache.hop.pipeline.transform.ITransform;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transform.TransformMetaDataCombi;
+
+/**
+ * The once-per-run preparation of transforms (truncating a table, creating or 
changing it, ...)
+ * that would otherwise run in every copy, Spark partition or Beam bundle, 
racing each other.
+ *
+ * <p>The local engine calls {@link #prepare(Pipeline)} after every copy is 
initialized. The native
+ * Spark and Beam engines call {@link #prepareOnDriver} before the job starts 
and hand the names of
+ * the prepared transforms to their partitions in {@link
+ * Const#INTERNAL_VARIABLE_PIPELINE_PREPARED_ONCE_TRANSFORMS}; the 
per-partition pipelines then only
+ * mark those transforms as prepared.
+ */
+public final class TransformPrepareOnce {
+
+  private TransformPrepareOnce() {}
+
+  /**
+   * Prepare the transforms of an initialized pipeline once each, on copy 0, 
and flag every copy. A
+   * pipeline that doesn't prepare itself (a per-partition pipeline of a 
distributed engine) only
+   * flags the transforms its engine prepared. A transactional pipeline (a 
connection group) leaves
+   * the work to the transforms' first row, inside the transaction.
+   */
+  public static void prepare(Pipeline pipeline) throws HopException {
+    List<TransformMetaDataCombi> combis = pipeline.getTransforms();
+    if (combis == null) {
+      return;
+    }
+    if (!pipeline.isPreparingTransformsOnce()) {
+      Set<String> prepared =
+          preparedOnDriver(
+              
pipeline.getVariable(Const.INTERNAL_VARIABLE_PIPELINE_PREPARED_ONCE_TRANSFORMS));
+      for (TransformMetaDataCombi combi : combis) {
+        if (prepared.contains(combi.transformName)) {
+          combi.transform.setPreparedOnce(true);
+        }
+      }
+      return;
+    }
+    if (pipeline.getExtensionDataMap().get(Const.CONNECTION_GROUP) != null) {
+      return;
+    }
+    Map<String, List<TransformMetaDataCombi>> copies = new LinkedHashMap<>();
+    for (TransformMetaDataCombi combi : combis) {
+      copies.computeIfAbsent(combi.transformName, n -> new 
ArrayList<>()).add(combi);
+    }
+    for (List<TransformMetaDataCombi> transformCopies : copies.values()) {
+      TransformMetaDataCombi first = transformCopies.get(0);
+      for (TransformMetaDataCombi combi : transformCopies) {
+        if (combi.copy == 0) {
+          first = combi;
+        }
+      }
+      IVariables variables = first.transform instanceof IVariables v ? v : 
pipeline;
+      if (!first.meta.needsPrepareOnce(variables)) {
+        continue;
+      }
+      prepare(first.transform, pipeline.getPipelineMeta(), 
first.transformMeta, variables);
+      for (TransformMetaDataCombi combi : transformCopies) {
+        combi.transform.setPreparedOnce(true);
+      }
+    }
+  }
+
+  /**
+   * On the driver of a distributed engine: prepare every transform of {@code 
transforms} that needs
+   * it, each in a pipeline of its own that only initializes that transform.
+   *
+   * @return the names of the prepared transforms, to hand to the partitions 
with {@link
+   *     #preparedOnDriverValue(Collection)}
+   */
+  public static List<String> prepareOnDriver(
+      PipelineMeta pipelineMeta,
+      Collection<TransformMeta> transforms,
+      IVariables variables,
+      IHopMetadataProvider metadataProvider,
+      ILoggingObject parent)
+      throws HopException {
+    List<String> prepared = new ArrayList<>();
+    for (TransformMeta transformMeta : transforms) {
+      if (transformMeta.getTransform() == null
+          || !transformMeta.getTransform().needsPrepareOnce(variables)) {
+        continue;
+      }
+      TransformMeta copy = (TransformMeta) transformMeta.clone();
+      copy.setCopiesString("1");
+      PipelineMeta single = new PipelineMeta();
+      single.setName(pipelineMeta.getName());
+      single.setFilename(pipelineMeta.getFilename());
+      single.setMetadataProvider(metadataProvider);
+      single.addTransform(copy);
+      LocalPipelineEngine pipeline = new LocalPipelineEngine(single, 
variables, parent);
+      pipeline.setMetadataProvider(metadataProvider);
+      pipeline.setPreparingTransformsOnce(false);
+      try {
+        pipeline.prepareExecution();
+        ITransform transform = pipeline.getTransform(copy.getName(), 0);
+        prepare(transform, pipelineMeta, transformMeta, variables);
+      } finally {
+        pipeline.disposeInitializedTransforms();
+      }
+      prepared.add(transformMeta.getName());
+    }
+    return prepared;
+  }
+
+  /**
+   * {@link #prepareOnDriver(PipelineMeta, Collection, IVariables, 
IHopMetadataProvider,
+   * ILoggingObject)} for the transforms that run: the ones an enabled hop 
connects (a lone
+   * transform in a pipeline without hops runs too).
+   */
+  public static List<String> prepareOnDriver(
+      PipelineMeta pipelineMeta,
+      IVariables variables,
+      IHopMetadataProvider metadataProvider,
+      ILoggingObject parent)
+      throws HopException {
+    Set<TransformMeta> active = new java.util.LinkedHashSet<>();
+    for (PipelineHopMeta hop : pipelineMeta.getPipelineHops()) {
+      if (hop != null && hop.isEnabled()) {
+        if (hop.getFromTransform() != null) {
+          active.add(hop.getFromTransform());
+        }
+        if (hop.getToTransform() != null) {
+          active.add(hop.getToTransform());
+        }
+      }
+    }
+    if (active.isEmpty() && pipelineMeta.nrTransforms() == 1) {
+      active.add(pipelineMeta.getTransform(0));
+    }
+    return prepareOnDriver(pipelineMeta, active, variables, metadataProvider, 
parent);
+  }
+
+  /** The value of {@link 
Const#INTERNAL_VARIABLE_PIPELINE_PREPARED_ONCE_TRANSFORMS}. */
+  public static String preparedOnDriverValue(Collection<String> 
transformNames) {
+    return String.join("\n", transformNames);
+  }
+
+  static Set<String> preparedOnDriver(String value) {
+    Set<String> names = new HashSet<>();
+    if (StringUtils.isNotEmpty(value)) {
+      for (String name : value.split("\n")) {
+        if (!name.isEmpty()) {
+          names.add(name);
+        }
+      }
+    }
+    return names;
+  }
+
+  private static void prepare(
+      ITransform transform,
+      PipelineMeta pipelineMeta,
+      TransformMeta transformMeta,
+      IVariables variables)
+      throws HopException {
+    if (transform instanceof BaseTransform<?, ?> baseTransform
+        && baseTransform.getInputRowMeta() == null) {
+      // The rows haven't arrived yet: the layout the previous transforms 
declare, as the SQL a
+      // transform dialog shows is built from
+      
baseTransform.setInputRowMeta(pipelineMeta.getPrevTransformFields(variables, 
transformMeta));

Review Comment:
   **Blocker.** This row meta stays on copy 0 for the whole run. 
`BaseTransform.obtainInputRowMeta()` only takes the row set's meta when 
`inputRowMeta` is null (or when there are several previous transforms). So copy 
0 reads every row with the declared layout, not the real one, and writes wrong 
values or fails whenever the two differ.
   
   Example: Get rows from result declares its fields only for design time and 
passes the result rows with their own layout. Get rows from result → Table 
output with "Truncate table" then binds values by the declared positions.
   
   A scratch test on the PR head shows it: an Injector declares `id, name`, the 
rows arrive as `name, id, extra`. In `processRow()` the transform sees `[id, 
name]`, and reading `id` throws a `HopValueException`.
   
   Suggested fix: set it back to null after `prepareOnce()`:
   
   ```java
       BaseTransform<?, ?> preset = null;
       if (transform instanceof BaseTransform<?, ?> baseTransform
           && baseTransform.getInputRowMeta() == null) {
         
baseTransform.setInputRowMeta(pipelineMeta.getPrevTransformFields(variables, 
transformMeta));
         preset = baseTransform;
       }
       try {
         transform.prepareOnce();
       } catch (HopException e) {
         throw new HopException(
             "Unable to prepare transform '" + transformMeta.getName() + "' for 
this run", e);
       } finally {
         if (preset != null) {
           preset.setInputRowMeta(null);
         }
       }
   ```
   
   With this change the same test sees `[name, id, extra]`, and your tests 
still pass. `integration-tests/database` already covers several copies with 
truncate (`0020-table-output-insert-multiple-copies.hpl`). A pipeline next to 
it would cover this case: Get rows from result, with its fields declared in a 
different order than the result rows → Table output, 4 copies, "Truncate 
table". It should check that every column holds the right values.



##########
engine/src/main/java/org/apache/hop/pipeline/TransformPrepareOnce.java:
##########
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.pipeline.transform.BaseTransform;
+import org.apache.hop.pipeline.transform.ITransform;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transform.TransformMetaDataCombi;
+
+/**
+ * The once-per-run preparation of transforms (truncating a table, creating or 
changing it, ...)
+ * that would otherwise run in every copy, Spark partition or Beam bundle, 
racing each other.
+ *
+ * <p>The local engine calls {@link #prepare(Pipeline)} after every copy is 
initialized. The native
+ * Spark and Beam engines call {@link #prepareOnDriver} before the job starts 
and hand the names of
+ * the prepared transforms to their partitions in {@link
+ * Const#INTERNAL_VARIABLE_PIPELINE_PREPARED_ONCE_TRANSFORMS}; the 
per-partition pipelines then only
+ * mark those transforms as prepared.
+ */
+public final class TransformPrepareOnce {
+
+  private TransformPrepareOnce() {}
+
+  /**
+   * Prepare the transforms of an initialized pipeline once each, on copy 0, 
and flag every copy. A
+   * pipeline that doesn't prepare itself (a per-partition pipeline of a 
distributed engine) only
+   * flags the transforms its engine prepared. A transactional pipeline (a 
connection group) leaves
+   * the work to the transforms' first row, inside the transaction.
+   */
+  public static void prepare(Pipeline pipeline) throws HopException {
+    List<TransformMetaDataCombi> combis = pipeline.getTransforms();
+    if (combis == null) {
+      return;
+    }
+    if (!pipeline.isPreparingTransformsOnce()) {
+      Set<String> prepared =
+          preparedOnDriver(
+              
pipeline.getVariable(Const.INTERNAL_VARIABLE_PIPELINE_PREPARED_ONCE_TRANSFORMS));
+      for (TransformMetaDataCombi combi : combis) {
+        if (prepared.contains(combi.transformName)) {
+          combi.transform.setPreparedOnce(true);
+        }
+      }
+      return;
+    }
+    if (pipeline.getExtensionDataMap().get(Const.CONNECTION_GROUP) != null) {
+      return;
+    }
+    Map<String, List<TransformMetaDataCombi>> copies = new LinkedHashMap<>();
+    for (TransformMetaDataCombi combi : combis) {
+      copies.computeIfAbsent(combi.transformName, n -> new 
ArrayList<>()).add(combi);
+    }
+    for (List<TransformMetaDataCombi> transformCopies : copies.values()) {
+      TransformMetaDataCombi first = transformCopies.get(0);
+      for (TransformMetaDataCombi combi : transformCopies) {
+        if (combi.copy == 0) {
+          first = combi;
+        }
+      }
+      IVariables variables = first.transform instanceof IVariables v ? v : 
pipeline;
+      if (!first.meta.needsPrepareOnce(variables)) {

Review Comment:
   **Should fix.** This also moves the work up front for a transform that runs 
in a single copy, where nothing races. For Table output with "Automatically 
update table structure", the table is then created and changed from the 
declared fields instead of the first row. That changes single-copy pipelines 
that work today. For example, with *Drop columns* on, a column that the rows 
carry but the declared layout leaves out gets dropped, data included.
   
   Suggested fix: on the local engine, only prepare when the transform runs in 
more than one copy:
   
   ```java
         if (transformCopies.size() < 2 || 
!first.meta.needsPrepareOnce(variables)) {
   ```
   
   A single copy then behaves exactly as on main, and Spark and Beam are 
unaffected because they prepare on the driver. The docs line in 
`tableoutput.adoc` about declared fields would then apply only to several 
copies, Spark and Beam.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to