hansva commented on code in PR #9341:
URL: https://github.com/apache/hop/pull/9341#discussion_r4536886760


##########
plugins/tech/datahub/src/main/java/org/apache/hop/lineage/datahub/DataHubEmitter.java:
##########
@@ -0,0 +1,479 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.lineage.datahub;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.openlineage.client.OpenLineage.RunEvent;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.lineage.datahub.DataHubSinkConfig.SchemaPolicy;
+import org.apache.hop.lineage.datahub.DataHubTranslator.LifecycleChange;
+import org.apache.hop.lineage.datahub.DataHubTranslator.Mcp;
+import org.apache.hop.lineage.datahub.DataHubTranslator.SchemaCandidate;
+import org.apache.hop.lineage.datahub.DataHubTranslator.StaleCheck;
+import org.apache.hop.lineage.datahub.DataHubTranslator.Translation;
+import org.apache.hop.lineage.openlineage.OpenLineageEmitter;
+
+/**
+ * Delivers mapped lineage events to DataHub: each event is translated into 
metadata change
+ * proposals ({@link DataHubTranslator}) and written in one {@code 
ingestProposalBatch} request.
+ *
+ * <p>DataHub is written to directly rather than through its OpenLineage 
endpoint. That endpoint
+ * needs server-wide settings that also change how every other OpenLineage 
producer is ingested,
+ * ingests asynchronously, and gives every transform a pipeline of its own; 
writing the proposals
+ * ourselves leaves the server as it is and makes Hop the only writer of the 
entities it describes.
+ *
+ * <p>The emitter runs on the dispatcher's worker thread, one event at a time, 
and is the same for a
+ * live event and one replayed from the spool.
+ */
+class DataHubEmitter implements OpenLineageEmitter {
+
+  /** Proposals per request; a large pipeline's terminal event can carry many 
datasets. */
+  static final int MAX_BATCH = 200;
+
+  /** How long what DataHub said about a dataset's schema is trusted before it 
is read again. */
+  static final long SCHEMA_STATE_TTL_MS = 10 * 60_000L;
+
+  /**
+   * How long a proposal this process wrote is not written again unchanged. 
Most of what a run
+   * writes is the same every time — the pipeline, its transforms, their 
datasets, often the job's
+   * inputs and outputs — and every aspect costs DataHub a database write; a 
Pipeline Executor
+   * running a pipeline per row would otherwise rewrite all of it for every 
row. The window is
+   * short, so an entity deleted in DataHub in the meantime comes back soon 
enough.
+   */
+  static final long WRITTEN_TTL_MS = 10 * 60_000L;
+
+  private static final int MAX_SCHEMA_STATES = 4_096;
+  private static final int MAX_WRITTEN = 16_384;
+  private static final ObjectMapper MAPPER = new ObjectMapper();
+  static final String FLOW_JOBS_QUERY =
+      "query($urns: [String!]!) { entities(urns: $urns) { urn ... on DataFlow 
{"
+          + " jobs: relationships(input: {types: [\"IsPartOf\"], direction: 
INCOMING, start: 0,"
+          + " count: 1000}) { relationships { entity { urn ... on DataJob { 
status { removed }"
+          + " properties { customProperties { key value } } } } } } } } }";
+  static final String DATASETS_QUERY =
+      "query($urns: [String!]!) { entities(urns: $urns) { urn ... on Dataset { 
exists"
+          + " schemaMetadata { name } deprecation { deprecated note } } } }";
+
+  private final DataHubTranslator translator;
+  private final DataHubClient client;
+  private final SchemaPolicy schemaPolicy;
+  private final String actor;
+  private final int maxRetries;
+  private final long retryBackoffMs;
+  private final ILogChannel log;
+  private final Map<String, SchemaState> schemaStates =
+      new LinkedHashMap<>(16, 0.75f, true) {
+        @Override
+        protected boolean removeEldestEntry(Map.Entry<String, SchemaState> 
eldest) {
+          return size() > MAX_SCHEMA_STATES;
+        }
+      };
+  private final Map<String, Long> written =
+      new LinkedHashMap<>(16, 0.75f, true) {
+        @Override
+        protected boolean removeEldestEntry(Map.Entry<String, Long> eldest) {
+          return size() > MAX_WRITTEN;
+        }
+      };
+  private boolean readFailureReported;
+
+  /** Whether Hop may write a dataset's schema, as last read from DataHub. */
+  private record SchemaState(boolean writable, long readAtMs) {}
+
+  /** What DataHub holds for a dataset, as far as the sink's decisions go. */
+  private record DatasetState(boolean hasForeignSchema, boolean deprecated, 
String note) {}
+
+  DataHubEmitter(
+      DataHubTranslator translator,
+      DataHubClient client,
+      SchemaPolicy schemaPolicy,
+      String actor,
+      int maxRetries,
+      long retryBackoffMs,
+      ILogChannel log) {
+    this.translator = translator;
+    this.client = client;
+    this.schemaPolicy = schemaPolicy;
+    this.actor = actor;
+    this.maxRetries = maxRetries;
+    this.retryBackoffMs = retryBackoffMs;
+    this.log = log;
+  }
+
+  @Override
+  public Result emit(RunEvent event) {
+    return emitAll(List.of(event)).get(0);
+  }
+
+  /**
+   * Writes the events in one go: their proposals, with one read of DataHub 
for the decisions they
+   * need, in as few requests as {@link #MAX_BATCH} allows. They succeed or 
wait together; an event
+   * that cannot be translated is refused on its own.
+   */
+  @Override
+  public List<Result> emitAll(List<RunEvent> events) {
+    Result[] results = new Result[events.size()];
+    List<Mcp> proposals = new ArrayList<>();
+    List<SchemaCandidate> schemas = new ArrayList<>();
+    List<LifecycleChange> lifecycle = new ArrayList<>();
+    List<StaleCheck> staleChecks = new ArrayList<>();
+    proposals.add(DataHubTranslator.hopPlatform());
+    for (int i = 0; i < events.size(); i++) {
+      try {
+        Translation translation = translator.translate(events.get(i));
+        proposals.addAll(translation.proposals());
+        schemas.addAll(translation.schemas());
+        lifecycle.addAll(translation.lifecycle());
+        staleChecks.addAll(translation.staleChecks());
+      } catch (RuntimeException e) {
+        log.logError("Unable to translate a lineage event for DataHub; it is 
skipped", e);
+        results[i] = Result.REJECTED;
+      }
+    }
+    Result outcome;
+    try {
+      proposals.addAll(decide(schemas, lifecycle));
+      List<Mcp> retirements = retire(staleChecks);
+      proposals.addAll(retirements);
+      long now = System.currentTimeMillis();
+      proposals.removeIf(mcp -> writtenRecently(mcp, now));

Review Comment:
   The 10-minute 'written recently' cache suppresses UPSERTs and PATCHes on the 
same `dataJobInputOutput` aspect independently. A root transform's UPSERT 
(`DataHubTranslator.java:215`) replaces `inputDatajobEdges`, and 
`linkToParentJob` (`DataHubTranslator.java:320-335`) PATCHes the parent edge 
back. Their cache entries were written at different times, so once the UPSERT's 
entry expires it is re-sent and wipes the edge, while the identical PATCH is 
still 'written recently' and skipped. For a child pipeline called from a 
Pipeline Executor or from two workflow actions, the workflow-to-pipeline edge 
then disappears for up to 10 minutes (longer for the other parent action). 
Maybe always send the PATCH after a re-sent UPSERT, or key the cache per aspect 
instead of per proposal?



##########
plugins/tech/datahub/src/main/java/org/apache/hop/lineage/datahub/DataHubUrns.java:
##########
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.lineage.datahub;
+
+import java.util.Locale;
+import java.util.UUID;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * DataHub URNs for what Hop reports.
+ *
+ * <p>An id that is part of a tuple URN ({@code urn:li:dataJob:(flow,id)}) 
cannot hold the
+ * characters that delimit the tuple. DataHub rejects such a URN outright 
("urns parts and key
+ * fields do not have same length"), and its own clients percent-encode those 
characters, which is
+ * what {@link #encode} does too: a transform called {@code Read (a, b)} 
becomes {@code Read %28a%2C
+ * b%29}. The readable name travels in the entity's properties.
+ */
+final class DataHubUrns {
+
+  /** The data platform Hop's pipelines and workflows belong to. */
+  static final String HOP_PLATFORM = "hop";
+
+  static final String HOP_PLATFORM_URN = "urn:li:dataPlatform:" + HOP_PLATFORM;
+
+  private DataHubUrns() {}
+
+  /** Percent-encodes what would otherwise end or split a tuple URN part. */
+  static String encode(String id) {
+    StringBuilder encoded = new StringBuilder(id.length());
+    for (char c : id.toCharArray()) {
+      switch (c) {
+        case '%' -> encoded.append("%25");
+        case ',' -> encoded.append("%2C");
+        case '(' -> encoded.append("%28");
+        case ')' -> encoded.append("%29");
+        default -> encoded.append(c);
+      }
+    }
+    return encoded.toString();
+  }
+
+  /**
+   * A pipeline or workflow. With a platform instance the flow id is prefixed 
with it, the way
+   * DataHub's own orchestrator integrations keep several installations apart.
+   */
+  static String dataFlow(String platformInstance, String name, String cluster) 
{

Review Comment:
   DataFlow and DataJob URNs are keyed only on the pipeline/workflow name (plus 
env and platform instance), so different Hop files with the same name merge 
into one DataHub flow. Two projects or folders each with a pipeline `load` 
containing `Table output` both write `urn:li:dataFlow:(hop,load,prod)` and the 
same dataJob URN; `dataFlowInfo`, `dataJobInputOutput` and the filename 
property flip with each run and the lineage of the two pipelines is mixed. 
Could the project name or the file path be part of the flow id?



##########
plugins/tech/datahub/src/main/java/org/apache/hop/lineage/datahub/DataHubEmitter.java:
##########
@@ -0,0 +1,479 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.lineage.datahub;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.openlineage.client.OpenLineage.RunEvent;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.lineage.datahub.DataHubSinkConfig.SchemaPolicy;
+import org.apache.hop.lineage.datahub.DataHubTranslator.LifecycleChange;
+import org.apache.hop.lineage.datahub.DataHubTranslator.Mcp;
+import org.apache.hop.lineage.datahub.DataHubTranslator.SchemaCandidate;
+import org.apache.hop.lineage.datahub.DataHubTranslator.StaleCheck;
+import org.apache.hop.lineage.datahub.DataHubTranslator.Translation;
+import org.apache.hop.lineage.openlineage.OpenLineageEmitter;
+
+/**
+ * Delivers mapped lineage events to DataHub: each event is translated into 
metadata change
+ * proposals ({@link DataHubTranslator}) and written in one {@code 
ingestProposalBatch} request.
+ *
+ * <p>DataHub is written to directly rather than through its OpenLineage 
endpoint. That endpoint
+ * needs server-wide settings that also change how every other OpenLineage 
producer is ingested,
+ * ingests asynchronously, and gives every transform a pipeline of its own; 
writing the proposals
+ * ourselves leaves the server as it is and makes Hop the only writer of the 
entities it describes.
+ *
+ * <p>The emitter runs on the dispatcher's worker thread, one event at a time, 
and is the same for a
+ * live event and one replayed from the spool.
+ */
+class DataHubEmitter implements OpenLineageEmitter {
+
+  /** Proposals per request; a large pipeline's terminal event can carry many 
datasets. */
+  static final int MAX_BATCH = 200;
+
+  /** How long what DataHub said about a dataset's schema is trusted before it 
is read again. */
+  static final long SCHEMA_STATE_TTL_MS = 10 * 60_000L;
+
+  /**
+   * How long a proposal this process wrote is not written again unchanged. 
Most of what a run
+   * writes is the same every time — the pipeline, its transforms, their 
datasets, often the job's
+   * inputs and outputs — and every aspect costs DataHub a database write; a 
Pipeline Executor
+   * running a pipeline per row would otherwise rewrite all of it for every 
row. The window is
+   * short, so an entity deleted in DataHub in the meantime comes back soon 
enough.
+   */
+  static final long WRITTEN_TTL_MS = 10 * 60_000L;
+
+  private static final int MAX_SCHEMA_STATES = 4_096;
+  private static final int MAX_WRITTEN = 16_384;
+  private static final ObjectMapper MAPPER = new ObjectMapper();
+  static final String FLOW_JOBS_QUERY =
+      "query($urns: [String!]!) { entities(urns: $urns) { urn ... on DataFlow 
{"
+          + " jobs: relationships(input: {types: [\"IsPartOf\"], direction: 
INCOMING, start: 0,"
+          + " count: 1000}) { relationships { entity { urn ... on DataJob { 
status { removed }"
+          + " properties { customProperties { key value } } } } } } } } }";
+  static final String DATASETS_QUERY =
+      "query($urns: [String!]!) { entities(urns: $urns) { urn ... on Dataset { 
exists"
+          + " schemaMetadata { name } deprecation { deprecated note } } } }";
+
+  private final DataHubTranslator translator;
+  private final DataHubClient client;
+  private final SchemaPolicy schemaPolicy;
+  private final String actor;
+  private final int maxRetries;
+  private final long retryBackoffMs;
+  private final ILogChannel log;
+  private final Map<String, SchemaState> schemaStates =
+      new LinkedHashMap<>(16, 0.75f, true) {
+        @Override
+        protected boolean removeEldestEntry(Map.Entry<String, SchemaState> 
eldest) {
+          return size() > MAX_SCHEMA_STATES;
+        }
+      };
+  private final Map<String, Long> written =
+      new LinkedHashMap<>(16, 0.75f, true) {
+        @Override
+        protected boolean removeEldestEntry(Map.Entry<String, Long> eldest) {
+          return size() > MAX_WRITTEN;
+        }
+      };
+  private boolean readFailureReported;
+
+  /** Whether Hop may write a dataset's schema, as last read from DataHub. */
+  private record SchemaState(boolean writable, long readAtMs) {}
+
+  /** What DataHub holds for a dataset, as far as the sink's decisions go. */
+  private record DatasetState(boolean hasForeignSchema, boolean deprecated, 
String note) {}
+
+  DataHubEmitter(
+      DataHubTranslator translator,
+      DataHubClient client,
+      SchemaPolicy schemaPolicy,
+      String actor,
+      int maxRetries,
+      long retryBackoffMs,
+      ILogChannel log) {
+    this.translator = translator;
+    this.client = client;
+    this.schemaPolicy = schemaPolicy;
+    this.actor = actor;
+    this.maxRetries = maxRetries;
+    this.retryBackoffMs = retryBackoffMs;
+    this.log = log;
+  }
+
+  @Override
+  public Result emit(RunEvent event) {
+    return emitAll(List.of(event)).get(0);
+  }
+
+  /**
+   * Writes the events in one go: their proposals, with one read of DataHub 
for the decisions they
+   * need, in as few requests as {@link #MAX_BATCH} allows. They succeed or 
wait together; an event
+   * that cannot be translated is refused on its own.
+   */
+  @Override
+  public List<Result> emitAll(List<RunEvent> events) {
+    Result[] results = new Result[events.size()];
+    List<Mcp> proposals = new ArrayList<>();
+    List<SchemaCandidate> schemas = new ArrayList<>();
+    List<LifecycleChange> lifecycle = new ArrayList<>();
+    List<StaleCheck> staleChecks = new ArrayList<>();
+    proposals.add(DataHubTranslator.hopPlatform());
+    for (int i = 0; i < events.size(); i++) {
+      try {
+        Translation translation = translator.translate(events.get(i));
+        proposals.addAll(translation.proposals());
+        schemas.addAll(translation.schemas());
+        lifecycle.addAll(translation.lifecycle());
+        staleChecks.addAll(translation.staleChecks());
+      } catch (RuntimeException e) {
+        log.logError("Unable to translate a lineage event for DataHub; it is 
skipped", e);
+        results[i] = Result.REJECTED;
+      }
+    }
+    Result outcome;
+    try {
+      proposals.addAll(decide(schemas, lifecycle));
+      List<Mcp> retirements = retire(staleChecks);
+      proposals.addAll(retirements);
+      long now = System.currentTimeMillis();
+      proposals.removeIf(mcp -> writtenRecently(mcp, now));
+      outcome = proposals.isEmpty() ? Result.SENT : send(proposals);
+      if (outcome == Result.SENT) {
+        for (Mcp mcp : retirements) {
+          // What this process wrote for a retired task no longer holds: when 
the task comes back,
+          // its next start has to write it again.
+          String prefix = mcp.entityUrn() + '\u0000';
+          written.keySet().removeIf(key -> key.startsWith(prefix));
+        }
+        for (Mcp mcp : proposals) {
+          written.put(key(mcp), now);
+        }
+        for (SchemaCandidate schema : schemas) {
+          if (proposals.contains(schema.schemaMetadata())) {
+            remember(schema.datasetUrn(), true);
+          }
+        }
+      }
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      outcome = Result.RETRY;
+    }
+    for (int i = 0; i < results.length; i++) {
+      if (results[i] == null) {
+        results[i] = outcome;
+      }
+    }
+    return Arrays.asList(results);
+  }
+
+  /**
+   * Soft-deletes the tasks of each flow that its run no longer has: a 
transform or action removed
+   * from the pipeline or workflow, or renamed. Only tasks written for the 
same Hop file are
+   * candidates, so a pipeline of the same name in another project cannot 
retire this one's tasks.
+   * DataHub keeps a soft-deleted task, and the task's next start restores it.
+   */
+  private List<Mcp> retire(List<StaleCheck> checks) throws 
InterruptedException {
+    if (checks.isEmpty()) {
+      return List.of();
+    }
+    Map<String, StaleCheck> byFlow = new LinkedHashMap<>();
+    checks.forEach(check -> byFlow.put(check.flowUrn(), check));
+    ObjectNode variables = MAPPER.createObjectNode();
+    byFlow.keySet().forEach(variables.putArray("urns")::add);
+    JsonNode data;
+    try {
+      data = client.graphql(FLOW_JOBS_QUERY, variables);
+    } catch (IOException e) {
+      reportReadFailure(e);
+      return List.of();
+    }
+    List<Mcp> retirements = new ArrayList<>();
+    for (JsonNode flow : data.path("entities")) {
+      StaleCheck check = byFlow.get(flow.path("urn").asText());
+      if (check == null) {
+        continue;
+      }
+      for (JsonNode relationship : flow.path("jobs").path("relationships")) {
+        JsonNode job = relationship.path("entity");
+        String urn = job.path("urn").asText();
+        // A soft-deleted task still "exists" for DataHub; its status says it 
was retired already.
+        if (job.path("status").path("removed").asBoolean(false)

Review Comment:
   `retire()` soft-deletes tasks by comparing DataHub's current jobs with the 
member list of whatever COMPLETE event it's handed, so a stale event replayed 
out of order retires tasks that are still live. Example: v1 of a pipeline 
completes while DataHub is down, so its COMPLETE is spooled (and `settle()` 
puts failed events at the back). The user adds a 'Sort rows' transform, v2 runs 
and is delivered. When the old v1 COMPLETE is replayed, its member list doesn't 
contain 'Sort rows', the filename matches, so the live task gets 
`status.removed=true`. The replayed UPSERTs of `dataFlowInfo` / 
`dataJobInputOutput` also overwrite the newer v2 aspects. Could retirement (and 
these UPSERTs) be skipped for events older than what DataHub already has, e.g. 
by comparing the run's event time?



##########
plugins/tech/datahub/src/main/java/org/apache/hop/lineage/datahub/DataHubEmitter.java:
##########
@@ -0,0 +1,479 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.lineage.datahub;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.openlineage.client.OpenLineage.RunEvent;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.lineage.datahub.DataHubSinkConfig.SchemaPolicy;
+import org.apache.hop.lineage.datahub.DataHubTranslator.LifecycleChange;
+import org.apache.hop.lineage.datahub.DataHubTranslator.Mcp;
+import org.apache.hop.lineage.datahub.DataHubTranslator.SchemaCandidate;
+import org.apache.hop.lineage.datahub.DataHubTranslator.StaleCheck;
+import org.apache.hop.lineage.datahub.DataHubTranslator.Translation;
+import org.apache.hop.lineage.openlineage.OpenLineageEmitter;
+
+/**
+ * Delivers mapped lineage events to DataHub: each event is translated into 
metadata change
+ * proposals ({@link DataHubTranslator}) and written in one {@code 
ingestProposalBatch} request.
+ *
+ * <p>DataHub is written to directly rather than through its OpenLineage 
endpoint. That endpoint
+ * needs server-wide settings that also change how every other OpenLineage 
producer is ingested,
+ * ingests asynchronously, and gives every transform a pipeline of its own; 
writing the proposals
+ * ourselves leaves the server as it is and makes Hop the only writer of the 
entities it describes.
+ *
+ * <p>The emitter runs on the dispatcher's worker thread, one event at a time, 
and is the same for a
+ * live event and one replayed from the spool.
+ */
+class DataHubEmitter implements OpenLineageEmitter {
+
+  /** Proposals per request; a large pipeline's terminal event can carry many 
datasets. */
+  static final int MAX_BATCH = 200;
+
+  /** How long what DataHub said about a dataset's schema is trusted before it 
is read again. */
+  static final long SCHEMA_STATE_TTL_MS = 10 * 60_000L;
+
+  /**
+   * How long a proposal this process wrote is not written again unchanged. 
Most of what a run
+   * writes is the same every time — the pipeline, its transforms, their 
datasets, often the job's
+   * inputs and outputs — and every aspect costs DataHub a database write; a 
Pipeline Executor
+   * running a pipeline per row would otherwise rewrite all of it for every 
row. The window is
+   * short, so an entity deleted in DataHub in the meantime comes back soon 
enough.
+   */
+  static final long WRITTEN_TTL_MS = 10 * 60_000L;
+
+  private static final int MAX_SCHEMA_STATES = 4_096;
+  private static final int MAX_WRITTEN = 16_384;
+  private static final ObjectMapper MAPPER = new ObjectMapper();
+  static final String FLOW_JOBS_QUERY =
+      "query($urns: [String!]!) { entities(urns: $urns) { urn ... on DataFlow 
{"
+          + " jobs: relationships(input: {types: [\"IsPartOf\"], direction: 
INCOMING, start: 0,"
+          + " count: 1000}) { relationships { entity { urn ... on DataJob { 
status { removed }"
+          + " properties { customProperties { key value } } } } } } } } }";
+  static final String DATASETS_QUERY =
+      "query($urns: [String!]!) { entities(urns: $urns) { urn ... on Dataset { 
exists"
+          + " schemaMetadata { name } deprecation { deprecated note } } } }";
+
+  private final DataHubTranslator translator;
+  private final DataHubClient client;
+  private final SchemaPolicy schemaPolicy;
+  private final String actor;
+  private final int maxRetries;
+  private final long retryBackoffMs;
+  private final ILogChannel log;
+  private final Map<String, SchemaState> schemaStates =
+      new LinkedHashMap<>(16, 0.75f, true) {
+        @Override
+        protected boolean removeEldestEntry(Map.Entry<String, SchemaState> 
eldest) {
+          return size() > MAX_SCHEMA_STATES;
+        }
+      };
+  private final Map<String, Long> written =
+      new LinkedHashMap<>(16, 0.75f, true) {
+        @Override
+        protected boolean removeEldestEntry(Map.Entry<String, Long> eldest) {
+          return size() > MAX_WRITTEN;
+        }
+      };
+  private boolean readFailureReported;
+
+  /** Whether Hop may write a dataset's schema, as last read from DataHub. */
+  private record SchemaState(boolean writable, long readAtMs) {}
+
+  /** What DataHub holds for a dataset, as far as the sink's decisions go. */
+  private record DatasetState(boolean hasForeignSchema, boolean deprecated, 
String note) {}
+
+  DataHubEmitter(
+      DataHubTranslator translator,
+      DataHubClient client,
+      SchemaPolicy schemaPolicy,
+      String actor,
+      int maxRetries,
+      long retryBackoffMs,
+      ILogChannel log) {
+    this.translator = translator;
+    this.client = client;
+    this.schemaPolicy = schemaPolicy;
+    this.actor = actor;
+    this.maxRetries = maxRetries;
+    this.retryBackoffMs = retryBackoffMs;
+    this.log = log;
+  }
+
+  @Override
+  public Result emit(RunEvent event) {
+    return emitAll(List.of(event)).get(0);
+  }
+
+  /**
+   * Writes the events in one go: their proposals, with one read of DataHub 
for the decisions they
+   * need, in as few requests as {@link #MAX_BATCH} allows. They succeed or 
wait together; an event
+   * that cannot be translated is refused on its own.
+   */
+  @Override
+  public List<Result> emitAll(List<RunEvent> events) {
+    Result[] results = new Result[events.size()];
+    List<Mcp> proposals = new ArrayList<>();
+    List<SchemaCandidate> schemas = new ArrayList<>();
+    List<LifecycleChange> lifecycle = new ArrayList<>();
+    List<StaleCheck> staleChecks = new ArrayList<>();
+    proposals.add(DataHubTranslator.hopPlatform());
+    for (int i = 0; i < events.size(); i++) {
+      try {
+        Translation translation = translator.translate(events.get(i));
+        proposals.addAll(translation.proposals());
+        schemas.addAll(translation.schemas());
+        lifecycle.addAll(translation.lifecycle());
+        staleChecks.addAll(translation.staleChecks());
+      } catch (RuntimeException e) {
+        log.logError("Unable to translate a lineage event for DataHub; it is 
skipped", e);
+        results[i] = Result.REJECTED;
+      }
+    }
+    Result outcome;
+    try {
+      proposals.addAll(decide(schemas, lifecycle));
+      List<Mcp> retirements = retire(staleChecks);
+      proposals.addAll(retirements);
+      long now = System.currentTimeMillis();
+      proposals.removeIf(mcp -> writtenRecently(mcp, now));
+      outcome = proposals.isEmpty() ? Result.SENT : send(proposals);
+      if (outcome == Result.SENT) {
+        for (Mcp mcp : retirements) {
+          // What this process wrote for a retired task no longer holds: when 
the task comes back,
+          // its next start has to write it again.
+          String prefix = mcp.entityUrn() + '\u0000';
+          written.keySet().removeIf(key -> key.startsWith(prefix));
+        }
+        for (Mcp mcp : proposals) {
+          written.put(key(mcp), now);
+        }
+        for (SchemaCandidate schema : schemas) {
+          if (proposals.contains(schema.schemaMetadata())) {
+            remember(schema.datasetUrn(), true);
+          }
+        }
+      }
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      outcome = Result.RETRY;
+    }
+    for (int i = 0; i < results.length; i++) {
+      if (results[i] == null) {
+        results[i] = outcome;
+      }
+    }
+    return Arrays.asList(results);
+  }
+
+  /**
+   * Soft-deletes the tasks of each flow that its run no longer has: a 
transform or action removed
+   * from the pipeline or workflow, or renamed. Only tasks written for the 
same Hop file are
+   * candidates, so a pipeline of the same name in another project cannot 
retire this one's tasks.
+   * DataHub keeps a soft-deleted task, and the task's next start restores it.
+   */
+  private List<Mcp> retire(List<StaleCheck> checks) throws 
InterruptedException {
+    if (checks.isEmpty()) {
+      return List.of();
+    }
+    Map<String, StaleCheck> byFlow = new LinkedHashMap<>();
+    checks.forEach(check -> byFlow.put(check.flowUrn(), check));
+    ObjectNode variables = MAPPER.createObjectNode();
+    byFlow.keySet().forEach(variables.putArray("urns")::add);
+    JsonNode data;
+    try {
+      data = client.graphql(FLOW_JOBS_QUERY, variables);
+    } catch (IOException e) {
+      reportReadFailure(e);
+      return List.of();
+    }
+    List<Mcp> retirements = new ArrayList<>();
+    for (JsonNode flow : data.path("entities")) {
+      StaleCheck check = byFlow.get(flow.path("urn").asText());
+      if (check == null) {
+        continue;
+      }
+      for (JsonNode relationship : flow.path("jobs").path("relationships")) {
+        JsonNode job = relationship.path("entity");
+        String urn = job.path("urn").asText();
+        // A soft-deleted task still "exists" for DataHub; its status says it 
was retired already.
+        if (job.path("status").path("removed").asBoolean(false)
+            || check.currentJobUrns().contains(urn)
+            || !check.filename().equals(customProperty(job, 
DataHubTranslator.FILENAME_PROPERTY))) {
+          continue;
+        }
+        ObjectNode status = MAPPER.createObjectNode();
+        status.put("removed", true);
+        retirements.add(Mcp.upsert("dataJob", urn, "status", status));
+        log.logBasic(
+            "Retiring DataHub task " + urn + ": its pipeline or workflow no 
longer has it");
+      }
+    }
+    return retirements;
+  }
+
+  private static String customProperty(JsonNode entity, String key) {
+    for (JsonNode property : 
entity.path("properties").path("customProperties")) {
+      if (key.equals(property.path("key").asText())) {
+        return property.path("value").asText(null);
+      }
+    }
+    return null;
+  }
+
+  private void reportReadFailure(IOException e) {
+    if (!readFailureReported) {
+      readFailureReported = true;
+      log.logError(
+          "Unable to read from DataHub, so dataset schemas, deprecations and 
retired tasks are"
+              + " left alone (reported once): "
+              + e.getMessage());
+    }
+  }
+
+  private boolean writtenRecently(Mcp mcp, long now) {
+    Long at = written.get(key(mcp));
+    return at != null && now - at < WRITTEN_TTL_MS;
+  }
+
+  private static String key(Mcp mcp) {
+    return mcp.entityUrn()
+        + '\u0000'
+        + mcp.aspectName()
+        + '\u0000'
+        + mcp.changeType()
+        + '\u0000'
+        + Integer.toHexString(mcp.value().toString().hashCode());
+  }
+
+  // 
---------------------------------------------------------------------------------------------
+  // Decisions that depend on what DataHub holds
+  // 
---------------------------------------------------------------------------------------------
+
+  /**
+   * The schema and lifecycle proposals that DataHub's current state allows. A 
dataset another
+   * source describes (DataHub's own database ingestion, say) keeps that 
source's schema; a
+   * deprecation a person wrote is never lifted. When DataHub cannot be read, 
both are left out: the
+   * lineage itself does not wait for them.
+   */
+  private List<Mcp> decide(List<SchemaCandidate> candidates, 
List<LifecycleChange> lifecycle)
+      throws InterruptedException {
+    List<SchemaCandidate> schemas = schemaPolicy == SchemaPolicy.NEVER ? 
List.of() : candidates;
+    if (schemas.isEmpty() && lifecycle.isEmpty()) {
+      return List.of();
+    }
+    Set<String> toRead = new LinkedHashSet<>();
+    long now = System.currentTimeMillis();
+    if (schemaPolicy == SchemaPolicy.MISSING) {
+      for (SchemaCandidate schema : schemas) {
+        SchemaState known = schemaStates.get(schema.datasetUrn());
+        if (known == null || now - known.readAtMs() > SCHEMA_STATE_TTL_MS) {
+          toRead.add(schema.datasetUrn());
+        }
+      }
+    }
+    for (LifecycleChange change : lifecycle) {
+      toRead.add(change.datasetUrn());
+    }
+    Map<String, DatasetState> states = toRead.isEmpty() ? Map.of() : 
read(toRead);
+    if (states == null) {

Review Comment:
   When the GraphQL read fails, `decide()` / `retire()` return no proposals, so 
schema, deprecation and retirement changes are dropped, but the event can still 
be reported SENT and removed from the spool, so they're never retried. That 
happens with a token without GraphQL read rights, a proxy blocking 
`/api/graphql`, or a single timeout; e.g. the lifecycle DEPRECATE for a deleted 
file or the `schemaMetadata` for a new table is lost for good. Also 
`readFailureReported` (line 252) is never reset, so every later failure is 
silent. I think a failed read should make the event RETRY.



##########
plugins/tech/datahub/src/main/java/org/apache/hop/lineage/datahub/DataHubEmitter.java:
##########
@@ -0,0 +1,479 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.lineage.datahub;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.openlineage.client.OpenLineage.RunEvent;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.lineage.datahub.DataHubSinkConfig.SchemaPolicy;
+import org.apache.hop.lineage.datahub.DataHubTranslator.LifecycleChange;
+import org.apache.hop.lineage.datahub.DataHubTranslator.Mcp;
+import org.apache.hop.lineage.datahub.DataHubTranslator.SchemaCandidate;
+import org.apache.hop.lineage.datahub.DataHubTranslator.StaleCheck;
+import org.apache.hop.lineage.datahub.DataHubTranslator.Translation;
+import org.apache.hop.lineage.openlineage.OpenLineageEmitter;
+
+/**
+ * Delivers mapped lineage events to DataHub: each event is translated into 
metadata change
+ * proposals ({@link DataHubTranslator}) and written in one {@code 
ingestProposalBatch} request.
+ *
+ * <p>DataHub is written to directly rather than through its OpenLineage 
endpoint. That endpoint
+ * needs server-wide settings that also change how every other OpenLineage 
producer is ingested,
+ * ingests asynchronously, and gives every transform a pipeline of its own; 
writing the proposals
+ * ourselves leaves the server as it is and makes Hop the only writer of the 
entities it describes.
+ *
+ * <p>The emitter runs on the dispatcher's worker thread, one event at a time, 
and is the same for a
+ * live event and one replayed from the spool.
+ */
+class DataHubEmitter implements OpenLineageEmitter {
+
+  /** Proposals per request; a large pipeline's terminal event can carry many 
datasets. */
+  static final int MAX_BATCH = 200;
+
+  /** How long what DataHub said about a dataset's schema is trusted before it 
is read again. */
+  static final long SCHEMA_STATE_TTL_MS = 10 * 60_000L;
+
+  /**
+   * How long a proposal this process wrote is not written again unchanged. 
Most of what a run
+   * writes is the same every time — the pipeline, its transforms, their 
datasets, often the job's
+   * inputs and outputs — and every aspect costs DataHub a database write; a 
Pipeline Executor
+   * running a pipeline per row would otherwise rewrite all of it for every 
row. The window is
+   * short, so an entity deleted in DataHub in the meantime comes back soon 
enough.
+   */
+  static final long WRITTEN_TTL_MS = 10 * 60_000L;
+
+  private static final int MAX_SCHEMA_STATES = 4_096;
+  private static final int MAX_WRITTEN = 16_384;
+  private static final ObjectMapper MAPPER = new ObjectMapper();
+  static final String FLOW_JOBS_QUERY =
+      "query($urns: [String!]!) { entities(urns: $urns) { urn ... on DataFlow 
{"
+          + " jobs: relationships(input: {types: [\"IsPartOf\"], direction: 
INCOMING, start: 0,"
+          + " count: 1000}) { relationships { entity { urn ... on DataJob { 
status { removed }"
+          + " properties { customProperties { key value } } } } } } } } }";
+  static final String DATASETS_QUERY =
+      "query($urns: [String!]!) { entities(urns: $urns) { urn ... on Dataset { 
exists"
+          + " schemaMetadata { name } deprecation { deprecated note } } } }";
+
+  private final DataHubTranslator translator;
+  private final DataHubClient client;
+  private final SchemaPolicy schemaPolicy;
+  private final String actor;
+  private final int maxRetries;
+  private final long retryBackoffMs;
+  private final ILogChannel log;
+  private final Map<String, SchemaState> schemaStates =
+      new LinkedHashMap<>(16, 0.75f, true) {
+        @Override
+        protected boolean removeEldestEntry(Map.Entry<String, SchemaState> 
eldest) {
+          return size() > MAX_SCHEMA_STATES;
+        }
+      };
+  private final Map<String, Long> written =
+      new LinkedHashMap<>(16, 0.75f, true) {
+        @Override
+        protected boolean removeEldestEntry(Map.Entry<String, Long> eldest) {
+          return size() > MAX_WRITTEN;
+        }
+      };
+  private boolean readFailureReported;
+
+  /** Whether Hop may write a dataset's schema, as last read from DataHub. */
+  private record SchemaState(boolean writable, long readAtMs) {}
+
+  /** What DataHub holds for a dataset, as far as the sink's decisions go. */
+  private record DatasetState(boolean hasForeignSchema, boolean deprecated, 
String note) {}
+
+  DataHubEmitter(
+      DataHubTranslator translator,
+      DataHubClient client,
+      SchemaPolicy schemaPolicy,
+      String actor,
+      int maxRetries,
+      long retryBackoffMs,
+      ILogChannel log) {
+    this.translator = translator;
+    this.client = client;
+    this.schemaPolicy = schemaPolicy;
+    this.actor = actor;
+    this.maxRetries = maxRetries;
+    this.retryBackoffMs = retryBackoffMs;
+    this.log = log;
+  }
+
+  @Override
+  public Result emit(RunEvent event) {
+    return emitAll(List.of(event)).get(0);
+  }
+
+  /**
+   * Writes the events in one go: their proposals, with one read of DataHub 
for the decisions they
+   * need, in as few requests as {@link #MAX_BATCH} allows. They succeed or 
wait together; an event
+   * that cannot be translated is refused on its own.
+   */
+  @Override
+  public List<Result> emitAll(List<RunEvent> events) {
+    Result[] results = new Result[events.size()];
+    List<Mcp> proposals = new ArrayList<>();
+    List<SchemaCandidate> schemas = new ArrayList<>();
+    List<LifecycleChange> lifecycle = new ArrayList<>();
+    List<StaleCheck> staleChecks = new ArrayList<>();
+    proposals.add(DataHubTranslator.hopPlatform());
+    for (int i = 0; i < events.size(); i++) {
+      try {
+        Translation translation = translator.translate(events.get(i));
+        proposals.addAll(translation.proposals());
+        schemas.addAll(translation.schemas());
+        lifecycle.addAll(translation.lifecycle());
+        staleChecks.addAll(translation.staleChecks());
+      } catch (RuntimeException e) {
+        log.logError("Unable to translate a lineage event for DataHub; it is 
skipped", e);
+        results[i] = Result.REJECTED;
+      }
+    }
+    Result outcome;
+    try {
+      proposals.addAll(decide(schemas, lifecycle));
+      List<Mcp> retirements = retire(staleChecks);
+      proposals.addAll(retirements);
+      long now = System.currentTimeMillis();
+      proposals.removeIf(mcp -> writtenRecently(mcp, now));
+      outcome = proposals.isEmpty() ? Result.SENT : send(proposals);
+      if (outcome == Result.SENT) {
+        for (Mcp mcp : retirements) {
+          // What this process wrote for a retired task no longer holds: when 
the task comes back,
+          // its next start has to write it again.
+          String prefix = mcp.entityUrn() + '\u0000';
+          written.keySet().removeIf(key -> key.startsWith(prefix));
+        }
+        for (Mcp mcp : proposals) {
+          written.put(key(mcp), now);
+        }
+        for (SchemaCandidate schema : schemas) {
+          if (proposals.contains(schema.schemaMetadata())) {
+            remember(schema.datasetUrn(), true);
+          }
+        }
+      }
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      outcome = Result.RETRY;
+    }
+    for (int i = 0; i < results.length; i++) {
+      if (results[i] == null) {
+        results[i] = outcome;
+      }
+    }
+    return Arrays.asList(results);
+  }
+
+  /**
+   * Soft-deletes the tasks of each flow that its run no longer has: a 
transform or action removed
+   * from the pipeline or workflow, or renamed. Only tasks written for the 
same Hop file are
+   * candidates, so a pipeline of the same name in another project cannot 
retire this one's tasks.
+   * DataHub keeps a soft-deleted task, and the task's next start restores it.
+   */
+  private List<Mcp> retire(List<StaleCheck> checks) throws 
InterruptedException {
+    if (checks.isEmpty()) {
+      return List.of();
+    }
+    Map<String, StaleCheck> byFlow = new LinkedHashMap<>();
+    checks.forEach(check -> byFlow.put(check.flowUrn(), check));
+    ObjectNode variables = MAPPER.createObjectNode();
+    byFlow.keySet().forEach(variables.putArray("urns")::add);
+    JsonNode data;
+    try {
+      data = client.graphql(FLOW_JOBS_QUERY, variables);
+    } catch (IOException e) {
+      reportReadFailure(e);
+      return List.of();
+    }
+    List<Mcp> retirements = new ArrayList<>();
+    for (JsonNode flow : data.path("entities")) {
+      StaleCheck check = byFlow.get(flow.path("urn").asText());
+      if (check == null) {
+        continue;
+      }
+      for (JsonNode relationship : flow.path("jobs").path("relationships")) {
+        JsonNode job = relationship.path("entity");
+        String urn = job.path("urn").asText();
+        // A soft-deleted task still "exists" for DataHub; its status says it 
was retired already.
+        if (job.path("status").path("removed").asBoolean(false)
+            || check.currentJobUrns().contains(urn)
+            || !check.filename().equals(customProperty(job, 
DataHubTranslator.FILENAME_PROPERTY))) {
+          continue;
+        }
+        ObjectNode status = MAPPER.createObjectNode();
+        status.put("removed", true);
+        retirements.add(Mcp.upsert("dataJob", urn, "status", status));
+        log.logBasic(
+            "Retiring DataHub task " + urn + ": its pipeline or workflow no 
longer has it");
+      }
+    }
+    return retirements;
+  }
+
+  private static String customProperty(JsonNode entity, String key) {
+    for (JsonNode property : 
entity.path("properties").path("customProperties")) {
+      if (key.equals(property.path("key").asText())) {
+        return property.path("value").asText(null);
+      }
+    }
+    return null;
+  }
+
+  private void reportReadFailure(IOException e) {
+    if (!readFailureReported) {
+      readFailureReported = true;
+      log.logError(
+          "Unable to read from DataHub, so dataset schemas, deprecations and 
retired tasks are"
+              + " left alone (reported once): "
+              + e.getMessage());
+    }
+  }
+
+  private boolean writtenRecently(Mcp mcp, long now) {
+    Long at = written.get(key(mcp));
+    return at != null && now - at < WRITTEN_TTL_MS;
+  }
+
+  private static String key(Mcp mcp) {
+    return mcp.entityUrn()
+        + '\u0000'
+        + mcp.aspectName()
+        + '\u0000'
+        + mcp.changeType()
+        + '\u0000'
+        + Integer.toHexString(mcp.value().toString().hashCode());
+  }
+
+  // 
---------------------------------------------------------------------------------------------
+  // Decisions that depend on what DataHub holds
+  // 
---------------------------------------------------------------------------------------------
+
+  /**
+   * The schema and lifecycle proposals that DataHub's current state allows. A 
dataset another
+   * source describes (DataHub's own database ingestion, say) keeps that 
source's schema; a
+   * deprecation a person wrote is never lifted. When DataHub cannot be read, 
both are left out: the
+   * lineage itself does not wait for them.
+   */
+  private List<Mcp> decide(List<SchemaCandidate> candidates, 
List<LifecycleChange> lifecycle)
+      throws InterruptedException {
+    List<SchemaCandidate> schemas = schemaPolicy == SchemaPolicy.NEVER ? 
List.of() : candidates;
+    if (schemas.isEmpty() && lifecycle.isEmpty()) {
+      return List.of();
+    }
+    Set<String> toRead = new LinkedHashSet<>();
+    long now = System.currentTimeMillis();
+    if (schemaPolicy == SchemaPolicy.MISSING) {
+      for (SchemaCandidate schema : schemas) {
+        SchemaState known = schemaStates.get(schema.datasetUrn());
+        if (known == null || now - known.readAtMs() > SCHEMA_STATE_TTL_MS) {
+          toRead.add(schema.datasetUrn());
+        }
+      }
+    }
+    for (LifecycleChange change : lifecycle) {
+      toRead.add(change.datasetUrn());
+    }
+    Map<String, DatasetState> states = toRead.isEmpty() ? Map.of() : 
read(toRead);
+    if (states == null) {
+      return List.of();
+    }
+
+    List<Mcp> proposals = new ArrayList<>();
+    for (SchemaCandidate schema : schemas) {
+      DatasetState state = states.get(schema.datasetUrn());
+      if (state != null) {
+        remember(schema.datasetUrn(), !state.hasForeignSchema());
+      }
+      SchemaState known = schemaStates.get(schema.datasetUrn());
+      if (schemaPolicy == SchemaPolicy.ALWAYS || (known != null && 
known.writable())) {
+        proposals.add(schema.schemaMetadata());
+      }
+    }
+    // In event order, each change seeing the ones before it: a run that 
writes a file and then
+    // moves it away lifts Hop's deprecation and then sets a new one, in that 
order.
+    Map<String, DatasetState> current = new HashMap<>(states);
+    for (LifecycleChange change : lifecycle) {
+      DatasetState state = current.get(change.datasetUrn());
+      Mcp deprecation = deprecation(change, state);
+      if (deprecation != null) {
+        proposals.add(deprecation);
+        current.put(
+            change.datasetUrn(),
+            new DatasetState(
+                state != null && state.hasForeignSchema(),
+                deprecation.value().path("deprecated").asBoolean(),
+                change.note()));
+      }
+    }
+    return proposals;
+  }
+
+  private Mcp deprecation(LifecycleChange change, DatasetState state) {
+    boolean deprecated = state != null && state.deprecated();
+    ObjectNode aspect = MAPPER.createObjectNode();
+    if (change.kind() == LifecycleChange.Kind.DEPRECATE) {
+      if (deprecated) {
+        return null; // already retired, by Hop or by a person: keep the note 
that is there
+      }
+      aspect.put("deprecated", true);
+      aspect.put("decommissionTime", change.timeMs());
+      if (change.replacementUrn() != null) {
+        aspect.put("replacement", change.replacementUrn());
+      }
+    } else {
+      boolean byHop =
+          state != null
+              && state.note() != null
+              && state.note().startsWith(DataHubTranslator.HOP_NOTE_PREFIX);
+      if (!deprecated || !byHop) {
+        return null; // not deprecated, or deprecated by a person: a write 
does not undo that
+      }
+      aspect.put("deprecated", false);
+    }
+    aspect.put("note", change.note());
+    aspect.put("actor", actor);
+    return Mcp.upsert("dataset", change.datasetUrn(), "deprecation", aspect);
+  }
+
+  private void remember(String datasetUrn, boolean writable) {
+    schemaStates.put(datasetUrn, new SchemaState(writable, 
System.currentTimeMillis()));
+  }
+
+  /** What DataHub holds for the datasets, or {@code null} when it could not 
be read. */
+  private Map<String, DatasetState> read(Collection<String> urns) throws 
InterruptedException {
+    ObjectNode variables = MAPPER.createObjectNode();
+    urns.forEach(variables.putArray("urns")::add);
+    JsonNode data;
+    try {
+      data = client.graphql(DATASETS_QUERY, variables);
+    } catch (IOException e) {
+      reportReadFailure(e);
+      return null;
+    }
+    Map<String, DatasetState> states = new HashMap<>();
+    for (JsonNode entity : data.path("entities")) {
+      String schemaName = 
entity.path("schemaMetadata").path("name").asText(null);
+      boolean hasSchema =
+          !entity.path("schemaMetadata").isMissingNode() && 
!entity.path("schemaMetadata").isNull();
+      JsonNode deprecation = entity.path("deprecation");
+      states.put(
+          entity.path("urn").asText(),
+          new DatasetState(
+              hasSchema && 
!DataHubTranslator.HOP_SCHEMA_NAME.equals(schemaName),
+              deprecation.path("deprecated").asBoolean(false),
+              deprecation.path("note").asText(null)));
+    }
+    return states;
+  }
+
+  // 
---------------------------------------------------------------------------------------------
+  // Delivery
+  // 
---------------------------------------------------------------------------------------------
+
+  /**
+   * Writes the proposals in batches. A batch DataHub refuses as a whole (one 
invalid proposal fails
+   * the request) is retried one proposal at a time, so a single bad aspect 
costs that aspect and
+   * not the run's lineage.
+   */
+  private Result send(List<Mcp> proposals) throws InterruptedException {
+    int rejected = 0;
+    for (int start = 0; start < proposals.size(); start += MAX_BATCH) {
+      List<Mcp> batch = proposals.subList(start, Math.min(proposals.size(), 
start + MAX_BATCH));
+      DataHubClient.Response response = ingestWithRetries(batch);
+      if (response.ok()) {
+        continue;
+      }
+      if (OpenLineageEmitter.failureOf(response.status()) == Result.RETRY) {
+        logFailure(response, batch.size());
+        return Result.RETRY;
+      }
+      for (Mcp mcp : batch) {

Review Comment:
   When DataHub refuses a batch with a non-transient 4xx, it's re-sent one 
proposal at a time, each through `ingestWithRetries`, all synchronously on the 
single worker thread (same blocking-retry pattern as in #9340). A 200-proposal 
batch refused with 400/413 becomes up to 200 sequential requests, and any 5xx 
along the way adds backoff sleeps of `retryBackoffMs << attempt` (shift up to 
16, i.e. up to 65536x the backoff). The queue backs up, the shutdown drain 
window expires, and `close()` leaves the worker mid-send. Capping the shift and 
bounding the per-proposal fallback would help.



##########
plugins/tech/datahub/src/main/java/org/apache/hop/lineage/datahub/DataHubSinkConfig.java:
##########
@@ -0,0 +1,312 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.lineage.datahub;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
+import java.util.Locale;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.variables.Variable;
+import org.apache.hop.core.variables.VariableScope;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.lineage.openlineage.OpenLineageSinkConfig;
+
+/**
+ * Configuration of the DataHub sink, read from {@code HOP_LINEAGE_DATAHUB_*} 
variables.
+ *
+ * <p>The delivery settings the sink shares with the OpenLineage sink (queue, 
spool, retries,
+ * timeouts, TLS, SQL parsing) keep their OpenLineage names under the DataHub 
prefix: {@code
+ * HOP_LINEAGE_DATAHUB_SPOOL_DIR} configures what {@code 
HOP_LINEAGE_OPENLINEAGE_SPOOL_DIR} does for
+ * that sink.
+ */
+public final class DataHubSinkConfig {
+
+  @Variable(
+      scope = VariableScope.ENGINE,
+      value = "",
+      description =
+          "DataHub GMS URL the DataHub lineage sink writes to, e.g. 
http://localhost:8080 or"
+              + " https://your-company.acryl.io/gms. Required when the sink is 
enabled.")
+  public static final String HOP_LINEAGE_DATAHUB_URL = 
"HOP_LINEAGE_DATAHUB_URL";
+
+  @Variable(
+      scope = VariableScope.ENGINE,
+      value = "",
+      description =
+          "DataHub personal access token for the DataHub lineage sink, sent as 
a bearer token."
+              + " Needed when DataHub has authentication enabled. May be 
encrypted with hop-encrypt.")
+  public static final String HOP_LINEAGE_DATAHUB_TOKEN = 
"HOP_LINEAGE_DATAHUB_TOKEN";
+
+  @Variable(
+      scope = VariableScope.ENGINE,
+      value = "PROD",
+      description =
+          "DataHub environment (fabric) of the datasets, pipelines and 
workflows Hop reports,"
+              + " e.g. PROD, DEV or TEST. Use the same value as the other 
sources that describe"
+              + " those datasets, or their lineage does not join up.")
+  public static final String HOP_LINEAGE_DATAHUB_ENV = 
"HOP_LINEAGE_DATAHUB_ENV";
+
+  @Variable(
+      scope = VariableScope.ENGINE,
+      value = "",
+      description =
+          "Optional DataHub platform instance for Hop: keeps the pipelines and 
workflows of"
+              + " several Hop installations apart in one DataHub.")
+  public static final String HOP_LINEAGE_DATAHUB_PLATFORM_INSTANCE =
+      "HOP_LINEAGE_DATAHUB_PLATFORM_INSTANCE";
+
+  @Variable(
+      scope = VariableScope.ENGINE,
+      value = "missing",
+      description =
+          "When the DataHub lineage sink writes a dataset's schema: 'missing' 
(only when DataHub"
+              + " has none, or the one it has came from Hop), 'always' or 
'never'.")
+  public static final String HOP_LINEAGE_DATAHUB_DATASET_SCHEMA =
+      "HOP_LINEAGE_DATAHUB_DATASET_SCHEMA";
+
+  @Variable(
+      scope = VariableScope.ENGINE,
+      value = "N",
+      description =
+          "Set to Y to let the DataHub lineage sink deprecate datasets a run 
deleted or moved, and"
+              + " clear that deprecation when a later run writes them again.")
+  public static final String HOP_LINEAGE_DATAHUB_LIFECYCLE = 
"HOP_LINEAGE_DATAHUB_LIFECYCLE";
+
+  @Variable(
+      scope = VariableScope.ENGINE,
+      value = "urn:li:corpuser:datahub",
+      description =
+          "DataHub user the DataHub lineage sink names as the actor of the 
runs and deprecations"
+              + " it writes.")
+  public static final String HOP_LINEAGE_DATAHUB_ACTOR = 
"HOP_LINEAGE_DATAHUB_ACTOR";
+
+  @Variable(
+      scope = VariableScope.ENGINE,
+      value = "",
+      description =
+          "Folder where the DataHub lineage sink keeps events until DataHub 
accepted them, so an"
+              + " outage or a restart delays lineage instead of losing it. One 
folder per Hop"
+              + " process; empty keeps events in memory only.")
+  public static final String HOP_LINEAGE_DATAHUB_SPOOL_DIR = 
"HOP_LINEAGE_DATAHUB_SPOOL_DIR";
+
+  /**
+   * How a dataset's {@code schemaMetadata} is written; see {@link
+   * #HOP_LINEAGE_DATAHUB_DATASET_SCHEMA}.
+   */
+  public enum SchemaPolicy {
+    /** Only when DataHub has no schema for the dataset, or the one it has was 
written by Hop. */
+    MISSING,
+    ALWAYS,
+    NEVER
+  }
+
+  private static final String DATAHUB_PREFIX = "HOP_LINEAGE_DATAHUB_";
+  private static final String OPENLINEAGE_PREFIX = "HOP_LINEAGE_OPENLINEAGE_";
+
+  /**
+   * The settings shared with the OpenLineage sink, passed through under the 
same suffix. The
+   * mapping options (transform jobs, parent runs) are not among them: DataHub 
needs them one way.
+   */
+  private static final List<String> SHARED_SUFFIXES =
+      List.of(
+          "CONNECT_TIMEOUT_MS",
+          "READ_TIMEOUT_MS",
+          "BUFFER_SIZE",
+          "MAX_RETRIES",
+          "RETRY_BACKOFF_MS",
+          "OVERFLOW_POLICY",
+          "ENQUEUE_TIMEOUT_MS",
+          "SHUTDOWN_DRAIN_MS",
+          "METRICS_INTERVAL_MS",
+          "TRUSTSTORE_PATH",
+          "TRUSTSTORE_PASSWORD",
+          "KEYSTORE_TYPE",
+          "SQL_PARSE",
+          "SPOOL_DIR",
+          "SPOOL_MAX_BYTES");
+
+  /**
+   * The path of DataHub's OpenLineage endpoint. An earlier version of this 
sink was configured with
+   * it; the URL still works, the path is dropped.
+   */
+  static final String OPENLINEAGE_ENDPOINT_PATH = 
"/openapi/openlineage/api/v1/lineage";
+
+  private final String gmsUrl;
+  private final String token;
+  private final String env;
+  private final String platformInstance;
+  private final SchemaPolicy schemaPolicy;
+  private final boolean lifecycleEnabled;
+  private final String actor;
+  private final IVariables source;
+
+  private DataHubSinkConfig(
+      String gmsUrl,
+      String token,
+      String env,
+      String platformInstance,
+      SchemaPolicy schemaPolicy,
+      boolean lifecycleEnabled,
+      String actor,
+      IVariables source) {
+    this.gmsUrl = gmsUrl;
+    this.token = token;
+    this.env = env;
+    this.platformInstance = platformInstance;
+    this.schemaPolicy = schemaPolicy;
+    this.lifecycleEnabled = lifecycleEnabled;
+    this.actor = actor;
+    this.source = source;
+  }
+
+  /**
+   * Reads the configuration.
+   *
+   * @throws IllegalArgumentException when the URL is missing or not an 
absolute http(s) URL, or a
+   *     setting has a value it does not accept
+   */
+  public static DataHubSinkConfig fromVariables(IVariables variables) {
+    String url = value(variables, HOP_LINEAGE_DATAHUB_URL);
+    if (url == null) {
+      throw new IllegalArgumentException(
+          HOP_LINEAGE_DATAHUB_URL + " is required when the DataHub lineage 
sink is enabled");
+    }
+    String env = StringUtils.defaultIfBlank(value(variables, 
HOP_LINEAGE_DATAHUB_ENV), "PROD");
+    return new DataHubSinkConfig(
+        normalizeUrl(url),
+        OpenLineageSinkConfig.secret(value(variables, 
HOP_LINEAGE_DATAHUB_TOKEN)),
+        env.toUpperCase(Locale.ROOT),
+        value(variables, HOP_LINEAGE_DATAHUB_PLATFORM_INSTANCE),
+        schemaPolicy(value(variables, HOP_LINEAGE_DATAHUB_DATASET_SCHEMA)),
+        parseBoolean(value(variables, HOP_LINEAGE_DATAHUB_LIFECYCLE), false),
+        StringUtils.defaultIfBlank(
+            value(variables, HOP_LINEAGE_DATAHUB_ACTOR), 
"urn:li:corpuser:datahub"),
+        variables);
+  }
+
+  private static String value(IVariables variables, String name) {
+    String value = variables.getVariable(name, null);
+    return StringUtils.isBlank(value) ? null : value.trim();
+  }
+
+  static SchemaPolicy schemaPolicy(String value) {
+    if (value == null) {
+      return SchemaPolicy.MISSING;
+    }
+    try {
+      return SchemaPolicy.valueOf(value.toUpperCase(Locale.ROOT));
+    } catch (IllegalArgumentException e) {
+      throw new IllegalArgumentException(
+          HOP_LINEAGE_DATAHUB_DATASET_SCHEMA
+              + " must be 'missing', 'always' or 'never', was: "
+              + value,
+          e);
+    }
+  }
+
+  static boolean parseBoolean(String value, boolean defaultValue) {
+    if (StringUtils.isBlank(value)) {
+      return defaultValue;
+    }
+    return switch (value.trim().toLowerCase(Locale.ROOT)) {
+      case "y", "yes", "true", "1", "on" -> true;
+      case "n", "no", "false", "0", "off" -> false;
+      default -> defaultValue;
+    };
+  }
+
+  /**
+   * The GMS URL without a trailing slash, and without the OpenLineage 
endpoint path an earlier
+   * version of this sink was configured with. A path of its own (DataHub 
Cloud serves GMS under
+   * {@code /gms}) is kept.
+   */
+  static String normalizeUrl(String raw) {
+    URI uri;
+    try {
+      uri = new URI(raw);
+    } catch (URISyntaxException e) {
+      throw new IllegalArgumentException(
+          HOP_LINEAGE_DATAHUB_URL + " is not a valid URL: " + raw, e);
+    }
+    String scheme = uri.getScheme();
+    if (scheme == null
+        || (!scheme.equalsIgnoreCase("http") && 
!scheme.equalsIgnoreCase("https"))
+        || uri.getHost() == null) {
+      throw new IllegalArgumentException(
+          HOP_LINEAGE_DATAHUB_URL + " must be an absolute http(s) URL, was: " 
+ raw);
+    }
+    String url = StringUtils.removeEnd(raw, "/");
+    url = StringUtils.removeEnd(url, OPENLINEAGE_ENDPOINT_PATH);
+    return StringUtils.removeEnd(url, "/");
+  }
+
+  /**
+   * The OpenLineage settings the sink runs its mapping, queue and spool with: 
the shared settings
+   * passed through, and the mapping DataHub needs — one job per transform, 
with parent runs.
+   */
+  public OpenLineageSinkConfig toOpenLineageConfig() {
+    Variables overlay = new Variables();
+    // Not posted to: the DataHub emitter writes to GMS itself. The 
OpenLineage settings need a URL.
+    overlay.setVariable(OpenLineageSinkConfig.VAR_URL, gmsUrl);
+    overlay.setVariable(OpenLineageSinkConfig.VAR_TRANSFORM_JOBS, "Y");
+    overlay.setVariable(OpenLineageSinkConfig.VAR_PARENT_RUNS, "Y");
+    for (String suffix : SHARED_SUFFIXES) {
+      String value = value(source, DATAHUB_PREFIX + suffix);
+      if (value != null) {
+        overlay.setVariable(OPENLINEAGE_PREFIX + suffix, value);
+      }
+    }
+    return OpenLineageSinkConfig.fromVariables(overlay);
+  }
+
+  public String getGmsUrl() {

Review Comment:
   Minor: these getters (lines 279-311) are hand-written; per the project 
conventions we use Lombok `@Getter` for this. Also `DataHubLineageSink.java:94` 
uses a fully qualified `java.util.Locale` instead of an import.



##########
plugins/tech/datahub/src/main/java/org/apache/hop/lineage/datahub/DataHubTranslator.java:
##########
@@ -0,0 +1,961 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.lineage.datahub;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.openlineage.client.OpenLineage;
+import io.openlineage.client.OpenLineage.RunEvent;
+import io.openlineage.client.OpenLineage.RunEvent.EventType;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.lineage.LineageRunLifecycleEmitter;
+import org.apache.hop.lineage.openlineage.OpenLineageEventMapper;
+
+/**
+ * Turns the OpenLineage events the mapper produces into DataHub metadata 
change proposals.
+ *
+ * <p>The model is DataHub's own for an orchestrator:
+ *
+ * <ul>
+ *   <li>a pipeline or workflow is a <em>DataFlow</em>;
+ *   <li>a transform or action is a <em>DataJob</em> of that flow, with the 
datasets it read and
+ *       wrote and, for a transform, the transforms that feed it as upstream 
jobs (the hop graph);
+ *   <li>every run is a <em>DataProcessInstance</em> of its flow or job, named 
for humans ({@code
+ *       nightly 2026-10-18T07:34Z}), with its parent run and its outcome.
+ * </ul>
+ *
+ * <p>A pipeline started by an action (or by a Pipeline Executor transform) is 
linked to it through
+ * its first transforms, which list that action as an upstream job, so the 
lineage graph runs from
+ * the workflow into the pipeline.
+ *
+ * <p>Everything here is a pure function of the event: an event replayed from 
the spool by a later
+ * process is translated exactly like a live one. What depends on the state of 
DataHub — whether a
+ * dataset's schema may be written, whether a deprecation may be lifted — is 
returned as a decision
+ * for {@link DataHubEmitter} to make.
+ */
+final class DataHubTranslator {
+
+  static final String HOP_PLATFORM_LOGO = 
"https://hop.apache.org/img/hop-logo.svg";;
+
+  /** Marks the schemas Hop writes, so a later write only replaces Hop's own. 
*/
+  static final String HOP_SCHEMA_NAME = "hop";
+
+  /** Custom property naming the Hop file a flow or task comes from. */
+  static final String FILENAME_PROPERTY = "filename";
+
+  /** Marks the deprecations Hop writes, so a later write only lifts Hop's 
own. */
+  static final String HOP_NOTE_PREFIX = "Hop: ";
+
+  private static final double COLUMN_LINEAGE_CONFIDENCE = 0.5;
+  private static final DateTimeFormatter RUN_NAME_TIME =
+      DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm'Z'", 
Locale.ROOT).withZone(ZoneOffset.UTC);
+  private static final ObjectMapper MAPPER = new ObjectMapper();
+
+  /** One proposal: an aspect for an entity, as {@code UPSERT} or as a JSON 
{@code PATCH}. */
+  record Mcp(
+      String entityType, String entityUrn, String changeType, String 
aspectName, JsonNode value) {
+    static Mcp upsert(String entityType, String urn, String aspectName, 
JsonNode value) {
+      return new Mcp(entityType, urn, "UPSERT", aspectName, value);
+    }
+
+    static Mcp patch(String entityType, String urn, String aspectName, 
ArrayNode operations) {
+      return new Mcp(entityType, urn, "PATCH", aspectName, operations);
+    }
+  }
+
+  /** A dataset schema Hop knows; whether it is written depends on what 
DataHub already has. */
+  record SchemaCandidate(String datasetUrn, Mcp schemaMetadata) {}
+
+  /** A dataset an event deleted or moved, or wrote again after Hop had 
retired it. */
+  record LifecycleChange(
+      Kind kind, String datasetUrn, String replacementUrn, String note, long 
timeMs) {
+    enum Kind {
+      DEPRECATE,
+      CLEAR
+    }
+  }
+
+  /**
+   * The tasks a flow is made of now, after a successful run: the emitter 
retires the tasks DataHub
+   * still has for the same file that are not among them (a transform or 
action removed or renamed).
+   */
+  record StaleCheck(String flowUrn, Set<String> currentJobUrns, String 
filename) {}
+
+  /** What one event comes down to. */
+  record Translation(
+      List<Mcp> proposals,
+      List<SchemaCandidate> schemas,
+      List<LifecycleChange> lifecycle,
+      List<StaleCheck> staleChecks) {}
+
+  private final String env;
+  private final String cluster;
+  private final String platformInstance;
+  private final String actor;
+  private final boolean lifecycleEnabled;
+
+  DataHubTranslator(DataHubSinkConfig config) {
+    this(
+        config.getEnv(),
+        config.getCluster(),
+        config.getPlatformInstance(),
+        config.getActor(),
+        config.isLifecycleEnabled());
+  }
+
+  DataHubTranslator(
+      String env, String cluster, String platformInstance, String actor, 
boolean lifecycleEnabled) {
+    this.env = env;
+    this.cluster = cluster;
+    this.platformInstance = platformInstance;
+    this.actor = actor;
+    this.lifecycleEnabled = lifecycleEnabled;
+  }
+
+  /** The {@code hop} data platform, so Hop's pipelines show under their own 
name and logo. */
+  static Mcp hopPlatform() {
+    ObjectNode info = MAPPER.createObjectNode();
+    info.put("name", DataHubUrns.HOP_PLATFORM);
+    info.put("displayName", "Apache Hop");
+    info.put("type", "OTHERS");
+    info.put("datasetNameDelimiter", "/");
+    info.put("logoUrl", HOP_PLATFORM_LOGO);
+    return Mcp.upsert("dataPlatform", DataHubUrns.HOP_PLATFORM_URN, 
"dataPlatformInfo", info);
+  }
+
+  Translation translate(RunEvent event) {
+    Subject subject = Subject.of(event);
+    List<Mcp> proposals = new ArrayList<>();
+    List<SchemaCandidate> schemas = new ArrayList<>();
+    List<LifecycleChange> lifecycle = new ArrayList<>();
+    List<StaleCheck> staleChecks = new ArrayList<>();
+    if (subject == null) {
+      return new Translation(proposals, schemas, lifecycle, staleChecks);
+    }
+    String flowUrn = DataHubUrns.dataFlow(platformInstance, 
subject.flowName(), cluster);
+    String jobUrn =
+        subject.jobName() == null ? null : DataHubUrns.dataJob(flowUrn, 
subject.jobName());
+    EventType type = event.getEventType();
+    long time = eventTime(event);
+
+    if (type == EventType.START) {
+      proposals.addAll(flowAspects(flowUrn, subject, jobUrn == null ? 
description(event) : null));
+      if (jobUrn != null) {
+        proposals.addAll(jobAspects(jobUrn, flowUrn, subject, 
description(event)));
+      }
+      proposals.addAll(runStarted(event, subject, jobUrn != null ? jobUrn : 
flowUrn, time));

Review Comment:
   The run's DataProcessInstance is written as STARTED right away on START, and 
nothing closes it when the engine never sends a terminal event (an action that 
throws in `execute()`, see #9339, or a FINISHED dropped by the bounded hub 
queue). DataHub then shows that run as running forever. On shutdown, 
`flushObservations` only emits OTHER events for dataset observations; could 
open runs be ended there with a FAILURE result?



##########
plugins/tech/datahub/src/main/java/org/apache/hop/lineage/datahub/DataHubEmitter.java:
##########
@@ -0,0 +1,479 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.lineage.datahub;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.openlineage.client.OpenLineage.RunEvent;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.lineage.datahub.DataHubSinkConfig.SchemaPolicy;
+import org.apache.hop.lineage.datahub.DataHubTranslator.LifecycleChange;
+import org.apache.hop.lineage.datahub.DataHubTranslator.Mcp;
+import org.apache.hop.lineage.datahub.DataHubTranslator.SchemaCandidate;
+import org.apache.hop.lineage.datahub.DataHubTranslator.StaleCheck;
+import org.apache.hop.lineage.datahub.DataHubTranslator.Translation;
+import org.apache.hop.lineage.openlineage.OpenLineageEmitter;
+
+/**
+ * Delivers mapped lineage events to DataHub: each event is translated into 
metadata change
+ * proposals ({@link DataHubTranslator}) and written in one {@code 
ingestProposalBatch} request.
+ *
+ * <p>DataHub is written to directly rather than through its OpenLineage 
endpoint. That endpoint
+ * needs server-wide settings that also change how every other OpenLineage 
producer is ingested,
+ * ingests asynchronously, and gives every transform a pipeline of its own; 
writing the proposals
+ * ourselves leaves the server as it is and makes Hop the only writer of the 
entities it describes.
+ *
+ * <p>The emitter runs on the dispatcher's worker thread, one event at a time, 
and is the same for a
+ * live event and one replayed from the spool.
+ */
+class DataHubEmitter implements OpenLineageEmitter {
+
+  /** Proposals per request; a large pipeline's terminal event can carry many 
datasets. */
+  static final int MAX_BATCH = 200;
+
+  /** How long what DataHub said about a dataset's schema is trusted before it 
is read again. */
+  static final long SCHEMA_STATE_TTL_MS = 10 * 60_000L;
+
+  /**
+   * How long a proposal this process wrote is not written again unchanged. 
Most of what a run
+   * writes is the same every time — the pipeline, its transforms, their 
datasets, often the job's
+   * inputs and outputs — and every aspect costs DataHub a database write; a 
Pipeline Executor
+   * running a pipeline per row would otherwise rewrite all of it for every 
row. The window is
+   * short, so an entity deleted in DataHub in the meantime comes back soon 
enough.
+   */
+  static final long WRITTEN_TTL_MS = 10 * 60_000L;
+
+  private static final int MAX_SCHEMA_STATES = 4_096;
+  private static final int MAX_WRITTEN = 16_384;
+  private static final ObjectMapper MAPPER = new ObjectMapper();
+  static final String FLOW_JOBS_QUERY =
+      "query($urns: [String!]!) { entities(urns: $urns) { urn ... on DataFlow 
{"
+          + " jobs: relationships(input: {types: [\"IsPartOf\"], direction: 
INCOMING, start: 0,"
+          + " count: 1000}) { relationships { entity { urn ... on DataJob { 
status { removed }"
+          + " properties { customProperties { key value } } } } } } } } }";
+  static final String DATASETS_QUERY =
+      "query($urns: [String!]!) { entities(urns: $urns) { urn ... on Dataset { 
exists"
+          + " schemaMetadata { name } deprecation { deprecated note } } } }";
+
+  private final DataHubTranslator translator;
+  private final DataHubClient client;
+  private final SchemaPolicy schemaPolicy;
+  private final String actor;
+  private final int maxRetries;
+  private final long retryBackoffMs;
+  private final ILogChannel log;
+  private final Map<String, SchemaState> schemaStates =
+      new LinkedHashMap<>(16, 0.75f, true) {
+        @Override
+        protected boolean removeEldestEntry(Map.Entry<String, SchemaState> 
eldest) {
+          return size() > MAX_SCHEMA_STATES;
+        }
+      };
+  private final Map<String, Long> written =
+      new LinkedHashMap<>(16, 0.75f, true) {
+        @Override
+        protected boolean removeEldestEntry(Map.Entry<String, Long> eldest) {
+          return size() > MAX_WRITTEN;
+        }
+      };
+  private boolean readFailureReported;
+
+  /** Whether Hop may write a dataset's schema, as last read from DataHub. */
+  private record SchemaState(boolean writable, long readAtMs) {}
+
+  /** What DataHub holds for a dataset, as far as the sink's decisions go. */
+  private record DatasetState(boolean hasForeignSchema, boolean deprecated, 
String note) {}
+
+  DataHubEmitter(
+      DataHubTranslator translator,
+      DataHubClient client,
+      SchemaPolicy schemaPolicy,
+      String actor,
+      int maxRetries,
+      long retryBackoffMs,
+      ILogChannel log) {
+    this.translator = translator;
+    this.client = client;
+    this.schemaPolicy = schemaPolicy;
+    this.actor = actor;
+    this.maxRetries = maxRetries;
+    this.retryBackoffMs = retryBackoffMs;
+    this.log = log;
+  }
+
+  @Override
+  public Result emit(RunEvent event) {
+    return emitAll(List.of(event)).get(0);
+  }
+
+  /**
+   * Writes the events in one go: their proposals, with one read of DataHub 
for the decisions they
+   * need, in as few requests as {@link #MAX_BATCH} allows. They succeed or 
wait together; an event
+   * that cannot be translated is refused on its own.
+   */
+  @Override
+  public List<Result> emitAll(List<RunEvent> events) {
+    Result[] results = new Result[events.size()];
+    List<Mcp> proposals = new ArrayList<>();
+    List<SchemaCandidate> schemas = new ArrayList<>();
+    List<LifecycleChange> lifecycle = new ArrayList<>();
+    List<StaleCheck> staleChecks = new ArrayList<>();
+    proposals.add(DataHubTranslator.hopPlatform());
+    for (int i = 0; i < events.size(); i++) {
+      try {
+        Translation translation = translator.translate(events.get(i));
+        proposals.addAll(translation.proposals());
+        schemas.addAll(translation.schemas());
+        lifecycle.addAll(translation.lifecycle());
+        staleChecks.addAll(translation.staleChecks());
+      } catch (RuntimeException e) {
+        log.logError("Unable to translate a lineage event for DataHub; it is 
skipped", e);
+        results[i] = Result.REJECTED;
+      }
+    }
+    Result outcome;
+    try {
+      proposals.addAll(decide(schemas, lifecycle));
+      List<Mcp> retirements = retire(staleChecks);
+      proposals.addAll(retirements);
+      long now = System.currentTimeMillis();
+      proposals.removeIf(mcp -> writtenRecently(mcp, now));
+      outcome = proposals.isEmpty() ? Result.SENT : send(proposals);
+      if (outcome == Result.SENT) {
+        for (Mcp mcp : retirements) {
+          // What this process wrote for a retired task no longer holds: when 
the task comes back,
+          // its next start has to write it again.
+          String prefix = mcp.entityUrn() + '\u0000';
+          written.keySet().removeIf(key -> key.startsWith(prefix));
+        }
+        for (Mcp mcp : proposals) {
+          written.put(key(mcp), now);
+        }
+        for (SchemaCandidate schema : schemas) {
+          if (proposals.contains(schema.schemaMetadata())) {
+            remember(schema.datasetUrn(), true);
+          }
+        }
+      }
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      outcome = Result.RETRY;
+    }
+    for (int i = 0; i < results.length; i++) {
+      if (results[i] == null) {
+        results[i] = outcome;
+      }
+    }
+    return Arrays.asList(results);
+  }
+
+  /**
+   * Soft-deletes the tasks of each flow that its run no longer has: a 
transform or action removed
+   * from the pipeline or workflow, or renamed. Only tasks written for the 
same Hop file are
+   * candidates, so a pipeline of the same name in another project cannot 
retire this one's tasks.
+   * DataHub keeps a soft-deleted task, and the task's next start restores it.
+   */
+  private List<Mcp> retire(List<StaleCheck> checks) throws 
InterruptedException {
+    if (checks.isEmpty()) {
+      return List.of();
+    }
+    Map<String, StaleCheck> byFlow = new LinkedHashMap<>();
+    checks.forEach(check -> byFlow.put(check.flowUrn(), check));
+    ObjectNode variables = MAPPER.createObjectNode();
+    byFlow.keySet().forEach(variables.putArray("urns")::add);
+    JsonNode data;
+    try {
+      data = client.graphql(FLOW_JOBS_QUERY, variables);
+    } catch (IOException e) {
+      reportReadFailure(e);
+      return List.of();
+    }
+    List<Mcp> retirements = new ArrayList<>();
+    for (JsonNode flow : data.path("entities")) {
+      StaleCheck check = byFlow.get(flow.path("urn").asText());
+      if (check == null) {
+        continue;
+      }
+      for (JsonNode relationship : flow.path("jobs").path("relationships")) {
+        JsonNode job = relationship.path("entity");
+        String urn = job.path("urn").asText();
+        // A soft-deleted task still "exists" for DataHub; its status says it 
was retired already.
+        if (job.path("status").path("removed").asBoolean(false)
+            || check.currentJobUrns().contains(urn)
+            || !check.filename().equals(customProperty(job, 
DataHubTranslator.FILENAME_PROPERTY))) {
+          continue;
+        }
+        ObjectNode status = MAPPER.createObjectNode();
+        status.put("removed", true);
+        retirements.add(Mcp.upsert("dataJob", urn, "status", status));
+        log.logBasic(
+            "Retiring DataHub task " + urn + ": its pipeline or workflow no 
longer has it");
+      }
+    }
+    return retirements;
+  }
+
+  private static String customProperty(JsonNode entity, String key) {
+    for (JsonNode property : 
entity.path("properties").path("customProperties")) {
+      if (key.equals(property.path("key").asText())) {
+        return property.path("value").asText(null);
+      }
+    }
+    return null;
+  }
+
+  private void reportReadFailure(IOException e) {
+    if (!readFailureReported) {
+      readFailureReported = true;
+      log.logError(
+          "Unable to read from DataHub, so dataset schemas, deprecations and 
retired tasks are"
+              + " left alone (reported once): "
+              + e.getMessage());
+    }
+  }
+
+  private boolean writtenRecently(Mcp mcp, long now) {
+    Long at = written.get(key(mcp));
+    return at != null && now - at < WRITTEN_TTL_MS;
+  }
+
+  private static String key(Mcp mcp) {
+    return mcp.entityUrn()
+        + '\u0000'
+        + mcp.aspectName()
+        + '\u0000'
+        + mcp.changeType()
+        + '\u0000'
+        + Integer.toHexString(mcp.value().toString().hashCode());

Review Comment:
   The dedupe key only uses the 32-bit `String.hashCode()` of the aspect JSON, 
so a changed aspect whose hash collides with the previous value within 10 
minutes is skipped as 'written recently' and DataHub keeps the stale value. 
Storing the full value or a strong digest (SHA-256) avoids that.



##########
docs/hop-user-manual/modules/ROOT/pages/technology/datahub/index.adoc:
##########
@@ -0,0 +1,188 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+  http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:documentationPath: /technology/datahub/
+:language: en_US
+:description: The DataHub technology plugin is a Hop lineage sink that writes 
pipelines, workflows, their runs and the datasets they read and write to 
DataHub.
+:page-marketplace: hop-tech-datahub

Review Comment:
   This declares `:page-marketplace: hop-tech-datahub`, but the plugin isn't 
added to the marketplace's `optional-plugins.yaml`, so it can't be installed 
from the marketplace on a slim client. Also nothing records that it needs the 
openlineage plugin at runtime: `dependencies.xml` points at `../openlineage`, 
and `DataHubLineageSink` extends `OpenLineageLineageSink`, so installing the 
zip without `plugins/tech/openlineage` fails with NoClassDefFoundError.



##########
plugins/tech/datahub/src/main/java/org/apache/hop/lineage/datahub/DataHubEmitter.java:
##########
@@ -0,0 +1,479 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.lineage.datahub;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.openlineage.client.OpenLineage.RunEvent;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.lineage.datahub.DataHubSinkConfig.SchemaPolicy;
+import org.apache.hop.lineage.datahub.DataHubTranslator.LifecycleChange;
+import org.apache.hop.lineage.datahub.DataHubTranslator.Mcp;
+import org.apache.hop.lineage.datahub.DataHubTranslator.SchemaCandidate;
+import org.apache.hop.lineage.datahub.DataHubTranslator.StaleCheck;
+import org.apache.hop.lineage.datahub.DataHubTranslator.Translation;
+import org.apache.hop.lineage.openlineage.OpenLineageEmitter;
+
+/**
+ * Delivers mapped lineage events to DataHub: each event is translated into 
metadata change
+ * proposals ({@link DataHubTranslator}) and written in one {@code 
ingestProposalBatch} request.
+ *
+ * <p>DataHub is written to directly rather than through its OpenLineage 
endpoint. That endpoint
+ * needs server-wide settings that also change how every other OpenLineage 
producer is ingested,
+ * ingests asynchronously, and gives every transform a pipeline of its own; 
writing the proposals
+ * ourselves leaves the server as it is and makes Hop the only writer of the 
entities it describes.
+ *
+ * <p>The emitter runs on the dispatcher's worker thread, one event at a time, 
and is the same for a
+ * live event and one replayed from the spool.
+ */
+class DataHubEmitter implements OpenLineageEmitter {
+
+  /** Proposals per request; a large pipeline's terminal event can carry many 
datasets. */
+  static final int MAX_BATCH = 200;
+
+  /** How long what DataHub said about a dataset's schema is trusted before it 
is read again. */
+  static final long SCHEMA_STATE_TTL_MS = 10 * 60_000L;
+
+  /**
+   * How long a proposal this process wrote is not written again unchanged. 
Most of what a run
+   * writes is the same every time — the pipeline, its transforms, their 
datasets, often the job's
+   * inputs and outputs — and every aspect costs DataHub a database write; a 
Pipeline Executor
+   * running a pipeline per row would otherwise rewrite all of it for every 
row. The window is
+   * short, so an entity deleted in DataHub in the meantime comes back soon 
enough.
+   */
+  static final long WRITTEN_TTL_MS = 10 * 60_000L;
+
+  private static final int MAX_SCHEMA_STATES = 4_096;
+  private static final int MAX_WRITTEN = 16_384;
+  private static final ObjectMapper MAPPER = new ObjectMapper();
+  static final String FLOW_JOBS_QUERY =
+      "query($urns: [String!]!) { entities(urns: $urns) { urn ... on DataFlow 
{"
+          + " jobs: relationships(input: {types: [\"IsPartOf\"], direction: 
INCOMING, start: 0,"
+          + " count: 1000}) { relationships { entity { urn ... on DataJob { 
status { removed }"
+          + " properties { customProperties { key value } } } } } } } } }";
+  static final String DATASETS_QUERY =
+      "query($urns: [String!]!) { entities(urns: $urns) { urn ... on Dataset { 
exists"
+          + " schemaMetadata { name } deprecation { deprecated note } } } }";
+
+  private final DataHubTranslator translator;
+  private final DataHubClient client;
+  private final SchemaPolicy schemaPolicy;
+  private final String actor;
+  private final int maxRetries;
+  private final long retryBackoffMs;
+  private final ILogChannel log;
+  private final Map<String, SchemaState> schemaStates =
+      new LinkedHashMap<>(16, 0.75f, true) {
+        @Override
+        protected boolean removeEldestEntry(Map.Entry<String, SchemaState> 
eldest) {
+          return size() > MAX_SCHEMA_STATES;
+        }
+      };
+  private final Map<String, Long> written =
+      new LinkedHashMap<>(16, 0.75f, true) {
+        @Override
+        protected boolean removeEldestEntry(Map.Entry<String, Long> eldest) {
+          return size() > MAX_WRITTEN;
+        }
+      };
+  private boolean readFailureReported;
+
+  /** Whether Hop may write a dataset's schema, as last read from DataHub. */
+  private record SchemaState(boolean writable, long readAtMs) {}
+
+  /** What DataHub holds for a dataset, as far as the sink's decisions go. */
+  private record DatasetState(boolean hasForeignSchema, boolean deprecated, 
String note) {}
+
+  DataHubEmitter(
+      DataHubTranslator translator,
+      DataHubClient client,
+      SchemaPolicy schemaPolicy,
+      String actor,
+      int maxRetries,
+      long retryBackoffMs,
+      ILogChannel log) {
+    this.translator = translator;
+    this.client = client;
+    this.schemaPolicy = schemaPolicy;
+    this.actor = actor;
+    this.maxRetries = maxRetries;
+    this.retryBackoffMs = retryBackoffMs;
+    this.log = log;
+  }
+
+  @Override
+  public Result emit(RunEvent event) {
+    return emitAll(List.of(event)).get(0);
+  }
+
+  /**
+   * Writes the events in one go: their proposals, with one read of DataHub 
for the decisions they
+   * need, in as few requests as {@link #MAX_BATCH} allows. They succeed or 
wait together; an event
+   * that cannot be translated is refused on its own.
+   */
+  @Override
+  public List<Result> emitAll(List<RunEvent> events) {
+    Result[] results = new Result[events.size()];
+    List<Mcp> proposals = new ArrayList<>();
+    List<SchemaCandidate> schemas = new ArrayList<>();
+    List<LifecycleChange> lifecycle = new ArrayList<>();
+    List<StaleCheck> staleChecks = new ArrayList<>();
+    proposals.add(DataHubTranslator.hopPlatform());
+    for (int i = 0; i < events.size(); i++) {
+      try {
+        Translation translation = translator.translate(events.get(i));
+        proposals.addAll(translation.proposals());
+        schemas.addAll(translation.schemas());
+        lifecycle.addAll(translation.lifecycle());
+        staleChecks.addAll(translation.staleChecks());
+      } catch (RuntimeException e) {
+        log.logError("Unable to translate a lineage event for DataHub; it is 
skipped", e);
+        results[i] = Result.REJECTED;
+      }
+    }
+    Result outcome;
+    try {
+      proposals.addAll(decide(schemas, lifecycle));
+      List<Mcp> retirements = retire(staleChecks);
+      proposals.addAll(retirements);
+      long now = System.currentTimeMillis();
+      proposals.removeIf(mcp -> writtenRecently(mcp, now));
+      outcome = proposals.isEmpty() ? Result.SENT : send(proposals);
+      if (outcome == Result.SENT) {
+        for (Mcp mcp : retirements) {
+          // What this process wrote for a retired task no longer holds: when 
the task comes back,
+          // its next start has to write it again.
+          String prefix = mcp.entityUrn() + '\u0000';
+          written.keySet().removeIf(key -> key.startsWith(prefix));
+        }
+        for (Mcp mcp : proposals) {
+          written.put(key(mcp), now);
+        }
+        for (SchemaCandidate schema : schemas) {
+          if (proposals.contains(schema.schemaMetadata())) {
+            remember(schema.datasetUrn(), true);
+          }
+        }
+      }
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      outcome = Result.RETRY;
+    }
+    for (int i = 0; i < results.length; i++) {

Review Comment:
   `emitAll()` gives every event in a dispatcher batch the same outcome. With 
the default (no `HOP_LINEAGE_DATAHUB_SPOOL_DIR`), a batch of 50 events that 
produces 3 ingest requests, where request 1 succeeds and request 2 gets a 503 
after retries, marks all 50 as RETRY, and `settle()` only keeps spooled events, 
so all 50 are lost, including the ones already written. With a spool they go to 
the back of the backlog and get reordered behind newer events (which feeds the 
stale-retire issue on line 227). Could the outcome be tracked per event?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to