bamaer commented on code in PR #9545:
URL: https://github.com/apache/hop/pull/9545#discussion_r4619032391
##########
plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileInputMeta.java:
##########
@@ -1100,4 +1104,38 @@ public void convertLegacyXml(Node node) {
}
}
}
+
+ /**
+ * The files this transform reads, as they are now, with the layout of its
fields; none when it
+ * takes the file names from another transform.
+ */
+ @Override
+ public List<IDeclaresLineage.Declaration> declaredLineage(
+ IVariables variables, IRowMeta inputRowMeta) {
+ if (getFileInput().isAcceptingFilenames()) {
+ return List.of();
+ }
+ IRowMeta layout = new RowMeta();
+ for (TextFileInputField field : inputFields) {
+ try {
+ IValueMeta value = ValueMetaFactory.createValueMeta(field.getName(),
field.getType());
+ value.setLength(field.getLength());
+ value.setPrecision(field.getPrecision());
+ layout.addValueMeta(value);
+ } catch (HopPluginException e) {
+ // A type that is not installed: the layout goes without it
+ }
+ }
+ List<IDeclaresLineage.Declaration> declarations = new ArrayList<>();
+ for (String path : getFilePaths(variables)) {
+ String extension = path.contains(".") ?
path.substring(path.lastIndexOf('.') + 1) : null;
Review Comment:
Done: the extension now comes from the base name, so `/data/v1.2/customers`
has none (`textFileInputTakesTheFormatFromTheFileNameNotItsFolder`).
##########
plugins/tech/datahub/src/main/java/org/apache/hop/lineage/datahub/DataHubPublisher.java:
##########
@@ -0,0 +1,339 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.lineage.datahub;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import lombok.Setter;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSelectInfo;
+import org.apache.commons.vfs2.FileSelector;
+import org.apache.commons.vfs2.FileType;
+import org.apache.hop.base.AbstractMeta;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.parameters.UnknownParamException;
+import org.apache.hop.core.util.Utils;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.lineage.LineageDefinitions;
+import org.apache.hop.lineage.hub.LineageSinkTarget;
+import org.apache.hop.lineage.metadata.LineageSink;
+import org.apache.hop.lineage.model.LineageEvent;
+import org.apache.hop.lineage.spi.LineageSinkHealth;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.workflow.WorkflowMeta;
+
+/**
+ * Publishes the pipelines and workflows of a folder to DataHub without
running them: each as a flow
+ * with its transforms or actions as tasks, the tables and files they declare
as their inputs and
+ * outputs, and the layouts of those files as dataset schemas. It writes
through a DataHub {@link
+ * LineageSink}, so the entities are the ones the runs report to.
+ *
+ * <p>With {@code includingDocumentation} set it also publishes what {@code
hop doc} writes for each
+ * pipeline and workflow as a DataHub document (DataHub 1.7 and later),
related to its flow and
+ * tasks, under a document of the project.
+ */
+public final class DataHubPublisher {
+
+ /** How long to wait for DataHub to accept what was published. */
+ static final long DELIVERY_TIMEOUT_MS = 120_000L;
+
+ /** How long to wait for DataHub to take the first document, before the
others are sent. */
+ static final long FIRST_DOCUMENT_TIMEOUT_MS = 30_000L;
+
+ /** The name of the project, set when a project is used. */
+ static final String VAR_PROJECT_NAME = "HOP_PROJECT_NAME";
+
+ /**
+ * What a publication did.
+ *
+ * @param pipelines the pipelines published
+ * @param workflows the workflows published
+ * @param unreadable the files that could not be read, and were skipped
+ * @param documentationNotPublished why the documentation was not published,
or null
+ * @param health how the delivery to DataHub went
+ */
+ public record Result(
+ int pipelines,
+ int workflows,
+ List<String> unreadable,
+ String documentationNotPublished,
+ LineageSinkHealth health) {
+
+ /** Whether everything was read and DataHub accepted all of it. */
+ public boolean isSuccess() {
+ return unreadable.isEmpty()
+ && documentationNotPublished == null
+ && health != null
+ && health.waiting() == 0
+ && health.rejected() == 0
+ && health.dropped() == 0;
+ }
+ }
+
+ private final IVariables variables;
+ private final IHopMetadataProvider metadataProvider;
+ private final ILogChannel log;
+
+ /**
+ * Whether to publish what {@code hop doc} writes for each pipeline and
workflow as well, as
+ * DataHub documents: DataHub 1.7 or later, an earlier version refuses them.
+ */
+ @Setter private boolean includingDocumentation;
+
+ /** Whether the documentation lists the parameters, as {@code hop doc -ip}
does. */
+ @Setter private boolean includingParameters;
+
+ /** Whether the documentation has the text of the notes, as {@code hop doc
-in} does. */
+ @Setter private boolean includingNotes;
+
+ public DataHubPublisher(
+ IVariables variables, IHopMetadataProvider metadataProvider, ILogChannel
log) {
+ this.variables = variables;
+ this.metadataProvider = metadataProvider;
+ this.log = log;
+ }
+
+ /**
+ * Publishes every pipeline and workflow in {@code folder} and its
subfolders.
+ *
+ * @param sinkName the name of a lineage sink of type DataHub
+ * @param folder the folder, variables allowed
+ * @throws HopException when the sink can't be used, or the folder doesn't
exist
+ */
+ public Result publish(String sinkName, String folder) throws HopException {
+ DataHubLineageSink sink = startSink(sinkName);
+ int pipelines = 0;
+ int workflows = 0;
+ List<String> unreadable = new ArrayList<>();
+ List<LineageEvent> documents = new ArrayList<>();
+ String documentationNotPublished = null;
+ LineageSinkHealth health;
+ try {
+ FileObject root = folder(folder);
+ DocumentationWriter documentation = includingDocumentation ?
documentationWriter() : null;
+ String project = variables.getVariable(VAR_PROJECT_NAME);
+ for (FileObject file : hopFiles(root)) {
+ String filename = HopVfs.getFilename(file);
+ try {
+ String relativeFile = root.getName().getRelativeName(file.getName());
+ if (filename.endsWith(".hpl")) {
Review Comment:
Done: it checks the VFS extension, lower-cased
(`anUpperCaseExtensionIsAPipeline`).
##########
engine/src/main/java/org/apache/hop/lineage/LineageSinkRoutes.java:
##########
@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.lineage;
+
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Supplier;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.util.Utils;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.lineage.hub.LineageHub;
+import org.apache.hop.lineage.metadata.LineageSink;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.config.PipelineRunConfiguration;
+import org.apache.hop.pipeline.engine.IPipelineEngine;
+import org.apache.hop.pipeline.transform.ITransform;
+import org.apache.hop.workflow.WorkflowMeta;
+import org.apache.hop.workflow.action.IAction;
+import org.apache.hop.workflow.config.WorkflowRunConfiguration;
+import org.apache.hop.workflow.engine.IWorkflowEngine;
+
+/**
+ * The lineage sinks a run reports to, besides the ones configured with
variables: the sink its run
+ * configuration selects, and the sinks of the run that started it. Worked out
once per run, when it
+ * is first asked, and kept with the run. The run holds those sinks in the
{@link LineageHub} until
+ * it ends and calls {@link #release(IPipelineEngine)}.
+ */
+public final class LineageSinkRoutes {
+
+ /** Where a run keeps its sinks, in its extension data. */
+ static final String EXTENSION_DATA_KEY = "lineage.sinks";
+
+ /** Set in a run's extension data once it let go of its sinks. */
+ static final String RELEASED_KEY = "lineage.sinks.released";
+
+ private LineageSinkRoutes() {}
+
+ /**
+ * Set to {@code Y} on a pipeline run that runs a unit test, and on the runs
it starts (the
+ * testing plugin's {@code __UnitTest_Run__}).
+ */
+ public static final String VAR_RUN_UNIT_TEST = "__UnitTest_Run__";
+
+ /**
+ * The keys of the sinks a pipeline run reports to; empty when there are
none, or when the run
+ * runs a unit test. Such a run runs a copy of the pipeline, with data sets
in place of transforms
+ * and no file name: its lineage is not the pipeline's. Its sinks get only
the test's results, see
+ * {@link #ofUnitTest}.
+ */
+ public static Set<String> of(IPipelineEngine<PipelineMeta> pipeline) {
+ return ofRun(pipeline, LineageHub.getInstance());
+ }
+
+ static Set<String> ofRun(IPipelineEngine<PipelineMeta> pipeline, LineageHub
hub) {
+ // Held now, so they are released when the run ends, like the sinks of any
run
+ Set<String> sinks = of(pipeline, hub);
+ return isUnitTestRun(pipeline) ? Set.of() : sinks;
+ }
+
+ /** The keys of the sinks the results of the unit test a pipeline run runs
report to. */
+ public static Set<String> ofUnitTest(IPipelineEngine<PipelineMeta> pipeline)
{
+ return of(pipeline, LineageHub.getInstance());
+ }
+
+ static boolean isUnitTestRun(IPipelineEngine<PipelineMeta> pipeline) {
+ return pipeline != null &&
"Y".equalsIgnoreCase(pipeline.getVariable(VAR_RUN_UNIT_TEST));
+ }
+
+ /** The keys of the sinks a workflow run reports to; empty when there are
none. */
+ public static Set<String> of(IWorkflowEngine<WorkflowMeta> workflow) {
+ return of(workflow, LineageHub.getInstance());
+ }
+
+ /**
+ * Whether lineage is observed for a pipeline run: lineage is enabled with
variables, or the run
+ * reports to a sink of its own. Callers that do extra work only to fill in
an event check this.
+ */
+ public static boolean isEnabled(IPipelineEngine<PipelineMeta> pipeline) {
+ return LineageHub.getInstance().isEnabled() || !of(pipeline).isEmpty();
+ }
+
+ /** Whether lineage is observed for a workflow run; see {@link
#isEnabled(IPipelineEngine)}. */
+ public static boolean isEnabled(IWorkflowEngine<WorkflowMeta> workflow) {
+ return LineageHub.getInstance().isEnabled() || !of(workflow).isEmpty();
+ }
+
+ /**
+ * Whether lineage is observed for the run of a transform; see {@link
+ * #isEnabled(IPipelineEngine)}.
+ */
+ public static boolean isEnabled(ITransform transform) {
+ return isEnabled(transform == null ? null : transform.getPipeline());
+ }
+
+ static Set<String> of(IPipelineEngine<PipelineMeta> pipeline, LineageHub
hub) {
+ if (pipeline == null) {
+ return Set.of();
+ }
+ return cached(
+ pipeline.getExtensionDataMap(),
+ hub,
+ () -> {
+ Set<String> parents = parentSinks(pipeline, hub);
+ PipelineRunConfiguration runConfiguration =
pipeline.getPipelineRunConfiguration();
+ String own =
+ useSink(
+ runConfiguration == null ? null :
runConfiguration.getLineageSinkName(),
+ pipeline,
+ pipeline.getMetadataProvider(),
+ pipeline.getLogChannel(),
+ hub);
+ return withParents(parents, own, hub);
+ });
+ }
+
+ static Set<String> of(IWorkflowEngine<WorkflowMeta> workflow, LineageHub
hub) {
+ if (workflow == null) {
+ return Set.of();
+ }
+ return cached(
+ workflow.getExtensionDataMap(),
+ hub,
+ () -> {
+ Set<String> parents = parentSinks(workflow, hub);
+ WorkflowRunConfiguration runConfiguration =
workflow.getWorkflowRunConfiguration();
+ String own =
+ useSink(
+ runConfiguration == null ? null :
runConfiguration.getLineageSinkName(),
+ workflow,
+ workflow.getMetadataProvider(),
+ workflow.getLogChannel(),
+ hub);
+ return withParents(parents, own, hub);
+ });
+ }
+
+ /**
+ * Lets go of the sinks a pipeline run held, once it ended and emitted its
last event: a sink that
+ * was replaced by a newer version of itself can then be shut down.
+ */
+ public static void release(IPipelineEngine<PipelineMeta> pipeline) {
+ release(pipeline == null ? null : pipeline.getExtensionDataMap(),
LineageHub.getInstance());
+ }
+
+ /** Lets go of the sinks a workflow run held; see {@link
#release(IPipelineEngine)}. */
+ public static void release(IWorkflowEngine<WorkflowMeta> workflow) {
+ release(workflow == null ? null : workflow.getExtensionDataMap(),
LineageHub.getInstance());
+ }
+
+ static void release(Map<String, Object> extensionData, LineageHub hub) {
+ if (extensionData == null) {
+ return;
+ }
+ Set<String> sinks;
+ synchronized (extensionData) {
+ if (!(extensionData.get(EXTENSION_DATA_KEY) instanceof Set<?> known)
+ || extensionData.putIfAbsent(RELEASED_KEY, Boolean.TRUE) != null) {
+ return;
+ }
+ @SuppressWarnings("unchecked")
+ Set<String> held = (Set<String>) known;
+ sinks = held;
+ }
+ hub.release(sinks);
+ }
+
+ /**
+ * The sinks of the run that started this one and the run's own, each held
once for this run, as
+ * {@link #release} lets go of each once. The parent's are safe to hold now:
the parent holds them
+ * until it ends, after this run.
+ *
+ * @param own the run's own sink, which {@link LineageHub#useSink} already
held, or null
+ */
+ private static Set<String> withParents(Set<String> parentSinks, String own,
LineageHub hub) {
+ Set<String> inherited = new LinkedHashSet<>(parentSinks);
+ if (own != null) {
+ inherited.remove(own);
+ }
+ hub.hold(inherited);
+ Set<String> sinks = new LinkedHashSet<>(parentSinks);
+ if (own != null) {
+ sinks.add(own);
+ }
+ return sinks;
+ }
+
+ private static Set<String> parentSinks(IPipelineEngine<PipelineMeta>
pipeline, LineageHub hub) {
+ if (pipeline.getParentPipeline() != null) {
+ return of(pipeline.getParentPipeline(), hub);
+ }
+ if (pipeline.getParentWorkflow() != null) {
+ return of(pipeline.getParentWorkflow(), hub);
+ }
+ return of(actionWorkflow(pipeline.getParent()), hub);
+ }
+
+ private static Set<String> parentSinks(IWorkflowEngine<WorkflowMeta>
workflow, LineageHub hub) {
+ if (workflow.getParentPipeline() != null) {
+ return of(workflow.getParentPipeline(), hub);
+ }
+ if (workflow.getParentWorkflow() != null) {
+ return of(workflow.getParentWorkflow(), hub);
+ }
+ return of(actionWorkflow(workflow.getParent()), hub);
+ }
+
+ /**
+ * The workflow of an action that started a run without naming its workflow
as the parent, such as
+ * the pipelines Run pipeline unit tests starts.
+ */
+ private static IWorkflowEngine<WorkflowMeta> actionWorkflow(Object parent) {
+ return parent instanceof IAction action ? action.getParentWorkflow() :
null;
+ }
+
+ /**
+ * Uses and holds the sink of a run configuration. A sink that can't be
loaded or resolved is
+ * logged on the run and left out: lineage must not stop a run.
+ *
+ * @return the key of the sink, or null when there is none
+ */
+ private static String useSink(
+ String sinkName,
+ IVariables variables,
+ IHopMetadataProvider metadataProvider,
+ ILogChannel log,
+ LineageHub hub) {
+ String name = variables.resolve(sinkName);
+ if (Utils.isEmpty(name) || metadataProvider == null) {
+ return null;
+ }
+ try {
+ LineageSink sink =
metadataProvider.getSerializer(LineageSink.class).load(name);
+ if (sink == null) {
+ throw new IllegalArgumentException("lineage sink '" + name + "' does
not exist");
+ }
+ String key = hub.useSink(sink.toTarget(variables, metadataProvider));
+ String failure = hub.failure(key);
+ if (failure != null && log != null) {
+ log.logError(
+ "Lineage sink '"
+ + name
+ + "' failed to start ("
+ + failure
+ + "). It is tried again a minute after it failed; until then
this run's lineage"
+ + " doesn't reach it.");
+ }
+ return key;
+ } catch (Exception e) {
+ if (log != null) {
+ log.logError("Unable to use lineage sink '" + name + "', this run
doesn't report to it", e);
+ }
+ return null;
+ }
+ }
+
+ private static Set<String> cached(
+ Map<String, Object> extensionData, LineageHub hub, Supplier<Set<String>>
resolver) {
+ if (extensionData == null) {
+ return Collections.unmodifiableSet(resolver.get());
Review Comment:
Done: without an extension data map the sinks are released right after
they're resolved, so no hold is left behind
(`aRunWithoutExtensionDataHoldsNoSinks`).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]