This is an automated email from the ASF dual-hosted git repository.
mattcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 105de798ee Fixes #8646: Update and expand the SDK documentation (#8647)
105de798ee is described below
commit 105de798eec41464a55da21c5f436c08d22a5415
Author: Matt Casters <[email protected]>
AuthorDate: Mon Sep 28 12:16:21 2026 +0200
Fixes #8646: Update and expand the SDK documentation (#8647)
* Fixes #8646: Update and expand the SDK documentation
* Issue #8646 : Correct the SDK documentation samples from review
Align the samples with the current APIs, point the dialog sections at the
annotation-derived widgets guide, and replace codebase paths that do not exist.
---
docs/hop-dev-manual/modules/ROOT/nav.adoc | 25 +-
.../modules/ROOT/pages/plugin-development.adoc | 2 +
.../modules/ROOT/pages/sdk/hop-sdk.adoc | 279 +--------------------
.../modules/ROOT/pages/sdk/index.adoc | 148 ++++++++++-
.../modules/ROOT/pages/sdk/logging.adoc | 131 ++++++++++
.../modules/ROOT/pages/sdk/metadata.adoc | 271 ++++++++++++++++++++
.../modules/ROOT/pages/sdk/pipelines.adoc | 214 ++++++++++++++++
.../modules/ROOT/pages/sdk/plugins/actions.adoc | 124 +++++++++
.../modules/ROOT/pages/sdk/plugins/commands.adoc | 94 +++++++
.../ROOT/pages/sdk/plugins/configuration.adoc | 106 ++++++++
.../modules/ROOT/pages/sdk/plugins/databases.adoc | 116 +++++++++
.../ROOT/pages/sdk/plugins/execution-info.adoc | 131 ++++++++++
.../ROOT/pages/sdk/plugins/extension-points.adoc | 77 ++++++
.../modules/ROOT/pages/sdk/plugins/file-types.adoc | 107 ++++++++
.../modules/ROOT/pages/sdk/plugins/gui.adoc | 122 +++++++++
.../modules/ROOT/pages/sdk/plugins/index.adoc | 148 +++++++++++
.../modules/ROOT/pages/sdk/plugins/metadata.adoc | 110 ++++++++
.../ROOT/pages/sdk/plugins/perspectives.adoc | 133 ++++++++++
.../modules/ROOT/pages/sdk/plugins/servlets.adoc | 104 ++++++++
.../modules/ROOT/pages/sdk/plugins/transforms.adoc | 199 +++++++++++++++
.../ROOT/pages/sdk/plugins/value-types.adoc | 100 ++++++++
.../ROOT/pages/sdk/plugins/variable-resolvers.adoc | 98 ++++++++
.../modules/ROOT/pages/sdk/plugins/vfs.adoc | 81 ++++++
.../hop-dev-manual/modules/ROOT/pages/sdk/vfs.adoc | 215 ++++++++++++++++
.../modules/ROOT/pages/sdk/workflows.adoc | 197 +++++++++++++++
25 files changed, 3055 insertions(+), 277 deletions(-)
diff --git a/docs/hop-dev-manual/modules/ROOT/nav.adoc
b/docs/hop-dev-manual/modules/ROOT/nav.adoc
index dc29c9e4d3..3e39cf3cd3 100644
--- a/docs/hop-dev-manual/modules/ROOT/nav.adoc
+++ b/docs/hop-dev-manual/modules/ROOT/nav.adoc
@@ -67,8 +67,29 @@ under the License.
** xref:testing/ui-testing.adoc[UI tests with SWTBot]
* Embedding and interfaces
-** xref:sdk/index.adoc[SDK]
-*** xref:sdk/hop-sdk.adoc[The Hop SDK]
+** xref:sdk/index.adoc[The Hop SDK]
+*** xref:sdk/pipelines.adoc[Pipelines API]
+*** xref:sdk/workflows.adoc[Workflows API]
+*** xref:sdk/metadata.adoc[Metadata and Providers]
+*** xref:sdk/vfs.adoc[File Handling with HopVFS]
+*** xref:sdk/logging.adoc[Logging Subsystem]
+*** xref:sdk/plugins/index.adoc[Plugin Development Guide]
+**** xref:sdk/plugins/transforms.adoc[Transforms]
+**** xref:sdk/plugins/actions.adoc[Actions]
+**** xref:sdk/plugins/metadata.adoc[Metadata Elements]
+**** xref:sdk/plugins/databases.adoc[Database Dialects]
+**** xref:sdk/plugins/gui.adoc[GUI Plugins]
+**** xref:sdk/plugins/file-types.adoc[Hop File Types]
+**** xref:sdk/plugins/variable-resolvers.adoc[Variable Resolvers]
+**** xref:sdk/plugins/extension-points.adoc[Extension Points]
+**** xref:sdk/plugins/commands.adoc[Hop Commands]
+**** xref:sdk/plugins/configuration.adoc[Configuration Plugins]
+**** xref:sdk/plugins/perspectives.adoc[Perspectives]
+**** xref:sdk/plugins/vfs.adoc[VFS Plugins]
+**** xref:sdk/plugins/value-types.adoc[Value Types]
+**** xref:sdk/plugins/servlets.adoc[Hop Server Servlets]
+**** xref:sdk/plugins/execution-info.adoc[Execution Information Locations]
+*** xref:sdk/hop-sdk.adoc[Legacy SDK Guide]
** xref:hopweb/index.adoc[Hop Web]
*** xref:hopweb/developer-guide.adoc[Hop Web Developer Guide]
*** xref:hopweb/hopweb-antipatterns.adoc[Hop Web Antipatterns]
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/plugin-development.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/plugin-development.adoc
index 348ff029a2..bf3b2fadb8 100644
--- a/docs/hop-dev-manual/modules/ROOT/pages/plugin-development.adoc
+++ b/docs/hop-dev-manual/modules/ROOT/pages/plugin-development.adoc
@@ -20,6 +20,8 @@ under the License.
This page explains how to develop new plugins with references to make
development easy.
+For detailed tutorials and concrete codebase examples for 15 primary plugin
types (transforms, actions, metadata, databases, GUI, file types, variable
resolvers, extension points, commands, and more), see
xref:sdk/plugins/index.adoc[**Plugin Development Guide**].
+
For contributing **toolbar buttons** and other GUI-only extensions
(`@GuiPlugin`, `@GuiToolbarElement`, `@GuiToolbarElementFilter`), see
xref:gui-plugins-toolbars.adoc[GUI plugins and toolbars].
That page covers the TextComposite and TableView extension points (SQL
formatters, JSON tools, table export, and similar).
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/sdk/hop-sdk.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/hop-sdk.adoc
index 64b2259ea0..3f8193d351 100644
--- a/docs/hop-dev-manual/modules/ROOT/pages/sdk/hop-sdk.adoc
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/hop-sdk.adoc
@@ -14,275 +14,18 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
////
-:description: First, we need to initialize the Hop API. This means we load
configuration details, search for and load plugins and so on.
-:toc:
-
+:description: The Hop SDK documentation has been reorganized into modular
guides.
+[[Hop-Sdk-Redirect]]
= The Hop SDK
-== Initializing
-
-First, we need to initialize the Hop API.
-This means we load configuration details, search for and load plugins and so
on.
-
-You can do this with:
-
-[source,java]
-----
-HopEnvironment.init();
-----
-
-== Plugin folders
-
-If you need to load plugins from different folders you can set system property
`HOP_PLUGIN_BASE_FOLDERS`.
-The default value is `plugins`, the relative path to the installation folder
plugins folder.
-You can add your own plugin folders with commas (`,`) separating the values.
-
-== Hop Metadata Providers
-
-Shared metadata in Hop is handled by a HopMetadataProvider.
-When using the Hop GUI you store it in a project in the ```metadata/``` folder.
-In that case you can point to such a folder with class
```JsonMetadataProvider```.
-Note that you can serialize such a metadata collection into a single JSON
string using ```SerializableMetadataProvider```.
-This can be used to send metadata to remote servers and locations.
-
-When you have a provider you can ask it to give you a serializer with which
you can add and retrieve all sorts of metadata objects.
-
-== Variables
-
-If you want to work with variables in your pipelines and workflows it makes
sense to create a top level IVariables object.
-The easiest way to do this is with:
-
-[source,java]
-----
-IVariables variables = Variables.getADefaultVariableSpace();
-----
-
-This method also takes into account variables which are configured in
hop-config.json (if that file can be found);
-
-== Pipelines
-
-=== Loading pipeline metadata from a file
-
-You can get the pipeline metadata from a `.hpl` XML file using:
-
-[source,java]
-----
-PipelineMeta pipelineMeta = new PipelineMeta(
- "path-to-your-filename.hpl", // The filename
- metadataProvider, // See above
- true, // set internal variables
- variables // see above
-);
-----
-
-=== Loading pipeline metadata from an input stream
-
-[source,java]
-----
-PipelineMeta pipelineMeta = new PipelineMeta(
- inputStream, // The stream to load from
- metadataProvider, // See above
- true, // set internal variables
- variables // see above
-);
-----
-
-=== Construct pipeline metadata with the Hop API
-
-Obviously you can start with an empty pipeline and add the transforms, hops
you like:
-
-[source,java]
-----
-PipelineMeta pipelineMeta = new PipelineMeta();
-
-// Generate 1M empty rows
-//
-RowGeneratorMeta rowGeneratorMeta = new RowGeneratorMeta();
-rowGeneratorMeta.setRowLimit("1000000");
-
-TransformMeta rowGenerator = new TransformMeta("1M", rowGeneratorMeta);
-rowGenerator.setLocation(50, 50);
-pipelineMeta.addTransform(rowGenerator);
-
-// Just a dummy placeholder for testing
-//
-DummyMeta dummyMeta = new DummyMeta();
-TransformMeta dummy = new TransformMeta("Output", dummyMeta);
-dummy.setLocation(250, 50);
-pipelineMeta.addTransform(dummy);
-
-// Add a hop between both
-//
-PipelineHopMeta generatorDummyHop = new PipelineHopMeta(rowGenerator, dummy);
-pipelineMeta.addPipelineHop(generatorDummyHop);
-
-----
-
-=== Pipeline execution
-
-The way a pipeline is executed depends on the run configuration you specify.
-To make it easy to get the engine we created a factory for you:
-
-[source,java]
-----
-IPipelineEngine pipelineEngine = PipelineEngineFactory.createPipelineEngine(
- variables, // see above
- "local", // The name of the run configuration defined in the
metadata
- metadataProvider, // The metadata provider to resolve the run configuration
details
- pipelineMeta // The pipeline metadata
-);
-
-// We can now simply execute this engine...
-//
-pipelineEngine.execute();
-
-// This execution runs in the background but we can wait for it to finish:
-//
-pipelineEngine.waitUnitlFinished();
-
-// When it's done we can evalute the results:
-//
-Result result = pipelineEngine.getResult();
-
-----
-
-=== Injecting data into a pipeline
-
-You can only inject data into a `LocalPipelineEngine`.
-Do so using the `addRowProducer`.
-Call this method after your run `prepareExecution()` so that the row producer
can be attached to the correct transform copy.
-After starting the execution of the pipeline you can then use the
`RowProducer` to put rows into the pipeline using `putRow()`.
-Make sure to call `setFinished()` when you're done feeding rows into the
pipeline.
-
-=== Retrieving rows from a pipeline
-
-This again is only supported on the local pipeline engine
`LocalPipelineEngine`.
-After `prepareExecution()` you can add row listeners to the various transforms:
-
-[source,java]
-----
-ITransform transform = localPipeline.getTransform("transform-name", 0);
-transform.addRowListener(new RowAdapter() {
- void rowWrittenEvent( IRowMeta rowMeta, Object[] row ) throws
HopTransformException {
- // A row was written during execution
- }
-});
-----
-
-== Workflows
-
-=== Loading workflow metadata from a file
-
-You can get the workflow metadata from a `.hwf` XML file using:
-
-[source,java]
-----
-WorkflowMeta workflowMeta = new WorkflowMeta(
- variables, // see above
- "path-to-your-filename.hwf", // The filename
- metadataProvider // See above
-);
-----
-
-=== Loading workflow metadata from an input stream
-
-[source,java]
-----
-WorkflowMeta workflowMeta = new WorkflowMeta(
- inputStream, // the inputstream to read the metadata from
- metadataProvider, // See above
- variables // see above
-);
-----
-
-=== Construct workflow metadata with the Hop API
-
-You typically start with an empty workflow and then add the actions and hops
you want:
-
-[source,java]
-----
-WorkflowMeta workflowMeta = new WorkflowMeta();
-
-// Add the Start action
-//
-ActionStart actionStart = new ActionStart("Start");
-ActionMeta startMeta = new ActionMeta(actionStart);
-startMeta.setLocation(50, 50);
-workflowMeta.addAction(startMeta);
-
-// Just a dummy placeholder for testing
-//
-ActionDummy actionDummy = new ActionDummy("Dummy");
-ActionMeta dummyMeta = new ActionMeta(dummyMeta);
-dummyMeta.setLocation(250, 50);
-workflowMeta.addAction(dummyMeta);
-
-// Add a hop between both
-//
-WorkflowHopMeta startDummyHop = new WorkflowHopMeta(startMeta, dummyMeta);
-workflowMeta.addWorkflowHop(generatorDummyHop);
-
-----
-
-=== Workflow execution
-
-Workflow engines are also plugins.
-Which plugin is used to execute your workflow metadata is specified in a
xref:manual::workflow/workflow-run-configurations/workflow-run-configurations.adoc[Workflow
Run Configuration].
-
-To make it easy to get the engine we created a factory for you:
-
-[source,java]
-----
-IWorkflowEngine workflowEngine = WorkflowEngineFactory.createWorkflowEngine(
- variables, // see above
- "local", // The name of the run configuration defined in the
metadata
- metadataProvider, // The metadata provider to resolve the run configuration
details
- workflowMeta, // The workflow metadata
- parentLogging // The parent logging object
-);
-
-// We can now execute this engine...
-// This execution does not run in the background.
-// When you get the result, the execution has completed.
-//
-Result result = workflowEngine.startExecution();
-
-----
-
-## Logging
-
-### Logging Registry
-
-Everything that executes something worth our time is registering its own Log
Channel in the hop `LoggingRegistry`.
-Every log channel gets its own unique ID with which we can see where a log
line came from.
-You can access the Logging Registry using `LoggingRegistry.getInstance()`.
-It contains the execution hierarchy of Hop work.
-For example if you have the log channel ID of a parent you can see all its
children with `getLogChannelChildren()` which will give you all the IDs of the
child log channels.
-What we get is in effect the execution lineage.
-
-## Log lines
-
-Whenever a log channel logs something using `logBasic()` or other logging
variants, that text along with some basic information is kept in the Hop Log
Store.
-You can get that one with `HopLogStore.getInstance()`.
-The logging lines are kept as logging events or class `HopLoggingEvent` in a
logging buffer `LoggingBuffer` which you can access using
`HopLogStore.getInstance().getAppender()`.
-
-If you want to grab the logging output of a pipeline, transform, workflow,
action, ... you need to start with the ID of the log channel associated with
that runtime object.
-Usually you can do `getLogChannel()` and then get the ID or the shortcut:
`getLogChannelId()`.
-
-If you want to get detailed information about every logging event you can ask
for a list with:
-
-[source,java]
-----
-int lastNr = HopLogStore.getLastBufferLineNr();
-List<HopLoggingEvent> events = getLogBufferFromTo( logChannelId, false, 0,
lastNr);
-----
-
-The details allow you to see which line was an error, what the timestamp was,
to which executable it belonged and so on.
-
-If you just want to see the flattened logging text you can ask the appender
for the information:
+The Hop SDK documentation has been updated and organized into dedicated,
modular guides:
-[source,java]
-----
-StringBuffer loggingText = HopLogStore.getAppender().getBuffer(logChannelId);
-----
+* xref:sdk/index.adoc[**The Hop Java SDK Overview**] — Maven coordinates,
environment initialization, plugin base folders, and variable management.
+* xref:sdk/pipelines.adoc[**Pipelines API**] — Loading, dynamically
assembling, executing pipelines, row injection, and listening to streams.
+* xref:sdk/workflows.adoc[**Workflows API**] — Loading, assembling, and
executing workflows.
+* xref:sdk/metadata.adoc[**Metadata and Providers**] — Metadata architecture,
providers (`JsonMetadataProvider`, `MemoryMetadataProvider`,
`MultiMetadataProvider`, `SerializableMetadataProvider`), and GUI widgets
(`MetaSelectionLine`).
+* xref:sdk/vfs.adoc[**File Handling with HopVFS**] — Unified file access via
Apache Commons VFS, avoiding `java.io.File`, closing resources, and streaming.
+* xref:sdk/logging.adoc[**Logging Subsystem**] — Logging registry, log
channels, buffers, and capturing log output programmatically.
+* xref:sdk/plugins/index.adoc[**Plugin Development Guide**] — Tutorials and
reference implementations for all 15 Hop plugin types.
+Please update your bookmarks to use xref:sdk/index.adoc[The Hop Java SDK
Overview].
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/sdk/index.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/index.adoc
index 32c05aceda..fc3138d545 100644
--- a/docs/hop-dev-manual/modules/ROOT/pages/sdk/index.adoc
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/index.adoc
@@ -14,15 +14,149 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
////
-:description: Driving Hop from your own Java code: initializing the API,
loading metadata, and running pipelines and workflows.
+:description: Driving Hop from your own Java code: initializing the API,
configuring plugin folders, managing variables, loading metadata, and running
pipelines and workflows.
[[Sdk-Index]]
-= SDK
+= The Hop Java SDK
-Hop is a set of Java libraries before it is a GUI or a command line tool, so a
pipeline or workflow can be executed from your own application.
+Apache Hop is a set of Java libraries before it is a GUI or command-line tool.
+The Hop Java Software Development Kit (SDK) allows you to embed Hop's data
orchestration engine directly into your own Java applications, execute
pipelines and workflows programmatically, manipulate metadata, stream rows
dynamically, and build custom plugins.
-* xref:sdk/hop-sdk.adoc[The Hop SDK] — initializing the API, plugin folders,
metadata providers, variables, and running pipelines and workflows.
+== Maven Dependencies
-Related:
+To embed Hop in your Java application, include the core and engine artifacts
in your Maven `pom.xml`:
-* xref:{page-component-version}@manual::hop-server/json-api.adoc[The Hop
Server JSON API] — the same capability over HTTP, when embedding the libraries
is not an option.
-* xref:architecture/execution-model.adoc[How a pipeline and a workflow run] —
what the SDK is driving underneath.
+[source,xml]
+----
+<dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-core</artifactId>
+ <version>${hop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-engine</artifactId>
+ <version>${hop.version}</version>
+ </dependency>
+ </dependencies>
+</dependencyManagement>
+
+<dependencies>
+ <dependency>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-engine</artifactId>
+ </dependency>
+</dependencies>
+----
+
+If your application interacts with the Hop GUI or needs to instantiate UI
dialogs and widgets, also include `hop-ui`:
+
+[source,xml]
+----
+<dependency>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-ui</artifactId>
+ <version>${hop.version}</version>
+</dependency>
+----
+
+== Initializing the Hop Runtime
+
+Before calling any Hop API methods that resolve plugins, load metadata, or
execute pipelines, you must initialize the Hop environment.
+Hop provides three initialization tiers depending on your runtime context:
+
+=== 1. HopEnvironment (Standard Engine Runtime)
+
+This is the standard entry point for embedding Hop when you need to load
metadata, run pipelines, and execute workflows:
+
+[source,java]
+----
+import org.apache.hop.core.HopEnvironment;
+
+// Initializes logging, registry, metadata providers, and all engine plugins
+HopEnvironment.init();
+----
+
+Calling `HopEnvironment.init()` loads core system configurations, populates
the plugin registry, and prepares runtime factories.
+
+=== 2. HopClientEnvironment (Lightweight Core Runtime)
+
+If your application only needs basic Hop utilities (such as variable
management, password encryption, basic VFS file handling, or logging) and will
not execute pipelines or workflows, use `HopClientEnvironment`:
+
+[source,java]
+----
+import org.apache.hop.core.HopClientEnvironment;
+
+// Initializes core plugins (VFS, password encoders, logging, value types)
+HopClientEnvironment.init();
+----
+
+=== 3. HopGuiEnvironment (GUI & Hop Web Runtime)
+
+When running the Hop GUI or Hop Web, `HopGuiEnvironment` initializes
UI-specific plugins (perspectives, dialogs, toolbar elements):
+
+[source,java]
+----
+import org.apache.hop.ui.hopgui.HopGuiEnvironment;
+
+HopGuiEnvironment.init();
+----
+
+== Configuring Plugin Folders
+
+Hop discovers plugins dynamically using Jandex annotation indexes.
+By default, Hop scans jars on the application classpath and looks for external
plugins in the relative `plugins` folder.
+
+If your application bundles plugins in a different directory or across
multiple directories, set the `HOP_PLUGIN_BASE_FOLDERS` variable or system
property before initializing:
+
+[source,java]
+----
+// Configure comma-separated plugin root directories
+System.setProperty("HOP_PLUGIN_BASE_FOLDERS",
"/opt/hop/plugins,/custom/hop-plugins");
+
+HopEnvironment.init();
+----
+
+== Working with Variables
+
+Variables parameterize pipelines, workflows, database connections, and file
paths.
+When using the SDK, always create a top-level `IVariables` instance to manage
variable resolution:
+
+[source,java]
+----
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.variables.Variables;
+
+// Create a default variable space populated with system and hop-config.json
properties
+IVariables variables = Variables.getADefaultVariableSpace();
+
+// Set custom runtime variables
+variables.setVariable("INPUT_DIR", "/data/incoming");
+variables.setVariable("BATCH_ID", "2026-09-27");
+
+// Resolve variable expressions
+String resolvedPath = variables.resolve("${INPUT_DIR}/records.csv");
+----
+
+Variable spaces can be chained hierarchically: a child variable space inherits
variables from its parent, and local modifications do not overwrite parent
values.
+
+== SDK Documentation Sections
+
+Explore the detailed guides below to learn how to work with specific areas of
the Hop SDK:
+
+* xref:sdk/pipelines.adoc[Pipelines API] — Loading, constructing, executing
pipelines, injecting rows, and listening to output streams.
+* xref:sdk/workflows.adoc[Workflows API] — Loading, constructing, executing
workflows, and evaluating task results.
+* xref:sdk/metadata.adoc[Metadata and Providers] — Working with metadata
elements, providers (`JsonMetadataProvider`, `MemoryMetadataProvider`,
`MultiMetadataProvider`), serializers, and GUI widgets (`MetaSelectionLine`).
+* xref:sdk/vfs.adoc[File Handling with HopVFS] — Accessing files via Apache
Commons VFS, avoiding `java.io.File`, closing resources, and streaming.
+* xref:sdk/logging.adoc[Logging Subsystem] — Channel hierarchies, the log
store, capturing execution logs, and tracing execution lineage.
+* xref:sdk/plugins/index.adoc[Plugin Development Guide] — Complete reference
and tutorials for developing all 15 Hop plugin types with codebase examples.
+
+== Related Topics
+
+* xref:{page-component-version}@manual::hop-server/json-api.adoc[Hop Server
JSON API] — Driving Hop pipelines and workflows over HTTP without embedding
Java libraries.
+* xref:architecture/execution-model.adoc[Execution Model] — Architectural deep
dive into how pipelines and workflows execute underneath the SDK.
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/sdk/logging.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/logging.adoc
new file mode 100644
index 0000000000..440a42bbec
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/logging.adoc
@@ -0,0 +1,131 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: Understanding Hop's hierarchical logging registry, capturing log
events, and extracting execution logs programmatically.
+[[Sdk-Logging]]
+= Logging Subsystem
+
+Every executable component in Apache Hop—pipelines, workflows, transforms,
actions, and engines—registers a unique **Log Channel** in a centralized
hierarchical registry.
+When embedding Hop, you can tap into this logging architecture to monitor
executions in real time, extract structured execution events, or redirect log
streams into your application's logging framework.
+
+== The Logging Registry
+
+The `LoggingRegistry` tracks all active and completed logging channels across
the runtime.
+Each log channel has a globally unique identifier (UUID string) and maintains
parent-child relationships with other channels.
+
+=== Inspecting the Hierarchy & Execution Lineage
+
+[source,java]
+----
+import org.apache.hop.core.logging.LoggingRegistry;
+
+LoggingRegistry registry = LoggingRegistry.getInstance();
+
+// Retrieve the channel ID of a running pipeline or workflow:
+String parentChannelId = pipelineEngine.getLogChannelId();
+
+// Discover all child log channels (e.g., all individual transform copies):
+List<String> childChannelIds = registry.getLogChannelChildren(parentChannelId);
+
+for (String childId : childChannelIds) {
+ String subject = registry.getLoggingObject(childId).getObjectName();
+ System.out.println("Child component: " + subject + " (channel: " + childId +
")");
+}
+----
+
+Because every transform copy and sub-pipeline registers its parent ID upon
creation, traversing child channel IDs yields the exact execution lineage tree
of the execution.
+
+== Hop Log Store & In-Memory Buffers
+
+All log statements emitted via `ILogChannel` (such as `logBasic()`,
`logError()`, or `logDebug()`) are forwarded to the centralized `HopLogStore`.
+The store maintains log events in an in-memory ring buffer (`LoggingBuffer`).
+
+=== Extracting Flattened Log Text
+
+To retrieve all text output logged by a pipeline, workflow, or specific
transform:
+
+[source,java]
+----
+import org.apache.hop.core.logging.HopLogStore;
+
+String channelId = pipelineEngine.getLogChannelId();
+
+// Extract the text buffer for the pipeline and optionally its children
+StringBuffer textBuffer = HopLogStore.getAppender().getBuffer(channelId, true);
+
+System.out.println("=== Execution Log ===");
+System.out.println(textBuffer.toString());
+----
+
+=== Extracting Structured Log Events
+
+For structured monitoring or streaming to external SIEM/observability systems,
query `HopLoggingEvent` objects directly:
+
+[source,java]
+----
+import org.apache.hop.core.logging.HopLoggingEvent;
+import org.apache.hop.core.logging.HopLogStore;
+import org.apache.hop.core.logging.LogLevel;
+
+String channelId = pipelineEngine.getLogChannelId();
+int startLine = 0;
+int lastLine = HopLogStore.getLastBufferLineNr();
+
+// Retrieve structured events for the channel hierarchy:
+List<HopLoggingEvent> events = HopLogStore.getLogBufferFromTo(
+ channelId,
+ true, // include children
+ startLine,
+ lastLine
+);
+
+for (HopLoggingEvent event : events) {
+ long timestamp = event.getTimeStamp();
+ LogLevel level = event.getLevel();
+ String message = event.getMessage().toString();
+
+ System.out.printf("[%tF %<tT] [%s] %s%n", timestamp, level.getCode(),
message);
+}
+----
+
+== Logging from Custom Code
+
+When writing custom transforms, actions, or embedding logic, always log
through an `ILogChannel` rather than calling `System.out.println`:
+
+[source,java]
+----
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.logging.LogChannel;
+
+// Create a named log channel:
+ILogChannel log = new LogChannel("MyApplication");
+
+// Emit messages at appropriate log levels:
+log.logBasic("Starting data ingestion process...");
+log.logDetailed("Connecting to remote endpoint: https://api.example.com";);
+log.logDebug("Payload size: 4096 bytes");
+
+try {
+ // perform task
+} catch (Exception e) {
+ log.logError("Failed to ingest records", e);
+}
+----
+
+== Related Topics
+
+* xref:architecture/logging-and-debugging.adoc[Logging and Debugging
Architecture] — Deep dive into the Hop logging subsystem internals.
+* xref:sdk/plugins/execution-info.adoc[Execution Information Locations] —
Persisting logs, metrics, and state to databases and remote storage.
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/sdk/metadata.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/metadata.adoc
new file mode 100644
index 0000000000..f04867fe5d
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/metadata.adoc
@@ -0,0 +1,271 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: Working with Hop metadata elements, metadata providers (Json,
Memory, Multi, Serializable), serializers, JSON persistence, and GUI
integration with MetaSelectionLine.
+[[Sdk-Metadata]]
+= Metadata and Providers
+
+Metadata objects in Apache Hop define shared, reusable configurations:
relational database connections, pipeline run configurations, execution
information locations, dataset definitions, variable resolvers, and dozens of
other components.
+Decoupling metadata from pipelines and workflows allows workflows to run
across development, testing, and production environments simply by pointing to
different metadata providers.
+
+== The Metadata Model
+
+A metadata element is a Plain Old Java Object (POJO) implementing
`IHopMetadata` (typically by extending `HopMetadataBase`) and annotated with
`@HopMetadata` and `@HopMetadataProperty`.
+
+=== Defining a Metadata Object
+
+[source,java]
+----
+import org.apache.hop.metadata.api.HopMetadata;
+import org.apache.hop.metadata.api.HopMetadataBase;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.IHopMetadata;
+import lombok.Getter;
+import lombok.Setter;
+
+@HopMetadata(
+ key = "custom-connection", // Kebab-case directory
identifier
+ legacyKeys = {"CustomConnection"}, // Backward compatibility keys
+ name = "Custom Connection",
+ description = "Defines a connection to an external API",
+ image = "custom-connection.svg",
+ category = "Connection"
+)
+@Getter
+@Setter
+public class CustomConnection extends HopMetadataBase implements IHopMetadata {
+
+ @HopMetadataProperty
+ private String hostname;
+
+ @HopMetadataProperty
+ private int port = 8080;
+
+ @HopMetadataProperty(password = true)
+ private String apiKey;
+
+ public CustomConnection() {
+ super();
+ }
+
+ public CustomConnection(String name, String hostname, int port, String
apiKey) {
+ this.name = name;
+ this.hostname = hostname;
+ this.port = port;
+ this.apiKey = apiKey;
+ }
+}
+----
+
+=== Key Naming Conventions
+* **`key`**: Always use lower-case kebab-case (`mail-server-connection`,
`pipeline-run-configuration`). This determines the folder name under
`metadata/<key>/` in the project.
+* **`legacyKeys`**: When renaming a key, always retain the old keys in
`legacyKeys` so existing installations do not silently lose their saved
metadata.
+
+== Providers and Serializers
+
+Hop manages metadata persistence through the `IHopMetadataProvider` interface.
+A provider acts as a factory for `IHopMetadataSerializer<T>`, which provides
type-safe CRUD operations.
+
+=== Performing CRUD Operations
+
+[source,java]
+----
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.api.IHopMetadataSerializer;
+
+IHopMetadataProvider provider = ...;
+
+// Obtain a serializer for your metadata class
+IHopMetadataSerializer<CustomConnection> serializer =
+ provider.getSerializer(CustomConnection.class);
+
+// 1. Create and Save
+CustomConnection conn = new CustomConnection("AnalyticsServer",
"analytics.internal", 9000, "secret123");
+serializer.save(conn);
+
+// 2. Check Existence
+boolean exists = serializer.exists("AnalyticsServer");
+
+// 3. Load by Name
+CustomConnection loaded = serializer.load("AnalyticsServer");
+
+// 4. List All Names
+List<String> names = serializer.listObjectNames();
+
+// 5. Load All Objects
+List<CustomConnection> allConnections = serializer.loadAll();
+
+// 6. Delete
+serializer.delete("AnalyticsServer");
+----
+
+== The Four Metadata Providers
+
+Hop ships four core metadata provider implementations suited for different
runtime environments:
+
+=== 1. JsonMetadataProvider (Filesystem / Project)
+The standard provider used by the Hop GUI and Hop CLI.
+It persists each metadata element as an individual JSON file under
`metadata/<key>/<name>.json`:
+
+[source,java]
+----
+import org.apache.hop.metadata.serializer.json.JsonMetadataProvider;
+
+// Point to the root metadata folder of a project
+String metadataRootFolder = "/home/dev/projects/sales-etl/metadata";
+JsonMetadataProvider jsonProvider = new JsonMetadataProvider(
+ twoWayPasswordEncoder,
+ metadataRootFolder,
+ variables
+);
+----
+
+=== 2. MemoryMetadataProvider (In-Memory / Unit Tests)
+Stores metadata entirely in memory in Java hash maps without touching disk.
+Ideal for:
+* Unit and integration testing where test isolation is critical.
+* Ephemeral serverless executions where disk writes are restricted.
+* Dynamically injecting temporary connections during runtime.
+
+[source,java]
+----
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+
+MemoryMetadataProvider memoryProvider = new MemoryMetadataProvider(
+ twoWayPasswordEncoder,
+ variables
+);
+
+// Populate with mock connections
+IHopMetadataSerializer<CustomConnection> serializer =
+ memoryProvider.getSerializer(CustomConnection.class);
+serializer.save(new CustomConnection("TestConn", "localhost", 8080,
"test-key"));
+----
+
+=== 3. MultiMetadataProvider (Hierarchical / Layered)
+Delegates across multiple metadata providers.
+Hop uses `MultiMetadataProvider` to combine project-specific metadata with
global shared metadata.
+Lookups walk the list from the end, so the **last** provider that has an
object wins.
+A save that does not name a provider writes to that same last provider.
+Put parent or shared providers first and the project provider last.
+A provider added after the project overrides the project on read and becomes
the save target.
+
+[source,java]
+----
+import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
+
+List<IHopMetadataProvider> providers = List.of(
+ sharedGlobalProvider, // Earlier: used when no later provider has this name
+ projectJsonProvider // Last: wins on read and receives default saves
+);
+
+MultiMetadataProvider multiProvider = new MultiMetadataProvider(
+ twoWayPasswordEncoder,
+ providers,
+ variables
+);
+
+// Load walks backwards. The project provider is checked before the shared one.
+CustomConnection conn =
multiProvider.getSerializer(CustomConnection.class).load("CentralDb");
+
+// Save with no metadata provider name on the object writes to the last
provider.
+multiProvider.getSerializer(CustomConnection.class).save(newConnection);
+----
+
+=== 4. SerializableMetadataProvider (Remote Serialization)
+Bundles all metadata from a provider into a single serializable JSON string.
+This is used by Hop clients to bundle and transmit project metadata across
network boundaries to Hop Server or remote containerized runners:
+
+[source,java]
+----
+import org.apache.hop.core.metadata.SerializableMetadataProvider;
+
+// Serialize all metadata in a provider into a JSON string:
+SerializableMetadataProvider serializableProvider =
+ new SerializableMetadataProvider(multiProvider);
+String metadataJson = serializableProvider.toJson();
+
+// On the remote engine, reconstitute the metadata provider:
+SerializableMetadataProvider remoteProvider =
+ new SerializableMetadataProvider(metadataJson);
+----
+
+== GUI Integration and Dialogs
+
+When writing transform, action, or metadata dialogs, metadata selection is a
common requirement.
+
+=== 1. MetaSelectionLine in Hand-Coded SWT Dialogs
+`MetaSelectionLine<T>` is a composite widget that renders a combo box listing
all available metadata objects of type `T` from the current
`IHopMetadataProvider`. It includes buttons to create a **New** instance,
**Edit** the selected instance, or **Refresh** the list:
+
+[source,java]
+----
+import org.apache.hop.ui.core.widget.MetaSelectionLine;
+
+MetaSelectionLine<CustomConnection> wConnection = new MetaSelectionLine<>(
+ variables,
+ metadataProvider,
+ CustomConnection.class,
+ shell,
+ SWT.NONE,
+ "Custom Connection",
+ "Select the external API connection to use"
+);
+
+// Populate the combo with available objects from the provider:
+wConnection.fillItems();
+
+// Select an active item:
+wConnection.setText("AnalyticsServer");
+
+// Retrieve selected item:
+String selectedName = wConnection.getText();
+----
+
+=== 2. Declarative Selection with GuiCompositeWidgets
+In modern Hop dialogs, avoid manual widget layouts.
+Annotate your Meta class with `@GuiWidgetElement(type =
GuiElementType.METADATA)`:
+
+[source,java]
+----
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.gui.plugin.GuiWidgetGroupType;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+
+@GuiPlugin
+public class MyTransformMeta extends BaseTransformMeta<MyTransform,
MyTransformData> {
+
+ public static final String GUI_PLUGIN_ELEMENT_PARENT_ID =
"MyTransformDialog-Parent";
+
+ @GuiWidgetElement(
+ id = "connection",
+ order = "0100",
+ type = GuiElementType.METADATA,
+ metadata = CustomConnection.class,
+ label = "i18n::MyTransform.Connection.Label",
+ toolTip = "i18n::MyTransform.Connection.Tooltip",
+ parentId = GUI_PLUGIN_ELEMENT_PARENT_ID,
+ groupType = GuiWidgetGroupType.BOXES,
+ group = "Connection Settings"
+ )
+ @HopMetadataProperty(key = "connection")
+ private String connectionName;
+}
+----
+
+When `GuiCompositeWidgets.addScrolledComposite()` renders this group, it
instantiates and wires a `MetaSelectionLine<CustomConnection>`.
+The dialog layout around that call is
xref:annotation-derived-widgets.adoc[Annotation derived widgets].
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/sdk/pipelines.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/pipelines.adoc
new file mode 100644
index 0000000000..f3bfda955b
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/pipelines.adoc
@@ -0,0 +1,214 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: Loading, dynamically constructing, executing pipelines
programmatically with the Hop SDK, injecting rows, and consuming real-time row
streams.
+[[Sdk-Pipelines]]
+= Pipelines API
+
+Pipelines are the core data processing units in Hop.
+With the Hop SDK, you can load pipelines from existing `.hpl` definition
files, construct pipeline topologies entirely in memory using the Java API,
execute pipelines using configurable engines, inject rows from external
sources, and stream output rows back into your application.
+
+== Loading Pipeline Metadata
+
+Pipeline metadata describes the structure, transforms, and hops of a pipeline.
+You can load metadata from either a file path or an arbitrary `InputStream`.
+
+=== Loading from a File
+
+[source,java]
+----
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.PipelineMeta;
+
+IVariables variables = Variables.getADefaultVariableSpace();
+
+// Path can be a local path or any HopVFS URI (e.g., s3://bucket/pipeline.hpl)
+String filename = "/path/to/my-pipeline.hpl";
+
+PipelineMeta pipelineMeta = new PipelineMeta(
+ filename,
+ metadataProvider, // IHopMetadataProvider (e.g. JsonMetadataProvider)
+ variables // parent variable space
+);
+----
+
+=== Loading from an InputStream
+
+If your application retrieves pipelines from a database, a repository, or a
web service, load directly from an `InputStream`:
+
+[source,java]
+----
+try (InputStream inputStream =
HopVfs.getInputStream("s3://bucket/my-pipeline.hpl", variables)) {
+ PipelineMeta pipelineMeta = new PipelineMeta(
+ inputStream,
+ metadataProvider,
+ variables
+ );
+}
+----
+
+== Constructing Pipelines Programmatically
+
+You can construct an entire pipeline from scratch using Java code without any
`.hpl` file:
+
+[source,java]
+----
+import org.apache.hop.pipeline.PipelineHopMeta;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.dummy.DummyMeta;
+import org.apache.hop.pipeline.transforms.rowgenerator.RowGeneratorMeta;
+
+PipelineMeta pipelineMeta = new PipelineMeta();
+pipelineMeta.setName("DynamicPipeline");
+
+// 1. Create a Row Generator transform
+RowGeneratorMeta rowGeneratorMeta = new RowGeneratorMeta();
+rowGeneratorMeta.setRowLimit("100000");
+
+TransformMeta rowGenerator = new TransformMeta("GenerateRows",
rowGeneratorMeta);
+rowGenerator.setLocation(100, 100);
+pipelineMeta.addTransform(rowGenerator);
+
+// 2. Create a Dummy transform (placeholder / pass-through)
+DummyMeta dummyMeta = new DummyMeta();
+TransformMeta dummy = new TransformMeta("CollectRows", dummyMeta);
+dummy.setLocation(300, 100);
+pipelineMeta.addTransform(dummy);
+
+// 3. Add a hop connecting both transforms
+PipelineHopMeta hop = new PipelineHopMeta(rowGenerator, dummy);
+pipelineMeta.addPipelineHop(hop);
+----
+
+== Executing Pipelines
+
+Pipelines are executed through the `IPipelineEngine` interface.
+Engines are plugins: the `PipelineEngineFactory` inspects the specified
**Pipeline Run Configuration** metadata object to instantiate the appropriate
engine (such as `LocalPipelineEngine`, `RemotePipelineEngine`, or
`BeamPipelineEngine`).
+
+[source,java]
+----
+import org.apache.hop.core.Result;
+import org.apache.hop.pipeline.engine.IPipelineEngine;
+import org.apache.hop.pipeline.engine.PipelineEngineFactory;
+
+// Create the pipeline engine using the "local" run configuration
+IPipelineEngine<PipelineMeta> pipelineEngine =
PipelineEngineFactory.createPipelineEngine(
+ variables,
+ "local", // Name of the Pipeline Run Configuration metadata
+ metadataProvider, // Metadata provider holding the run configuration
+ pipelineMeta
+);
+
+// Execute the pipeline (starts execution in background threads)
+pipelineEngine.execute();
+
+// Wait for all transforms to finish processing
+pipelineEngine.waitUntilFinished();
+
+// Evaluate the final execution result
+Result result = pipelineEngine.getResult();
+System.out.println("Pipeline completed. Errors: " + result.getNrErrors());
+System.out.println("Lines input: " + result.getNrLinesInput());
+System.out.println("Lines written: " + result.getNrLinesWritten());
+----
+
+== Injecting Data into a Running Pipeline
+
+When embedding Hop, you often want to stream rows generated by your host
application directly into a pipeline.
+Data injection is supported on `LocalPipelineEngine` using `RowProducer`:
+
+[source,java]
+----
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.pipeline.RowProducer;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+
+LocalPipelineEngine localPipeline = (LocalPipelineEngine) pipelineEngine;
+
+// Prepare execution before attaching row producers
+localPipeline.prepareExecution();
+
+// Attach a RowProducer to the target transform copy (transform name, copy
index 0)
+RowProducer rowProducer = localPipeline.addRowProducer("CollectRows", 0);
+
+// Start the transform threads
+localPipeline.startThreads();
+
+// Define row layout
+IRowMeta rowMeta = new RowMeta();
+rowMeta.addValueMeta(new ValueMetaInteger("id"));
+rowMeta.addValueMeta(new ValueMetaString("customer_name"));
+
+// Push rows into the transform input buffer
+rowProducer.putRow(rowMeta, new Object[] {1L, "Alice"});
+rowProducer.putRow(rowMeta, new Object[] {2L, "Bob"});
+
+// Signal that all rows have been fed into the transform
+rowProducer.finished();
+
+// Await pipeline termination
+localPipeline.waitUntilFinished();
+----
+
+== Consuming Output Rows from a Pipeline
+
+You can capture rows emitted by any transform during execution by registering
an `IRowListener` via `RowAdapter`:
+
+[source,java]
+----
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.pipeline.transform.ITransform;
+import org.apache.hop.pipeline.transform.RowAdapter;
+
+// Prepare execution
+localPipeline.prepareExecution();
+
+// Obtain reference to the transform instance
+ITransform outputTransform = localPipeline.getTransform("CollectRows", 0);
+
+// Register a row listener to receive emitted rows
+outputTransform.addRowListener(new RowAdapter() {
+ @Override
+ public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) throws
HopTransformException {
+ String id = rowMeta.getString(row, 0);
+ String name = rowMeta.getString(row, 1);
+ System.out.println("Captured output row: id=" + id + ", name=" + name);
+ }
+});
+
+// Start threads and await finish
+localPipeline.startThreads();
+localPipeline.waitUntilFinished();
+----
+
+== Stopping Pipelines Programmatically
+
+To stop or abort a running pipeline cleanly from external monitoring or
lifecycle triggers:
+
+[source,java]
+----
+if (pipelineEngine.isRunning()) {
+ // Gracefully stop all transforms
+ pipelineEngine.stopAll();
+}
+----
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/actions.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/actions.adoc
new file mode 100644
index 0000000000..4b952f0458
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/actions.adoc
@@ -0,0 +1,124 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: Building custom workflow actions in Apache Hop: the two-class
pattern, execution lifecycle, Result passing, and GUI dialog integration.
+[[Sdk-Plugins-Actions]]
+= Action Plugins
+
+Actions are the task-level building blocks of Hop workflows.
+Unlike transforms (which continuously process streams of rows in parallel),
actions execute sequentially or conditionally as individual tasks: verifying
files, executing scripts, checking database connectivity, or sending
notifications.
+
+== The Two-Class Pattern
+
+An action plugin requires only two classes:
+1. `ActionFoo`: Extends `ActionBase` and implements `IAction`. Serves as both
design-time metadata and runtime execution unit.
+2. `ActionFooDialog`: Extends `ActionBaseDialog` and implements
`IActionDialog`. The user interface configuration dialog.
+
+== 1. The Action Class (`ActionFoo`)
+
+Annotate the class with `@Action` and decorate configurable properties with
`@HopMetadataProperty`:
+
+[source,java]
+----
+import org.apache.hop.core.Result;
+import org.apache.hop.core.annotations.Action;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.workflow.action.ActionBase;
+import org.apache.hop.workflow.action.IAction;
+import lombok.Getter;
+import lombok.Setter;
+
+@Action(
+ id = "PingServer",
+ name = "i18n::PingServer.Name",
+ description = "i18n::PingServer.Description",
+ image = "ping-server.svg",
+ categoryDescription =
"i18n:org.apache.hop.workflow:ActionCategory.Category.Utility",
+ documentationUrl = "/workflow/actions/ping-server.html"
+)
+@GuiPlugin
+@Getter
+@Setter
+public class ActionPingServer extends ActionBase implements IAction {
+
+ @HopMetadataProperty(key = "hostname")
+ private String hostname;
+
+ @HopMetadataProperty(key = "timeout_ms")
+ private int timeoutMs = 3000;
+
+ public ActionPingServer() {
+ this("");
+ }
+
+ public ActionPingServer(String name) {
+ super(name, "");
+ }
+
+ @Override
+ public Result execute(Result prevResult, int nr) {
+ Result result = prevResult;
+ result.setNrErrors(0);
+
+ // Resolve variables in hostname
+ String targetHost = resolve(hostname);
+
+ try {
+ logBasic("Pinging host: " + targetHost);
+ boolean reachable =
InetAddress.getByName(targetHost).isReachable(timeoutMs);
+
+ if (reachable) {
+ logBasic("Host " + targetHost + " is reachable.");
+ result.setResult(true);
+ } else {
+ logError("Host " + targetHost + " is unreachable.");
+ result.setResult(false);
+ result.setNrErrors(1);
+ }
+ } catch (Exception e) {
+ logError("Error pinging " + targetHost, e);
+ result.setResult(false);
+ result.setNrErrors(1);
+ }
+
+ return result;
+ }
+}
+----
+
+=== Key Action Methods
+* `execute(Result prevResult, int nr)`: The core execution entry point. Always
set `result.setResult(true/false)` and update `result.setNrErrors(...)`.
+* `isEvaluation()`: Return `true` if this action can evaluate to true/false
(allowing conditional hops).
+* `isUnconditional()`: Return `true` if hops leaving this action are
unconditional (like START).
+
+== 2. The GUI Dialog (`ActionFooDialog`)
+
+The dialog extends `ActionDialog`.
+The name field is `wName`.
+Annotate the action fields and build the dialog the same way as a transform.
+See xref:annotation-derived-widgets.adoc[Annotation derived widgets], under
"Transform and action dialogs".
+
+== Concrete Codebase Examples
+
+* `plugins/actions/eval`: Evaluates expressions with conditional
success/failure.
+* `plugins/actions/deletefile`: File deletion using `HopVfs`.
+*
`plugins/misc/mail/src/main/java/org/apache/hop/mail/workflow/actions/mail/ActionMail.java`:
Email notifications, attachments, and authentication.
+
+== See also
+
+* xref:plugin-types/pipeline-workflow.adoc[Pipelines and workflows]
+* xref:annotation-derived-widgets.adoc[Annotation derived widgets]
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/commands.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/commands.adoc
new file mode 100644
index 0000000000..dfd95a8b5b
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/commands.adoc
@@ -0,0 +1,94 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: Adding new command-line sub-commands to the Hop CLI using
@HopCommand and picocli.
+[[Sdk-Plugins-Commands]]
+= Hop Commands
+
+A `@HopCommand` plugin contributes an entire sub-command to the Hop command
line interface (`hop <command>`).
+All built-in Hop CLI sub-commands—including `hop run`, `hop conf`, `hop
server`, `hop encrypt`, `hop search`, and `hop export`—are implemented as
`@HopCommand` plugins.
+
+== Implementing a Command
+
+Annotate the class with `@HopCommand` and picocli's `@CommandLine.Command`,
and implement `IHopCommand` and `Runnable`:
+
+[source,java]
+----
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.hop.plugin.HopCommand;
+import org.apache.hop.hop.plugin.IHopCommand;
+import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+
+@HopCommand(
+ id = "audit",
+ description = "Scans project metadata and pipelines for policy compliance"
+)
+@Command(
+ name = "audit",
+ description = "Audit Hop project metadata"
+)
+public class HopCommandAudit implements IHopCommand, Runnable {
+
+ @Option(
+ names = {"-p", "--project"},
+ description = "The project name to audit"
+ )
+ private String projectName;
+
+ @Option(
+ names = {"-s", "--strict"},
+ description = "Fail with non-zero exit code if warnings are found"
+ )
+ private boolean strict;
+
+ private CommandLine cmd;
+ private IVariables variables;
+ private MultiMetadataProvider metadataProvider;
+
+ @Override
+ public void initialize(CommandLine cmd, IVariables variables,
+ MultiMetadataProvider metadataProvider) throws
HopException {
+ this.cmd = cmd;
+ this.variables = variables;
+ this.metadataProvider = metadataProvider;
+ }
+
+ @Override
+ public void run() {
+ System.out.println("Auditing project: " + projectName);
+ // Perform audit checks across metadataProvider
+ }
+}
+----
+
+== Key Architectural Methods
+
+* `initialize(...)`: Receives the parsed picocli `CommandLine`, current
`IVariables`, and the project's `MultiMetadataProvider`.
+* `run()`: Executes the command. Use standard Java exit conventions or throw
exceptions caught by the runner.
+
+== Concrete Codebase Examples
+
+* `engine/src/main/java/org/apache/hop/run/HopCommandRun.java`: The `hop run`
command implementation.
+* `engine/src/main/java/org/apache/hop/encryption/HopCommandEncrypt.java`: The
`hop encrypt` utility.
+* `engine/src/main/java/org/apache/hop/search/HopSearch.java`: The `hop
search` command.
+
+== See also
+
+* xref:plugin-types/metadata-configuration.adoc[Metadata and configuration]
diff --git
a/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/configuration.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/configuration.adoc
new file mode 100644
index 0000000000..72cfc3810b
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/configuration.adoc
@@ -0,0 +1,106 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: Adding options to existing CLI tools and configuration tabs to
the Hop GUI using @ConfigPlugin.
+[[Sdk-Plugins-Configuration]]
+= Configuration Plugins
+
+A `@ConfigPlugin` adds extra command-line options to an **existing** Hop tool
(such as `hop run`, `hop conf`, or `hop search`) via picocli mixins.
+Additionally, when combined with `@GuiPlugin` annotations, the same
configuration plugin automatically renders as a settings tab in the Hop GUI
Options dialog.
+
+== Implementing a Configuration Plugin
+
+Annotate the class with `@ConfigPlugin` and implement `IConfigOptions`:
+
+[source,java]
+----
+import org.apache.hop.core.config.plugin.ConfigPlugin;
+import org.apache.hop.core.config.plugin.IConfigOptions;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.metadata.api.IHasHopMetadataProvider;
+import picocli.CommandLine.Option;
+
+@ConfigPlugin(
+ id = "CustomRunOption",
+ description = "Adds notification webhook flag to hop run",
+ category = ConfigPlugin.CATEGORY_RUN // Tool targeting: CATEGORY_RUN,
CATEGORY_CONFIG, etc.
+)
+@GuiPlugin(
+ id = "CustomRunOptionGui",
+ description = "Custom Run Preferences"
+)
+public class CustomRunConfigPlugin implements IConfigOptions {
+
+ private static CustomRunConfigPlugin instance;
+
+ @Option(
+ names = {"-w", "--webhook"},
+ description = "Webhook URL to notify upon execution finish"
+ )
+ @GuiWidgetElement(
+ id = "webhook_url",
+ order = "0100",
+ type = GuiElementType.TEXT,
+ label = "Default Webhook Notification URL",
+ parentId = "CustomRunOptionGui"
+ )
+ private String webhookUrl;
+
+ public static CustomRunConfigPlugin getInstance() {
+ if (instance == null) {
+ instance = new CustomRunConfigPlugin();
+ }
+ return instance;
+ }
+
+ @Override
+ public boolean handleOption(
+ ILogChannel log, IHasHopMetadataProvider metadataProvider, IVariables
variables)
+ throws HopException {
+ if (webhookUrl != null) {
+ log.logBasic("Configured execution notification webhook: " + webhookUrl);
+ // Return true if handled completely, or false to proceed with standard
tool execution
+ }
+ return false;
+ }
+}
+----
+
+== Key Architectural Attributes
+
+* **`category`**: Determines which command mixes in the plugin:
+ * `CATEGORY_RUN`: `hop run`
+ * `CATEGORY_CONFIG`: `hop conf`
+ * `CATEGORY_SEARCH`: `hop search`
+ * `CATEGORY_SERVER`: `hop server`
+ * `CATEGORY_IMPORT`: `hop import`
+ * `CATEGORY_EXPORT`: `hop export`
+* **Dual CLI and GUI Role**: By adding `@GuiPlugin` and `@GuiWidgetElement` to
the same class along with a static `getInstance()` method, Hop GUI
automatically creates a tab in the Options dialog.
+
+== Concrete Codebase Examples
+
+*
`plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ManageProjectsOptionPlugin.java`:
Command-line project create and edit options, including `-p`.
+*
`ui/src/main/java/org/apache/hop/ui/hopgui/search/config/SearchConfigPlugin.java`:
Adds search configuration options to the CLI and the GUI settings dialog.
+
+== See also
+
+* xref:plugin-types/metadata-configuration.adoc[Metadata and configuration]
+* xref:annotation-derived-widgets.adoc[Annotation derived widgets]
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/databases.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/databases.adoc
new file mode 100644
index 0000000000..48bb4a2467
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/databases.adoc
@@ -0,0 +1,116 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: Developing relational database dialect plugins in Apache Hop:
JDBC URLs, SQL syntax generation, and column type mappings.
+[[Sdk-Plugins-Databases]]
+= Database Plugins and Dialects
+
+A database plugin teaches Apache Hop how to interact with a specific
relational database dialect: assembling JDBC URLs, formatting DDL statements,
mapping SQL data types to Hop's `IValueMeta` types, and identifying reserved
keywords.
+
+== The Plugin Class
+
+Annotate the class with `@DatabaseMetaPlugin` and extend `BaseDatabaseMeta`:
+
+[source,java]
+----
+import org.apache.hop.core.database.BaseDatabaseMeta;
+import org.apache.hop.core.database.DatabaseMetaPlugin;
+import org.apache.hop.core.database.IDatabase;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+
+@DatabaseMetaPlugin(
+ type = "CUSTOMDB",
+ typeDescription = "Custom Database Engine"
+)
+@GuiPlugin
+public class CustomDatabaseMeta extends BaseDatabaseMeta implements IDatabase {
+
+ @Override
+ public String getDriverClass() {
+ return "com.customdb.jdbc.CustomDriver";
+ }
+
+ @Override
+ public String getURL(String hostname, String port, String databaseName) {
+ if (Utils.isEmpty(port)) {
+ return "jdbc:customdb://" + hostname + "/" + databaseName;
+ }
+ return "jdbc:customdb://" + hostname + ":" + port + "/" + databaseName;
+ }
+
+ @Override
+ public int getDefaultDatabasePort() {
+ return 5432;
+ }
+
+ @Override
+ public String[] getAccessTypeList() {
+ return new String[] {"Native (JDBC)"};
+ }
+
+ @Override
+ public String getFieldDefinition(IValueMeta v, String tk, String pk,
+ boolean useAutoIncrement, boolean
addFieldName,
+ boolean addCr) {
+ // Translate Hop value types into SQL DDL column specifications
+ StringBuilder retval = new StringBuilder();
+ String fieldname = v.getName();
+ int length = v.getLength();
+ int precision = v.getPrecision();
+
+ if (addFieldName) {
+ retval.append(fieldname).append(" ");
+ }
+
+ switch (v.getType()) {
+ case IValueMeta.TYPE_INTEGER:
+ retval.append("BIGINT");
+ break;
+ case IValueMeta.TYPE_STRING:
+ if (length > 0 && length < 4000) {
+ retval.append("VARCHAR(").append(length).append(")");
+ } else {
+ retval.append("TEXT");
+ }
+ break;
+ default:
+ retval.append("TEXT");
+ break;
+ }
+ return retval.toString();
+ }
+
+ @Override
+ public String[] getReservedWords() {
+ return new String[] {"SELECT", "FROM", "WHERE", "ORDER", "GROUP", "BY",
"USER"};
+ }
+}
+----
+
+== JDBC Drivers and Classloading
+
+* **Driver Placement:** For licensing reasons, JDBC drivers should *not* be
bundled directly inside the plugin zip. Drivers are placed in `lib/jdbc` of the
Hop installation (or custom directories defined by `HOP_SHARED_JDBC_FOLDERS`).
+* **Bulk Loaders:** If your dialect plugin ships with dedicated bulk-loader
transforms (e.g., PostgreSQL COPY or MySQL Bulk Loader), declare the same
`classLoaderGroup` on both plugins so the transform sees the dialect's loaded
driver classes.
+
+== Concrete Codebase Examples
+
+*
`plugins/databases/postgresql/src/main/java/org/apache/hop/databases/postgresql/PostgreSqlDatabaseMeta.java`:
Comprehensive relational dialect implementation.
+*
`plugins/databases/duckdb/src/main/java/org/apache/hop/databases/duckdb/DuckDBDatabaseMeta.java`:
Embedded file-based database dialect.
+
+== See also
+
+* xref:database/index.adoc[Database plugins]
+* xref:database/creating-a-dialect.adoc[Creating a dialect]
diff --git
a/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/execution-info.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/execution-info.adoc
new file mode 100644
index 0000000000..55cdb98d82
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/execution-info.adoc
@@ -0,0 +1,131 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: Building custom execution information locations in Apache Hop:
persisting and querying execution logs, metrics, and state.
+[[Sdk-Plugins-ExecutionInfo]]
+= Execution Information Locations
+
+An execution information location is a pluggable storage target where Apache
Hop records runtime telemetry: pipeline and workflow execution records, status
changes, component metrics, execution logs, and captured row samples.
+The Hop GUI Execution perspective, Hop Server, and external monitoring
dashboards query these locations to visualize run history.
+
+== Implementing an Execution Information Location
+
+Annotate your class with `@ExecutionInfoLocationPlugin` and implement
`IExecutionInfoLocation`:
+
+[source,java]
+----
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.execution.Execution;
+import org.apache.hop.execution.ExecutionData;
+import org.apache.hop.execution.ExecutionInfoLocation;
+import org.apache.hop.execution.ExecutionType;
+import org.apache.hop.execution.ExecutionState;
+import org.apache.hop.execution.IExecutionInfoLocation;
+import org.apache.hop.execution.plugin.ExecutionInfoLocationPlugin;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import lombok.Getter;
+import lombok.Setter;
+
+@ExecutionInfoLocationPlugin(
+ id = "CustomElasticLocation",
+ name = "Custom Elasticsearch Location",
+ description = "Persists Hop execution history and metrics to Elasticsearch"
+)
+@GuiPlugin
+@Getter
+@Setter
+public class CustomElasticExecutionInfoLocation implements
IExecutionInfoLocation {
+
+ @GuiWidgetElement(
+ id = "cluster_url",
+ type = GuiElementType.TEXT,
+ label = "Elasticsearch URL",
+ parentId = ExecutionInfoLocation.GUI_PLUGIN_ELEMENT_PARENT_ID
+ )
+ @HopMetadataProperty(key = "cluster_url")
+ private String clusterUrl;
+
+ @Override
+ public void initialize(IVariables variables, IHopMetadataProvider
metadataProvider)
+ throws HopException {
+ // Connect to target storage
+ }
+
+ // --- Writing telemetry during execution ---
+
+ @Override
+ public void registerExecution(Execution execution) throws HopException {
+ // Write execution metadata (pipeline/workflow name, start time,
parameters)
+ }
+
+ @Override
+ public void updateExecutionState(ExecutionState executionState) throws
HopException {
+ // Update runtime metrics (lines read, written, errors, status changes)
+ }
+
+ @Override
+ public void registerData(ExecutionData executionData) throws HopException {
+ // Store sampled row data and execution logs
+ }
+
+ // --- Reading telemetry for the Hop GUI and API ---
+
+ @Override
+ public Execution getExecution(String executionId) throws HopException {
+ return fetchExecutionRecord(executionId);
+ }
+
+ @Override
+ public ExecutionState getExecutionState(String executionId) throws
HopException {
+ return fetchExecutionState(executionId);
+ }
+
+ @Override
+ public List<String> findChildIds(ExecutionType parentType, String
parentExecutionId)
+ throws HopException {
+ // Critical: returns child execution IDs (transforms or sub-workflows) for
drill-down
+ return queryChildren(parentExecutionId);
+ }
+
+ @Override
+ public void close() throws HopException {
+ // Flush pending buffers and close client connections
+ }
+}
+----
+
+This sample is abbreviated.
+`IExecutionInfoLocation` declares further methods with no default, including
`clone()`, the plugin id and name accessors, `clearCaches()`,
`unBuffer(String)`, `deleteExecution(String)`, `getExecutionState(String,
boolean)`, `getExecutionStateLoggingText(String, int)`,
`getExecutionIds(boolean, int)`, and the `findExecutions` lookups.
+Read the interface before implementing a location.
+
+
+
+== Key Architectural Rules
+
+* **Implement Both Halves:** An execution location is both a writer (called
continuously during runs) and a reader (queried by the GUI Execution
perspective). Both halves, including child lookups (`findChildIds`), must be
implemented for drill-down views to work.
+* **Buffering and Throughput:** Execution state updates occur frequently
during high-throughput pipeline runs. Implement in-memory buffering and batch
flushes to prevent I/O bottlenecks.
+* **Metadata Configuration:** Configuration settings reside in an `Execution
Information Location` metadata object. Annotate fields with
`@HopMetadataProperty` and `@GuiWidgetElement` using `parentId =
ExecutionInfoLocation.GUI_PLUGIN_ELEMENT_PARENT_ID`.
+
+== Concrete Codebase Examples
+
+*
`engine/src/main/java/org/apache/hop/execution/local/FileExecutionInfoLocation.java`:
Standard file-based execution telemetry storage.
+*
`plugins/misc/execution-database/src/main/java/org/apache/hop/execution/database/CachingDatabaseExecutionInfoLocation.java`:
Relational database execution storage.
+*
`engine/src/main/java/org/apache/hop/execution/caching/CachingFileExecutionInfoLocation.java`:
File-backed cache of execution telemetry.
+
+== See also
+
+* xref:plugin-types/execution-observability.adoc[Execution and observability]
diff --git
a/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/extension-points.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/extension-points.adoc
new file mode 100644
index 0000000000..c92ccab605
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/extension-points.adoc
@@ -0,0 +1,77 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: Hooking into runtime and GUI lifecycles in Apache Hop using
@ExtensionPoint plugins.
+[[Sdk-Plugins-ExtensionPoints]]
+= Extension Points
+
+An extension point is an event-driven hook in Apache Hop.
+Hop announces specific lifecycle events (such as `HopGuiStart`,
`PipelinePrepareExecution`, `TransformBeforeStart`, or `PipelineCompleted`),
and any registered plugin listening for that event is invoked synchronously.
+
+== Implementing an Extension Point
+
+Annotate your class with `@ExtensionPoint` and implement `IExtensionPoint<T>`:
+
+[source,java]
+----
+import org.apache.hop.core.extension.ExtensionPoint;
+import org.apache.hop.core.extension.IExtensionPoint;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.engine.IPipelineEngine;
+
+@ExtensionPoint(
+ id = "AuditPipelineCompleted",
+ extensionPointId = "PipelineCompleted",
+ description = "Logs auditing information when any pipeline finishes
execution"
+)
+public class AuditPipelineCompletedXp implements
IExtensionPoint<IPipelineEngine<PipelineMeta>> {
+
+ @Override
+ public void callExtensionPoint(ILogChannel log, IVariables variables,
+ IPipelineEngine<PipelineMeta> pipeline)
throws HopException {
+ log.logBasic("Pipeline completed: " + pipeline.getPipelineMeta().getName()
+
+ " with status: " + pipeline.getStatusDescription());
+ }
+}
+----
+
+== Key Architectural Rules
+
+=== 1. Bind to Interfaces, Not Implementations
+[IMPORTANT]
+====
+**Always bind to the engine interface (`IPipelineEngine` or
`IWorkflowEngine`), never concrete classes (`Pipeline` or `Workflow`).**
+Declaring `IExtensionPoint<Pipeline>` will function on the local engine but
crash with a `ClassCastException` on remote engines or Apache Beam engines.
+====
+
+=== 2. Inline Synchronous Execution
+Extension points execute on the thread that triggered the event.
+Keep your logic fast and non-blocking. If your extension point needs to
perform heavy I/O, network requests, or database writes, offload the workload
to a background worker queue.
+
+=== 3. Discovering Available Extension Points
+All core extension point IDs are defined as constants in the
`org.apache.hop.core.extension.HopExtensionPoint` enum.
+
+== Concrete Codebase Examples
+
+*
`engine/src/main/java/org/apache/hop/lineage/xp/LineageHubPipelineCompletedXp.java`:
Emits lineage events upon pipeline completion.
+*
`ui/src/main/java/org/apache/hop/ui/hopgui/notifications/NotificationSystemInitializer.java`:
Initializes notifications on the `HopGuiStart` extension point.
+*
`plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/PipelineStartCheckProjectExtensionPoint.java`:
Hooks `PipelinePrepareExecution` to check the active project.
+
+== See also
+
+* xref:plugin-types/execution-observability.adoc[Execution and observability]
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/file-types.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/file-types.adoc
new file mode 100644
index 0000000000..993973947a
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/file-types.adoc
@@ -0,0 +1,107 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: Registering custom file types in Apache Hop: extensions, icons,
file handlers, and opening files in the Hop GUI.
+[[Sdk-Plugins-FileTypes]]
+= Hop File Types
+
+A Hop file type plugin teaches the Hop GUI how to recognize, display, and open
specific file extensions (such as `.hpl`, `.hwf`, `.json`, `.csv`, `.sql`, or
`.parquet`).
+File types define the icon used in the Explorer perspective, the file filter
masks in file-open dialogs, and the editor opened when double-clicking a file.
+
+== Implementing a File Type
+
+Annotate your class with `@HopFileTypePlugin` and extend `HopFileTypeBase`:
+
+[source,java]
+----
+import java.util.Properties;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.ui.hopgui.HopGui;
+import org.apache.hop.ui.hopgui.file.HopFileTypeBase;
+import org.apache.hop.ui.hopgui.file.HopFileTypePlugin;
+import org.apache.hop.ui.hopgui.file.IHopFileType;
+import org.apache.hop.ui.hopgui.file.IHopFileTypeHandler;
+import org.apache.hop.ui.hopgui.file.empty.EmptyHopFileTypeHandler;
+
+@HopFileTypePlugin(
+ id = "CustomMarkdownFileType",
+ name = "Markdown File",
+ description = "Documentation and markdown files",
+ image = "markdown.svg"
+)
+public class MarkdownFileType extends HopFileTypeBase implements IHopFileType {
+
+ public static final String[] EXTENSIONS = new String[] {"*.md",
"*.markdown"};
+
+ @Override
+ public String getName() {
+ return "Markdown";
+ }
+
+ @Override
+ public String[] getFilterExtensions() {
+ return EXTENSIONS;
+ }
+
+ @Override
+ public String[] getFilterNames() {
+ return new String[] {"Markdown Files (*.md, *.markdown)"};
+ }
+
+ @Override
+ public String getDefaultFileExtension() {
+ return "md";
+ }
+
+ @Override
+ public Properties getCapabilities() {
+ Properties capabilities = new Properties();
+ capabilities.setProperty(IHopFileType.CAPABILITY_SAVE, "true");
+ capabilities.setProperty(IHopFileType.CAPABILITY_SAVE_AS, "true");
+ capabilities.setProperty(IHopFileType.CAPABILITY_CLOSE, "true");
+ return capabilities;
+ }
+
+ @Override
+ public IHopFileTypeHandler openFile(HopGui hopGui, String filename,
IVariables variables)
+ throws HopException {
+ // Create the handler for this file and open it in the perspective that
edits the type.
+ return new EmptyHopFileTypeHandler();
+ }
+}
+----
+
+`HopFileTypeBase.hasCapability(String)` reads that map.
+Capability names are the `CAPABILITY_*` constants on `IHopFileType`.
+Explorer file types build the same map with
`FileTypeCapabilities.getCapabilities(String...)` in
`org.apache.hop.ui.hopgui.perspective.explorer.file.capabilities`.
+
+== Key Methods
+
+* `getFilterExtensions()`: String array of wildcard patterns (e.g.,
`{"*.parquet"}`).
+* `openFile(HopGui, String, IVariables)`: Instantiates or activates the
appropriate editor or perspective tab.
+* `hasCapability(String)`: Informs the GUI whether the file supports an
operation such as save, close, or creation.
+* `getContextHandlers()`: Provides custom actions in right-click context menus
in the file explorer.
+
+== Concrete Codebase Examples
+
+*
`ui/src/main/java/org/apache/hop/ui/hopgui/file/pipeline/HopPipelineFileType.java`:
The core file type handling `.hpl` pipeline files.
+*
`ui/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/file/types/text/BaseTextExplorerFileType.java`:
Plain text file handler in the Explorer perspective.
+*
`ui/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/file/types/log/LogExplorerFileType.java`:
Log viewer file type.
+
+== See also
+
+* xref:plugin-types/gui.adoc[GUI]
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/gui.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/gui.adoc
new file mode 100644
index 0000000000..3e3899dcbf
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/gui.adoc
@@ -0,0 +1,122 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: Extending the Hop GUI with toolbar buttons, menus, context
actions, settings tabs, and composite widgets using @GuiPlugin.
+[[Sdk-Plugins-Gui]]
+= GUI Plugins (Toolbars, Menus, Panels)
+
+The `@GuiPlugin` annotation allows you to extend the Hop GUI without
subclassing existing GUI classes.
+A class marked with `@GuiPlugin` is scanned for contribution annotations that
attach buttons, menus, shortcuts, tabs, or custom composite panels to existing
GUI elements.
+
+== Common Contribution Annotations
+
+[cols="1,3", options="header"]
+|===
+|Annotation |Functionality
+
+|`@GuiToolbarElement`
+|Adds a button, toggle, or custom composite widget to an existing toolbar.
+
+|`@GuiToolbarElementFilter`
+|Conditionally shows or hides a toolbar element based on context.
+
+|`@GuiMenuElement`
+|Adds a menu entry into the top-level Hop GUI menu bar.
+
+|`@GuiContextAction`
+|Adds an action to canvas right-click context menus.
+
+|`@GuiWidgetElement`
+|Declares a settings widget (text, metadata combo, checkbox, file browser)
inside dialogs.
+
+|`@GuiTab`
+|Adds a new tab to an existing tab folder.
+|===
+
+== 1. Adding a Toolbar Button
+
+Toolbar elements bind to a parent container via `parentId` and sort
alphabetically by `id` (conventionally using numeric prefixes):
+
+[source,java]
+----
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.toolbar.GuiToolbarElement;
+import org.apache.hop.core.gui.plugin.toolbar.GuiToolbarElementType;
+import org.apache.hop.ui.hopgui.HopGui;
+
+@GuiPlugin
+public class CustomToolbarItem {
+
+ public static final String TOOLBAR_ITEM_ID = "10500-my-custom-action";
+
+ @GuiToolbarElement(
+ root = HopGui.ID_MAIN_TOOLBAR,
+ id = TOOLBAR_ITEM_ID,
+ type = GuiToolbarElementType.BUTTON,
+ image = "custom-action.svg",
+ toolTip = "Execute Custom Task"
+ )
+ public void executeAction() {
+ HopGui hopGui = HopGui.getInstance();
+ // Perform action
+ hopGui.getLog().logBasic("Custom toolbar action executed!");
+ }
+}
+----
+
+== 2. Adding Custom Panels and Widgets to Toolbars
+
+Toolbars can also host complex composites (such as combo boxes, search bars,
or progress meters) by specifying `type = GuiToolbarElementType.CUSTOM`:
+
+[source,java]
+----
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.widgets.Combo;
+import org.eclipse.swt.widgets.Composite;
+import org.eclipse.swt.widgets.Control;
+
+@GuiPlugin
+public class SearchLocationItem {
+
+ @GuiToolbarElement(
+ root = HopGui.ID_MAIN_TOOLBAR,
+ id = "20000-search-location",
+ type = GuiToolbarElementType.CUSTOM
+ )
+ public Control createSearchControl(Composite parent) {
+ Combo combo = new Combo(parent, SWT.DROP_DOWN | SWT.READ_ONLY);
+ combo.setItems("All", "Pipelines", "Workflows", "Metadata");
+ combo.select(0);
+ return combo;
+ }
+}
+----
+
+== 3. Declarative Dialog Widgets (`@GuiWidgetElement`)
+
+Annotate fields on the metadata class instead of laying out SWT controls by
hand.
+Field types, groups, and the dialog `open()` recipe are
xref:annotation-derived-widgets.adoc[Annotation derived widgets].
+
+== Concrete Codebase Examples
+
+*
`ui/src/main/java/org/apache/hop/ui/hopgui/notifications/NotificationToolbarItem.java`:
Toolbar button contributed with `@GuiToolbarElement`.
+* `ui/src/main/java/org/apache/hop/ui/core/gui/GuiCompositeWidgets.java`:
Turns `@GuiWidgetElement` annotations into SWT controls.
+
+== See also
+
+* xref:plugin-types/gui.adoc[GUI]
+* xref:gui-plugins-toolbars.adoc[GUI plugins and toolbars]
+* xref:annotation-derived-widgets.adoc[Annotation derived widgets]
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/index.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/index.adoc
new file mode 100644
index 0000000000..7a3ac8b22b
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/index.adoc
@@ -0,0 +1,148 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: An overview of developing Apache Hop plugins: discovery via
Jandex, registry, classloaders, project packaging, and supported plugin types.
+[[Sdk-Plugins-Index]]
+= Hop Plugin Development Guide
+
+Apache Hop is architected around an extensible plugin ecosystem.
+Nearly every functional capability in Hop is a plugin: the transforms that
process rows, the actions in workflows, the databases Hop connects to, the
metadata objects edited in the GUI, the GUI perspectives, the CLI sub-commands,
and the storage systems accessed via VFS.
+
+This guide provides practical, step-by-step instructions for implementing Hop
plugins across **15 primary plugin types**, with direct references to
real-world implementations in the Apache Hop repository.
+
+== How Hop Discovers Plugins
+
+Hop uses annotation-driven discovery powered by
https://smallrye.io/jandex/[Jandex] annotation indexes (`META-INF/jandex.idx`):
+
+1. **Native Classpath Plugins:** Discovered from jars on the application
classpath containing a Jandex index. Core plugins in `hop-core`, `hop-engine`,
and `hop-ui` are loaded this way.
+2. **Plugin Folder Plugins:** Discovered from jars inside the `plugins/`
directory (or custom folders configured via `HOP_PLUGIN_BASE_FOLDERS`). Each
plugin folder receives its own isolated `URLClassLoader` covering the plugin
jar and its private `lib/` directory.
+
+[IMPORTANT]
+====
+**No Jandex index = No plugin.**
+Every plugin jar must contain `META-INF/jandex.idx`.
+The root Hop POM binds the `jandex-maven-plugin` automatically for all modules
in the repository.
+If you develop a plugin outside the Hop repository, you must configure the
Jandex plugin in your `pom.xml`.
+====
+
+== Classloading and `classLoaderGroup`
+
+To avoid classpath conflicts, external plugins run in isolated classloaders.
+However, related plugins often need to share classes (for example, a database
dialect and its corresponding bulk-loader transform, or a VFS plugin and its
metadata connection).
+
+Set `classLoaderGroup` on the plugin annotation to group plugins into a shared
classloader:
+
+[source,java]
+----
+@VfsPlugin(
+ type = "azure",
+ typeDescription = "Azure VFS",
+ classLoaderGroup = "vfs-azure"
+)
+----
+
+== Project Structure & Packaging
+
+In the Hop codebase, plugins are organized under `plugins/` by category:
+
+* `plugins/transforms`: Pipeline transforms
+* `plugins/actions`: Workflow actions
+* `plugins/databases`: Database dialects
+* `plugins/tech`: Technology bundles (AWS, Azure, Google, Neo4j, and the VFS
providers)
+* `plugins/valuetypes`: Custom row data types
+* `plugins/resolvers`: Variable resolvers
+* `plugins/misc`: Miscellaneous plugins
+
+Adding a plugin requires:
+1. Creating a Maven module inheriting from the category POM (e.g.
`hop-plugins-transforms`).
+2. Adding `src/assembly/assembly.xml` referring to
`assemblies/shared/hop-plugin-libs.xml`.
+3. Adding `src/main/resources/version.xml` with `${project.version}`.
+4. Adding a `<dependency>` with `<type>zip</type>` in
`assemblies/plugins/pom.xml`.
+
+== Plugin Types Catalogue
+
+Explore the dedicated development guides below for each plugin type:
+
+[cols="1,2,3", options="header"]
+|===
+|Plugin Type |Annotation |Guide & Overview
+
+|**Transform**
+|`@Transform`
+|xref:sdk/plugins/transforms.adoc[Transform Plugins] — Units of work in
pipelines streaming rows.
+
+|**Action**
+|`@Action`
+|xref:sdk/plugins/actions.adoc[Action Plugins] — Sequential and conditional
tasks in workflows.
+
+|**Metadata Element**
+|`@HopMetadata`
+|xref:sdk/plugins/metadata.adoc[Metadata Element Plugins] — Shared
configurations persisted as JSON.
+
+|**Database Dialect**
+|`@DatabaseMetaPlugin`
+|xref:sdk/plugins/databases.adoc[Database Plugins] — Relational database
dialects and SQL syntax.
+
+|**GUI Contribution**
+|`@GuiPlugin`
+|xref:sdk/plugins/gui.adoc[GUI Plugins] — Toolbars, menus, settings tabs, and
composite widgets.
+
+|**File Type**
+|`@HopFileTypePlugin`
+|xref:sdk/plugins/file-types.adoc[Hop File Types] — File formats recognised
and opened by Hop GUI.
+
+|**Variable Resolver**
+|`@VariableResolverPlugin`
+|xref:sdk/plugins/variable-resolvers.adoc[Variable Resolvers] — Retrieving
secrets from external vaults.
+
+|**Extension Point**
+|`@ExtensionPoint`
+|xref:sdk/plugins/extension-points.adoc[Extension Points] — Hooking into
runtime and GUI lifecycles.
+
+|**Hop Command**
+|`@HopCommand`
+|xref:sdk/plugins/commands.adoc[Hop Commands] — New CLI sub-commands (`hop
<subcommand>`).
+
+|**Configuration**
+|`@ConfigPlugin`
+|xref:sdk/plugins/configuration.adoc[Configuration Plugins] — CLI option
mixins and GUI options tabs.
+
+|**Perspective**
+|`@HopPerspectivePlugin`
+|xref:sdk/plugins/perspectives.adoc[Perspectives] — Full-screen workbenches in
Hop GUI.
+
+|**VFS Provider**
+|`@VfsPlugin`
+|xref:sdk/plugins/vfs.adoc[VFS Plugins] — Storage URL schemes for Apache
Commons VFS.
+
+|**Value Metadata**
+|`@ValueMetaPlugin`
+|xref:sdk/plugins/value-types.adoc[Value Types] — Custom data types in the
pipeline row stream.
+
+|**Hop Server Servlet**
+|`@HopServerServlet`
+|xref:sdk/plugins/servlets.adoc[Server Servlets] — Custom HTTP endpoints on
Hop Server.
+
+|**Execution Info Location**
+|`@ExecutionInfoLocationPlugin`
+|xref:sdk/plugins/execution-info.adoc[Execution Information Locations] —
Storing runtime metrics and logs.
+|===
+
+== See also
+
+* xref:plugin-development.adoc[How plugins work]
+* xref:plugin-types/index.adoc[Plugin types] — every plugin type, including
the ones this guide does not walk through.
+* xref:annotation-derived-widgets.adoc[Annotation derived widgets]
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/metadata.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/metadata.adoc
new file mode 100644
index 0000000000..3b2da617db
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/metadata.adoc
@@ -0,0 +1,110 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: Building custom metadata element plugins in Apache Hop: the
@HopMetadata annotation, JSON serialization, and GUI editors.
+[[Sdk-Plugins-Metadata]]
+= Metadata Element Plugins
+
+Metadata element plugins add new types of reusable configurations to Apache
Hop: database connections, pipeline run configurations, execution information
locations, and custom API connections.
+These objects are edited in the Hop GUI Metadata perspective and stored as
JSON files under the project's `metadata/` directory.
+
+== 1. The Metadata Class
+
+Create a POJO extending `HopMetadataBase` and implementing `IHopMetadata`,
annotated with `@HopMetadata`:
+
+[source,java]
+----
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.metadata.api.HopMetadata;
+import org.apache.hop.metadata.api.HopMetadataBase;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.IHopMetadata;
+import lombok.Getter;
+import lombok.Setter;
+
+@HopMetadata(
+ key = "webhook-notification", // Storage folder name
(kebab-case)
+ legacyKeys = {"WebhookNotification"}, // Backward compatibility
+ name = "Webhook Notification",
+ description = "Defines a webhook endpoint for workflow notifications",
+ image = "webhook.svg",
+ category = "Alerts"
+)
+@GuiPlugin
+@Getter
+@Setter
+public class WebhookNotification extends HopMetadataBase implements
IHopMetadata {
+
+ public static final String GUI_PLUGIN_ELEMENT_PARENT_ID =
"WebhookNotification-Parent";
+
+ @HopMetadataProperty
+ private String url;
+
+ @HopMetadataProperty(password = true)
+ private String secretToken;
+
+ @HopMetadataProperty
+ private int retries = 3;
+
+ public WebhookNotification() {
+ super();
+ }
+
+ public WebhookNotification(String name, String url, String secretToken) {
+ this.name = name;
+ this.url = url;
+ this.secretToken = secretToken;
+ }
+}
+----
+
+=== Naming Rules for `@HopMetadata`
+* **`key`**: Must be lower-case kebab-case (`webhook-notification`). This
forms the folder path `metadata/webhook-notification/<name>.json`.
+* **`legacyKeys`**: If you ever rename a key, include the previous key in
`legacyKeys` so older projects continue loading their configurations seamlessly.
+
+== 2. The GUI Metadata Editor (`*Editor`)
+
+A metadata editor extends `MetadataEditor`.
+The editor creates the name field, and the other fields come from
`@GuiWidgetElement` on the metadata class.
+Pass that name control into `createCompositeWidgets`.
+The worked example is xref:annotation-derived-widgets.adoc[Annotation derived
widgets], under "Metadata editors".
+
+== 3. Selecting Metadata in Transform and Action Dialogs
+
+To let users select a metadata element in another dialog:
+
+* Use `MetaSelectionLine<T>` directly in SWT:
++
+[source,java]
+----
+MetaSelectionLine<WebhookNotification> wWebhook = new MetaSelectionLine<>(
+ variables, metadataProvider, WebhookNotification.class,
+ shell, SWT.NONE, "Webhook", "Select the webhook configuration"
+);
+wWebhook.fillItems();
+----
+* Or annotate the field with `@GuiWidgetElement(type =
GuiElementType.METADATA)`. The layout is
xref:annotation-derived-widgets.adoc[Annotation derived widgets].
+
+== Concrete Codebase Examples
+
+* `engine/src/main/java/org/apache/hop/partition/PartitionSchema.java` and
`ui/src/main/java/org/apache/hop/ui/partition/PartitionSchemaEditor.java`:
Standard in-tree metadata object with editor.
+*
`plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/metadata/SftpConnection.java`:
Connection metadata element.
+
+== See also
+
+* xref:metadata-plugins.adoc[Metadata plugins]
+* xref:metadata-serialization.adoc[Metadata serialization]
+* xref:annotation-derived-widgets.adoc[Annotation derived widgets]
diff --git
a/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/perspectives.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/perspectives.adoc
new file mode 100644
index 0000000000..d953156096
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/perspectives.adoc
@@ -0,0 +1,133 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: Building full-screen workbench perspectives in the Hop GUI using
@HopPerspectivePlugin.
+[[Sdk-Plugins-Perspectives]]
+= Perspectives
+
+A perspective represents an entire full-screen workbench view in the Hop GUI,
selected via the primary navigation icons in the left sidebar.
+The Data Orchestration, Metadata, File Explorer, and Execution Information
workbenches are all perspective plugins.
+
+== Implementing a Perspective
+
+Annotate your class with `@HopPerspectivePlugin` and implement
`IHopPerspective`:
+
+[source,java]
+----
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.ui.hopgui.HopGui;
+import org.apache.hop.ui.hopgui.file.IHopFileType;
+import org.apache.hop.ui.hopgui.perspective.HopPerspectivePlugin;
+import org.apache.hop.ui.hopgui.perspective.IHopPerspective;
+import java.util.Collections;
+import java.util.List;
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.layout.FillLayout;
+import org.eclipse.swt.widgets.Composite;
+import org.eclipse.swt.widgets.Control;
+
+@HopPerspectivePlugin(
+ id = "300-CustomPerspective", // Numeric prefix dictates
sidebar icon order
+ name = "Custom Workbench",
+ description = "Custom analytics workbench",
+ image = "custom-perspective.svg"
+)
+@GuiPlugin
+public class CustomPerspective implements IHopPerspective {
+
+ private static CustomPerspective instance;
+ private Composite composite;
+ private boolean initialized;
+
+ public CustomPerspective() {
+ instance = this;
+ }
+
+ public static CustomPerspective getInstance() {
+ return instance;
+ }
+
+ @Override
+ public String getId() {
+ return "300-CustomPerspective";
+ }
+
+ @Override
+ public void initialize(HopGui hopGui, Composite parent) {
+ this.composite = new Composite(parent, SWT.NONE);
+ this.composite.setLayout(new FillLayout());
+
+ // Build workbench UI controls inside this.composite...
+
+ this.initialized = true;
+ }
+
+ @Override
+ public void activate() {
+ // Called when the user clicks this perspective's icon in the sidebar
+ }
+
+ @Override
+ public void perspectiveActivated() {
+ // Post-activation logic
+ }
+
+ @Override
+ public boolean isActive() {
+ return HopGui.getInstance().isActivePerspective(this);
+ }
+
+ @Override
+ public Control getControl() {
+ return composite;
+ }
+
+ @Override
+ public List<IHopFileType> getSupportedHopFileTypes() {
+ return Collections.emptyList();
+ }
+
+ public boolean isInitialized() {
+ return initialized;
+ }
+}
+----
+
+This sample is abbreviated.
+`IHopPerspective` has more methods, and most of them have defaults.
+The methods without a default are `getId()`, `activate()`,
`perspectiveActivated()`, `isActive()`, `initialize(HopGui, Composite)`, and
`getControl()`.
+
+
+
+== Key Architectural Rules
+
+* **Numeric Ordering:** The `id` attribute should start with a 3-digit number
(e.g. `010-DataOrchestration`, `100-HopExplorerPerspective`, `300-...`) which
strictly determines its position in the left navigation sidebar.
+* **Initialization Guard:**
+[WARNING]
+====
+If a perspective is listed in a user's `disabledGuiElements` configuration,
Hop GUI will instantiate the object but *never call `initialize()`*.
+Always guard public methods and singleton accessors with an `isInitialized()`
check to prevent NullPointerExceptions.
+====
+
+== Concrete Codebase Examples
+
+*
`ui/src/main/java/org/apache/hop/ui/hopgui/perspective/explorer/ExplorerPerspective.java`:
File browser and viewer workbench.
+*
`ui/src/main/java/org/apache/hop/ui/hopgui/perspective/execution/ExecutionPerspective.java`:
Pipeline and workflow execution monitoring perspective.
+*
`ui/src/main/java/org/apache/hop/ui/hopgui/perspective/configuration/ConfigurationPerspective.java`:
Project and system configuration perspective.
+
+== See also
+
+* xref:plugin-types/gui.adoc[GUI]
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/servlets.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/servlets.adoc
new file mode 100644
index 0000000000..89e087312a
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/servlets.adoc
@@ -0,0 +1,104 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: Building custom HTTP REST and management endpoints on Hop Server
using @HopServerServlet plugins.
+[[Sdk-Plugins-Servlets]]
+= Hop Server Servlets
+
+A Hop Server servlet plugin exposes a custom HTTP endpoint on Hop Server.
+All built-in remote execution and monitoring endpoints—such as pipeline
execution, status checks, metric querying, and stop/pause operations—are
implemented as `@HopServerServlet` plugins.
+
+== Implementing a Server Servlet
+
+Annotate the class with `@HopServerServlet` and extend `BaseHttpServlet` while
implementing `IHopServerPlugin`:
+
+[source,java]
+----
+import org.apache.hop.core.annotations.HopServerServlet;
+import org.apache.hop.www.BaseHttpServlet;
+import org.apache.hop.www.IHopServerPlugin;
+import org.apache.hop.www.PipelineMap;
+import org.apache.hop.www.WorkflowMap;
+import jakarta.servlet.ServletException;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.servlet.http.HttpServletResponse;
+import java.io.IOException;
+
+@HopServerServlet(
+ id = "customHealthCheck",
+ name = "Custom Server Health Check Endpoint"
+)
+public class CustomHealthCheckServlet extends BaseHttpServlet implements
IHopServerPlugin {
+
+ public static final String CONTEXT_PATH = "/hop/health";
+
+ public CustomHealthCheckServlet() {
+ super();
+ }
+
+ public CustomHealthCheckServlet(PipelineMap pipelineMap, WorkflowMap
workflowMap) {
+ super(pipelineMap, workflowMap);
+ }
+
+ @Override
+ public String getContextPath() {
+ return CONTEXT_PATH;
+ }
+
+ @Override
+ public void doGet(HttpServletRequest request, HttpServletResponse response)
+ throws ServletException, IOException {
+ if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) {
+ return;
+ }
+
+ response.setContentType("application/json");
+ response.setStatus(HttpServletResponse.SC_OK);
+
+ int runningPipelines = getPipelineMap().getPipelineObjects().size();
+ int runningWorkflows = getWorkflowMap().getWorkflowObjects().size();
+
+ String json = String.format(
+ "{\"status\":\"UP\",\"activePipelines\":%d,\"activeWorkflows\":%d}",
+ runningPipelines, runningWorkflows
+ );
+ response.getWriter().write(json);
+ }
+}
+----
+
+== Key Architectural Rules
+
+* **`getContextPath()`**: Defines the URL mount point under the server root
(e.g. `/hop/health`).
+* **Authentication and Security**:
+[WARNING]
+====
+A servlet plugin adds to the attack surface of Hop Server.
+Hop Server authenticates the caller.
+What the endpoint returns, and what it does with request parameters, is the
plugin's responsibility.
+See xref:architecture/security.adoc[Security for developers].
+====
+
+== Concrete Codebase Examples
+
+* `engine/src/main/java/org/apache/hop/www/GetPipelineStatusServlet.java`:
Status report endpoint for running pipelines.
+* `engine/src/main/java/org/apache/hop/www/GetWorkflowStatusServlet.java`:
Workflow status reporting servlet.
+*
`engine/src/main/java/org/apache/hop/www/PrepareExecutionPipelineServlet.java`:
Pipeline initialization and execution endpoint.
+
+== See also
+
+* xref:plugin-types/execution-observability.adoc[Execution and observability]
+* xref:architecture/security.adoc[Security for developers]
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/transforms.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/transforms.adoc
new file mode 100644
index 0000000000..a37e121d17
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/transforms.adoc
@@ -0,0 +1,199 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: Building custom pipeline transforms in Apache Hop: the
four-class pattern, row processing lifecycle, metadata properties, and GUI
dialog integration.
+[[Sdk-Plugins-Transforms]]
+= Transform Plugins
+
+Transforms are the fundamental building blocks of Hop pipelines.
+A transform reads rows from incoming hops, performs operations (transforming,
filtering, enriching, validating, or writing), and emits rows to outgoing hops.
+
+== Core Architecture: The Four Classes
+
+Every transform plugin is implemented as a set of four collaborating classes
with a consistent naming convention:
+
+[cols="1,2,3", options="header"]
+|===
+|Class |Base Class / Interface |Responsibility
+
+|`FooMeta`
+|`BaseTransformMeta<Foo, FooData>`
+|**Design-time metadata:** Configuration settings, serialization
(`@HopMetadataProperty`), output row schema (`getFields`), validation (`check`).
+
+|`Foo`
+|`BaseTransform<FooMeta, FooData>`
+|**Runtime execution:** Processing rows (`processRow`), initialization
(`init`), and cleanup (`dispose`). One instance per running copy.
+
+|`FooData`
+|`BaseTransformData`
+|**Runtime state:** Mutable variables, cached field indexes, temporary
buffers. Separate from the transform to cleanly isolate multiple concurrent
copies.
+
+|`FooDialog`
+|`BaseTransformDialog`
+|**User interface:** Settings dialog in the Hop GUI.
+|===
+
+== 1. The Metadata Class (`FooMeta`)
+
+Annotate the metadata class with `@Transform` and decorate each configurable
field with `@HopMetadataProperty`:
+
+[source,java]
+----
+import org.apache.hop.core.annotations.Transform;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.pipeline.transform.BaseTransformMeta;
+import lombok.Getter;
+import lombok.Setter;
+
+@Transform(
+ id = "CustomFilter",
+ name = "i18n::CustomFilter.Name",
+ description = "i18n::CustomFilter.Description",
+ image = "custom-filter.svg",
+ categoryDescription =
"i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Transform",
+ documentationUrl = "/pipeline/transforms/custom-filter.html"
+)
+@GuiPlugin
+@Getter
+@Setter
+public class CustomFilterMeta extends BaseTransformMeta<CustomFilter,
CustomFilterData> {
+
+ @HopMetadataProperty(key = "target_field")
+ private String targetField;
+
+ @HopMetadataProperty(key = "threshold_value")
+ private int thresholdValue = 100;
+
+ @Override
+ public void setDefault() {
+ targetField = "";
+ thresholdValue = 100;
+ }
+}
+----
+
+=== Crucial Methods on the Meta Class
+
+* `getFields(IRowMeta inputRowMeta, String name, IRowMeta[] info,
TransformMeta nextStep, IVariables variables, IHopMetadataProvider
metadataProvider)`:
+Describes outgoing rows.
+If your transform adds, removes, or alters fields, modify `inputRowMeta` in
this method.
+Downstream transforms rely on this method during design time.
+* `check(...)`:
+Validates configuration when a user clicks "Verify pipeline" in the GUI.
+
+== 2. The Runtime Class (`Foo`)
+
+The runtime class executes the data processing logic:
+
+[source,java]
+----
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.pipeline.Pipeline;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.BaseTransform;
+import org.apache.hop.pipeline.transform.TransformMeta;
+
+public class CustomFilter extends BaseTransform<CustomFilterMeta,
CustomFilterData> {
+
+ public CustomFilter(TransformMeta meta, CustomFilterMeta metaData,
+ CustomFilterData data, int copyNr,
+ PipelineMeta pipelineMeta, Pipeline pipeline) {
+ super(meta, metaData, data, copyNr, pipelineMeta, pipeline);
+ }
+
+ @Override
+ public boolean init() {
+ if (!super.init()) {
+ return false;
+ }
+ // Initialize external connections or resources here
+ return true;
+ }
+
+ @Override
+ public boolean processRow() throws HopException {
+ // 1. Fetch next incoming row
+ Object[] row = getRow();
+
+ // If row is null, input stream is exhausted
+ if (row == null) {
+ setOutputDone();
+ return false; // Tells the engine to stop calling processRow()
+ }
+
+ // 2. Handle first row setup (resolve field indexes once)
+ if (first) {
+ first = false;
+ data.outputRowMeta = getInputRowMeta().clone();
+ meta.getFields(data.outputRowMeta, getTransformName(), null, null, this,
metadataProvider);
+
+ data.fieldIndex = getInputRowMeta().indexOfValue(meta.getTargetField());
+ if (data.fieldIndex < 0) {
+ throw new HopException("Field " + meta.getTargetField() + " not found
in input stream!");
+ }
+ }
+
+ // 3. Process the row
+ Long value = getInputRowMeta().getInteger(row, data.fieldIndex);
+ if (value != null && value >= meta.getThresholdValue()) {
+ // Emit row to outgoing hops
+ putRow(data.outputRowMeta, row);
+ }
+
+ return true; // Keep processing subsequent rows
+ }
+
+ @Override
+ public void dispose() {
+ // Clean up streams, sockets, or file handles
+ super.dispose();
+ }
+}
+----
+
+== 3. The Runtime Data Class (`FooData`)
+
+[source,java]
+----
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.pipeline.transform.BaseTransformData;
+
+public class CustomFilterData extends BaseTransformData {
+ public IRowMeta outputRowMeta;
+ public int fieldIndex = -1;
+
+ public CustomFilterData() {
+ super();
+ }
+}
+----
+
+== 4. The GUI Dialog (`FooDialog`)
+
+The dialog extends `BaseTransformDialog`.
+Annotate the metadata fields with `@GuiWidgetElement` and build them with
`GuiCompositeWidgets`.
+The `open()` recipe is xref:annotation-derived-widgets.adoc[Annotation derived
widgets], under "Transform and action dialogs".
+
+== Concrete Codebase Examples
+
+* `plugins/transforms/detectemptystream`: Minimal transform example.
+* `plugins/transforms/coalesce`: Modern transform demonstrating field mapping,
annotations, and unit tests.
+
+== See also
+
+* xref:plugin-types/pipeline-workflow.adoc[Pipelines and workflows]
+* xref:annotation-derived-widgets.adoc[Annotation derived widgets]
diff --git
a/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/value-types.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/value-types.adoc
new file mode 100644
index 0000000000..b85c8c2a61
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/value-types.adoc
@@ -0,0 +1,100 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: Implementing custom row data types in Apache Hop using
@ValueMetaPlugin and IValueMeta.
+[[Sdk-Plugins-ValueTypes]]
+= Value Metadata Plugins
+
+A value metadata plugin defines a new data type that flows across pipeline
hops in row streams (such as JSON, Avro, UUID, or XML).
+The plugin defines how values are stored in memory, converted to other data
types, compared, formatted, and serialized across network hops.
+
+== Implementing a Value Type
+
+Annotate your class with `@ValueMetaPlugin` and extend `ValueMetaBase`:
+
+[source,java]
+----
+import org.apache.hop.core.exception.HopValueException;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.value.ValueMetaBase;
+import org.apache.hop.core.row.value.ValueMetaPlugin;
+import java.util.UUID;
+
+@ValueMetaPlugin(
+ id = "32", // Permanent numeric ID constant
as a String
+ name = "UUID",
+ description = "Universally Unique Identifier",
+ image = "uuid.svg"
+)
+public class ValueMetaUuid extends ValueMetaBase implements IValueMeta {
+
+ public static final int TYPE_UUID = 32;
+
+ public ValueMetaUuid() {
+ this(null);
+ }
+
+ public ValueMetaUuid(String name) {
+ super(name, TYPE_UUID);
+ }
+
+ @Override
+ public int getType() {
+ return TYPE_UUID;
+ }
+
+ @Override
+ public Class<?> getNativeDataTypeClass() {
+ return UUID.class;
+ }
+
+ @Override
+ public String getString(Object object) throws HopValueException {
+ if (object == null) return null;
+ return object.toString();
+ }
+
+ @Override
+ public Object convertData(IValueMeta meta2, Object data2) throws
HopValueException {
+ if (data2 == null) return null;
+ if (data2 instanceof UUID) return data2;
+ if (data2 instanceof String str) {
+ return UUID.fromString(str);
+ }
+ return super.convertData(meta2, data2);
+ }
+
+ @Override
+ public Object clone() {
+ return new ValueMetaUuid(getName());
+ }
+}
+----
+
+== Key Architectural Rules
+
+* **Permanent Numeric ID:** The `id` string represents a permanent integer
constant serialized into `.hpl` pipeline files.
+* **Conversion Coverage:** Every value type must provide lossless conversions
to `String` and `Binary` so transforms that do not explicitly recognize your
custom type can still process or store the data.
+
+== Concrete Codebase Examples
+
+* `core/src/main/java/org/apache/hop/core/row/value/ValueMetaString.java`:
Core string value metadata implementation.
+* `core/src/main/java/org/apache/hop/core/row/value/ValueMetaJson.java`: JSON
value metadata. JSON is a core value type, not a plugin under
`plugins/valuetypes`.
+
+== See also
+
+* xref:value-types.adoc[Value types]
+* xref:plugin-types/data-connectivity.adoc[Data and connectivity]
diff --git
a/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/variable-resolvers.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/variable-resolvers.adoc
new file mode 100644
index 0000000000..5a3594ed24
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/variable-resolvers.adoc
@@ -0,0 +1,98 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: Developing variable resolver plugins in Apache Hop: integrating
external secrets managers and vaults via the #{resolver/path} syntax.
+[[Sdk-Plugins-VariableResolvers]]
+= Variable Resolvers
+
+A variable resolver fetches dynamic configuration values or sensitive
credentials on the fly from an external store: AWS Secrets Manager, Azure Key
Vault, Google Secret Manager, HashiCorp Vault, or another pipeline.
+This enables Hop workflows to use `#{resolver-name/secret-path}` syntax
without hardcoding secrets in metadata or file configurations.
+
+== Implementing a Variable Resolver
+
+Annotate your resolver class with `@VariableResolverPlugin` and implement
`IVariableResolver`:
+
+[source,java]
+----
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.variables.resolver.IVariableResolver;
+import org.apache.hop.core.variables.resolver.VariableResolver;
+import org.apache.hop.core.variables.resolver.VariableResolverPlugin;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import lombok.Getter;
+import lombok.Setter;
+
+@VariableResolverPlugin(
+ id = "custom-vault",
+ name = "Custom Vault Resolver",
+ description = "Resolves credentials from an external enterprise vault"
+)
+@GuiPlugin
+@Getter
+@Setter
+public class CustomVaultVariableResolver implements IVariableResolver {
+
+ @GuiWidgetElement(
+ id = "vault_url",
+ type = GuiElementType.TEXT,
+ label = "Vault Base URL",
+ parentId = VariableResolver.GUI_PLUGIN_ELEMENT_PARENT_ID
+ )
+ @HopMetadataProperty(key = "vault_url")
+ private String vaultUrl;
+
+ @GuiWidgetElement(
+ id = "vault_token",
+ type = GuiElementType.TEXT,
+ password = true,
+ label = "Vault Token",
+ parentId = VariableResolver.GUI_PLUGIN_ELEMENT_PARENT_ID
+ )
+ @HopMetadataProperty(key = "vault_token", password = true)
+ private String vaultToken;
+
+ @Override
+ public void init() {
+ // Validate connection to vault server
+ }
+
+ @Override
+ public String resolve(String secretPath, IVariables variables) throws
HopException {
+ // Query external vault using secretPath and return the secret value
+ return fetchSecretFromVault(vaultUrl, vaultToken, secretPath);
+ }
+}
+----
+
+== Key Architectural Rules
+
+* **Metadata Binding:** Variable resolver configuration is stored inside a
`Variable Resolver` metadata object. Decorate fields with
`@HopMetadataProperty` for JSON persistence and `@GuiWidgetElement` with
`parentId = VariableResolver.GUI_PLUGIN_ELEMENT_PARENT_ID` to automatically
appear in the GUI configuration dialog.
+* **Provider Context:** Variable resolution occurs in the context of the
running pipeline or workflow's `IHopMetadataProvider`. Never rely on static
global metadata singletons, or execution on remote Hop Servers will fail to
locate the resolver configuration.
+
+== Concrete Codebase Examples
+
+*
`plugins/tech/azure/src/main/java/org/apache/hop/core/variables/resolver/AzureKeyVaultVariableResolver.java`:
Azure Key Vault secret resolution.
+*
`plugins/tech/google/src/main/java/org/apache/hop/core/variables/resolver/GooleSecretManagerVariableResolver.java`:
Google Secret Manager integration. The class name is spelled that way in the
repository.
+*
`plugins/resolvers/pipeline/src/main/java/org/apache/hop/resolvers/pipeline/VariableResolverPipeline.java`:
Resolving variables dynamically by executing a pipeline.
+
+== See also
+
+* xref:plugin-types/metadata-configuration.adoc[Metadata and configuration]
+* xref:annotation-derived-widgets.adoc[Annotation derived widgets]
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/vfs.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/vfs.adoc
new file mode 100644
index 0000000000..a7d73d1ad7
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/plugins/vfs.adoc
@@ -0,0 +1,81 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: Implementing Virtual File System (VFS) plugins in Apache Hop:
adding custom storage providers and URL schemes to Apache Commons VFS.
+[[Sdk-Plugins-Vfs]]
+= VFS Plugins
+
+A VFS plugin teaches Apache Hop how to communicate with a new storage system
(such as Amazon S3, Google Cloud Storage, Azure Blob, SFTP, or WebDAV) by
registering custom URL schemes with the shared Apache Commons VFS manager.
+
+== Implementing a VFS Plugin
+
+Annotate the class with `@VfsPlugin` and implement `IVfs`:
+
+[source,java]
+----
+import org.apache.commons.vfs2.provider.FileProvider;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.vfs.plugin.IVfs;
+import org.apache.hop.core.vfs.plugin.VfsPlugin;
+import java.util.Map;
+
+@VfsPlugin(
+ type = "customcloud",
+ typeDescription = "Custom Cloud Storage VFS Provider",
+ classLoaderGroup = "vfs-customcloud"
+)
+public class CustomCloudVfsPlugin implements IVfs {
+
+ @Override
+ public String[] getUrlSchemes() {
+ // Return all URL prefixes claimed by this provider:
+ return new String[] {"customcloud", "ccfs"};
+ }
+
+ @Override
+ public FileProvider getProvider() {
+ // Return standard Commons VFS FileProvider implementation:
+ return new CustomCloudFileProvider();
+ }
+
+ @Override
+ public Map<String, FileProvider> getProviders(IVariables variables) {
+ // Return named providers based on configured metadata objects:
+ // e.g. "my-connection://path/to/file"
+ return loadNamedConnectionProviders(variables);
+ }
+}
+----
+
+== Key Architectural Rules
+
+* **Named Connections as URL Schemes:** Through `getProviders(IVariables)`,
each named metadata connection (such as an Azure or S3 connection object)
registers as its own global scheme (`my-connection://path`).
+* **`classLoaderGroup` is Mandatory:**
+[IMPORTANT]
+====
+Always specify `classLoaderGroup` on `@VfsPlugin` matching your metadata
connection classes and GUI editor plugins.
+Because named schemes register in a shared process-wide Commons VFS manager,
mismatched classloaders will result in `ClassCastException` when casting
configuration metadata.
+====
+
+== Concrete Codebase Examples
+
+*
`plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureVfsPlugin.java`:
Azure Blob and Data Lake Gen2 VFS provider.
+*
`plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/GoogleStorageVfsPlugin.java`:
Google Cloud Storage (GCS) provider.
+*
`plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/SftpVfsPlugin.java`:
SFTP file system provider.
+
+== See also
+
+* xref:plugin-types/data-connectivity.adoc[Data and connectivity]
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/sdk/vfs.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/vfs.adoc
new file mode 100644
index 0000000000..b3bad04135
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/vfs.adoc
@@ -0,0 +1,215 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: File handling in Apache Hop using Apache Commons VFS and HopVfs:
avoiding java.io.File, closing resources, and streaming across storage systems.
+[[Sdk-Vfs]]
+= File Handling with HopVFS
+
+Apache Hop uses **Apache Commons VFS** as its unified file system abstraction.
+Whether your data resides on a local NVMe drive, in AWS S3, Google Cloud
Storage, Azure Blob, an SFTP server, or inside a ZIP archive, Hop treats every
path through a single, consistent API: `org.apache.commons.vfs2.FileObject` and
the `org.apache.hop.core.vfs.HopVfs` helper class.
+
+== The Golden Rule: Avoid `java.io.File`
+
+[IMPORTANT]
+====
+**Unless there really is no other choice, never use `java.io.File` in Hop code
or plugins.**
+The default in Apache Hop is always to use
`org.apache.commons.vfs2.FileObject` via `HopVfs`.
+====
+
+Using `java.io.File` is one of the most common causes of bugs when running
pipelines outside a local developer workstation:
+
+* **Non-local Schemes:** `java.io.File` only understands the local operating
system's filesystem syntax. Passing a cloud path such as
`s3://my-bucket/orders.csv` or `gs://my-bucket/orders.csv` to `java.io.File`
will either throw a `FileNotFoundException` or corrupt the path into a
nonsensical local directory like `./s3:/my-bucket/orders.csv`.
+* **Portability across Runtimes:** Pipelines configured on a local machine
frequently execute on a remote Hop Server, in a Docker container, or on a
distributed cluster with Apache Beam / Spark / Flink. Using `HopVfs` ensures
that paths remain portable across all runtime targets.
+* **Variable Resolution:** `HopVfs` automatically expands variables (such as
`${PROJECT_HOME}/data/file.csv` or `${S3_BUCKET}/data.parquet`) before
resolving the underlying file system provider.
+
+== Core File Operations with HopVfs
+
+The `HopVfs` utility class provides static convenience methods for interacting
with files across all supported schemes.
+
+=== 1. Obtaining a `FileObject`
+
+[source,java]
+----
+import org.apache.commons.vfs2.FileObject;
+import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.core.variables.IVariables;
+
+IVariables variables = ...;
+String path = "${PROJECT_HOME}/input/sales_data.csv";
+
+// Resolves variables and obtains the Commons VFS FileObject
+FileObject fileObject = HopVfs.getFileObject(path, variables);
+
+if (fileObject.exists()) {
+ long sizeInBytes = fileObject.getContent().getSize();
+ boolean isFolder = fileObject.isFolder();
+}
+----
+
+=== 2. Reading Streams
+
+Always read file contents through `HopVfs.getInputStream()`:
+
+[source,java]
+----
+import java.io.InputStream;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+
+// Open stream directly from a path:
+try (InputStream inputStream =
HopVfs.getInputStream("s3://analytics-bucket/logs/app.log", variables);
+ BufferedReader reader = new BufferedReader(new
InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ // Process line
+ }
+}
+----
+
+You can also retrieve an `InputStream` directly from an existing `FileObject`:
+
+[source,java]
+----
+try (InputStream inputStream = HopVfs.getInputStream(fileObject)) {
+ // Read bytes
+}
+----
+
+=== 3. Writing Streams
+
+To create or overwrite a file, obtain an `OutputStream` via
`HopVfs.getOutputStream()`:
+
+[source,java]
+----
+import java.io.OutputStream;
+import java.io.BufferedWriter;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+
+String targetPath = "azure://data-container/output/results.json";
+boolean append = false;
+
+try (OutputStream outputStream = HopVfs.getOutputStream(targetPath, append,
variables);
+ BufferedWriter writer = new BufferedWriter(new
OutputStreamWriter(outputStream, StandardCharsets.UTF_8))) {
+ writer.write("{\"status\": \"OK\"}");
+ writer.flush();
+}
+----
+
+=== 4. Directory & File Management
+
+[source,java]
+----
+FileObject folder = HopVfs.getFileObject("s3://bucket/landing-zone/",
variables);
+
+// Create directory (and parents if needed)
+if (!folder.exists()) {
+ folder.createFolder();
+}
+
+// List child files
+FileObject[] children = folder.getChildren();
+for (FileObject child : children) {
+ if (child.isFile()) {
+ System.out.println("Found file: " + child.getName().getBaseName());
+ }
+}
+
+// Delete file or folder
+fileObject.delete();
+
+// Delete folder and all recursive contents
+folder.deleteAll();
+----
+
+== Closing Resources: Preventing Descriptor and Connection Leaks
+
+[CAUTION]
+====
+**Always close streams and `FileObject` handles.**
+Failing to close VFS resources leads to severe production issues:
+1. **Network socket exhaustion**: Cloud providers (AWS S3, Azure Blob, Google
Cloud Storage) and SFTP pools maintain underlying HTTP client connections and
SSH sessions. Leaving streams open drains the connection pool and stalls
subsequent requests.
+2. **File descriptor leaks**: On Linux/macOS, open file descriptors eventually
exceed the OS process limit (`Too many open files`).
+3. **File locking on Windows**: Windows locks open files, preventing
subsequent rename or delete operations in workflows.
+====
+
+=== Best Practices for Resource Management
+
+* **Use try-with-resources for all streams**:
++
+[source,java]
+----
+try (InputStream in = HopVfs.getInputStream(filename, variables)) {
+ // Read stream
+} // Automatically closed on exit
+----
+* **Close `FileObject` instances when finished**:
+`FileObject` implements `AutoCloseable`. When performing extensive file system
scans or operations, close the `FileObject` to release cached metadata and
network channel handles:
++
+[source,java]
+----
+try (FileObject file = HopVfs.getFileObject(remotePath, variables)) {
+ if (file.exists()) {
+ // perform operations
+ }
+} // file.close() releases underlying VFS resources
+----
+* **Use `file.close()` in transform `dispose()` methods**:
+If a transform opens a `FileObject` in `init()` or `processRow()`, ensure
`file.close()` is called in `dispose()`.
+
+== Supported URL Schemes and Related Docs
+
+Hop supports standard VFS schemes out of the box, with additional schemes
provided via plugins:
+
+|===
+|Scheme |Storage Provider |Plugin / Documentation
+
+|`file://`
+|Local file system
+|Built into `hop-core`
+
+|`s3://`
+|Amazon Simple Storage Service (S3)
+|xref:{page-component-version}@manual::vfs/aws-s3-vfs.adoc[AWS S3 VFS in User
Manual]
+
+|`gs://`
+|Google Cloud Storage (GCS)
+|xref:{page-component-version}@manual::vfs/google-cloud-storage-vfs.adoc[Google
Cloud Storage VFS]
+
+|`azure://` / `azfs://`
+|Microsoft Azure Blob & Data Lake
+|xref:{page-component-version}@manual::vfs/azure-blob-storage-vfs.adoc[Azure
Storage VFS]
+
+|`ftp://` / `sftp://`
+|FTP and Secure FTP (SSH)
+|`plugins/tech/ftp`, `plugins/tech/sftp`
+
+|`zip:` / `gz:` / `tar:`
+|Compressed archives
+|Apache Commons VFS core
+
+|`hdfs://`
+|Hadoop Distributed File System
+|xref:{page-component-version}@manual::vfs/hdfs-vfs.adoc[HDFS VFS]
+
+|`dbfs://`
+|Databricks Unity Catalog Volumes
+|xref:{page-component-version}@manual::vfs/databricks-vfs.adoc[Databricks VFS]
+|===
+
+* For configuring VFS connections in projects, see
xref:{page-component-version}@manual::vfs.adoc[Hop VFS in the User Manual].
+* For creating custom VFS schemes and storage providers, see
xref:sdk/plugins/vfs.adoc[VFS Plugins].
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/sdk/workflows.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/sdk/workflows.adoc
new file mode 100644
index 0000000000..10e0cc9ff7
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/sdk/workflows.adoc
@@ -0,0 +1,197 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: Loading, dynamically constructing, executing workflows
programmatically with the Hop SDK, and evaluating task execution results.
+[[Sdk-Workflows]]
+= Workflows API
+
+Workflows coordinate high-level sequential and parallel tasks: checking
prerequisites, executing pipelines, transferring files, managing errors, and
sending alerts.
+With the Hop SDK, you can load workflows from `.hwf` files, build workflow
topologies dynamically in memory, execute them synchronously or asynchronously,
and inspect execution outcomes.
+
+== Loading Workflow Metadata
+
+Workflow metadata defines the actions, hops, execution conditions, and
parameters.
+You can load workflow metadata from a file path or an arbitrary `InputStream`.
+
+=== Loading from a File
+
+[source,java]
+----
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.workflow.WorkflowMeta;
+
+IVariables variables = Variables.getADefaultVariableSpace();
+
+// Path can be a local filesystem path or any HopVFS scheme (s3://, etc.)
+String filename = "/path/to/my-workflow.hwf";
+
+WorkflowMeta workflowMeta = new WorkflowMeta(
+ variables,
+ filename,
+ metadataProvider // IHopMetadataProvider (e.g., JsonMetadataProvider)
+);
+----
+
+=== Loading from an InputStream
+
+[source,java]
+----
+try (InputStream inputStream =
HopVfs.getInputStream("gs://bucket/workflows/nightly.hwf", variables)) {
+ WorkflowMeta workflowMeta = new WorkflowMeta(
+ inputStream,
+ metadataProvider,
+ variables
+ );
+}
+----
+
+== Constructing Workflows Programmatically
+
+You can assemble a complete workflow using the Java API:
+
+[source,java]
+----
+import org.apache.hop.workflow.WorkflowHopMeta;
+import org.apache.hop.workflow.WorkflowMeta;
+import org.apache.hop.workflow.action.ActionMeta;
+import org.apache.hop.workflow.actions.dummy.ActionDummy;
+import org.apache.hop.workflow.actions.start.ActionStart;
+
+WorkflowMeta workflowMeta = new WorkflowMeta();
+workflowMeta.setName("DynamicWorkflow");
+
+// 1. Add the START action (entry point)
+ActionStart actionStart = new ActionStart("Start");
+ActionMeta startMeta = new ActionMeta(actionStart);
+startMeta.setLocation(100, 100);
+workflowMeta.addAction(startMeta);
+
+// 2. Add a subsequent task (e.g. Dummy action)
+ActionDummy actionDummy = new ActionDummy("Complete");
+ActionMeta dummyMeta = new ActionMeta(actionDummy);
+dummyMeta.setLocation(300, 100);
+workflowMeta.addAction(dummyMeta);
+
+// 3. Connect actions with a workflow hop
+WorkflowHopMeta hop = new WorkflowHopMeta(startMeta, dummyMeta);
+// Hop condition: unconditional (true), or conditional based on evaluation
success
+hop.setUnconditional(true);
+workflowMeta.addWorkflowHop(hop);
+----
+
+=== Hop Evaluation Modes
+
+A `WorkflowHopMeta` between action A and action B can behave in one of three
ways:
+
+* **Unconditional**: Action B executes regardless of whether Action A
succeeded or failed:
++
+[source,java]
+----
+hop.setUnconditional(true);
+----
+* **On Success (Conditional)**: Action B executes only if Action A returned a
successful result (`result.getResult() == true`):
++
+[source,java]
+----
+hop.setUnconditional(false);
+hop.setEvaluation(true);
+----
+* **On Failure (Conditional)**: Action B executes only if Action A failed
(`result.getResult() == false`):
++
+[source,java]
+----
+hop.setUnconditional(false);
+hop.setEvaluation(false);
+----
+
+== Executing Workflows
+
+Workflows are executed by an `IWorkflowEngine`.
+Like pipeline engines, workflow engines are created via the
`WorkflowEngineFactory` using a named **Workflow Run Configuration**:
+
+[source,java]
+----
+import org.apache.hop.core.Result;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.logging.LoggingObject;
+import org.apache.hop.workflow.engine.IWorkflowEngine;
+import org.apache.hop.workflow.engine.WorkflowEngineFactory;
+
+// The factory parent is an ILoggingObject. LogChannel implements ILogChannel
only.
+ILoggingObject loggingObject = new LoggingObject("WorkflowRunner");
+
+// Create the workflow engine using the "local" run configuration
+IWorkflowEngine<WorkflowMeta> workflowEngine =
WorkflowEngineFactory.createWorkflowEngine(
+ variables,
+ "local", // Name of the Workflow Run Configuration metadata
+ metadataProvider, // Metadata provider containing the configuration
+ workflowMeta,
+ loggingObject // Parent logging object
+);
+
+// Execute the workflow synchronously
+Result result = workflowEngine.startExecution();
+
+// Inspect outcome
+if (result.getResult() && result.getNrErrors() == 0) {
+ System.out.println("Workflow succeeded!");
+} else {
+ System.err.println("Workflow failed with " + result.getNrErrors() + "
errors.");
+}
+----
+
+== Passing Parameters and Variables
+
+You can pass execution variables and parameters directly to the workflow
engine before starting execution:
+
+[source,java]
+----
+// Inject runtime variables
+workflowEngine.setVariable("CUSTOMER_ID", "10492");
+workflowEngine.setVariable("EXECUTION_DATE", "2026-09-27");
+
+// If workflow parameters are declared on WorkflowMeta:
+workflowEngine.setParameterValue("ENVIRONMENT", "PRODUCTION");
+
+// Start execution
+Result result = workflowEngine.startExecution();
+----
+
+== Asynchronous Execution & Interruption
+
+If you prefer non-blocking execution or need to monitor the workflow
asynchronously:
+
+[source,java]
+----
+// Start workflow in background thread
+workflowEngine.fireWorkflowStartedListeners();
+Thread runner = new Thread(() -> {
+ try {
+ Result res = workflowEngine.startExecution();
+ } catch (Exception e) {
+ workflowEngine.getLogChannel().logError("Workflow error", e);
+ }
+});
+runner.start();
+
+// Check if running
+if (workflowEngine.isActive()) {
+ // To stop the workflow gracefully:
+ workflowEngine.stopExecution();
+}
+----
