This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 0d18753202 Fixes #8732 : Lint in the project context: -j and -e for
hop lint, and the project index for Lint Project and Lint Selected Folder
(#8742)
0d18753202 is described below
commit 0d187532020bbb14feb0b77cf7e020fdb3fdb94e
Author: Bart Maertens <[email protected]>
AuthorDate: Sat Oct 3 11:36:23 2026 +0200
Fixes #8732 : Lint in the project context: -j and -e for hop lint, and the
project index for Lint Project and Lint Selected Folder (#8742)
---
.../hop/core/config/plugin/ConfigPlugin.java | 1 +
.../modules/ROOT/pages/hop-tools/hop-lint.adoc | 15 ++
.../org/apache/hop/lint/CustomRuleExecutor.java | 5 +
.../org/apache/hop/lint/ExplorerLintGuiPlugin.java | 87 +++----
.../main/java/org/apache/hop/lint/HopLinter.java | 56 ++++-
.../main/java/org/apache/hop/lint/LintCommand.java | 174 +++++++++++---
.../java/org/apache/hop/lint/LintPathUtils.java | 13 ++
.../org/apache/hop/lint/ProjectContextTest.java | 250 +++++++++++++++++++++
.../projects/config/ProjectsLintOptionPlugin.java | 27 +++
9 files changed, 559 insertions(+), 69 deletions(-)
diff --git
a/core/src/main/java/org/apache/hop/core/config/plugin/ConfigPlugin.java
b/core/src/main/java/org/apache/hop/core/config/plugin/ConfigPlugin.java
index 9143f53802..89b58180b4 100644
--- a/core/src/main/java/org/apache/hop/core/config/plugin/ConfigPlugin.java
+++ b/core/src/main/java/org/apache/hop/core/config/plugin/ConfigPlugin.java
@@ -37,6 +37,7 @@ public @interface ConfigPlugin {
String CATEGORY_DOC = "doc";
String CATEGORY_PYTHON = "python";
String CATEGORY_NAMING = "naming";
+ String CATEGORY_LINT = "lint";
String CATEGORY_GUI = "gui";
String CATEGORY_EXPORT = "export";
diff --git a/docs/hop-user-manual/modules/ROOT/pages/hop-tools/hop-lint.adoc
b/docs/hop-user-manual/modules/ROOT/pages/hop-tools/hop-lint.adoc
index c1f121a34b..becac14f5e 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/hop-tools/hop-lint.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/hop-tools/hop-lint.adoc
@@ -36,16 +36,31 @@ See xref:linting/index.adoc[Linting] for what the linter
does, and xref:linting/
hop lint # lint the current directory
hop lint pipelines/load.hpl # lint one file
hop lint /path/to/project # lint a folder
+hop lint -e dev /path/to/project # lint with the variables of an
environment
hop lint --list-rules # show the effective rule set
hop lint --help # full option list
----
+== Project and environment
+
+`hop lint` takes the project and environment options of `hop run`.
+Hop's own checks then see the variables a pipeline is run with, so a transform
that reads `'${INPUT_FOLDER}'` from the environment is checked against that
folder rather than reported as unable to find it.
+Without `-j` or `-e` the default project and environment from the Hop
configuration apply, but only to what lies inside that project.
+Anything else, such as another project's folder or a commit checked by the
pre-commit hook, is linted with the `metadata` folder found above it.
+A project chosen with `-j` or `-e` applies whatever is linted; for a target
outside it, `hop lint` prints a warning.
+
== Options
[cols="1,3", options="header"]
|===
|Option |Description
+|`-j`, `--project <name>`
+|Lint with this project's variables and metadata
+
+|`-e`, `--environment <name>`
+|Lint with this environment's variables, and its project's
+
|`--format <format>`
|Report format: `text` (default), `json` or `sarif`
diff --git
a/plugins/misc/lint/src/main/java/org/apache/hop/lint/CustomRuleExecutor.java
b/plugins/misc/lint/src/main/java/org/apache/hop/lint/CustomRuleExecutor.java
index 1fe5f6be25..faeec3e56e 100644
---
a/plugins/misc/lint/src/main/java/org/apache/hop/lint/CustomRuleExecutor.java
+++
b/plugins/misc/lint/src/main/java/org/apache/hop/lint/CustomRuleExecutor.java
@@ -87,6 +87,11 @@ public class CustomRuleExecutor {
}
}
+ /** Whether a project index is in place for the rules evaluated on this
thread. */
+ public static boolean hasProjectIndex() {
+ return PROJECT_INDEX.get().isPopulated();
+ }
+
/**
* Make the file being linted available to the rules evaluated on this
thread.
*
diff --git
a/plugins/misc/lint/src/main/java/org/apache/hop/lint/ExplorerLintGuiPlugin.java
b/plugins/misc/lint/src/main/java/org/apache/hop/lint/ExplorerLintGuiPlugin.java
index 4c7988d96d..a7c311ee31 100644
---
a/plugins/misc/lint/src/main/java/org/apache/hop/lint/ExplorerLintGuiPlugin.java
+++
b/plugins/misc/lint/src/main/java/org/apache/hop/lint/ExplorerLintGuiPlugin.java
@@ -540,47 +540,58 @@ public class ExplorerLintGuiPlugin {
List<LintResult> results = new java.util.ArrayList<>();
int processedFilesCount = 0;
- for (String filePath : hopFilePaths) {
- File file = new File(filePath);
- if (progressDialog.isCancelled()) {
- log.logDetailed("Folder linting cancelled by user");
- return;
- }
+ // The unreferenced pipeline and workflow rules need the whole
project, not just the
+ // folder: a pipeline in it may be called from anywhere in the
project.
+ CustomRuleExecutor.setProjectIndex(
+ linter.buildProjectIndex(folderPath, metadataProvider,
variables));
+ try {
+
+ for (String filePath : hopFilePaths) {
+ File file = new File(filePath);
+ if (progressDialog.isCancelled()) {
+ log.logDetailed("Folder linting cancelled by user");
+ return;
+ }
- try {
- progressDialog.updateProgress(
- "Processing: " + file.getName(), processedFilesCount,
hopFilePaths.size());
- String normalizedPath =
LintPathUtils.normalizePath(file.getAbsolutePath());
- List<LintResult> fileResults;
- if (normalizedPath.toLowerCase().endsWith(".hpl")) {
- PipelineMeta pipelineMeta =
- new PipelineMeta(file.getAbsolutePath(),
metadataProvider, variables);
- fileResults =
- PipelineLintResultsBuilder.build(
- pipelineMeta, normalizedPath, metadataProvider,
variables);
- } else if
(HopMetadataFileLoader.isMetadataJsonFile(normalizedPath)) {
- fileResults = linter.processFile(file, metadataProvider,
variables);
- } else {
- fileResults = linter.processFile(file, metadataProvider,
variables);
+ try {
+ progressDialog.updateProgress(
+ "Processing: " + file.getName(), processedFilesCount,
hopFilePaths.size());
+ String normalizedPath =
LintPathUtils.normalizePath(file.getAbsolutePath());
+ List<LintResult> fileResults;
+ if (normalizedPath.toLowerCase().endsWith(".hpl")) {
+ PipelineMeta pipelineMeta =
+ new PipelineMeta(file.getAbsolutePath(),
metadataProvider, variables);
+ fileResults =
+ PipelineLintResultsBuilder.build(
+ pipelineMeta, normalizedPath, metadataProvider,
variables);
+ } else if
(HopMetadataFileLoader.isMetadataJsonFile(normalizedPath)) {
+ fileResults = linter.processFile(file, metadataProvider,
variables);
+ } else {
+ fileResults = linter.processFile(file, metadataProvider,
variables);
+ }
+ results.addAll(fileResults);
+ LintResultsManager.getInstance()
+ .updateResultsForFile(normalizedPath, fileResults);
+ processedFilesCount++;
+ } catch (Exception e) {
+ log.logError("Error processing file: " +
file.getAbsolutePath(), e);
+ LintResult errorResult =
+ new LintResult(
+ "SYSTEM-001",
+ "File Processing Error",
+ "ERROR",
+ "Failed to process file: " + e.getMessage(),
+ LintPathUtils.normalizePath(file.getAbsolutePath()));
+ results.add(errorResult);
+ LintResultsManager.getInstance()
+ .updateResultsForFile(
+ LintPathUtils.normalizePath(file.getAbsolutePath()),
+ List.of(errorResult));
+ processedFilesCount++;
}
- results.addAll(fileResults);
-
LintResultsManager.getInstance().updateResultsForFile(normalizedPath,
fileResults);
- processedFilesCount++;
- } catch (Exception e) {
- log.logError("Error processing file: " +
file.getAbsolutePath(), e);
- LintResult errorResult =
- new LintResult(
- "SYSTEM-001",
- "File Processing Error",
- "ERROR",
- "Failed to process file: " + e.getMessage(),
- LintPathUtils.normalizePath(file.getAbsolutePath()));
- results.add(errorResult);
- LintResultsManager.getInstance()
- .updateResultsForFile(
- LintPathUtils.normalizePath(file.getAbsolutePath()),
List.of(errorResult));
- processedFilesCount++;
}
+ } finally {
+ CustomRuleExecutor.setProjectIndex(null);
}
progressDialog.setComplete("Completed. Found " + results.size() +
" issues");
diff --git a/plugins/misc/lint/src/main/java/org/apache/hop/lint/HopLinter.java
b/plugins/misc/lint/src/main/java/org/apache/hop/lint/HopLinter.java
index 23b41b9f77..455128b7c9 100644
--- a/plugins/misc/lint/src/main/java/org/apache/hop/lint/HopLinter.java
+++ b/plugins/misc/lint/src/main/java/org/apache/hop/lint/HopLinter.java
@@ -32,6 +32,7 @@ import org.apache.hop.core.IProgressMonitor;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.logging.ILogChannel;
import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.variables.Variables;
import org.apache.hop.lint.registry.EffectiveRuleSet;
@@ -195,6 +196,21 @@ public class HopLinter {
List<LintResult> allResults = new ArrayList<>();
long startTime = System.currentTimeMillis();
+ if (variables == null) {
+ variables = Variables.getADefaultVariableSpace();
+ }
+ // Lint Project in Hop Gui called this without an index, so the rules that
need the whole
+ // project (STRUCT-004 and STRUCT-005, unreferenced pipelines and
workflows) were skipped there
+ // while the CLI, which built its own, reported them.
+ boolean ownIndex = false;
+ if (!CustomRuleExecutor.hasProjectIndex()) {
+ LintProjectIndex index = buildProjectIndex(projectPath,
metadataProvider, variables);
+ if (index != null) {
+ CustomRuleExecutor.setProjectIndex(index);
+ ownIndex = true;
+ }
+ }
+
try {
log.logBasic("Starting linter with project path: " + projectPath);
@@ -220,12 +236,6 @@ public class HopLinter {
+ fileDiscoveryTime
+ "ms)");
- // Create default variables if not provided
- if (variables == null) {
- log.logBasic("Variables is null, creating default variable space");
- variables = Variables.getADefaultVariableSpace();
- }
-
// Process each file
long fileProcessingStart = System.currentTimeMillis();
int processedFiles = 0;
@@ -317,11 +327,45 @@ public class HopLinter {
"ERROR",
"Failed to complete linting: " + e.getMessage(),
"system"));
+ } finally {
+ if (ownIndex) {
+ CustomRuleExecutor.setProjectIndex(null);
+ }
}
return allResults;
}
+ /**
+ * The references in the project a lint target belongs to, for the rules
that need the whole
+ * project to answer.
+ *
+ * <p>The index covers {@code PROJECT_HOME} when the target lies inside it,
so a folder or a file
+ * is judged against everything that could call it, not only its neighbours.
Outside a project a
+ * folder is indexed on its own; a single file outside a project gets no
index, and those rules
+ * stay quiet rather than guess.
+ *
+ * @return the index, or null when the target has no project to index
+ */
+ public LintProjectIndex buildProjectIndex(
+ String targetPath, IHopMetadataProvider metadataProvider, IVariables
variables) {
+ File target = new File(targetPath).getAbsoluteFile();
+ String root = null;
+ String projectHome = variables == null ? null :
variables.getVariable("PROJECT_HOME");
+ if (!Utils.isEmpty(projectHome) && LintPathUtils.isWithin(target, new
File(projectHome))) {
+ root = new File(projectHome).getAbsolutePath();
+ } else if (target.isDirectory()) {
+ root = target.getPath();
+ }
+ if (root == null) {
+ return null;
+ }
+ LintProjectIndex index =
+ LintProjectIndex.build(findHopFiles(root), metadataProvider,
variables);
+ log.logDetailed("Indexed " + index.getIndexedFiles().size() + " file(s)
under " + root);
+ return index;
+ }
+
/**
* Find all .hpl and .hwf files in the project directory
*
diff --git
a/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintCommand.java
b/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintCommand.java
index 356ca8ba96..f481f76ae8 100644
--- a/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintCommand.java
+++ b/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintCommand.java
@@ -37,6 +37,8 @@ import lombok.Setter;
import org.apache.hop.core.Const;
import org.apache.hop.core.HopEnvironment;
import org.apache.hop.core.HopVersionProvider;
+import org.apache.hop.core.config.plugin.ConfigPlugin;
+import org.apache.hop.core.config.plugin.IConfigOptions;
import org.apache.hop.core.encryption.Encr;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.logging.DefaultLogLevel;
@@ -50,10 +52,12 @@ import org.apache.hop.core.plugins.TransformPluginType;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.variables.Variables;
+import org.apache.hop.hop.Hop;
import org.apache.hop.hop.plugin.HopCommand;
import org.apache.hop.hop.plugin.IHopCommand;
import org.apache.hop.lint.registry.EffectiveRuleSet;
import org.apache.hop.lint.registry.RuleRegistry;
+import org.apache.hop.metadata.api.IHasHopMetadataProvider;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.metadata.serializer.json.JsonMetadataProvider;
import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
@@ -80,7 +84,7 @@ import picocli.CommandLine.Parameters;
+ "reaches the --fail-on threshold (ERROR by default) or warnings
exceed "
+ "--max-warnings.")
@HopCommand(id = "lint", description = "Check Hop files against the lint
rules")
-public class LintCommand implements Callable<Integer>, IHopCommand {
+public class LintCommand implements Callable<Integer>, IHopCommand,
IHasHopMetadataProvider {
// Deliberately no static ILogChannel field here. Touching LogChannel loads
Hop's configuration
// during class initialisation, which prints to stdout before main() gets a
chance to run — and
@@ -189,14 +193,28 @@ public class LintCommand implements Callable<Integer>,
IHopCommand {
private IVariables commandVariables;
private MultiMetadataProvider metadataProvider;
+ /** Whether -j, -e or a default project enabled a project, whose metadata
then applies. */
+ private boolean projectEnabled;
+
+ /** Whether the project was asked for with -j or -e, here or on the hop
command itself. */
+ private boolean projectChosen;
+
+ /** The variables as they were before a project was enabled, to fall back
to. */
+ private IVariables variablesBeforeProject;
+
+ private boolean prepared;
+
@Override
public void initialize(
- CommandLine cmd, IVariables variables, MultiMetadataProvider
metadataProvider) {
+ CommandLine cmd, IVariables variables, MultiMetadataProvider
metadataProvider)
+ throws HopException {
this.cmd = cmd;
this.commandVariables = variables;
this.metadataProvider = metadataProvider;
// The hand-rolled parser upper-cased these, so "--severity warning" has
to keep working.
cmd.setCaseInsensitiveEnumValuesAllowed(true);
+ // -j and -e, from the projects plugin, as hop run and the other commands
have them.
+ Hop.addMixinPlugins(cmd, ConfigPlugin.CATEGORY_LINT);
}
// The stack trace is only printed when the user asks for it with --verbose
@@ -232,17 +250,17 @@ public class LintCommand implements Callable<Integer>,
IHopCommand {
}
if (!Utils.isEmpty(listFieldsFor)) {
- initializeHopEnvironment();
+ prepare();
return printFields(listFieldsFor);
}
if (listMetadataTypes) {
- initializeHopEnvironment();
+ prepare();
return printMetadataTypes();
}
if (listRules) {
- initializeHopEnvironment();
+ prepare();
printRuleList();
return 0;
}
@@ -265,7 +283,8 @@ public class LintCommand implements Callable<Integer>,
IHopCommand {
try {
printRunHeader(target);
- initializeHopEnvironment();
+ prepare();
+ applyProjectTo(target);
HopLinter linter = new HopLinter();
loadConfiguration(linter, target);
@@ -429,8 +448,9 @@ public class LintCommand implements Callable<Integer>,
IHopCommand {
*/
private int printMetadataTypes() {
try {
- IVariables variables = Variables.getADefaultVariableSpace();
String targetPath = Utils.isEmpty(target) ? userDirectory() : target;
+ applyProjectTo(targetPath);
+ IVariables variables = variables();
IHopMetadataProvider provider = resolveMetadataProvider(new
File(targetPath), variables);
if (provider == null) {
System.err.println("No metadata provider available; cannot list
metadata types.");
@@ -565,7 +585,7 @@ public class LintCommand implements Callable<Integer>,
IHopCommand {
throw new IllegalArgumentException("--pre-commit requires --staged-file
<path>");
}
- initializeHopEnvironment();
+ prepare();
List<File> stagedFiles =
PreCommitLintService.readStagedFiles(stagedFileList, new
File(userDirectory()));
if (stagedFiles.isEmpty()) {
@@ -580,8 +600,9 @@ public class LintCommand implements Callable<Integer>,
IHopCommand {
// The lint target for path-relative purposes is the project the staged
files live in.
target = projectRootOf(stagedFiles.get(0));
+ applyProjectTo(stagedFiles.get(0).getPath());
- IVariables variables = Variables.getADefaultVariableSpace();
+ IVariables variables = variables();
// Without a metadata provider a pipeline will not load at all, and
connection rules cannot
// resolve — the hook would pass commits it should have blocked.
IHopMetadataProvider metadataProvider =
resolveMetadataProvider(stagedFiles.get(0), variables);
@@ -671,6 +692,103 @@ public class LintCommand implements Callable<Integer>,
IHopCommand {
System.out.println();
}
+ /**
+ * Start Hop and apply {@code -j} / {@code -e}, once.
+ *
+ * <p>Without this the run used an empty set of variables: a Get File Names
transform reading
+ * {@code ${JDBC_PROPERTIES_FOLDER}} from the project's environment reported
"No files can be
+ * found to read", where Verify in Hop Gui, with the environment active, was
clean. As with {@code
+ * hop run}, the default project and environment from hop-config.json are
enabled when neither
+ * option is given; {@link #applyProjectTo} then decides whether they apply
to what is linted.
+ */
+ private void prepare() throws HopException {
+ if (prepared) {
+ return;
+ }
+ prepared = true;
+ initializeHopEnvironment();
+ if (cmd == null) {
+ return;
+ }
+ IVariables variables = variables();
+ // "hop -j x lint" enables the project before this command runs, and
leaves its name behind.
+ projectChosen =
+ !Utils.isEmpty(variables.getVariable("HOP_PROJECT_NAME"))
+ || optionGiven("-j", "--project", "-e", "--environment");
+ variablesBeforeProject = new Variables();
+ variablesBeforeProject.copyFrom(variables);
+ for (Object mixin : cmd.getMixins().values()) {
+ if (mixin instanceof IConfigOptions options
+ && options.handleOption(LogChannel.GENERAL, this, variables)) {
+ projectEnabled = true;
+ }
+ }
+ }
+
+ private boolean optionGiven(String... names) {
+ CommandLine.ParseResult parseResult = cmd.getParseResult();
+ if (parseResult == null) {
+ return false;
+ }
+ for (String name : names) {
+ if (parseResult.hasMatchedOption(name)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /** The variables Hop was started with, with the project's and environment's
on top. */
+ private IVariables variables() {
+ if (commandVariables == null) {
+ commandVariables = Variables.getADefaultVariableSpace();
+ }
+ return commandVariables;
+ }
+
+ /**
+ * Keep the enabled project's variables and metadata only where they belong:
when -j or -e asked
+ * for it, or when what is linted lies inside it.
+ *
+ * <p>A stock hop-config.json has a default project. Applied to everything,
it made {@code hop
+ * lint /path/to/other-project} and the pre-commit hook read the default
project's connections
+ * instead of the other project's own {@code metadata/} folder, and report
connections that exist
+ * as missing. A default project that does not contain the target is
dropped, and the target's own
+ * metadata folder is used as before.
+ */
+ private void applyProjectTo(String path) {
+ if (!projectEnabled) {
+ return;
+ }
+ String projectHome = variables().getVariable("PROJECT_HOME");
+ boolean contains =
+ !Utils.isEmpty(projectHome)
+ && !Utils.isEmpty(path)
+ && LintPathUtils.isWithin(new File(path), new File(projectHome));
+ if (contains) {
+ return;
+ }
+ if (projectChosen) {
+ System.err.println(
+ "Warning: "
+ + path
+ + " is outside the project in use ("
+ + projectHome
+ + "); its variables and metadata apply.");
+ return;
+ }
+ commandVariables = variablesBeforeProject;
+ projectEnabled = false;
+ if (verbose) {
+ System.out.println(
+ "Not using the default project ("
+ + projectHome
+ + "): it does not contain "
+ + path
+ + ". Choose a project with -j or -e.");
+ }
+ }
+
/**
* Bring up Hop far enough to load pipelines and workflows properly.
*
@@ -726,33 +844,30 @@ public class LintCommand implements Callable<Integer>,
IHopCommand {
private List<LintResult> runLinting(HopLinter linter, String targetPath)
throws Exception {
File targetFile = new File(targetPath);
- IVariables variables = Variables.getADefaultVariableSpace();
+ IVariables variables = variables();
IHopMetadataProvider metadataProvider =
resolveMetadataProvider(targetFile, variables);
if (targetFile.isFile()) {
if (verbose) {
System.out.println("Linting file: " + targetPath);
}
- return new ArrayList<>(linter.processFile(targetFile, metadataProvider,
variables));
+ // A file in a project is judged against the whole project, so it can be
reported as
+ // called by nothing; outside one there is nothing to judge it against.
+ CustomRuleExecutor.setProjectIndex(
+ linter.buildProjectIndex(targetPath, metadataProvider, variables));
+ try {
+ return new ArrayList<>(linter.processFile(targetFile,
metadataProvider, variables));
+ } finally {
+ CustomRuleExecutor.setProjectIndex(null);
+ }
}
if (targetFile.isDirectory()) {
if (verbose) {
System.out.println("Linting directory: " + targetPath);
}
- // Index the project's references first, so that rules which depend on
the project as a whole
- // — whether a pipeline is called by anything, whether a connection is
used — have something
- // to read. Only a directory lint can build this; a single file has no
project to see.
- List<String> projectFiles = linter.findHopFiles(targetPath);
- LintProjectIndex index = LintProjectIndex.build(projectFiles,
metadataProvider, variables);
- if (verbose) {
- System.out.println("Indexed " + index.getIndexedFiles().size() + "
file(s) for references");
- }
- CustomRuleExecutor.setProjectIndex(index);
- try {
- return new ArrayList<>(linter.run(targetPath, metadataProvider,
variables, null));
- } finally {
- CustomRuleExecutor.setProjectIndex(null);
- }
+ // The run indexes the project's references itself, for the rules that
need the project
+ // as a whole: whether a pipeline is called by anything, whether a
connection is used.
+ return new ArrayList<>(linter.run(targetPath, metadataProvider,
variables, null));
}
throw new IllegalArgumentException("Target does not exist: " + targetPath);
}
@@ -766,6 +881,15 @@ public class LintCommand implements Callable<Integer>,
IHopCommand {
* about it: point the linter at a pipeline deep in a project and it still
finds the project.
*/
private IHopMetadataProvider resolveMetadataProvider(File target, IVariables
variables) {
+ // An enabled project brings its own metadata, parent projects included,
from wherever its
+ // configuration says it lives.
+ if (projectEnabled && metadataProvider != null) {
+ if (verbose) {
+ System.out.println(
+ "Using the metadata of project home: " +
variables.getVariable("PROJECT_HOME"));
+ }
+ return metadataProvider;
+ }
File metadataFolder = findMetadataFolder(target);
if (metadataFolder == null) {
if (verbose) {
diff --git
a/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintPathUtils.java
b/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintPathUtils.java
index 913e28bac2..177b38fd6f 100644
--- a/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintPathUtils.java
+++ b/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintPathUtils.java
@@ -17,6 +17,7 @@
package org.apache.hop.lint;
import java.io.File;
+import java.io.IOException;
import org.apache.commons.vfs2.FileObject;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.vfs.HopVfs;
@@ -74,6 +75,18 @@ public final class LintPathUtils {
return normalizedA.equalsIgnoreCase(normalizedB);
}
+ /** Whether a file or folder is the folder given or lies somewhere below it.
*/
+ public static boolean isWithin(File path, File folder) {
+ if (path == null || folder == null) {
+ return false;
+ }
+ try {
+ return
path.getCanonicalFile().toPath().startsWith(folder.getCanonicalFile().toPath());
+ } catch (IOException e) {
+ return
path.getAbsoluteFile().toPath().startsWith(folder.getAbsoluteFile().toPath());
+ }
+ }
+
/** Explorer passes the full VFS path as {@code path} and the basename as
{@code name}. */
public static String resolveExplorerFilePath(String path, String name) {
if (Utils.isEmpty(path)) {
diff --git
a/plugins/misc/lint/src/test/java/org/apache/hop/lint/ProjectContextTest.java
b/plugins/misc/lint/src/test/java/org/apache/hop/lint/ProjectContextTest.java
new file mode 100644
index 0000000000..51179d9feb
--- /dev/null
+++
b/plugins/misc/lint/src/test/java/org/apache/hop/lint/ProjectContextTest.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.lint;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import org.apache.hop.core.config.plugin.IConfigOptions;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.metadata.api.IHasHopMetadataProvider;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import picocli.CommandLine;
+
+/**
+ * Linting in the context of a project.
+ *
+ * <p>{@code hop lint} ignored the project and its environment, so variables
they define were
+ * unresolved. Lint Project and Lint Selected Folder in Hop Gui left out the
rules that need the
+ * whole project, STRUCT-004 and STRUCT-005, which the CLI reported.
+ *
+ * @see <a href="https://github.com/apache/hop/issues/8732";>#8732</a>
+ */
+public class ProjectContextTest {
+
+ @TempDir private Path dir;
+
+ private Path project;
+
+ @BeforeEach
+ void createProject() throws Exception {
+ project = dir.resolve("project");
+ Files.createDirectories(project.resolve("pipelines/load"));
+ Files.writeString(
+ project.resolve("hop-lint.yml"), "rules:\n STRUCT-004:\n enabled:
true\n");
+ writePipeline(project.resolve("pipelines/load/customers.hpl"));
+ writePipeline(project.resolve("pipelines/report.hpl"));
+ }
+
+ @AfterEach
+ void clearIndex() {
+ CustomRuleExecutor.setProjectIndex(null);
+ }
+
+ private static void writePipeline(Path file) throws Exception {
+ PipelineMeta pipelineMeta = new PipelineMeta();
+ pipelineMeta.setName(file.getFileName().toString().replace(".hpl", ""));
+ Files.writeString(file,
pipelineMeta.getXml(Variables.getADefaultVariableSpace()));
+ }
+
+ private IVariables projectVariables() {
+ IVariables variables = Variables.getADefaultVariableSpace();
+ variables.setVariable("PROJECT_HOME", project.toString());
+ return variables;
+ }
+
+ private static long count(List<LintResult> results, String ruleId) {
+ return results.stream().filter(r -> ruleId.equals(r.getRuleId())).count();
+ }
+
+ /** What Lint Project in Hop Gui does: run() with no index of its own. */
+ @Test
+ public void aProjectLintReportsUnreferencedPipelines() {
+ List<LintResult> results =
+ new HopLinter()
+ .run(project.toString(), new MemoryMetadataProvider(),
projectVariables(), null);
+
+ assertEquals(2, count(results, "STRUCT-004"), results.toString());
+ assertFalse(CustomRuleExecutor.hasProjectIndex(), "the run leaves no index
behind");
+ }
+
+ /** A folder is indexed against the whole project, not only itself. */
+ @Test
+ public void aFolderInAProjectIsIndexedAgainstTheProject() {
+ LintProjectIndex index =
+ new HopLinter()
+ .buildProjectIndex(
+ project.resolve("pipelines/load").toString(), null,
projectVariables());
+
+ assertTrue(
+ index.getIndexedFiles().stream().anyMatch(f ->
f.endsWith("pipelines/report.hpl")),
+ index.getIndexedFiles().toString());
+ }
+
+ @Test
+ public void aFolderOutsideAProjectIsIndexedOnItsOwn() {
+ LintProjectIndex index =
+ new HopLinter()
+ .buildProjectIndex(
+ project.resolve("pipelines/load").toString(),
+ null,
+ Variables.getADefaultVariableSpace());
+
+ assertEquals(1, index.getIndexedFiles().size(),
index.getIndexedFiles().toString());
+ }
+
+ @Test
+ public void aSingleFileOutsideAProjectHasNoIndex() {
+ assertNull(
+ new HopLinter()
+ .buildProjectIndex(
+ project.resolve("pipelines/report.hpl").toString(),
+ null,
+ Variables.getADefaultVariableSpace()));
+ }
+
+ // ------------------------------------------------------------------ hop
lint -j / -e
+
+ /**
+ * Stands in for the projects plugin's -j / -e. It enables its project
whether or not -j is given,
+ * as the real one enables the default project of hop-config.json.
+ */
+ public static class FakeProjectOptions implements IConfigOptions {
+ @CommandLine.Option(
+ names = {"-j", "--project"},
+ description = "The project")
+ private String projectName;
+
+ private final String projectHome;
+ private final MultiMetadataProvider projectMetadata;
+
+ FakeProjectOptions(String projectHome, MultiMetadataProvider
projectMetadata) {
+ this.projectHome = projectHome;
+ this.projectMetadata = projectMetadata;
+ }
+
+ @Override
+ public boolean handleOption(
+ ILogChannel log, IHasHopMetadataProvider hasHopMetadataProvider,
IVariables variables) {
+ variables.setVariable("PROJECT_HOME", projectHome);
+ hasHopMetadataProvider.setMetadataProvider(projectMetadata);
+ return true;
+ }
+ }
+
+ /** Runs hop lint with the stand-in project; returns stdout and stderr. */
+ private String[] lint(String projectHome, String... args) throws Exception {
+ LintCommand command = new LintCommand();
+ CommandLine commandLine = new CommandLine(command);
+ command.initialize(
+ commandLine,
+ Variables.getADefaultVariableSpace(),
+ new MultiMetadataProvider(Variables.getADefaultVariableSpace()));
+ MultiMetadataProvider projectMetadata =
+ new MultiMetadataProvider(Variables.getADefaultVariableSpace());
+ commandLine.addMixin("project", new FakeProjectOptions(projectHome,
projectMetadata));
+
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ByteArrayOutputStream err = new ByteArrayOutputStream();
+ PrintStream systemOut = System.out;
+ PrintStream systemErr = System.err;
+ try {
+ System.setOut(new PrintStream(out, true, StandardCharsets.UTF_8));
+ System.setErr(new PrintStream(err, true, StandardCharsets.UTF_8));
+ commandLine.execute(args);
+ } finally {
+ System.setOut(systemOut);
+ System.setErr(systemErr);
+ }
+ return new String[] {
+ out.toString(StandardCharsets.UTF_8),
err.toString(StandardCharsets.UTF_8)
+ };
+ }
+
+ @Test
+ public void theProjectsMetadataIsUsed() throws Exception {
+ String[] output = lint(project.toString(), "-v",
project.resolve("pipelines").toString());
+
+ assertTrue(output[0].contains("Using the metadata of project home"),
output[0]);
+ assertFalse(output[1].contains("outside the project"), output[1]);
+ }
+
+ /** Another project, with its own metadata folder, outside the default one.
*/
+ private Path otherProject() throws Exception {
+ Path other = dir.resolve("other");
+ Files.createDirectories(other.resolve("metadata/rdbms"));
+ writePipeline(other.resolve("orders.hpl"));
+ return other;
+ }
+
+ /**
+ * A stock hop-config.json has a default project. Applied to a project it
does not contain, it
+ * replaced that project's own metadata and reported its connections as
missing.
+ */
+ @Test
+ public void aDefaultProjectDoesNotApplyToAnotherProject() throws Exception {
+ Path other = otherProject();
+
+ String[] output = lint(project.toString(), "-v", other.toString());
+
+ assertTrue(
+ output[0].contains("Using metadata folder: " +
other.resolve("metadata")), output[0]);
+ assertFalse(output[0].contains("Using the metadata of project home"),
output[0]);
+ assertFalse(output[1].contains("outside the project"), output[1]);
+ }
+
+ @Test
+ public void theHookUsesTheMetadataOfTheProjectItCommitsTo() throws Exception
{
+ Path other = otherProject();
+ Path staged = dir.resolve("staged.txt");
+ Files.writeString(staged, other.resolve("orders.hpl") + "\n");
+
+ String[] output =
+ lint(project.toString(), "-v", "--pre-commit", "--staged-file",
staged.toString());
+
+ assertTrue(
+ output[0].contains("Using metadata folder: " +
other.resolve("metadata")), output[0]);
+ assertFalse(output[0].contains("Using the metadata of project home"),
output[0]);
+ }
+
+ /** A project asked for with -j applies wherever the target is, with a
warning. */
+ @Test
+ public void aChosenProjectAppliesOutsideItsHomeWithAWarning() throws
Exception {
+ Path other = otherProject();
+
+ String[] output = lint(project.toString(), "-v", "-j", "default",
other.toString());
+
+ assertTrue(output[0].contains("Using the metadata of project home"),
output[0]);
+ assertTrue(output[1].contains("is outside the project in use"), output[1]);
+ }
+}
diff --git
a/plugins/misc/projects/src/main/java/org/apache/hop/projects/config/ProjectsLintOptionPlugin.java
b/plugins/misc/projects/src/main/java/org/apache/hop/projects/config/ProjectsLintOptionPlugin.java
new file mode 100644
index 0000000000..0b2b1a5e2a
--- /dev/null
+++
b/plugins/misc/projects/src/main/java/org/apache/hop/projects/config/ProjectsLintOptionPlugin.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.projects.config;
+
+import org.apache.hop.core.config.plugin.ConfigPlugin;
+import org.apache.hop.core.config.plugin.IConfigOptions;
+
+@ConfigPlugin(
+ id = "ProjectsLintOptionPlugin",
+ description = "Project and Environment configuration options for hop lint",
+ category = ConfigPlugin.CATEGORY_LINT)
+public class ProjectsLintOptionPlugin extends ProjectsOptionPlugin implements
IConfigOptions {}
