This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 897b81481ec camel-jbang-plugin-k: infer the dependencies by inspecting
the route (#13193)
897b81481ec is described below
commit 897b81481eca7f211821efa188d80e3f96a312b3
Author: Claudio Miranda <clau...@claudius.com.br>
AuthorDate: Wed Feb 21 05:26:52 2024 -0300
camel-jbang-plugin-k: infer the dependencies by inspecting the route
(#13193)
---
.../dsl/jbang/core/commands/k/IntegrationRun.java | 162 +++++++++++++++++----
.../jbang/core/commands/k/IntegrationRunTest.java | 58 ++++++++
.../dsl/jbang/core/commands/k/KubeBaseTest.java | 13 ++
.../src/test/resources/Sample.java | 43 ++++++
.../src/test/resources/route-deps.yaml | 36 +++++
5 files changed, 281 insertions(+), 31 deletions(-)
diff --git
a/dsl/camel-jbang/camel-jbang-plugin-k/src/main/java/org/apache/camel/dsl/jbang/core/commands/k/IntegrationRun.java
b/dsl/camel-jbang/camel-jbang-plugin-k/src/main/java/org/apache/camel/dsl/jbang/core/commands/k/IntegrationRun.java
index 6d5b5a267ca..526bff0bc64 100644
---
a/dsl/camel-jbang/camel-jbang-plugin-k/src/main/java/org/apache/camel/dsl/jbang/core/commands/k/IntegrationRun.java
+++
b/dsl/camel-jbang/camel-jbang-plugin-k/src/main/java/org/apache/camel/dsl/jbang/core/commands/k/IntegrationRun.java
@@ -16,6 +16,7 @@
*/
package org.apache.camel.dsl.jbang.core.commands.k;
+import java.io.File;
import java.io.FileInputStream;
import java.util.ArrayList;
import java.util.Arrays;
@@ -25,7 +26,9 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import java.util.StringJoiner;
+import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
@@ -42,6 +45,8 @@ import org.apache.camel.github.GistResourceResolver;
import org.apache.camel.github.GitHubResourceResolver;
import org.apache.camel.impl.DefaultCamelContext;
import org.apache.camel.impl.engine.DefaultResourceResolvers;
+import org.apache.camel.main.KameletMain;
+import org.apache.camel.main.download.DownloadListener;
import org.apache.camel.spi.ResourceResolver;
import org.apache.camel.util.FileUtil;
import org.apache.camel.util.IOHelper;
@@ -67,6 +72,11 @@ import picocli.CommandLine.Command;
@Command(name = "run", description = "Run Camel integrations on Kubernetes",
sortOptions = false)
public class IntegrationRun extends KubeBaseCommand {
+ // ignored list of dependencies, can be either groupId or artifactId
+ // as camel-k loads dependencies from the catalog produced by
camel-k-runtime, some camel core dependencies
+ // are not available, so we have to skip them
+ private static final String[] SKIP_DEPS = new String[] {
"camel-core-languages", "camel-endpointdsl" };
+
@CommandLine.Parameters(description = "The Camel file(s) to run.",
arity = "0..9", paramLabel = "<files>")
String[] filePaths;
@@ -170,6 +180,7 @@ public class IntegrationRun extends KubeBaseCommand {
public IntegrationRun(CamelJBangMain main) {
super(main);
+ Arrays.sort(SKIP_DEPS);
}
public Integer doCall() throws Exception {
@@ -182,17 +193,6 @@ public class IntegrationRun extends KubeBaseCommand {
integration.getMetadata()
.setName(getIntegrationName(integrationSources));
- if (dependencies != null && dependencies.length > 0) {
- List<String> deps = new ArrayList<>();
- for (String dependency : dependencies) {
- String normalized = normalizeDependency(dependency);
- validateDependency(normalized, printer());
- deps.add(normalized);
- }
-
- integration.getSpec().setDependencies(deps);
- }
-
if (kit != null) {
IntegrationKit integrationKit = new IntegrationKit();
integrationKit.setName(kit);
@@ -246,6 +246,17 @@ public class IntegrationRun extends KubeBaseCommand {
} else {
List<Source> resolvedSources = resolveSources(integrationSources);
+ Set<String> intDependencies =
calculateDependencies(resolvedSources);
+ if (dependencies != null && dependencies.length > 0) {
+ for (String dependency : dependencies) {
+ String normalized = normalizeDependency(dependency);
+ validateDependency(normalized, printer());
+ intDependencies.add(normalized);
+ }
+ }
+ List<String> deps = new ArrayList<>(intDependencies);
+ integration.getSpec().setDependencies(deps);
+
List<Flows> flows = new ArrayList<>();
List<Sources> sources = new ArrayList<>();
for (Source source : resolvedSources) {
@@ -404,22 +415,22 @@ public class IntegrationRun extends KubeBaseCommand {
}
}
- private List<Source> resolveSources(List<String> sources) {
+ private List<Source> resolveSources(List<String> sourcePaths) {
List<Source> resolved = new ArrayList<>();
- for (String source : sources) {
- SourceScheme sourceScheme = SourceScheme.fromUri(source);
- String fileExtension = FileUtil.onlyExt(source);
- String fileName = SourceScheme.onlyName(FileUtil.onlyName(source))
+ "." + fileExtension;
+ for (String sourcePath : sourcePaths) {
+ SourceScheme sourceScheme = SourceScheme.fromUri(sourcePath);
+ String fileExtension = FileUtil.onlyExt(sourcePath);
+ String fileName =
SourceScheme.onlyName(FileUtil.onlyName(sourcePath)) + "." + fileExtension;
try {
switch (sourceScheme) {
case GIST -> {
StringJoiner all = new StringJoiner(",");
- GistHelper.fetchGistUrls(source, all);
+ GistHelper.fetchGistUrls(sourcePath, all);
try (ResourceResolver resolver = new
GistResourceResolver()) {
for (String uri : all.toString().split(",")) {
resolved.add(new Source(
- fileName,
+ fileName, sourcePath,
IOHelper.loadText(resolver.resolve(uri).getInputStream()),
fileExtension, compression, false));
}
@@ -428,24 +439,24 @@ public class IntegrationRun extends KubeBaseCommand {
case HTTP -> {
try (ResourceResolver resolver = new
DefaultResourceResolvers.HttpResolver()) {
resolved.add(new Source(
- fileName,
-
IOHelper.loadText(resolver.resolve(source).getInputStream()),
+ fileName, sourcePath,
+
IOHelper.loadText(resolver.resolve(sourcePath).getInputStream()),
fileExtension, compression, false));
}
}
case HTTPS -> {
try (ResourceResolver resolver = new
DefaultResourceResolvers.HttpsResolver()) {
resolved.add(new Source(
- fileName,
-
IOHelper.loadText(resolver.resolve(source).getInputStream()),
+ fileName, sourcePath,
+
IOHelper.loadText(resolver.resolve(sourcePath).getInputStream()),
fileExtension, compression, false));
}
}
case FILE -> {
try (ResourceResolver resolver = new
DefaultResourceResolvers.FileResolver()) {
resolved.add(new Source(
- fileName,
-
IOHelper.loadText(resolver.resolve(source).getInputStream()),
+ fileName, sourcePath,
+
IOHelper.loadText(resolver.resolve(sourcePath).getInputStream()),
fileExtension, compression, true));
}
}
@@ -453,27 +464,28 @@ public class IntegrationRun extends KubeBaseCommand {
try (ResourceResolver resolver = new
DefaultResourceResolvers.ClasspathResolver()) {
resolver.setCamelContext(new
DefaultCamelContext());
resolved.add(new Source(
- fileName,
-
IOHelper.loadText(resolver.resolve(source).getInputStream()),
+ fileName, sourcePath,
+
IOHelper.loadText(resolver.resolve(sourcePath).getInputStream()),
fileExtension, compression, true));
}
}
case GITHUB, RAW_GITHUB -> {
StringJoiner all = new StringJoiner(",");
- GitHubHelper.fetchGithubUrls(source, all);
+ GitHubHelper.fetchGithubUrls(sourcePath, all);
try (ResourceResolver resolver = new
GitHubResourceResolver()) {
for (String uri : all.toString().split(",")) {
resolved.add(new Source(
- fileName,
+ fileName, sourcePath,
IOHelper.loadText(resolver.resolve(uri).getInputStream()),
fileExtension, compression, false));
}
}
}
case UNKNOWN -> {
- try (FileInputStream fis = new
FileInputStream(source)) {
- resolved.add(new Source(fileName,
IOHelper.loadText(fis), fileExtension, compression, true));
+ try (FileInputStream fis = new
FileInputStream(sourcePath)) {
+ resolved.add(
+ new Source(fileName, sourcePath,
IOHelper.loadText(fis), fileExtension, compression, true));
}
}
}
@@ -540,7 +552,7 @@ public class IntegrationRun extends KubeBaseCommand {
}
}
- private record Source(String name, String content, String extension,
boolean compressed, boolean local) {
+ private record Source(String name, String path, String content, String
extension, boolean compressed, boolean local) {
/**
* Provides source contant and automatically handles compression of
content when enabled.
@@ -568,4 +580,92 @@ public class IntegrationRun extends KubeBaseCommand {
}
}
+ private Set<String> calculateDependencies(List<Source> resolvedSources)
throws Exception {
+ List<String> files = new ArrayList<>();
+ for (Source s : resolvedSources) {
+ if (s.local && !s.path.startsWith("classpath:")) {
+ // get the absolute path for a local file
+ files.add("file://" + new File(s.path).getAbsolutePath());
+ } else {
+ files.add(s.path);
+ }
+ }
+ final KameletMain main = new KameletMain();
+ // main.setDownload(false);
+ main.setFresh(false);
+ RunDownloadListener downloadListener = new
RunDownloadListener(resolvedSources);
+ main.setDownloadListener(downloadListener);
+ main.setSilent(true);
+ // enable stub in silent mode so we do not use real components
+ main.setStubPattern("*");
+ // do not run for very long in silent run
+ main.addInitialProperty("camel.main.autoStartup", "false");
+ main.addInitialProperty("camel.main.durationMaxSeconds", "1");
+ main.addInitialProperty("camel.jbang.verbose", "false");
+ main.addInitialProperty("camel.main.routesIncludePattern",
String.join(",", files));
+
+ main.start();
+ main.run();
+
+ main.stop();
+ main.shutdown();
+ return downloadListener.getDependencies();
+ }
+
+ private static class RunDownloadListener implements DownloadListener {
+
+ final Set<String> dependencies = new TreeSet<>();
+ private final List<Source> resolvedSources;
+
+ public RunDownloadListener(List<Source> resolvedSources) {
+ this.resolvedSources = resolvedSources;
+ }
+
+ @Override
+ public void onDownloadDependency(String groupId, String artifactId,
String version) {
+ if (!skipArtifact(groupId, artifactId)) {
+ // format: camel:<component name>
+ // KameletMain is used to resolve the dependencies and it
already contains
+ // camel-kamelets and camel-rest artifacts, then the source
code must be inspected
+ // to actually add them if they are used in the route.
+ if ("camel-rest".equals(artifactId) &&
routeContainsEndpoint("rest")) {
+ dependencies.add("camel:" + artifactId.replace("camel-",
""));
+ }
+ if (("camel-kamelet".equals(artifactId) ||
"camel-yaml-dsl".equals(artifactId))
+ && routeContainsEndpoint("kamelet")) {
+ dependencies.add("camel:" + artifactId.replace("camel-",
""));
+ }
+ if (!"camel-rest".equals(artifactId) &&
!"camel-kamelet".equals(artifactId)
+ && !"camel-yaml-dsl".equals(artifactId)) {
+ dependencies.add("camel:" + artifactId.replace("camel-",
""));
+ }
+ }
+ }
+
+ private boolean skipArtifact(String groupId, String artifactId) {
+ return Arrays.binarySearch(SKIP_DEPS, artifactId) >= 0 ||
Arrays.binarySearch(SKIP_DEPS, groupId) >= 0;
+ }
+
+ // inspect the source code to determine if it contains a specific
endpoint
+ private boolean routeContainsEndpoint(String componentName) {
+ boolean contains = false;
+ for (Source source : resolvedSources) {
+ // find if the route contains the component with the format:
<component>:
+ if (source.content.contains(componentName + ":")) {
+ contains = true;
+ break;
+ }
+ }
+ return contains;
+ }
+
+ @Override
+ public void onAlreadyDownloadedDependency(String groupId, String
artifactId, String version) {
+ }
+
+ private Set<String> getDependencies() {
+ return dependencies;
+ }
+ }
+
}
diff --git
a/dsl/camel-jbang/camel-jbang-plugin-k/src/test/java/org/apache/camel/dsl/jbang/core/commands/k/IntegrationRunTest.java
b/dsl/camel-jbang/camel-jbang-plugin-k/src/test/java/org/apache/camel/dsl/jbang/core/commands/k/IntegrationRunTest.java
index aaf3e182098..d32c97a2815 100644
---
a/dsl/camel-jbang/camel-jbang-plugin-k/src/test/java/org/apache/camel/dsl/jbang/core/commands/k/IntegrationRunTest.java
+++
b/dsl/camel-jbang/camel-jbang-plugin-k/src/test/java/org/apache/camel/dsl/jbang/core/commands/k/IntegrationRunTest.java
@@ -17,6 +17,7 @@
package org.apache.camel.dsl.jbang.core.commands.k;
+import java.util.Arrays;
import java.util.regex.Pattern;
import org.apache.camel.RuntimeCamelException;
@@ -199,6 +200,63 @@ class IntegrationRunTest extends KubeBaseTest {
traits: {}""", printer.getOutput());
}
+ @Test
+ public void shouldInferDependenciesYamlRoute() throws Exception {
+ IntegrationRun command = createCommand();
+ command.filePaths = new String[] { "classpath:route-deps.yaml" };
+ command.output = "yaml";
+ command.doCall();
+
+ String[] specDependencies = getDependencies(printer.getOutput());
+ String[] deps = new String[] {
+ "camel:caffeine", "camel:http", "camel:jackson",
"camel:jsonpath", "camel:log", "camel:timer" };
+ Assertions.assertArrayEquals(deps, specDependencies, "Dependencies
don't match: " + Arrays.toString(specDependencies));
+ }
+
+ @Test
+ public void shouldAddComplexDependenciesYamlRoute() throws Exception {
+ IntegrationRun command = createCommand();
+ command.filePaths = new String[] { "classpath:route-deps.yaml" };
+ command.dependencies = new String[] { "camel-twitter",
"mvn:foo:bar:1.0" };
+ command.output = "yaml";
+ command.doCall();
+
+ String[] specDependencies = getDependencies(printer.getOutput());
+ String[] deps = new String[] {
+ "camel:caffeine", "camel:http", "camel:jackson",
"camel:jsonpath", "camel:log", "camel:timer", "camel:twitter",
+ "mvn:foo:bar:1.0" };
+ Assertions.assertArrayEquals(deps, specDependencies, "Dependencies
don't match: " + Arrays.toString(specDependencies));
+ }
+
+ @Test
+ public void shouldInferDependenciesJavaRoute() throws Exception {
+ IntegrationRun command = createCommand();
+ command.filePaths = new String[] { "classpath:Sample.java" };
+ command.output = "yaml";
+ command.doCall();
+
+ String[] specDependencies = getDependencies(printer.getOutput());
+ String[] deps = new String[] {
+ "camel:aws2-s3", "camel:caffeine", "camel:dropbox",
"camel:jacksonxml", "camel:java-joor-dsl", "camel:kafka",
+ "camel:mongodb", "camel:telegram", "camel:zipfile" };
+ Assertions.assertArrayEquals(deps, specDependencies, "Dependencies
don't match: " + Arrays.toString(specDependencies));
+ }
+
+ @Test
+ public void shouldAddComplexDependenciesJavaRoute() throws Exception {
+ IntegrationRun command = createCommand();
+ command.filePaths = new String[] { "classpath:Sample.java" };
+ command.dependencies = new String[] { "camel-twitter",
"mvn:foo:bar:1.0" };
+ command.output = "yaml";
+ command.doCall();
+
+ String[] specDependencies = getDependencies(printer.getOutput());
+ String[] deps = new String[] {
+ "camel:aws2-s3", "camel:caffeine", "camel:dropbox",
"camel:jacksonxml", "camel:java-joor-dsl", "camel:kafka",
+ "camel:mongodb", "camel:telegram", "camel:twitter",
"camel:zipfile", "mvn:foo:bar:1.0" };
+ Assertions.assertArrayEquals(deps, specDependencies, "Dependencies
don't match: " + Arrays.toString(specDependencies));
+ }
+
@Test
public void shouldAddEnvVars() throws Exception {
IntegrationRun command = createCommand();
diff --git
a/dsl/camel-jbang/camel-jbang-plugin-k/src/test/java/org/apache/camel/dsl/jbang/core/commands/k/KubeBaseTest.java
b/dsl/camel-jbang/camel-jbang-plugin-k/src/test/java/org/apache/camel/dsl/jbang/core/commands/k/KubeBaseTest.java
index ab3dde9278f..bcc1e0929c7 100644
---
a/dsl/camel-jbang/camel-jbang-plugin-k/src/test/java/org/apache/camel/dsl/jbang/core/commands/k/KubeBaseTest.java
+++
b/dsl/camel-jbang/camel-jbang-plugin-k/src/test/java/org/apache/camel/dsl/jbang/core/commands/k/KubeBaseTest.java
@@ -19,6 +19,8 @@ package org.apache.camel.dsl.jbang.core.commands.k;
import java.io.IOException;
import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.server.mock.KubernetesCrudDispatcher;
@@ -36,6 +38,7 @@ import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.TestInstance;
+import org.yaml.snakeyaml.Yaml;
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public class KubeBaseTest {
@@ -90,4 +93,14 @@ public class KubeBaseTest {
return created;
}
+ protected String[] getDependencies(String yamlSource) {
+ Yaml yaml = new Yaml();
+ Map<String, Object> obj = yaml.load(yamlSource);
+ //noinspection unchecked
+ obj = (Map<String, Object>) obj.get("spec");
+ //noinspection unchecked
+ List<String> specDeps = (List<String>) obj.get("dependencies");
+ return specDeps.toArray(new String[specDeps.size()]);
+ }
+
}
diff --git
a/dsl/camel-jbang/camel-jbang-plugin-k/src/test/resources/Sample.java
b/dsl/camel-jbang/camel-jbang-plugin-k/src/test/resources/Sample.java
new file mode 100644
index 00000000000..1e1ba626c7e
--- /dev/null
+++ b/dsl/camel-jbang/camel-jbang-plugin-k/src/test/resources/Sample.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.lang.Exception;
+import java.lang.Override;
+import org.apache.camel.Exchange;
+import org.apache.camel.builder.RouteBuilder;
+
+public class Sample extends RouteBuilder {
+
+ String s = "random";
+
+ @Override
+ public void configure() throws Exception {
+ String params = "random_string";
+ from("telegram:" + params)
+ .setHeader("test",simple("${in.header.fileName}"))
+ .pollEnrich()
+
.simple("aws2-s3://data?fileName=${in.header.fileName}&deleteAfterRead=false")
+ .unmarshal().jacksonXml()
+ .to("mongodb:test")
+ .end()
+ .unmarshal().zipFile()
+ .to("dropbox:random")
+ .toD("caffeine-cache:" + s)
+ .to("kafka:test");
+ }
+}
+
diff --git
a/dsl/camel-jbang/camel-jbang-plugin-k/src/test/resources/route-deps.yaml
b/dsl/camel-jbang/camel-jbang-plugin-k/src/test/resources/route-deps.yaml
new file mode 100644
index 00000000000..d7eb7a9c4e1
--- /dev/null
+++ b/dsl/camel-jbang/camel-jbang-plugin-k/src/test/resources/route-deps.yaml
@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+- from:
+ uri: "timer:sample"
+ parameters:
+ period: "2s"
+ steps:
+ - toD: "caffeine-cache:cache-${routeId}?key=lastUpdate"
+ - toD: "https://test/fdsnws/event/1/query";
+ - unmarshal:
+ json: {}
+ - claimCheck:
+ operation: Push
+ - toD: "caffeine-cache:cache-${routeId}?key=lastUpdate"
+ - split:
+ jsonpath: "$.features"
+ steps:
+ - marshal:
+ json: {}
+ - to: "log:info"
+
