This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push: new 897b81481ec camel-jbang-plugin-k: infer the dependencies by inspecting the route (#13193) 897b81481ec is described below commit 897b81481eca7f211821efa188d80e3f96a312b3 Author: Claudio Miranda <clau...@claudius.com.br> AuthorDate: Wed Feb 21 05:26:52 2024 -0300 camel-jbang-plugin-k: infer the dependencies by inspecting the route (#13193) --- .../dsl/jbang/core/commands/k/IntegrationRun.java | 162 +++++++++++++++++---- .../jbang/core/commands/k/IntegrationRunTest.java | 58 ++++++++ .../dsl/jbang/core/commands/k/KubeBaseTest.java | 13 ++ .../src/test/resources/Sample.java | 43 ++++++ .../src/test/resources/route-deps.yaml | 36 +++++ 5 files changed, 281 insertions(+), 31 deletions(-) diff --git a/dsl/camel-jbang/camel-jbang-plugin-k/src/main/java/org/apache/camel/dsl/jbang/core/commands/k/IntegrationRun.java b/dsl/camel-jbang/camel-jbang-plugin-k/src/main/java/org/apache/camel/dsl/jbang/core/commands/k/IntegrationRun.java index 6d5b5a267ca..526bff0bc64 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-k/src/main/java/org/apache/camel/dsl/jbang/core/commands/k/IntegrationRun.java +++ b/dsl/camel-jbang/camel-jbang-plugin-k/src/main/java/org/apache/camel/dsl/jbang/core/commands/k/IntegrationRun.java @@ -16,6 +16,7 @@ */ package org.apache.camel.dsl.jbang.core.commands.k; +import java.io.File; import java.io.FileInputStream; import java.util.ArrayList; import java.util.Arrays; @@ -25,7 +26,9 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.StringJoiner; +import java.util.TreeSet; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -42,6 +45,8 @@ import org.apache.camel.github.GistResourceResolver; import org.apache.camel.github.GitHubResourceResolver; import org.apache.camel.impl.DefaultCamelContext; import org.apache.camel.impl.engine.DefaultResourceResolvers; +import org.apache.camel.main.KameletMain; +import org.apache.camel.main.download.DownloadListener; import org.apache.camel.spi.ResourceResolver; import org.apache.camel.util.FileUtil; import org.apache.camel.util.IOHelper; @@ -67,6 +72,11 @@ import picocli.CommandLine.Command; @Command(name = "run", description = "Run Camel integrations on Kubernetes", sortOptions = false) public class IntegrationRun extends KubeBaseCommand { + // ignored list of dependencies, can be either groupId or artifactId + // as camel-k loads dependencies from the catalog produced by camel-k-runtime, some camel core dependencies + // are not available, so we have to skip them + private static final String[] SKIP_DEPS = new String[] { "camel-core-languages", "camel-endpointdsl" }; + @CommandLine.Parameters(description = "The Camel file(s) to run.", arity = "0..9", paramLabel = "<files>") String[] filePaths; @@ -170,6 +180,7 @@ public class IntegrationRun extends KubeBaseCommand { public IntegrationRun(CamelJBangMain main) { super(main); + Arrays.sort(SKIP_DEPS); } public Integer doCall() throws Exception { @@ -182,17 +193,6 @@ public class IntegrationRun extends KubeBaseCommand { integration.getMetadata() .setName(getIntegrationName(integrationSources)); - if (dependencies != null && dependencies.length > 0) { - List<String> deps = new ArrayList<>(); - for (String dependency : dependencies) { - String normalized = normalizeDependency(dependency); - validateDependency(normalized, printer()); - deps.add(normalized); - } - - integration.getSpec().setDependencies(deps); - } - if (kit != null) { IntegrationKit integrationKit = new IntegrationKit(); integrationKit.setName(kit); @@ -246,6 +246,17 @@ public class IntegrationRun extends KubeBaseCommand { } else { List<Source> resolvedSources = resolveSources(integrationSources); + Set<String> intDependencies = calculateDependencies(resolvedSources); + if (dependencies != null && dependencies.length > 0) { + for (String dependency : dependencies) { + String normalized = normalizeDependency(dependency); + validateDependency(normalized, printer()); + intDependencies.add(normalized); + } + } + List<String> deps = new ArrayList<>(intDependencies); + integration.getSpec().setDependencies(deps); + List<Flows> flows = new ArrayList<>(); List<Sources> sources = new ArrayList<>(); for (Source source : resolvedSources) { @@ -404,22 +415,22 @@ public class IntegrationRun extends KubeBaseCommand { } } - private List<Source> resolveSources(List<String> sources) { + private List<Source> resolveSources(List<String> sourcePaths) { List<Source> resolved = new ArrayList<>(); - for (String source : sources) { - SourceScheme sourceScheme = SourceScheme.fromUri(source); - String fileExtension = FileUtil.onlyExt(source); - String fileName = SourceScheme.onlyName(FileUtil.onlyName(source)) + "." + fileExtension; + for (String sourcePath : sourcePaths) { + SourceScheme sourceScheme = SourceScheme.fromUri(sourcePath); + String fileExtension = FileUtil.onlyExt(sourcePath); + String fileName = SourceScheme.onlyName(FileUtil.onlyName(sourcePath)) + "." + fileExtension; try { switch (sourceScheme) { case GIST -> { StringJoiner all = new StringJoiner(","); - GistHelper.fetchGistUrls(source, all); + GistHelper.fetchGistUrls(sourcePath, all); try (ResourceResolver resolver = new GistResourceResolver()) { for (String uri : all.toString().split(",")) { resolved.add(new Source( - fileName, + fileName, sourcePath, IOHelper.loadText(resolver.resolve(uri).getInputStream()), fileExtension, compression, false)); } @@ -428,24 +439,24 @@ public class IntegrationRun extends KubeBaseCommand { case HTTP -> { try (ResourceResolver resolver = new DefaultResourceResolvers.HttpResolver()) { resolved.add(new Source( - fileName, - IOHelper.loadText(resolver.resolve(source).getInputStream()), + fileName, sourcePath, + IOHelper.loadText(resolver.resolve(sourcePath).getInputStream()), fileExtension, compression, false)); } } case HTTPS -> { try (ResourceResolver resolver = new DefaultResourceResolvers.HttpsResolver()) { resolved.add(new Source( - fileName, - IOHelper.loadText(resolver.resolve(source).getInputStream()), + fileName, sourcePath, + IOHelper.loadText(resolver.resolve(sourcePath).getInputStream()), fileExtension, compression, false)); } } case FILE -> { try (ResourceResolver resolver = new DefaultResourceResolvers.FileResolver()) { resolved.add(new Source( - fileName, - IOHelper.loadText(resolver.resolve(source).getInputStream()), + fileName, sourcePath, + IOHelper.loadText(resolver.resolve(sourcePath).getInputStream()), fileExtension, compression, true)); } } @@ -453,27 +464,28 @@ public class IntegrationRun extends KubeBaseCommand { try (ResourceResolver resolver = new DefaultResourceResolvers.ClasspathResolver()) { resolver.setCamelContext(new DefaultCamelContext()); resolved.add(new Source( - fileName, - IOHelper.loadText(resolver.resolve(source).getInputStream()), + fileName, sourcePath, + IOHelper.loadText(resolver.resolve(sourcePath).getInputStream()), fileExtension, compression, true)); } } case GITHUB, RAW_GITHUB -> { StringJoiner all = new StringJoiner(","); - GitHubHelper.fetchGithubUrls(source, all); + GitHubHelper.fetchGithubUrls(sourcePath, all); try (ResourceResolver resolver = new GitHubResourceResolver()) { for (String uri : all.toString().split(",")) { resolved.add(new Source( - fileName, + fileName, sourcePath, IOHelper.loadText(resolver.resolve(uri).getInputStream()), fileExtension, compression, false)); } } } case UNKNOWN -> { - try (FileInputStream fis = new FileInputStream(source)) { - resolved.add(new Source(fileName, IOHelper.loadText(fis), fileExtension, compression, true)); + try (FileInputStream fis = new FileInputStream(sourcePath)) { + resolved.add( + new Source(fileName, sourcePath, IOHelper.loadText(fis), fileExtension, compression, true)); } } } @@ -540,7 +552,7 @@ public class IntegrationRun extends KubeBaseCommand { } } - private record Source(String name, String content, String extension, boolean compressed, boolean local) { + private record Source(String name, String path, String content, String extension, boolean compressed, boolean local) { /** * Provides source contant and automatically handles compression of content when enabled. @@ -568,4 +580,92 @@ public class IntegrationRun extends KubeBaseCommand { } } + private Set<String> calculateDependencies(List<Source> resolvedSources) throws Exception { + List<String> files = new ArrayList<>(); + for (Source s : resolvedSources) { + if (s.local && !s.path.startsWith("classpath:")) { + // get the absolute path for a local file + files.add("file://" + new File(s.path).getAbsolutePath()); + } else { + files.add(s.path); + } + } + final KameletMain main = new KameletMain(); + // main.setDownload(false); + main.setFresh(false); + RunDownloadListener downloadListener = new RunDownloadListener(resolvedSources); + main.setDownloadListener(downloadListener); + main.setSilent(true); + // enable stub in silent mode so we do not use real components + main.setStubPattern("*"); + // do not run for very long in silent run + main.addInitialProperty("camel.main.autoStartup", "false"); + main.addInitialProperty("camel.main.durationMaxSeconds", "1"); + main.addInitialProperty("camel.jbang.verbose", "false"); + main.addInitialProperty("camel.main.routesIncludePattern", String.join(",", files)); + + main.start(); + main.run(); + + main.stop(); + main.shutdown(); + return downloadListener.getDependencies(); + } + + private static class RunDownloadListener implements DownloadListener { + + final Set<String> dependencies = new TreeSet<>(); + private final List<Source> resolvedSources; + + public RunDownloadListener(List<Source> resolvedSources) { + this.resolvedSources = resolvedSources; + } + + @Override + public void onDownloadDependency(String groupId, String artifactId, String version) { + if (!skipArtifact(groupId, artifactId)) { + // format: camel:<component name> + // KameletMain is used to resolve the dependencies and it already contains + // camel-kamelets and camel-rest artifacts, then the source code must be inspected + // to actually add them if they are used in the route. + if ("camel-rest".equals(artifactId) && routeContainsEndpoint("rest")) { + dependencies.add("camel:" + artifactId.replace("camel-", "")); + } + if (("camel-kamelet".equals(artifactId) || "camel-yaml-dsl".equals(artifactId)) + && routeContainsEndpoint("kamelet")) { + dependencies.add("camel:" + artifactId.replace("camel-", "")); + } + if (!"camel-rest".equals(artifactId) && !"camel-kamelet".equals(artifactId) + && !"camel-yaml-dsl".equals(artifactId)) { + dependencies.add("camel:" + artifactId.replace("camel-", "")); + } + } + } + + private boolean skipArtifact(String groupId, String artifactId) { + return Arrays.binarySearch(SKIP_DEPS, artifactId) >= 0 || Arrays.binarySearch(SKIP_DEPS, groupId) >= 0; + } + + // inspect the source code to determine if it contains a specific endpoint + private boolean routeContainsEndpoint(String componentName) { + boolean contains = false; + for (Source source : resolvedSources) { + // find if the route contains the component with the format: <component>: + if (source.content.contains(componentName + ":")) { + contains = true; + break; + } + } + return contains; + } + + @Override + public void onAlreadyDownloadedDependency(String groupId, String artifactId, String version) { + } + + private Set<String> getDependencies() { + return dependencies; + } + } + } diff --git a/dsl/camel-jbang/camel-jbang-plugin-k/src/test/java/org/apache/camel/dsl/jbang/core/commands/k/IntegrationRunTest.java b/dsl/camel-jbang/camel-jbang-plugin-k/src/test/java/org/apache/camel/dsl/jbang/core/commands/k/IntegrationRunTest.java index aaf3e182098..d32c97a2815 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-k/src/test/java/org/apache/camel/dsl/jbang/core/commands/k/IntegrationRunTest.java +++ b/dsl/camel-jbang/camel-jbang-plugin-k/src/test/java/org/apache/camel/dsl/jbang/core/commands/k/IntegrationRunTest.java @@ -17,6 +17,7 @@ package org.apache.camel.dsl.jbang.core.commands.k; +import java.util.Arrays; import java.util.regex.Pattern; import org.apache.camel.RuntimeCamelException; @@ -199,6 +200,63 @@ class IntegrationRunTest extends KubeBaseTest { traits: {}""", printer.getOutput()); } + @Test + public void shouldInferDependenciesYamlRoute() throws Exception { + IntegrationRun command = createCommand(); + command.filePaths = new String[] { "classpath:route-deps.yaml" }; + command.output = "yaml"; + command.doCall(); + + String[] specDependencies = getDependencies(printer.getOutput()); + String[] deps = new String[] { + "camel:caffeine", "camel:http", "camel:jackson", "camel:jsonpath", "camel:log", "camel:timer" }; + Assertions.assertArrayEquals(deps, specDependencies, "Dependencies don't match: " + Arrays.toString(specDependencies)); + } + + @Test + public void shouldAddComplexDependenciesYamlRoute() throws Exception { + IntegrationRun command = createCommand(); + command.filePaths = new String[] { "classpath:route-deps.yaml" }; + command.dependencies = new String[] { "camel-twitter", "mvn:foo:bar:1.0" }; + command.output = "yaml"; + command.doCall(); + + String[] specDependencies = getDependencies(printer.getOutput()); + String[] deps = new String[] { + "camel:caffeine", "camel:http", "camel:jackson", "camel:jsonpath", "camel:log", "camel:timer", "camel:twitter", + "mvn:foo:bar:1.0" }; + Assertions.assertArrayEquals(deps, specDependencies, "Dependencies don't match: " + Arrays.toString(specDependencies)); + } + + @Test + public void shouldInferDependenciesJavaRoute() throws Exception { + IntegrationRun command = createCommand(); + command.filePaths = new String[] { "classpath:Sample.java" }; + command.output = "yaml"; + command.doCall(); + + String[] specDependencies = getDependencies(printer.getOutput()); + String[] deps = new String[] { + "camel:aws2-s3", "camel:caffeine", "camel:dropbox", "camel:jacksonxml", "camel:java-joor-dsl", "camel:kafka", + "camel:mongodb", "camel:telegram", "camel:zipfile" }; + Assertions.assertArrayEquals(deps, specDependencies, "Dependencies don't match: " + Arrays.toString(specDependencies)); + } + + @Test + public void shouldAddComplexDependenciesJavaRoute() throws Exception { + IntegrationRun command = createCommand(); + command.filePaths = new String[] { "classpath:Sample.java" }; + command.dependencies = new String[] { "camel-twitter", "mvn:foo:bar:1.0" }; + command.output = "yaml"; + command.doCall(); + + String[] specDependencies = getDependencies(printer.getOutput()); + String[] deps = new String[] { + "camel:aws2-s3", "camel:caffeine", "camel:dropbox", "camel:jacksonxml", "camel:java-joor-dsl", "camel:kafka", + "camel:mongodb", "camel:telegram", "camel:twitter", "camel:zipfile", "mvn:foo:bar:1.0" }; + Assertions.assertArrayEquals(deps, specDependencies, "Dependencies don't match: " + Arrays.toString(specDependencies)); + } + @Test public void shouldAddEnvVars() throws Exception { IntegrationRun command = createCommand(); diff --git a/dsl/camel-jbang/camel-jbang-plugin-k/src/test/java/org/apache/camel/dsl/jbang/core/commands/k/KubeBaseTest.java b/dsl/camel-jbang/camel-jbang-plugin-k/src/test/java/org/apache/camel/dsl/jbang/core/commands/k/KubeBaseTest.java index ab3dde9278f..bcc1e0929c7 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-k/src/test/java/org/apache/camel/dsl/jbang/core/commands/k/KubeBaseTest.java +++ b/dsl/camel-jbang/camel-jbang-plugin-k/src/test/java/org/apache/camel/dsl/jbang/core/commands/k/KubeBaseTest.java @@ -19,6 +19,8 @@ package org.apache.camel.dsl.jbang.core.commands.k; import java.io.IOException; import java.util.HashMap; +import java.util.List; +import java.util.Map; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.server.mock.KubernetesCrudDispatcher; @@ -36,6 +38,7 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestInstance; +import org.yaml.snakeyaml.Yaml; @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class KubeBaseTest { @@ -90,4 +93,14 @@ public class KubeBaseTest { return created; } + protected String[] getDependencies(String yamlSource) { + Yaml yaml = new Yaml(); + Map<String, Object> obj = yaml.load(yamlSource); + //noinspection unchecked + obj = (Map<String, Object>) obj.get("spec"); + //noinspection unchecked + List<String> specDeps = (List<String>) obj.get("dependencies"); + return specDeps.toArray(new String[specDeps.size()]); + } + } diff --git a/dsl/camel-jbang/camel-jbang-plugin-k/src/test/resources/Sample.java b/dsl/camel-jbang/camel-jbang-plugin-k/src/test/resources/Sample.java new file mode 100644 index 00000000000..1e1ba626c7e --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-plugin-k/src/test/resources/Sample.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.lang.Exception; +import java.lang.Override; +import org.apache.camel.Exchange; +import org.apache.camel.builder.RouteBuilder; + +public class Sample extends RouteBuilder { + + String s = "random"; + + @Override + public void configure() throws Exception { + String params = "random_string"; + from("telegram:" + params) + .setHeader("test",simple("${in.header.fileName}")) + .pollEnrich() + .simple("aws2-s3://data?fileName=${in.header.fileName}&deleteAfterRead=false") + .unmarshal().jacksonXml() + .to("mongodb:test") + .end() + .unmarshal().zipFile() + .to("dropbox:random") + .toD("caffeine-cache:" + s) + .to("kafka:test"); + } +} + diff --git a/dsl/camel-jbang/camel-jbang-plugin-k/src/test/resources/route-deps.yaml b/dsl/camel-jbang/camel-jbang-plugin-k/src/test/resources/route-deps.yaml new file mode 100644 index 00000000000..d7eb7a9c4e1 --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-plugin-k/src/test/resources/route-deps.yaml @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +- from: + uri: "timer:sample" + parameters: + period: "2s" + steps: + - toD: "caffeine-cache:cache-${routeId}?key=lastUpdate" + - toD: "https://test/fdsnws/event/1/query" + - unmarshal: + json: {} + - claimCheck: + operation: Push + - toD: "caffeine-cache:cache-${routeId}?key=lastUpdate" + - split: + jsonpath: "$.features" + steps: + - marshal: + json: {} + - to: "log:info" +