Repository: camel Updated Branches: refs/heads/master 43ae121e6 -> 2c3fa6c4b
CAMEL-10865: camel catalog - Add catalog for connectors Project: http://git-wip-us.apache.org/repos/asf/camel/repo Commit: http://git-wip-us.apache.org/repos/asf/camel/commit/2c3fa6c4 Tree: http://git-wip-us.apache.org/repos/asf/camel/tree/2c3fa6c4 Diff: http://git-wip-us.apache.org/repos/asf/camel/diff/2c3fa6c4 Branch: refs/heads/master Commit: 2c3fa6c4b1371b185d4deb8ce7a20350272d0e92 Parents: 2bf3ca5 Author: Claus Ibsen <davscl...@apache.org> Authored: Tue Feb 21 11:23:53 2017 +0100 Committer: Claus Ibsen <davscl...@apache.org> Committed: Tue Feb 21 11:26:08 2017 +0100 ---------------------------------------------------------------------- .../connector/CamelConnectorCatalog.java | 11 +- .../catalog/connector/ConnectorDataStore.java | 35 ++++- .../camel/catalog/connector/ConnectorDto.java | 5 + .../connector/DefaultCamelConnectorCatalog.java | 8 +- .../connector/MemoryConnectorDataStore.java | 75 +++++++-- .../connector/CamelConnectorCatalogTest.java | 60 ++++++-- platforms/catalog-nexus/pom.xml | 44 +++--- .../catalog/nexus/BaseNexusRepository.java | 3 + .../nexus/ComponentCatalogNexusRepository.java | 5 +- .../nexus/ConnectorCatalogNexusRepository.java | 153 +++++++++++++++++++ .../camel/catalog/nexus/ConnectorDataStore.java | 42 ----- .../ConnectorDataStoreNexusRepository.java | 151 ------------------ .../camel/catalog/nexus/ConnectorDto.java | 102 ------------- .../catalog/nexus/MemoryConnectorDataStore.java | 110 ------------- .../LocalFileConnectorNexusRepository.java | 2 +- .../LocalNexusConnectorRepositoryTest.java | 10 +- 16 files changed, 351 insertions(+), 465 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/camel/blob/2c3fa6c4/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/CamelConnectorCatalog.java ---------------------------------------------------------------------- diff --git a/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/CamelConnectorCatalog.java b/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/CamelConnectorCatalog.java index c7e8d03..6e41646 100644 --- a/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/CamelConnectorCatalog.java +++ b/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/CamelConnectorCatalog.java @@ -56,13 +56,18 @@ public interface CamelConnectorCatalog { /** * Finds all the connectors from the catalog + * + * @param latestVersionOnly whether to include only latest version of the connectors */ - List<ConnectorDto> findConnector(); + List<ConnectorDto> findConnector(boolean latestVersionOnly); /** - * Find all the connectors that matches the name, label or description from the catalog + * Find all the connectors that matches the maven coordinate, name, label or description from the catalog + * + * @param filter filter text + * @param latestVersionOnly whether to include only latest version of the connectors */ - List<ConnectorDto> findConnector(String filter); + List<ConnectorDto> findConnector(String filter, boolean latestVersionOnly); /** * Returns the <tt>camel-connector</tt> json file for the given connector with the Maven coordinate http://git-wip-us.apache.org/repos/asf/camel/blob/2c3fa6c4/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/ConnectorDataStore.java ---------------------------------------------------------------------- diff --git a/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/ConnectorDataStore.java b/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/ConnectorDataStore.java index 0243396..f3ae803 100644 --- a/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/ConnectorDataStore.java +++ b/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/ConnectorDataStore.java @@ -18,18 +18,47 @@ package org.apache.camel.catalog.connector; import java.util.List; +/** + * Data store for connector details to be used by the {@link CamelConnectorCatalog}. + */ public interface ConnectorDataStore { + /** + * Adds or updates the connector to the catalog + * + * @param dto the connector dto + * @param connectorJson the <tt>camel-connector</tt> json file + * @param connectorSchemaJson the <tt>camel-connector-schema</tt> json file + */ void addConnector(ConnectorDto dto, String connectorJson, String connectorSchemaJson); + /** + * Removes the connector from the catalog + * + * @param dto the connector dto + */ void removeConnector(ConnectorDto dto); - List<ConnectorDto> findConnector(); - - List<ConnectorDto> findConnector(String filter); + /** + * Find all the connectors that matches the maven coordinate, name, label or description from the catalog + * + * @param filter filter text + * @param latestVersionOnly whether to include only latest version of the connectors + */ + List<ConnectorDto> findConnector(String filter, boolean latestVersionOnly); + /** + * Returns the <tt>camel-connector</tt> json file for the given connector with the Maven coordinate + * + * @param dto the connector dto + */ String connectorJSon(ConnectorDto dto); + /** + * Returns the <tt>camel-connector-schema</tt> json file for the given connector with the Maven coordinate + * + * @param dto the connector dto + */ String connectorSchemaJSon(ConnectorDto dto); } http://git-wip-us.apache.org/repos/asf/camel/blob/2c3fa6c4/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/ConnectorDto.java ---------------------------------------------------------------------- diff --git a/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/ConnectorDto.java b/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/ConnectorDto.java index b83e74c..80ab8fb 100644 --- a/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/ConnectorDto.java +++ b/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/ConnectorDto.java @@ -29,6 +29,10 @@ public class ConnectorDto { private String artifactId; private String version; + public String getMavenGav() { + return groupId + ":" + artifactId + ":" + version; + } + public String getName() { return name; } @@ -114,4 +118,5 @@ public class ConnectorDto { + ", version='" + version + '\'' + ']'; } + } http://git-wip-us.apache.org/repos/asf/camel/blob/2c3fa6c4/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/DefaultCamelConnectorCatalog.java ---------------------------------------------------------------------- diff --git a/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/DefaultCamelConnectorCatalog.java b/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/DefaultCamelConnectorCatalog.java index 07d1447..d3272a2 100644 --- a/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/DefaultCamelConnectorCatalog.java +++ b/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/DefaultCamelConnectorCatalog.java @@ -54,13 +54,13 @@ public class DefaultCamelConnectorCatalog implements CamelConnectorCatalog { } @Override - public List<ConnectorDto> findConnector() { - return dataStore.findConnector(); + public List<ConnectorDto> findConnector(boolean latestVersionOnly) { + return findConnector(null, latestVersionOnly); } @Override - public List<ConnectorDto> findConnector(String filter) { - return dataStore.findConnector(filter); + public List<ConnectorDto> findConnector(String filter, boolean latestVersionOnly) { + return dataStore.findConnector(filter, latestVersionOnly); } @Override http://git-wip-us.apache.org/repos/asf/camel/blob/2c3fa6c4/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/MemoryConnectorDataStore.java ---------------------------------------------------------------------- diff --git a/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/MemoryConnectorDataStore.java b/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/MemoryConnectorDataStore.java index c9a9ef7..495a7f7 100644 --- a/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/MemoryConnectorDataStore.java +++ b/platforms/catalog-connector/src/main/java/org/apache/camel/catalog/connector/MemoryConnectorDataStore.java @@ -17,7 +17,9 @@ package org.apache.camel.catalog.connector; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; +import java.util.Locale; import java.util.Set; import java.util.concurrent.CopyOnWriteArraySet; @@ -43,16 +45,71 @@ public class MemoryConnectorDataStore implements ConnectorDataStore { } @Override - public List<ConnectorDto> findConnector() { - List<ConnectorDto> dtos = new ArrayList<>(); - store.forEach(e -> dtos.add(e.getDto())); - return dtos; - } + public List<ConnectorDto> findConnector(String filter, boolean latestVersionOnly) { + final List<ConnectorDto> answer = new ArrayList<>(); + + if (filter != null && !filter.isEmpty()) { + // search ignore case + filter = filter.toLowerCase(Locale.US); + + for (ConnectorDetails detail : store) { + ConnectorDto dto = detail.getDto(); + if (dto.getName().toLowerCase(Locale.US).contains(filter)) { + answer.add(dto); + } else if (dto.getDescription().toLowerCase(Locale.US).contains(filter)) { + answer.add(dto); + } else if (dto.getGroupId().toLowerCase(Locale.US).contains(filter)) { + answer.add(dto); + } else if (dto.getArtifactId().toLowerCase(Locale.US).contains(filter)) { + answer.add(dto); + } else if (dto.getVersion().toLowerCase(Locale.US).contains(filter)) { + answer.add(dto); + } else { + String labels = dto.getLabels(); + if (labels != null) { + String[] arr = labels.split(","); + for (String lab : arr) { + lab = lab.toLowerCase(Locale.US); + if (lab.contains(filter)) { + answer.add(dto); + break; + } + } + } + } + } + } else { + store.forEach(d -> answer.add(d.getDto())); + } - @Override - public List<ConnectorDto> findConnector(String filter) { - // TODO: collect - return null; + // filter only latest version + if (latestVersionOnly && answer.size() > 1) { + // sort first + answer.sort(Comparator.comparing(ConnectorDto::getMavenGav)); + + // keep only latest in each group + List<ConnectorDto> unique = new ArrayList<>(); + ConnectorDto prev = null; + + for (ConnectorDto dto : answer) { + if (prev == null + || (prev.getGroupId().equals(dto.getGroupId()) + && prev.getArtifactId().equals(dto.getArtifactId()))) { + prev = dto; + } else { + unique.add(prev); + prev = dto; + } + } + if (prev != null) { + // special for last element + unique.add(prev); + } + + return unique; + } + + return answer; } @Override http://git-wip-us.apache.org/repos/asf/camel/blob/2c3fa6c4/platforms/catalog-connector/src/test/java/org/apache/camel/catalog/connector/CamelConnectorCatalogTest.java ---------------------------------------------------------------------- diff --git a/platforms/catalog-connector/src/test/java/org/apache/camel/catalog/connector/CamelConnectorCatalogTest.java b/platforms/catalog-connector/src/test/java/org/apache/camel/catalog/connector/CamelConnectorCatalogTest.java index 66cf39a..972a2a6 100644 --- a/platforms/catalog-connector/src/test/java/org/apache/camel/catalog/connector/CamelConnectorCatalogTest.java +++ b/platforms/catalog-connector/src/test/java/org/apache/camel/catalog/connector/CamelConnectorCatalogTest.java @@ -27,41 +27,81 @@ public class CamelConnectorCatalogTest { public void testAddConnector() throws Exception { CamelConnectorCatalog catalog = new DefaultCamelConnectorCatalog(); - assertEquals(0, catalog.findConnector().size()); + assertEquals(0, catalog.findConnector(false).size()); catalog.addConnector("org.apache.camel", "myfoo-connector", "2.19.0", "MyFoo", "Something cool", "foo,timer", null, null); - assertEquals(1, catalog.findConnector().size()); + assertEquals(1, catalog.findConnector(false).size()); } @Test public void testRemoveConnector() throws Exception { CamelConnectorCatalog catalog = new DefaultCamelConnectorCatalog(); - assertEquals(0, catalog.findConnector().size()); + assertEquals(0, catalog.findConnector(false).size()); catalog.addConnector("org.apache.camel", "myfoo-connector", "2.19.0", "MyFoo", "Something cool", "foo,timer", null, null); - assertEquals(1, catalog.findConnector().size()); + assertEquals(1, catalog.findConnector(false).size()); catalog.removeConnector("org.apache.camel", "myfoo-connector", "2.19.0"); - assertEquals(0, catalog.findConnector().size()); + assertEquals(0, catalog.findConnector(false).size()); } - @Ignore("Not implemented yet") - public void testFindConnector() throws Exception { + @Test + public void testFindConnectorFilter() throws Exception { + CamelConnectorCatalog catalog = new DefaultCamelConnectorCatalog(); + + assertEquals(0, catalog.findConnector(false).size()); + + catalog.addConnector("org.apache.camel", "myfoo-connector", "2.19.0", + "MyFoo", "Something cool", "foo,timer", null, null); + + assertEquals(1, catalog.findConnector("foo", false).size()); + assertEquals(0, catalog.findConnector("bar", false).size()); + } + + @Test + public void testFindConnectorLatestVersionOnly() throws Exception { CamelConnectorCatalog catalog = new DefaultCamelConnectorCatalog(); - assertEquals(0, catalog.findConnector().size()); + assertEquals(0, catalog.findConnector(false).size()); catalog.addConnector("org.apache.camel", "myfoo-connector", "2.19.0", "MyFoo", "Something cool", "foo,timer", null, null); - assertEquals(1, catalog.findConnector("foo").size()); - assertEquals(0, catalog.findConnector("bar").size()); + catalog.addConnector("org.apache.camel", "myfoo-connector", "2.19.1", + "MyFoo", "Something more cool", "foo,timer", null, null); + + assertEquals(1, catalog.findConnector("foo", true).size()); + assertEquals(0, catalog.findConnector("bar", true).size()); + + assertEquals("2.19.1", catalog.findConnector("foo", true).get(0).getVersion()); + assertEquals("Something more cool", catalog.findConnector("foo", true).get(0).getDescription()); + } + + @Test + public void testFindConnectorNotLatestVersionOnly() throws Exception { + CamelConnectorCatalog catalog = new DefaultCamelConnectorCatalog(); + + assertEquals(0, catalog.findConnector(false).size()); + + catalog.addConnector("org.apache.camel", "myfoo-connector", "2.19.0", + "MyFoo", "Something cool", "foo,timer", null, null); + + catalog.addConnector("org.apache.camel", "myfoo-connector", "2.19.1", + "MyFoo", "Something more cool", "foo,timer", null, null); + + assertEquals(2, catalog.findConnector("foo", false).size()); + assertEquals(0, catalog.findConnector("bar", false).size()); + + assertEquals("2.19.0", catalog.findConnector("foo", false).get(0).getVersion()); + assertEquals("Something cool", catalog.findConnector("foo", false).get(0).getDescription()); + assertEquals("2.19.1", catalog.findConnector("foo", false).get(1).getVersion()); + assertEquals("Something more cool", catalog.findConnector("foo", false).get(1).getDescription()); } } http://git-wip-us.apache.org/repos/asf/camel/blob/2c3fa6c4/platforms/catalog-nexus/pom.xml ---------------------------------------------------------------------- diff --git a/platforms/catalog-nexus/pom.xml b/platforms/catalog-nexus/pom.xml index 071f334..09946cb 100644 --- a/platforms/catalog-nexus/pom.xml +++ b/platforms/catalog-nexus/pom.xml @@ -1,22 +1,23 @@ <?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> - <!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> + <modelVersion>4.0.0</modelVersion> <parent> @@ -42,16 +43,9 @@ <groupId>org.apache.camel</groupId> <artifactId>camel-catalog</artifactId> </dependency> - - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - <version>${jackson2-version}</version> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> + <groupId>org.apache.camel</groupId> + <artifactId>camel-catalog-connector</artifactId> </dependency> <!-- testing --> http://git-wip-us.apache.org/repos/asf/camel/blob/2c3fa6c4/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/BaseNexusRepository.java ---------------------------------------------------------------------- diff --git a/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/BaseNexusRepository.java b/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/BaseNexusRepository.java index 6dd8347..1962c09 100644 --- a/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/BaseNexusRepository.java +++ b/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/BaseNexusRepository.java @@ -40,6 +40,9 @@ import org.w3c.dom.NodeList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * A base class for scanning and index Maven Nexus repositories for artifacts which can be added to catalogs. + */ public abstract class BaseNexusRepository { final Logger log = LoggerFactory.getLogger(getClass()); http://git-wip-us.apache.org/repos/asf/camel/blob/2c3fa6c4/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/ComponentCatalogNexusRepository.java ---------------------------------------------------------------------- diff --git a/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/ComponentCatalogNexusRepository.java b/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/ComponentCatalogNexusRepository.java index 152b058..72bcc9e 100644 --- a/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/ComponentCatalogNexusRepository.java +++ b/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/ComponentCatalogNexusRepository.java @@ -42,6 +42,9 @@ public class ComponentCatalogNexusRepository extends BaseNexusRepository { return camelCatalog; } + /** + * Sets the {@link CamelCatalog} to be used. + */ public void setCamelCatalog(CamelCatalog camelCatalog) { this.camelCatalog = camelCatalog; } @@ -80,8 +83,8 @@ public class ComponentCatalogNexusRepository extends BaseNexusRepository { * @param json component json schema */ protected void addComponent(NexusArtifactDto dto, CamelCatalog camelCatalog, String scheme, String javaType, String json) { - log.info("Added component: {}:{}:{} to Camel Catalog", dto.getGroupId(), dto.getArtifactId(), dto.getVersion()); camelCatalog.addComponent(scheme, javaType, json); + log.info("Added component: {}:{}:{} to Camel Catalog", dto.getGroupId(), dto.getArtifactId(), dto.getVersion()); } /** http://git-wip-us.apache.org/repos/asf/camel/blob/2c3fa6c4/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/ConnectorCatalogNexusRepository.java ---------------------------------------------------------------------- diff --git a/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/ConnectorCatalogNexusRepository.java b/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/ConnectorCatalogNexusRepository.java new file mode 100644 index 0000000..b7ce96b --- /dev/null +++ b/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/ConnectorCatalogNexusRepository.java @@ -0,0 +1,153 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.catalog.nexus; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Iterator; +import java.util.Set; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.camel.catalog.CollectionStringBuffer; +import org.apache.camel.catalog.connector.CamelConnectorCatalog; + +import static org.apache.camel.catalog.CatalogHelper.loadText; + +/** + * Nexus repository that can scan for custom Camel connectors and add to the {@link CamelConnectorCatalog}. + */ +public class ConnectorCatalogNexusRepository extends BaseNexusRepository { + + private CamelConnectorCatalog camelConnectorCatalog; + + public ConnectorCatalogNexusRepository() { + super("connector"); + } + + public CamelConnectorCatalog getCamelConnectorCatalog() { + return camelConnectorCatalog; + } + + /** + * Sets the {@link CamelConnectorCatalog} to be used. + */ + public void setCamelConnectorCatalog(CamelConnectorCatalog camelConnectorCatalog) { + this.camelConnectorCatalog = camelConnectorCatalog; + } + + @Override + public void start() { + if (camelConnectorCatalog == null) { + throw new IllegalArgumentException("CamelConnectorCatalog must be configured"); + } + + super.start(); + } + + @Override + public void onNewArtifacts(Set<NexusArtifactDto> newArtifacts) { + // now download the new artifact JARs and look inside to find more details + for (NexusArtifactDto dto : newArtifacts) { + try { + log.debug("Processing new artifact: {}:{}:{}", dto.getGroupId(), dto.getArtifactId(), dto.getVersion()); + String url = createArtifactURL(dto); + URL jarUrl = new URL(url); + addCustomCamelConnectorFromArtifact(dto, jarUrl); + } catch (Throwable e) { + log.warn("Error downloading connector JAR " + dto.getArtifactLink() + ". This exception is ignored. " + e.getMessage()); + } + } + } + + /** + * Adds the connector to the data store + * + * @param dto the artifact + * @param name the name of connector + * @param description the description of connector + * @param labels the labels of connector + * @param connectorJson camel-connector JSon + * @param connectorSchemaJson camel-connector-schema JSon + */ + protected void addConnector(NexusArtifactDto dto, String name, String description, String labels, + String connectorJson, String connectorSchemaJson) { + + String groupId = dto.getGroupId(); + String artifactId = dto.getArtifactId(); + String version = dto.getVersion(); + + camelConnectorCatalog.addConnector(groupId, artifactId, version, name, description, labels, connectorJson, connectorSchemaJson); + log.info("Added connector: {}:{}:{}", dto.getGroupId(), dto.getArtifactId(), dto.getVersion()); + } + + /** + * Adds any discovered third party Camel connectors from the artifact. + */ + private void addCustomCamelConnectorFromArtifact(NexusArtifactDto dto, URL jarUrl) { + try (URLClassLoader classLoader = new URLClassLoader(new URL[] {jarUrl});) { + String[] json = loadConnectorJSonSchema(classLoader); + if (json != null) { + + ObjectMapper mapper = new ObjectMapper(); + JsonNode tree = mapper.readTree(json[0]); + String name = tree.get("name").textValue(); + String description = tree.get("description").textValue(); + Iterator<JsonNode> it = tree.withArray("labels").iterator(); + + CollectionStringBuffer csb = new CollectionStringBuffer(","); + while (it.hasNext()) { + String text = it.next().textValue(); + csb.append(text); + } + + addConnector(dto, name, description, csb.toString(), json[0], json[1]); + } + } catch (IOException e) { + log.warn("Error scanning JAR for custom Camel components", e); + } + } + + private String[] loadConnectorJSonSchema(URLClassLoader classLoader) { + String[] answer = new String[2]; + + String path = "camel-connector.json"; + try { + InputStream is = classLoader.getResourceAsStream(path); + if (is != null) { + answer[0] = loadText(is); + } + } catch (Throwable e) { + log.warn("Error loading " + path + " file", e); + } + + path = "camel-connector-schema.json"; + try { + InputStream is = classLoader.getResourceAsStream(path); + if (is != null) { + answer[1] = loadText(is); + } + } catch (Throwable e) { + log.warn("Error loading " + path + " file", e); + } + + return answer; + } + +} http://git-wip-us.apache.org/repos/asf/camel/blob/2c3fa6c4/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/ConnectorDataStore.java ---------------------------------------------------------------------- diff --git a/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/ConnectorDataStore.java b/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/ConnectorDataStore.java deleted file mode 100644 index e71099d..0000000 --- a/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/ConnectorDataStore.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.camel.catalog.nexus; - -import java.util.List; - -public interface ConnectorDataStore { - - /** - * Number of connectors in the data store. - */ - int size(); - - /** - * Adds a connector to the data store. - */ - void addConnector(ConnectorDto connector); - - /** - * Search for connectors in the data store. - * - * @param filter the filter connectors based on their names, description or labels. - * @param latestVersionOnly to only include the latest version of a given Maven <tt>groupId:artifactId</tt> - * @return the found connectors, or an empty list if none found - */ - List<ConnectorDto> searchConnectors(String filter, boolean latestVersionOnly); - -} http://git-wip-us.apache.org/repos/asf/camel/blob/2c3fa6c4/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/ConnectorDataStoreNexusRepository.java ---------------------------------------------------------------------- diff --git a/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/ConnectorDataStoreNexusRepository.java b/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/ConnectorDataStoreNexusRepository.java deleted file mode 100644 index b2177b7..0000000 --- a/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/ConnectorDataStoreNexusRepository.java +++ /dev/null @@ -1,151 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.camel.catalog.nexus; - -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.camel.catalog.CatalogHelper; -import org.apache.camel.catalog.CollectionStringBuffer; - -import static org.apache.camel.catalog.CatalogHelper.loadText; - -/** - * Nexus repository that can scan for custom Camel connectors and add to the {@link ConnectorDataStore}. - */ -public class ConnectorDataStoreNexusRepository extends BaseNexusRepository { - - private ConnectorDataStore connectorDataStore; - - public ConnectorDataStoreNexusRepository() { - super("connector"); - } - - public ConnectorDataStore getConnectorDataStore() { - return connectorDataStore; - } - - public void setConnectorDataStore(ConnectorDataStore connectorDataStore) { - this.connectorDataStore = connectorDataStore; - } - - @Override - public void start() { - if (connectorDataStore == null) { - throw new IllegalArgumentException("ConnectorDataStore must be configured"); - } - - super.start(); - } - - @Override - public void onNewArtifacts(Set<NexusArtifactDto> newArtifacts) { - // now download the new artifact JARs and look inside to find more details - for (NexusArtifactDto dto : newArtifacts) { - try { - log.debug("Processing new artifact: {}:{}:{}", dto.getGroupId(), dto.getArtifactId(), dto.getVersion()); - String url = createArtifactURL(dto); - URL jarUrl = new URL(url); - addCustomCamelConnectorFromArtifact(dto, jarUrl); - } catch (Throwable e) { - log.warn("Error downloading connector JAR " + dto.getArtifactLink() + ". This exception is ignored. " + e.getMessage()); - } - } - } - - /** - * Adds the connector to the data store - * - * @param dto the artifact - * @param name the name of connector - * @param description the description of connector - * @param labels the labels of connector - * @param connectorJson camel-connector JSon - * @param connectorSchemaJson camel-connector-schema JSon - */ - protected void addConnector(NexusArtifactDto dto, String name, String description, String labels, - String connectorJson, String connectorSchemaJson) { - if (connectorDataStore != null) { - ConnectorDto connector = new ConnectorDto(dto, name, description, labels, connectorJson, connectorSchemaJson); - log.info("Added connector: {}:{}:{}", dto.getGroupId(), dto.getArtifactId(), dto.getVersion()); - connectorDataStore.addConnector(connector); - } - } - - /** - * Adds any discovered third party Camel connectors from the artifact. - */ - private void addCustomCamelConnectorFromArtifact(NexusArtifactDto dto, URL jarUrl) { - try (URLClassLoader classLoader = new URLClassLoader(new URL[] {jarUrl});) { - String[] json = loadConnectorJSonSchema(classLoader); - if (json != null) { - - ObjectMapper mapper = new ObjectMapper(); - JsonNode tree = mapper.readTree(json[0]); - String name = tree.get("name").textValue(); - String description = tree.get("description").textValue(); - Iterator<JsonNode> it = tree.withArray("labels").iterator(); - - CollectionStringBuffer csb = new CollectionStringBuffer(","); - while (it.hasNext()) { - String text = it.next().textValue(); - csb.append(text); - } - - addConnector(dto, name, description, csb.toString(), json[0], json[1]); - } - } catch (IOException e) { - log.warn("Error scanning JAR for custom Camel components", e); - } - } - - private String[] loadConnectorJSonSchema(URLClassLoader classLoader) { - String[] answer = new String[2]; - - String path = "camel-connector.json"; - try { - InputStream is = classLoader.getResourceAsStream(path); - if (is != null) { - answer[0] = loadText(is); - } - } catch (Throwable e) { - log.warn("Error loading " + path + " file", e); - } - - path = "camel-connector-schema.json"; - try { - InputStream is = classLoader.getResourceAsStream(path); - if (is != null) { - answer[1] = loadText(is); - } - } catch (Throwable e) { - log.warn("Error loading " + path + " file", e); - } - - return answer; - } - -} http://git-wip-us.apache.org/repos/asf/camel/blob/2c3fa6c4/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/ConnectorDto.java ---------------------------------------------------------------------- diff --git a/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/ConnectorDto.java b/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/ConnectorDto.java deleted file mode 100644 index 33bea4a..0000000 --- a/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/ConnectorDto.java +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.camel.catalog.nexus; - -import java.io.Serializable; - -public class ConnectorDto implements Serializable { - - private NexusArtifactDto nexusArtifactDto; - private String name; - private String description; - private String labels; - private String connectorJson; - private String connectorSchemaJson; - - public ConnectorDto(NexusArtifactDto nexusArtifactDto, String name, String description, String labels, - String connectorJson, String connectorSchemaJson) { - this.nexusArtifactDto = nexusArtifactDto; - this.name = name; - this.description = description; - this.labels = labels; - this.connectorJson = connectorJson; - this.connectorSchemaJson = connectorSchemaJson; - } - - public NexusArtifactDto getNexusArtifactDto() { - return nexusArtifactDto; - } - - public String getName() { - return name; - } - - public String getDescription() { - return description; - } - - public String getLabels() { - return labels; - } - - public String getConnectorJson() { - return connectorJson; - } - - public String getConnectorSchemaJson() { - return connectorSchemaJson; - } - - public String getMavenGav() { - return nexusArtifactDto.getGroupId() + ":" + nexusArtifactDto.getArtifactId() + ":" + nexusArtifactDto.getVersion(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - ConnectorDto that = (ConnectorDto) o; - - if (!nexusArtifactDto.equals(that.nexusArtifactDto)) { - return false; - } - return name.equals(that.name); - } - - @Override - public int hashCode() { - int result = nexusArtifactDto.hashCode(); - result = 31 * result + name.hashCode(); - return result; - } - - @Override - public String toString() { - return "ConnectorDto[" - + "groupId='" + nexusArtifactDto.getGroupId() + '\'' - + ", artifactId='" + nexusArtifactDto.getArtifactId() + '\'' - + ", version='" + nexusArtifactDto.getVersion() + '\'' - + ", name='" + name + '\'' - + ']'; - } - -} http://git-wip-us.apache.org/repos/asf/camel/blob/2c3fa6c4/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/MemoryConnectorDataStore.java ---------------------------------------------------------------------- diff --git a/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/MemoryConnectorDataStore.java b/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/MemoryConnectorDataStore.java deleted file mode 100644 index 5be41b2..0000000 --- a/platforms/catalog-nexus/src/main/java/org/apache/camel/catalog/nexus/MemoryConnectorDataStore.java +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.camel.catalog.nexus; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Locale; -import java.util.Set; - -/** - * A in-memory based {@link ConnectorDataStore}. - */ -public class MemoryConnectorDataStore implements ConnectorDataStore { - - private final Set<ConnectorDto> connectors = new LinkedHashSet<>(); - - @Override - public int size() { - return connectors.size(); - } - - @Override - public void addConnector(ConnectorDto connector) { - connectors.add(connector); - } - - @Override - public List<ConnectorDto> searchConnectors(String filter, boolean latestVersionOnly) { - List<ConnectorDto> answer = new ArrayList<>(); - - if (filter == null || filter.isEmpty()) { - // return all of them - answer.addAll(connectors); - } else { - // search ignore case - filter = filter.toLowerCase(Locale.US); - for (ConnectorDto dto : connectors) { - if (dto.getName().toLowerCase(Locale.US).contains(filter)) { - answer.add(dto); - } else if (dto.getDescription().toLowerCase(Locale.US).contains(filter)) { - answer.add(dto); - } else if (dto.getNexusArtifactDto().getGroupId().toLowerCase(Locale.US).contains(filter)) { - answer.add(dto); - } else if (dto.getNexusArtifactDto().getArtifactId().toLowerCase(Locale.US).contains(filter)) { - answer.add(dto); - } else if (dto.getNexusArtifactDto().getVersion().toLowerCase(Locale.US).contains(filter)) { - answer.add(dto); - } else { - String labels = dto.getLabels(); - if (labels != null) { - String[] arr = labels.split(","); - for (String lab : arr) { - lab = lab.toLowerCase(Locale.US); - if (lab.contains(filter)) { - answer.add(dto); - break; - } - } - } - } - } - } - - // filter only latest version - if (latestVersionOnly && answer.size() > 1) { - // sort first - answer.sort(Comparator.comparing(ConnectorDto::getMavenGav)); - - // keep only latest in each group - List<ConnectorDto> unique = new ArrayList<>(); - ConnectorDto prev = null; - - for (ConnectorDto dto : answer) { - if (prev == null - || (prev.getNexusArtifactDto().getGroupId().equals(dto.getNexusArtifactDto().getGroupId()) - && prev.getNexusArtifactDto().getArtifactId().equals(dto.getNexusArtifactDto().getArtifactId())) ) { - prev = dto; - } else { - unique.add(prev); - prev = dto; - } - } - if (prev != null) { - // special for last element - unique.add(prev); - } - - answer = unique; - } - - return answer; - - } -} http://git-wip-us.apache.org/repos/asf/camel/blob/2c3fa6c4/platforms/catalog-nexus/src/test/java/org/apache/camel/catalog/nexus/LocalFileConnectorNexusRepository.java ---------------------------------------------------------------------- diff --git a/platforms/catalog-nexus/src/test/java/org/apache/camel/catalog/nexus/LocalFileConnectorNexusRepository.java b/platforms/catalog-nexus/src/test/java/org/apache/camel/catalog/nexus/LocalFileConnectorNexusRepository.java index 2a1ae96..ad697e7 100644 --- a/platforms/catalog-nexus/src/test/java/org/apache/camel/catalog/nexus/LocalFileConnectorNexusRepository.java +++ b/platforms/catalog-nexus/src/test/java/org/apache/camel/catalog/nexus/LocalFileConnectorNexusRepository.java @@ -20,7 +20,7 @@ import java.io.File; import java.net.MalformedURLException; import java.net.URL; -public class LocalFileConnectorNexusRepository extends ConnectorDataStoreNexusRepository { +public class LocalFileConnectorNexusRepository extends ConnectorCatalogNexusRepository { private Runnable onAddConnector; http://git-wip-us.apache.org/repos/asf/camel/blob/2c3fa6c4/platforms/catalog-nexus/src/test/java/org/apache/camel/catalog/nexus/LocalNexusConnectorRepositoryTest.java ---------------------------------------------------------------------- diff --git a/platforms/catalog-nexus/src/test/java/org/apache/camel/catalog/nexus/LocalNexusConnectorRepositoryTest.java b/platforms/catalog-nexus/src/test/java/org/apache/camel/catalog/nexus/LocalNexusConnectorRepositoryTest.java index 2bddf53..733b632 100644 --- a/platforms/catalog-nexus/src/test/java/org/apache/camel/catalog/nexus/LocalNexusConnectorRepositoryTest.java +++ b/platforms/catalog-nexus/src/test/java/org/apache/camel/catalog/nexus/LocalNexusConnectorRepositoryTest.java @@ -20,11 +20,13 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import junit.framework.TestCase; +import org.apache.camel.catalog.connector.CamelConnectorCatalog; +import org.apache.camel.catalog.connector.DefaultCamelConnectorCatalog; import org.junit.Test; public class LocalNexusConnectorRepositoryTest extends TestCase { - private MemoryConnectorDataStore dataStore = new MemoryConnectorDataStore(); + private CamelConnectorCatalog catalog = new DefaultCamelConnectorCatalog(); @Test public void testLocalNexus() throws Exception { @@ -32,12 +34,12 @@ public class LocalNexusConnectorRepositoryTest extends TestCase { repo.setInitialDelay(2); repo.setDelay(3); repo.setNexusUrl("dummy"); - repo.setConnectorDataStore(dataStore); + repo.setCamelConnectorCatalog(catalog); final CountDownLatch latch = new CountDownLatch(1); repo.setOnAddConnector(latch::countDown); - int before = dataStore.size(); + int before = catalog.findConnector(false).size(); repo.start(); @@ -45,7 +47,7 @@ public class LocalNexusConnectorRepositoryTest extends TestCase { repo.stop(); - int after = dataStore.size(); + int after = catalog.findConnector(false).size(); assertTrue("There should be 1 connector found", after - before == 1); }
