This is an automated email from the ASF dual-hosted git repository. riemer pushed a commit to branch support-service-selection-for-adapters in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 08c3bd956200ec35997b5fdd30bbb494b50427ba Author: Dominik Riemer <[email protected]> AuthorDate: Wed Apr 3 23:27:23 2024 +0200 feat: Support custom service tags and service selection for adapters --- .../apache/streampipes/commons/constants/Envs.java | 4 +- .../commons/environment/DefaultEnvironment.java | 5 + .../commons/environment/Environment.java | 2 + .../management/AdapterMasterManagement.java | 21 ++-- .../management/management/GuessManagement.java | 19 +++- .../management/management/WorkerRestClient.java | 4 +- .../management/management/WorkerUrlProvider.java | 42 ------- .../connect/management/util/WorkerPaths.java | 23 ---- .../model/connect/adapter/AdapterDescription.java | 15 +++ .../model/deployment/DeploymentConfiguration.java | 36 ++++-- .../extensions/svcdiscovery/SpServiceTag.java | 22 ++++ .../svcdiscovery/SpServiceTagPrefix.java | 1 + .../IExtensionsServiceEndpointGenerator.java | 45 ++++++++ .../streampipes/manager/assets/AssetFetcher.java | 2 +- .../ExtensionsServiceEndpointGenerator.java | 76 +++++++------ .../execution/task/DiscoverEndpointsTask.java | 9 +- .../manager/health/PipelineHealthCheck.java | 2 +- .../CustomTransformOutputSchemaGenerator.java | 6 +- .../manager/preview/PipelinePreview.java | 9 +- .../remote/ContainerProvidedOptionsHandler.java | 2 +- .../streampipes/rest/impl/ServiceTagResource.java | 43 +++++++ .../rest/impl/connect/DescriptionResource.java | 19 +++- .../impl/connect/RuntimeResolvableResource.java | 19 +++- .../svcdiscovery/api/ISpServiceDiscovery.java | 9 +- .../svcdiscovery/SpServiceDiscoveryCore.java | 27 +++-- .../extensions/CustomServiceTagResolver.java | 49 ++++++++ .../StreamPipesExtensionsServiceBase.java | 2 + .../src/lib/apis/service-tags.service.ts | 46 ++++++++ .../src/lib/model/gen/streampipes-model.ts | 49 ++++---- .../platform-services/src/public-api.ts | 1 + .../adapter-deployment-settings.component.html | 78 +++++++++++++ .../adapter-deployment-settings.component.scss | 0 .../adapter-deployment-settings.component.ts | 123 +++++++++++++++++++++ .../adapter-settings.component.html | 11 ++ ui/src/app/connect/connect.module.ts | 2 + 35 files changed, 636 insertions(+), 187 deletions(-) diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java index 4c58485e27..489c54d5a3 100644 --- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java @@ -97,7 +97,9 @@ public enum Envs { SP_NATS_HOST("SP_NATS_HOST", "nats"), SP_NATS_PORT("SP_NATS_PORT", "4222"), - SP_PULSAR_URL("SP_PULSAR_URL", "pulsar://localhost:6650"); + SP_PULSAR_URL("SP_PULSAR_URL", "pulsar://localhost:6650"), + + SP_SERVICE_TAGS("SP_SERVICE_TAGS", ""); private final String envVariableName; private String defaultValue; diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/DefaultEnvironment.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/DefaultEnvironment.java index 5e613e0909..3fef8c55f7 100644 --- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/DefaultEnvironment.java +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/DefaultEnvironment.java @@ -274,4 +274,9 @@ public class DefaultEnvironment implements Environment { return new StringEnvironmentVariable(Envs.SP_PULSAR_URL); } + @Override + public StringEnvironmentVariable getCustomServiceTags() { + return new StringEnvironmentVariable(Envs.SP_SERVICE_TAGS); + } + } diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/Environment.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/Environment.java index 092e6992d6..ceb2d2d1f2 100644 --- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/Environment.java +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/Environment.java @@ -135,4 +135,6 @@ public interface Environment { StringEnvironmentVariable getPulsarUrl(); + StringEnvironmentVariable getCustomServiceTags(); + } diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java index ec87909021..56220fde37 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java @@ -23,7 +23,7 @@ import org.apache.streampipes.commons.exceptions.SepaParseException; import org.apache.streampipes.commons.exceptions.connect.AdapterException; import org.apache.streampipes.commons.prometheus.adapter.AdapterMetrics; import org.apache.streampipes.connect.management.util.GroundingUtils; -import org.apache.streampipes.connect.management.util.WorkerPaths; +import org.apache.streampipes.manager.execution.endpoint.ExtensionsServiceEndpointGenerator; import org.apache.streampipes.manager.monitoring.pipeline.ExtensionsLogProvider; import org.apache.streampipes.manager.verification.DataStreamVerifier; import org.apache.streampipes.model.SpDataStream; @@ -32,12 +32,11 @@ import org.apache.streampipes.model.util.ElementIdGenerator; import org.apache.streampipes.resource.management.AdapterResourceManager; import org.apache.streampipes.resource.management.DataStreamResourceManager; import org.apache.streampipes.storage.api.IAdapterStorage; -import org.apache.streampipes.storage.management.StorageDispatcher; +import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.URISyntaxException; import java.util.List; import java.util.NoSuchElementException; @@ -54,12 +53,12 @@ public class AdapterMasterManagement { private final DataStreamResourceManager dataStreamResourceManager; - public AdapterMasterManagement(IAdapterStorage adapterStorage, + public AdapterMasterManagement(IAdapterStorage adapterInstanceStorage, AdapterResourceManager adapterResourceManager, DataStreamResourceManager dataStreamResourceManager, AdapterMetrics adapterMetrics ) { - this.adapterInstanceStorage = adapterStorage; + this.adapterInstanceStorage = adapterInstanceStorage; this.adapterMetrics = adapterMetrics; this.adapterResourceManager = adapterResourceManager; this.dataStreamResourceManager = dataStreamResourceManager; @@ -164,7 +163,11 @@ public class AdapterMasterManagement { try { // Find endpoint to start adapter on - var baseUrl = WorkerPaths.findEndpointUrl(ad.getAppId()); + var baseUrl = new ExtensionsServiceEndpointGenerator().getEndpointBaseUrl( + ad.getAppId(), + SpServiceUrlProvider.ADAPTER, + ad.getDeploymentConfiguration().getDesiredServiceTags() + ); // Update selected endpoint URL of adapter ad.setSelectedEndpointUrl(baseUrl); @@ -177,7 +180,7 @@ public class AdapterMasterManagement { adapterMetrics.register(ad.getElementId(), ad.getName()); LOG.info("Started adapter " + elementId + " on: " + baseUrl); - } catch (NoServiceEndpointsAvailableException | URISyntaxException e) { + } catch (NoServiceEndpointsAvailableException e) { throw new AdapterException("Could not start adapter due to unavailable service endpoint", e); } } @@ -192,8 +195,4 @@ public class AdapterMasterManagement { throw new AdapterException(); } } - - private IAdapterStorage getAdapterInstanceStorage() { - return StorageDispatcher.INSTANCE.getNoSqlStore().getAdapterInstanceStorage(); - } } diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/GuessManagement.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/GuessManagement.java index 104bae82e0..472cb7eca8 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/GuessManagement.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/GuessManagement.java @@ -24,11 +24,15 @@ import org.apache.streampipes.commons.exceptions.connect.ParseException; import org.apache.streampipes.connect.management.AdapterEventPreviewPipeline; import org.apache.streampipes.connect.management.util.WorkerPaths; import org.apache.streampipes.extensions.api.connect.exception.WorkerAdapterException; +import org.apache.streampipes.manager.api.extensions.IExtensionsServiceEndpointGenerator; import org.apache.streampipes.manager.execution.ExtensionServiceExecutions; +import org.apache.streampipes.manager.execution.endpoint.ExtensionsServiceEndpointGenerator; import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.connect.guess.AdapterEventPreview; import org.apache.streampipes.model.connect.guess.GuessSchema; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag; import org.apache.streampipes.serializers.json.JacksonSerializer; +import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -39,21 +43,25 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.Set; public class GuessManagement { private static final Logger LOG = LoggerFactory.getLogger(GuessManagement.class); - private final WorkerUrlProvider workerUrlProvider; + private final IExtensionsServiceEndpointGenerator endpointGenerator; private final ObjectMapper objectMapper; public GuessManagement() { - this.workerUrlProvider = new WorkerUrlProvider(); + this.endpointGenerator = new ExtensionsServiceEndpointGenerator(); this.objectMapper = JacksonSerializer.getObjectMapper(); } public GuessSchema guessSchema(AdapterDescription adapterDescription) throws ParseException, WorkerAdapterException, NoServiceEndpointsAvailableException, IOException { - var workerUrl = getWorkerUrl(adapterDescription.getAppId()); + var workerUrl = getWorkerUrl( + adapterDescription.getAppId(), + adapterDescription.getDeploymentConfiguration().getDesiredServiceTags() + ); var description = objectMapper.writeValueAsString(adapterDescription); LOG.info("Guess schema at: " + workerUrl); @@ -72,8 +80,9 @@ public class GuessManagement { } } - private String getWorkerUrl(String appId) throws NoServiceEndpointsAvailableException { - var baseUrl = workerUrlProvider.getWorkerBaseUrl(appId); + private String getWorkerUrl(String appId, + Set<SpServiceTag> customServiceTags) throws NoServiceEndpointsAvailableException { + var baseUrl = endpointGenerator.getEndpointBaseUrl(appId, SpServiceUrlProvider.ADAPTER, customServiceTags); return baseUrl + WorkerPaths.getGuessSchemaPath(); } diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java index c715459070..4104c82a4f 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java @@ -52,10 +52,10 @@ public class WorkerRestClient { private static final Logger LOG = LoggerFactory.getLogger(WorkerRestClient.class); - public static void invokeStreamAdapter(String endpointUrl, + public static void invokeStreamAdapter(String baseUrl, String elementId) throws AdapterException { var adapterStreamDescription = getAndDecryptAdapter(elementId); - var url = endpointUrl + WorkerPaths.getStreamInvokePath(); + var url = baseUrl + WorkerPaths.getStreamInvokePath(); startAdapter(url, adapterStreamDescription); updateStreamAdapterStatus(adapterStreamDescription.getElementId(), true); diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerUrlProvider.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerUrlProvider.java deleted file mode 100644 index b9fc970ab4..0000000000 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerUrlProvider.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.connect.management.management; - -import org.apache.streampipes.commons.exceptions.NoServiceEndpointsAvailableException; -import org.apache.streampipes.manager.execution.endpoint.ExtensionsServiceEndpointGenerator; -import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider; - -public class WorkerUrlProvider { - - - public WorkerUrlProvider() { - } - - public String getWorkerUrl(String appId) throws NoServiceEndpointsAvailableException { - return getEndpointGenerator(appId).getEndpointResourceUrl(); - } - - public String getWorkerBaseUrl(String appId) throws NoServiceEndpointsAvailableException { - return getEndpointGenerator(appId).getEndpointBaseUrl(); - } - - private ExtensionsServiceEndpointGenerator getEndpointGenerator(String appId) { - return new ExtensionsServiceEndpointGenerator(appId, SpServiceUrlProvider.ADAPTER); - } - -} diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/WorkerPaths.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/WorkerPaths.java index 29d89ebd1b..7bd2b0c1ed 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/WorkerPaths.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/WorkerPaths.java @@ -17,13 +17,6 @@ */ package org.apache.streampipes.connect.management.util; -import org.apache.streampipes.commons.exceptions.NoServiceEndpointsAvailableException; -import org.apache.streampipes.manager.execution.endpoint.ExtensionsServiceEndpointGenerator; -import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider; - -import java.net.URI; -import java.net.URISyntaxException; - public class WorkerPaths { private static final String WorkerMainPath = "/api/v1/worker"; @@ -36,14 +29,6 @@ public class WorkerPaths { return WorkerMainPath + "/stream/stop"; } - public static String getSetInvokePath() { - return WorkerMainPath + "/set/invoke"; - } - - public static String getSetStopPath() { - return WorkerMainPath + "/set/stop"; - } - public static String getRunningAdaptersPath() { return WorkerMainPath + "/running"; } @@ -56,12 +41,4 @@ public class WorkerPaths { return WorkerMainPath + "/guess/schema"; } - public static String findEndpointUrl(String appId) throws NoServiceEndpointsAvailableException, URISyntaxException { - SpServiceUrlProvider serviceUrlProvider = SpServiceUrlProvider.ADAPTER; - String endpointUrl = new ExtensionsServiceEndpointGenerator(appId, serviceUrlProvider).getEndpointResourceUrl(); - URI uri = new URI(endpointUrl); - return uri.getScheme() + "://" + uri.getAuthority(); - } - - } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/AdapterDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/AdapterDescription.java index 5fd6b21b3f..d8b549211d 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/AdapterDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/AdapterDescription.java @@ -25,6 +25,7 @@ import org.apache.streampipes.model.connect.rules.schema.SchemaTransformationRul import org.apache.streampipes.model.connect.rules.stream.StreamTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.AddTimestampRuleDescription; import org.apache.streampipes.model.connect.rules.value.ValueTransformationRuleDescription; +import org.apache.streampipes.model.deployment.DeploymentConfiguration; import org.apache.streampipes.model.grounding.EventGrounding; import org.apache.streampipes.model.schema.EventSchema; import org.apache.streampipes.model.shared.annotation.TsModel; @@ -55,6 +56,8 @@ public class AdapterDescription extends VersionedNamedStreamPipesEntity { // Is used to store where the adapter is running to stop it private String selectedEndpointUrl; + private DeploymentConfiguration deploymentConfiguration; + /** * This is used to identify all the service within the service group the adapter can be invoked in */ @@ -72,6 +75,7 @@ public class AdapterDescription extends VersionedNamedStreamPipesEntity { this.config = new ArrayList<>(); this.category = new ArrayList<>(); this.dataStream = new SpDataStream(); + this.deploymentConfiguration = new DeploymentConfiguration(); } public AdapterDescription(int version) { @@ -81,6 +85,7 @@ public class AdapterDescription extends VersionedNamedStreamPipesEntity { this.config = new ArrayList<>(); this.category = new ArrayList<>(); this.dataStream = new SpDataStream(); + this.deploymentConfiguration = new DeploymentConfiguration(); this.setVersion(version); } @@ -89,6 +94,7 @@ public class AdapterDescription extends VersionedNamedStreamPipesEntity { this.rules = new ArrayList<>(); this.category = new ArrayList<>(); this.dataStream = new SpDataStream(); + this.deploymentConfiguration = new DeploymentConfiguration(); } @@ -109,6 +115,7 @@ public class AdapterDescription extends VersionedNamedStreamPipesEntity { this.dataStream = new SpDataStream(other.getDataStream()); } this.running = other.isRunning(); + this.deploymentConfiguration = other.getDeploymentConfiguration(); } public String getRev() { @@ -259,4 +266,12 @@ public class AdapterDescription extends VersionedNamedStreamPipesEntity { public void setRunning(boolean running) { this.running = running; } + + public DeploymentConfiguration getDeploymentConfiguration() { + return deploymentConfiguration; + } + + public void setDeploymentConfiguration(DeploymentConfiguration deploymentConfiguration) { + this.deploymentConfiguration = deploymentConfiguration; + } } diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointProvider.java b/streampipes-model/src/main/java/org/apache/streampipes/model/deployment/DeploymentConfiguration.java similarity index 50% rename from streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointProvider.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/deployment/DeploymentConfiguration.java index 6e76fedfe7..3e44f8f5e6 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointProvider.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/deployment/DeploymentConfiguration.java @@ -16,17 +16,35 @@ * */ -package org.apache.streampipes.manager.execution.endpoint; +package org.apache.streampipes.model.deployment; -import org.apache.streampipes.commons.exceptions.NoServiceEndpointsAvailableException; -import org.apache.streampipes.model.base.InvocableStreamPipesEntity; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag; -public class ExtensionsServiceEndpointProvider { +import java.util.HashSet; +import java.util.Set; - public String findSelectedEndpoint(InvocableStreamPipesEntity g) throws NoServiceEndpointsAvailableException { - return new ExtensionsServiceEndpointGenerator( - g.getAppId(), - ExtensionsServiceEndpointUtils.getPipelineElementType(g)) - .getEndpointResourceUrl(); +public class DeploymentConfiguration { + + private Set<SpServiceTag> desiredServiceTags; + private String selectedEndpointUrl; + + public DeploymentConfiguration() { + this.desiredServiceTags = new HashSet<>(); + } + + public Set<SpServiceTag> getDesiredServiceTags() { + return desiredServiceTags; + } + + public void setDesiredServiceTags(Set<SpServiceTag> desiredServiceTags) { + this.desiredServiceTags = desiredServiceTags; + } + + public String getSelectedEndpointUrl() { + return selectedEndpointUrl; + } + + public void setSelectedEndpointUrl(String selectedEndpointUrl) { + this.selectedEndpointUrl = selectedEndpointUrl; } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTag.java b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTag.java index 71b7fe428e..572fc4a7f6 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTag.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTag.java @@ -17,6 +17,11 @@ */ package org.apache.streampipes.model.extensions.svcdiscovery; +import org.apache.streampipes.model.shared.annotation.TsModel; + +import java.util.Objects; + +@TsModel public class SpServiceTag { private static final String COLON = ":"; @@ -56,4 +61,21 @@ public class SpServiceTag { public void setValue(String value) { this.value = value; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SpServiceTag that = (SpServiceTag) o; + return prefix == that.prefix && Objects.equals(value, that.value); + } + + @Override + public int hashCode() { + return Objects.hash(prefix, value); + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTagPrefix.java b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTagPrefix.java index f412bdbe6f..568a8295cb 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTagPrefix.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTagPrefix.java @@ -19,6 +19,7 @@ package org.apache.streampipes.model.extensions.svcdiscovery; public enum SpServiceTagPrefix { SP_GROUP, + CUSTOM, ADAPTER, DATA_STREAM, DATA_PROCESSOR, diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/api/extensions/IExtensionsServiceEndpointGenerator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/api/extensions/IExtensionsServiceEndpointGenerator.java new file mode 100644 index 0000000000..7414c7bd26 --- /dev/null +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/api/extensions/IExtensionsServiceEndpointGenerator.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.manager.api.extensions; + +import org.apache.streampipes.commons.exceptions.NoServiceEndpointsAvailableException; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag; +import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider; + +import java.util.Collections; +import java.util.Set; + +public interface IExtensionsServiceEndpointGenerator { + + String getEndpointResourceUrl(String appId, + SpServiceUrlProvider spServiceUrlProvider, + Set<SpServiceTag> customServiceTags) + throws NoServiceEndpointsAvailableException; + + default String getEndpointResourceUrl(String appId, + SpServiceUrlProvider spServiceUrlProvider) + throws NoServiceEndpointsAvailableException { + return getEndpointResourceUrl(appId, spServiceUrlProvider, Collections.emptySet()); + } + + String getEndpointBaseUrl(String appId, + SpServiceUrlProvider spServiceUrlProvider, + Set<SpServiceTag> customServiceTags) + throws NoServiceEndpointsAvailableException; +} diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/assets/AssetFetcher.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/assets/AssetFetcher.java index 553762018e..25c3c9accf 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/assets/AssetFetcher.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/assets/AssetFetcher.java @@ -40,7 +40,7 @@ public class AssetFetcher { } public InputStream fetchPipelineElementAssets() throws IOException, NoServiceEndpointsAvailableException { - String endpointUrl = new ExtensionsServiceEndpointGenerator(appId, spServiceUrlProvider).getEndpointResourceUrl(); + String endpointUrl = new ExtensionsServiceEndpointGenerator().getEndpointResourceUrl(appId, spServiceUrlProvider); return Request .Get(endpointUrl + ASSET_ENDPOINT_APPENDIX) .execute() diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointGenerator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointGenerator.java index cb2d28408a..ae1b8227c7 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointGenerator.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint/ExtensionsServiceEndpointGenerator.java @@ -18,7 +18,8 @@ package org.apache.streampipes.manager.execution.endpoint; import org.apache.streampipes.commons.exceptions.NoServiceEndpointsAvailableException; -import org.apache.streampipes.model.base.NamedStreamPipesEntity; +import org.apache.streampipes.manager.api.extensions.IExtensionsServiceEndpointGenerator; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag; import org.apache.streampipes.svcdiscovery.SpServiceDiscovery; import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceTypes; import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider; @@ -26,59 +27,66 @@ import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collections; import java.util.List; +import java.util.Set; +import java.util.stream.Stream; -public class ExtensionsServiceEndpointGenerator { +public class ExtensionsServiceEndpointGenerator implements IExtensionsServiceEndpointGenerator { private static final Logger LOG = LoggerFactory.getLogger(ExtensionsServiceEndpointGenerator.class); - private final String appId; - private final SpServiceUrlProvider spServiceUrlProvider; - - public ExtensionsServiceEndpointGenerator(String appId, - SpServiceUrlProvider spServiceUrlProvider) { - this.appId = appId; - this.spServiceUrlProvider = spServiceUrlProvider; + public ExtensionsServiceEndpointGenerator() { } - public ExtensionsServiceEndpointGenerator(NamedStreamPipesEntity entity) { - this.appId = entity.getAppId(); - this.spServiceUrlProvider = ExtensionsServiceEndpointUtils.getPipelineElementType(entity); + public String getEndpointResourceUrl(String appId, + SpServiceUrlProvider spServiceUrlProvider, + Set<SpServiceTag> customServiceTags) + throws NoServiceEndpointsAvailableException { + return spServiceUrlProvider.getInvocationUrl(selectService(appId, spServiceUrlProvider, customServiceTags), appId); } - public String getEndpointResourceUrl() throws NoServiceEndpointsAvailableException { - return spServiceUrlProvider.getInvocationUrl(selectService(), appId); + public String getEndpointBaseUrl(String appId, + SpServiceUrlProvider spServiceUrlProvider, + Set<SpServiceTag> customServiceTags) throws NoServiceEndpointsAvailableException { + return selectService(appId, spServiceUrlProvider, customServiceTags); } - public String getEndpointBaseUrl() throws NoServiceEndpointsAvailableException { - return selectService(); + private String selectService(String appId, + SpServiceUrlProvider spServiceUrlProvider, + Set<SpServiceTag> customServiceTags) throws NoServiceEndpointsAvailableException { + List<String> serviceEndpoints = getServiceEndpoints(appId, spServiceUrlProvider, customServiceTags); + if (!serviceEndpoints.isEmpty()) { + return serviceEndpoints.get(0); + } else { + LOG.error("Could not find any service endpoints for appId {}, serviceTag {}", appId, + spServiceUrlProvider.getServiceTag(appId).asString()); + throw new NoServiceEndpointsAvailableException( + "Could not find any matching service endpoints - are all software components running?"); + } } - private List<String> getServiceEndpoints() { + private List<String> getServiceEndpoints(String appId, + SpServiceUrlProvider spServiceUrlProvider, + Set<SpServiceTag> customServiceTags) { return SpServiceDiscovery .getServiceDiscovery() .getServiceEndpoints( DefaultSpServiceTypes.EXT, true, - Collections - .singletonList( - this.spServiceUrlProvider - .getServiceTag(appId) - .asString() - ) + getDesiredServiceTags(appId, spServiceUrlProvider, customServiceTags) ); } - private String selectService() throws NoServiceEndpointsAvailableException { - List<String> serviceEndpoints = getServiceEndpoints(); - if (!serviceEndpoints.isEmpty()) { - return getServiceEndpoints().get(0); - } else { - LOG.error("Could not find any service endpoints for appId {}, serviceTag {}", appId, - this.spServiceUrlProvider.getServiceTag(appId).asString()); - throw new NoServiceEndpointsAvailableException( - "Could not find any matching service endpoints - are all software components running?"); - } + private List<String> getDesiredServiceTags(String appId, + SpServiceUrlProvider serviceUrlProvider, + Set<SpServiceTag> customServiceTags) { + return Stream.concat( + Stream.of( + serviceUrlProvider.getServiceTag(appId) + ), + customServiceTags.stream() + ) + .map(SpServiceTag::asString) + .toList(); } } diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/task/DiscoverEndpointsTask.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/task/DiscoverEndpointsTask.java index 896c704f04..40da134aa1 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/task/DiscoverEndpointsTask.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/task/DiscoverEndpointsTask.java @@ -20,7 +20,8 @@ package org.apache.streampipes.manager.execution.task; import org.apache.streampipes.commons.exceptions.NoServiceEndpointsAvailableException; import org.apache.streampipes.manager.execution.PipelineExecutionInfo; -import org.apache.streampipes.manager.execution.endpoint.ExtensionsServiceEndpointProvider; +import org.apache.streampipes.manager.execution.endpoint.ExtensionsServiceEndpointGenerator; +import org.apache.streampipes.manager.execution.endpoint.ExtensionsServiceEndpointUtils; import org.apache.streampipes.model.api.EndpointSelectable; import org.apache.streampipes.model.base.InvocableStreamPipesEntity; import org.apache.streampipes.model.pipeline.Pipeline; @@ -69,7 +70,11 @@ public class DiscoverEndpointsTask implements PipelineExecutionTask { private String findSelectedEndpoint(InvocableStreamPipesEntity pipelineElement) throws NoServiceEndpointsAvailableException { - return new ExtensionsServiceEndpointProvider().findSelectedEndpoint(pipelineElement); + return new ExtensionsServiceEndpointGenerator() + .getEndpointResourceUrl( + pipelineElement.getAppId(), + ExtensionsServiceEndpointUtils.getPipelineElementType(pipelineElement) + ); } } diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/PipelineHealthCheck.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/PipelineHealthCheck.java index 40216cc87b..84ab806c30 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/PipelineHealthCheck.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/health/PipelineHealthCheck.java @@ -139,7 +139,7 @@ public class PipelineHealthCheck implements Runnable { private String findEndpointUrl(InvocableStreamPipesEntity graph) throws NoServiceEndpointsAvailableException { SpServiceUrlProvider serviceUrlProvider = ExtensionsServiceEndpointUtils.getPipelineElementType(graph); - return new ExtensionsServiceEndpointGenerator(graph.getAppId(), serviceUrlProvider).getEndpointResourceUrl(); + return new ExtensionsServiceEndpointGenerator().getEndpointResourceUrl(graph.getAppId(), serviceUrlProvider); } private boolean shouldRetry(String instanceId) { diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/output/CustomTransformOutputSchemaGenerator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/output/CustomTransformOutputSchemaGenerator.java index 936adabb49..77551acb4a 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/output/CustomTransformOutputSchemaGenerator.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/output/CustomTransformOutputSchemaGenerator.java @@ -64,8 +64,10 @@ public class CustomTransformOutputSchemaGenerator extends OutputSchemaGenerator< private EventSchema makeRequest() { try { String httpRequestBody = JacksonSerializer.getObjectMapper().writeValueAsString(dataProcessorInvocation); - String endpointUrl = new ExtensionsServiceEndpointGenerator(dataProcessorInvocation.getAppId(), - SpServiceUrlProvider.DATA_PROCESSOR).getEndpointResourceUrl(); + String endpointUrl = new ExtensionsServiceEndpointGenerator().getEndpointResourceUrl( + dataProcessorInvocation.getAppId(), + SpServiceUrlProvider.DATA_PROCESSOR + ); Response httpResp = Request.Post(endpointUrl + "/output").bodyString(httpRequestBody, ContentType .APPLICATION_JSON).execute(); diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/preview/PipelinePreview.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/preview/PipelinePreview.java index 1113c9a4c8..903ecbeb75 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/preview/PipelinePreview.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/preview/PipelinePreview.java @@ -80,10 +80,11 @@ public class PipelinePreview { } private String findSelectedEndpoint(InvocableStreamPipesEntity g) throws NoServiceEndpointsAvailableException { - return new ExtensionsServiceEndpointGenerator( - g.getAppId(), - ExtensionsServiceEndpointUtils.getPipelineElementType(g)) - .getEndpointResourceUrl(); + return new ExtensionsServiceEndpointGenerator() + .getEndpointResourceUrl( + g.getAppId(), + ExtensionsServiceEndpointUtils.getPipelineElementType(g) + ); } private void invokeGraphs(List<InvocableStreamPipesEntity> graphs) { diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/remote/ContainerProvidedOptionsHandler.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/remote/ContainerProvidedOptionsHandler.java index fd1957f5cc..16acbba678 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/remote/ContainerProvidedOptionsHandler.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/remote/ContainerProvidedOptionsHandler.java @@ -55,6 +55,6 @@ public class ContainerProvidedOptionsHandler { private String getEndpointUrl(String appId) throws NoServiceEndpointsAvailableException { SpServiceUrlProvider provider = ExtensionsServiceEndpointUtils.getPipelineElementType(appId); - return new ExtensionsServiceEndpointGenerator(appId, provider).getEndpointResourceUrl() + "/configurations"; + return new ExtensionsServiceEndpointGenerator().getEndpointResourceUrl(appId, provider) + "/configurations"; } } diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/ServiceTagResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/ServiceTagResource.java new file mode 100644 index 0000000000..536933f950 --- /dev/null +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/ServiceTagResource.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.rest.impl; + +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag; +import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource; +import org.apache.streampipes.svcdiscovery.SpServiceDiscovery; +import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery; + +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Set; + +@RestController +@RequestMapping("/api/v2/service-tags") +public class ServiceTagResource extends AbstractAuthGuardedRestResource { + + private final ISpServiceDiscovery serviceDiscovery = SpServiceDiscovery.getServiceDiscovery(); + + @GetMapping(produces = MediaType.APPLICATION_JSON_VALUE) + public Set<SpServiceTag> getCustomServiceTags() { + return serviceDiscovery.getCustomServiceTags(true); + } +} diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/DescriptionResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/DescriptionResource.java index dd40dd8541..ab49f32a9c 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/DescriptionResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/DescriptionResource.java @@ -22,11 +22,14 @@ import org.apache.streampipes.commons.exceptions.NoServiceEndpointsAvailableExce import org.apache.streampipes.commons.exceptions.SpRuntimeException; import org.apache.streampipes.commons.exceptions.connect.AdapterException; import org.apache.streampipes.connect.management.management.DescriptionManagement; -import org.apache.streampipes.connect.management.management.WorkerUrlProvider; +import org.apache.streampipes.manager.api.extensions.IExtensionsServiceEndpointGenerator; +import org.apache.streampipes.manager.execution.endpoint.ExtensionsServiceEndpointGenerator; import org.apache.streampipes.model.connect.adapter.AdapterDescription; +import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.DeleteMapping; @@ -43,11 +46,11 @@ import java.util.Optional; public class DescriptionResource extends AbstractAdapterResource<DescriptionManagement> { private static final Logger LOG = LoggerFactory.getLogger(DescriptionResource.class); - private final WorkerUrlProvider workerUrlProvider; + private final IExtensionsServiceEndpointGenerator endpointGenerator; public DescriptionResource() { super(DescriptionManagement::new); - workerUrlProvider = new WorkerUrlProvider(); + endpointGenerator = new ExtensionsServiceEndpointGenerator(); } @GetMapping(path = "/adapters", produces = MediaType.APPLICATION_JSON_VALUE) @@ -65,7 +68,7 @@ public class DescriptionResource extends AbstractAdapterResource<DescriptionMana Optional<AdapterDescription> adapterDescriptionOptional = managementService.getAdapter(id); if (adapterDescriptionOptional.isPresent()) { AdapterDescription adapterDescription = adapterDescriptionOptional.get(); - String workerUrl = workerUrlProvider.getWorkerUrl(adapterDescription.getAppId()); + String workerUrl = getServiceResourceUrl(adapterDescription.getAppId()); result = managementService.getAssets(workerUrl); } @@ -93,7 +96,7 @@ public class DescriptionResource extends AbstractAdapterResource<DescriptionMana Optional<AdapterDescription> adapterDescriptionOptional = managementService.getAdapter(id); if (adapterDescriptionOptional.isPresent()) { AdapterDescription adapterDescription = adapterDescriptionOptional.get(); - String workerUrl = workerUrlProvider.getWorkerUrl(adapterDescription.getAppId()); + String workerUrl = getServiceResourceUrl(adapterDescription.getAppId()); result = managementService.getIconAsset(workerUrl); } @@ -120,7 +123,7 @@ public class DescriptionResource extends AbstractAdapterResource<DescriptionMana Optional<AdapterDescription> adapterDescriptionOptional = managementService.getAdapter(id); if (adapterDescriptionOptional.isPresent()) { AdapterDescription adapterDescription = adapterDescriptionOptional.get(); - String workerUrl = workerUrlProvider.getWorkerUrl(adapterDescription.getAppId()); + String workerUrl = getServiceResourceUrl(adapterDescription.getAppId()); result = managementService.getDocumentationAsset(workerUrl); } @@ -148,4 +151,8 @@ public class DescriptionResource extends AbstractAdapterResource<DescriptionMana return badRequest(e); } } + + private String getServiceResourceUrl(String appId) throws NoServiceEndpointsAvailableException { + return endpointGenerator.getEndpointResourceUrl(appId, SpServiceUrlProvider.ADAPTER); + } } diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/RuntimeResolvableResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/RuntimeResolvableResource.java index 6266439bb8..dc7d8c816d 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/RuntimeResolvableResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/connect/RuntimeResolvableResource.java @@ -24,15 +24,18 @@ import org.apache.streampipes.commons.exceptions.connect.AdapterException; import org.apache.streampipes.commons.prometheus.adapter.AdapterMetricsManager; import org.apache.streampipes.connect.management.management.WorkerAdministrationManagement; import org.apache.streampipes.connect.management.management.WorkerRestClient; -import org.apache.streampipes.connect.management.management.WorkerUrlProvider; +import org.apache.streampipes.manager.api.extensions.IExtensionsServiceEndpointGenerator; +import org.apache.streampipes.manager.execution.endpoint.ExtensionsServiceEndpointGenerator; import org.apache.streampipes.model.monitoring.SpLogMessage; import org.apache.streampipes.model.runtime.RuntimeOptionsRequest; import org.apache.streampipes.model.runtime.RuntimeOptionsResponse; import org.apache.streampipes.resource.management.SpResourceManager; import org.apache.streampipes.storage.management.StorageDispatcher; +import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PathVariable; @@ -41,23 +44,25 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import java.util.Collections; + @RestController @RequestMapping("/api/v2/connect/master/resolvable") public class RuntimeResolvableResource extends AbstractAdapterResource<WorkerAdministrationManagement> { private static final Logger LOG = LoggerFactory.getLogger(RuntimeResolvableResource.class); - private final WorkerUrlProvider workerUrlProvider; + private final IExtensionsServiceEndpointGenerator endpointGenerator; public RuntimeResolvableResource() { super(() -> new WorkerAdministrationManagement( StorageDispatcher.INSTANCE.getNoSqlStore() - .getAdapterInstanceStorage(), + .getAdapterInstanceStorage(), AdapterMetricsManager.INSTANCE.getAdapterMetrics(), new SpResourceManager().manageAdapters(), new SpResourceManager().manageDataStreams() )); - this.workerUrlProvider = new WorkerUrlProvider(); + this.endpointGenerator = new ExtensionsServiceEndpointGenerator(); } @PostMapping( @@ -68,7 +73,11 @@ public class RuntimeResolvableResource extends AbstractAdapterResource<WorkerAdm @RequestBody RuntimeOptionsRequest runtimeOptionsRequest) { try { - String workerEndpoint = workerUrlProvider.getWorkerBaseUrl(appId); + String workerEndpoint = endpointGenerator.getEndpointBaseUrl( + appId, + SpServiceUrlProvider.ADAPTER, + Collections.emptySet() + ); RuntimeOptionsResponse result = WorkerRestClient.getConfiguration(workerEndpoint, appId, runtimeOptionsRequest); return ok(result); diff --git a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/ISpServiceDiscovery.java b/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/ISpServiceDiscovery.java index 4f4350f7d8..1b648cef8a 100644 --- a/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/ISpServiceDiscovery.java +++ b/streampipes-service-discovery-api/src/main/java/org/apache/streampipes/svcdiscovery/api/ISpServiceDiscovery.java @@ -17,16 +17,19 @@ */ package org.apache.streampipes.svcdiscovery.api; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag; + import java.util.List; +import java.util.Set; public interface ISpServiceDiscovery { /** - * Get active pipeline element service endpoints + * Get custom service tags * - * @return list of pipeline element endpoints + * @return set of service tags */ - List<String> getActivePipelineElementEndpoints(); + Set<SpServiceTag> getCustomServiceTags(boolean restrictToHealthy); /** * Get service endpoints diff --git a/streampipes-service-discovery/src/main/java/org/apache/streampipes/svcdiscovery/SpServiceDiscoveryCore.java b/streampipes-service-discovery/src/main/java/org/apache/streampipes/svcdiscovery/SpServiceDiscoveryCore.java index b887dbd976..19f1479483 100644 --- a/streampipes-service-discovery/src/main/java/org/apache/streampipes/svcdiscovery/SpServiceDiscoveryCore.java +++ b/streampipes-service-discovery/src/main/java/org/apache/streampipes/svcdiscovery/SpServiceDiscoveryCore.java @@ -21,10 +21,10 @@ package org.apache.streampipes.svcdiscovery; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceStatus; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; import org.apache.streampipes.storage.api.CRUDStorage; import org.apache.streampipes.storage.management.StorageDispatcher; import org.apache.streampipes.svcdiscovery.api.ISpServiceDiscovery; -import org.apache.streampipes.svcdiscovery.api.model.DefaultSpServiceTypes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,18 +47,21 @@ public class SpServiceDiscoveryCore implements ISpServiceDiscovery { } @Override - public List<String> getActivePipelineElementEndpoints() { - LOG.info("Discovering active pipeline element service endpoints"); - return getServiceEndpoints(DefaultSpServiceTypes.EXT, - true, - List.of()); + public Set<SpServiceTag> getCustomServiceTags(boolean restrictToHealthy) { + var activeServices = findServices(0); + return activeServices + .stream() + .filter(service -> !restrictToHealthy || service.getStatus() != SpServiceStatus.UNHEALTHY) + .flatMap(service -> service.getTags().stream()) + .filter(serviceTag -> serviceTag.getPrefix() == SpServiceTagPrefix.CUSTOM) + .collect(Collectors.toSet()); } @Override public List<String> getServiceEndpoints(String serviceGroup, boolean restrictToHealthy, List<String> filterByTags) { - List<SpServiceRegistration> activeServices = findService(0); + List<SpServiceRegistration> activeServices = findServices(0); return activeServices .stream() @@ -73,9 +76,9 @@ public class SpServiceDiscoveryCore implements ISpServiceDiscovery { return service.getServiceUrl(); } -/** -* Checks if all the tags specified in the filter are supported by the service. -*/ + /** + * Checks if all the tags specified in the filter are supported by the service. + */ private boolean allFiltersSupported(SpServiceRegistration service, List<String> filterByTags) { if (filterByTags.isEmpty()) { @@ -88,7 +91,7 @@ public class SpServiceDiscoveryCore implements ISpServiceDiscovery { return serviceTags.containsAll(filterByTags); } - private List<SpServiceRegistration> findService(int retryCount) { + private List<SpServiceRegistration> findServices(int retryCount) { var services = serviceStorage.getAll(); if (services.isEmpty()) { if (retryCount < MAX_RETRIES) { @@ -96,7 +99,7 @@ public class SpServiceDiscoveryCore implements ISpServiceDiscovery { retryCount++; LOG.info("Could not find any extensions services, retrying ({}/{})", retryCount, MAX_RETRIES); TimeUnit.MILLISECONDS.sleep(1000); - return findService(retryCount); + return findServices(retryCount); } catch (InterruptedException e) { LOG.warn("Could not find a service currently due to exception {}", e.getMessage()); return Collections.emptyList(); diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/CustomServiceTagResolver.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/CustomServiceTagResolver.java new file mode 100644 index 0000000000..4033789c8c --- /dev/null +++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/CustomServiceTagResolver.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.service.extensions; + +import org.apache.streampipes.commons.environment.Environment; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Set; +import java.util.stream.Collectors; + +public class CustomServiceTagResolver { + + private final Environment env; + + public CustomServiceTagResolver(Environment env) { + this.env = env; + } + + public Set<SpServiceTag> getCustomServiceTags() { + if (env.getCustomServiceTags().exists()) { + var serviceTags = env.getCustomServiceTags().getValue(); + return Arrays.stream( + serviceTags.split(",") + ).map(serviceTagString -> SpServiceTag.create(SpServiceTagPrefix.CUSTOM, serviceTagString)) + .collect(Collectors.toSet()); + } else { + return Collections.emptySet(); + } + } +} diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/StreamPipesExtensionsServiceBase.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/StreamPipesExtensionsServiceBase.java index 81775f2271..0bdb9c63ed 100644 --- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/StreamPipesExtensionsServiceBase.java +++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/StreamPipesExtensionsServiceBase.java @@ -19,6 +19,7 @@ package org.apache.streampipes.service.extensions; import org.apache.streampipes.client.StreamPipesClient; +import org.apache.streampipes.commons.environment.Environments; import org.apache.streampipes.extensions.api.migration.IModelMigrator; import org.apache.streampipes.extensions.management.client.StreamPipesClientResolver; import org.apache.streampipes.extensions.management.init.DeclarersSingleton; @@ -130,6 +131,7 @@ public abstract class StreamPipesExtensionsServiceBase extends StreamPipesServic DeclarersSingleton.getInstance().getServiceDefinition().getServiceGroup())); } tags.addAll(getExtensionsServiceTags(extensions)); + tags.addAll(new CustomServiceTagResolver(Environments.getEnvironment()).getCustomServiceTags()); return tags; } diff --git a/ui/projects/streampipes/platform-services/src/lib/apis/service-tags.service.ts b/ui/projects/streampipes/platform-services/src/lib/apis/service-tags.service.ts new file mode 100644 index 0000000000..2dd4755377 --- /dev/null +++ b/ui/projects/streampipes/platform-services/src/lib/apis/service-tags.service.ts @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { Injectable } from '@angular/core'; +import { HttpClient } from '@angular/common/http'; +import { PlatformServicesCommons } from './commons.service'; +import { Observable } from 'rxjs'; +import { SpServiceTag } from '../model/gen/streampipes-model'; +import { map } from 'rxjs/operators'; + +@Injectable({ + providedIn: 'root', +}) +export class ServiceTagService { + constructor( + private http: HttpClient, + private platformServicesCommons: PlatformServicesCommons, + ) {} + + getCustomServiceTags(): Observable<SpServiceTag[]> { + return this.http + .get(`${this.platformServicesCommons.apiBasePath}/service-tags`) + .pipe( + map(response => { + return (response as any[]).map(p => + SpServiceTag.fromData(p), + ); + }), + ); + } +} diff --git a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts index b3b8efeadf..0e1b55b5c8 100644 --- a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts +++ b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts @@ -20,7 +20,7 @@ /* tslint:disable */ /* eslint-disable */ // @ts-nocheck -// Generated using typescript-generator version 3.2.1263 on 2024-03-28 15:50:50. +// Generated using typescript-generator version 3.2.1263 on 2024-04-02 22:53:18. export class NamedStreamPipesEntity { '@class': @@ -112,6 +112,7 @@ export class AdapterDescription extends VersionedNamedStreamPipesEntity { 'correspondingServiceGroup': string; 'createdAt': number; 'dataStream': SpDataStream; + 'deploymentConfiguration': DeploymentConfiguration; 'eventGrounding': EventGrounding; 'eventSchema': EventSchema; 'icon': string; @@ -142,6 +143,9 @@ export class AdapterDescription extends VersionedNamedStreamPipesEntity { instance.correspondingServiceGroup = data.correspondingServiceGroup; instance.createdAt = data.createdAt; instance.dataStream = SpDataStream.fromData(data.dataStream); + instance.deploymentConfiguration = DeploymentConfiguration.fromData( + data.deploymentConfiguration, + ); instance.eventGrounding = EventGrounding.fromData(data.eventGrounding); instance.eventSchema = EventSchema.fromData(data.eventSchema); instance.icon = data.icon; @@ -1364,6 +1368,26 @@ export class DeleteRuleDescription extends SchemaTransformationRuleDescription { } } +export class DeploymentConfiguration { + desiredServiceTags: SpServiceTag[]; + selectedEndpointUrl: string; + + static fromData( + data: DeploymentConfiguration, + target?: DeploymentConfiguration, + ): DeploymentConfiguration { + if (!data) { + return data; + } + const instance = target || new DeploymentConfiguration(); + instance.desiredServiceTags = __getCopyArrayFn(SpServiceTag.fromData)( + data.desiredServiceTags, + ); + instance.selectedEndpointUrl = data.selectedEndpointUrl; + return instance; + } +} + export class DomainStaticProperty extends StaticProperty { '@class': 'org.apache.streampipes.model.staticproperty.DomainStaticProperty'; 'requiredClass': string; @@ -2521,28 +2545,6 @@ export class PageResult extends DataSeries { } } -export class PersistedDataStream { - measureName: string; - pipelineId: string; - pipelineName: string; - schema: EventSchema; - - static fromData( - data: PersistedDataStream, - target?: PersistedDataStream, - ): PersistedDataStream { - if (!data) { - return data; - } - const instance = target || new PersistedDataStream(); - instance.measureName = data.measureName; - instance.pipelineId = data.pipelineId; - instance.pipelineName = data.pipelineName; - instance.schema = EventSchema.fromData(data.schema); - return instance; - } -} - export class Pipeline extends ElementComposition { _id: string; _rev: string; @@ -4198,6 +4200,7 @@ export type SpServiceStatus = export type SpServiceTagPrefix = | 'SP_GROUP' + | 'CUSTOM' | 'ADAPTER' | 'DATA_STREAM' | 'DATA_PROCESSOR' diff --git a/ui/projects/streampipes/platform-services/src/public-api.ts b/ui/projects/streampipes/platform-services/src/public-api.ts index 669bf4f238..dddd4ad0f8 100644 --- a/ui/projects/streampipes/platform-services/src/public-api.ts +++ b/ui/projects/streampipes/platform-services/src/public-api.ts @@ -45,6 +45,7 @@ export * from './lib/apis/pipeline-element-template.service'; export * from './lib/apis/pipeline-monitoring.service'; export * from './lib/apis/pipeline-template.service'; export * from './lib/apis/semantic-types-rest.service'; +export * from './lib/apis/service-tags.service'; export * from './lib/apis/user.service'; export * from './lib/apis/user-admin.service'; export * from './lib/apis/user-group.service'; diff --git a/ui/src/app/connect/components/adapter-configuration/adapter-settings/adapter-deployment-settings/adapter-deployment-settings.component.html b/ui/src/app/connect/components/adapter-configuration/adapter-settings/adapter-deployment-settings/adapter-deployment-settings.component.html new file mode 100644 index 0000000000..64d30f5980 --- /dev/null +++ b/ui/src/app/connect/components/adapter-configuration/adapter-settings/adapter-deployment-settings/adapter-deployment-settings.component.html @@ -0,0 +1,78 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + ~ + --> + +<div fxFlex="100" fxLayout="column"> + <mat-radio-group + [(ngModel)]="deploymentMode" + (change)="handleSelectionChange($event)" + > + <mat-radio-button value="all">Any available service</mat-radio-button> + <mat-radio-button + value="filter" + [disabled]=" + deploymentMode === 'all' && availableServiceTags.length === 0 + " + > + Restrict to service tags + </mat-radio-button> + </mat-radio-group> + + <div + *ngIf="deploymentMode === 'filter'" + fxFlex="100" + fxLayout="column" + class="mt-10" + > + <mat-form-field color="accent"> + <mat-label>Service Tags</mat-label> + <mat-chip-grid #chipGrid aria-label="Fruit selection"> + <mat-chip-row + *ngFor=" + let serviceTag of deploymentConfiguration.desiredServiceTags + " + (removed)="remove(serviceTag.value)" + > + {{ serviceTag.value }} + <button matChipRemove> + <mat-icon>cancel</mat-icon> + </button> + </mat-chip-row> + </mat-chip-grid> + <input + placeholder="Add service tag" + #serviceTagInput + [formControl]="serviceTagCtrl" + [matChipInputFor]="chipGrid" + [matAutocomplete]="auto" + [matChipInputSeparatorKeyCodes]="separatorKeysCodes" + (matChipInputTokenEnd)="add($event)" + /> + <mat-autocomplete + #auto="matAutocomplete" + (optionSelected)="selected($event)" + > + <mat-option + *ngFor="let serviceTag of filteredServiceTags | async" + [value]="serviceTag" + > + {{ serviceTag }} + </mat-option> + </mat-autocomplete> + </mat-form-field> + </div> +</div> diff --git a/ui/src/app/connect/components/adapter-configuration/adapter-settings/adapter-deployment-settings/adapter-deployment-settings.component.scss b/ui/src/app/connect/components/adapter-configuration/adapter-settings/adapter-deployment-settings/adapter-deployment-settings.component.scss new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ui/src/app/connect/components/adapter-configuration/adapter-settings/adapter-deployment-settings/adapter-deployment-settings.component.ts b/ui/src/app/connect/components/adapter-configuration/adapter-settings/adapter-deployment-settings/adapter-deployment-settings.component.ts new file mode 100644 index 0000000000..2047b223d0 --- /dev/null +++ b/ui/src/app/connect/components/adapter-configuration/adapter-settings/adapter-deployment-settings/adapter-deployment-settings.component.ts @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { Component, ElementRef, Input, OnInit, ViewChild } from '@angular/core'; +import { + DeploymentConfiguration, + ServiceTagService, + SpServiceTag, +} from '@streampipes/platform-services'; +import { Observable } from 'rxjs'; +import { FormControl } from '@angular/forms'; +import { COMMA, ENTER } from '@angular/cdk/keycodes'; +import { MatChipInputEvent } from '@angular/material/chips'; +import { map, startWith } from 'rxjs/operators'; +import { MatAutocompleteSelectedEvent } from '@angular/material/autocomplete'; +import { MatRadioChange } from '@angular/material/radio'; + +@Component({ + selector: 'sp-adapter-deployment-settings', + templateUrl: './adapter-deployment-settings.component.html', + styleUrls: ['./adapter-deployment-settings.component.scss'], +}) +export class SpAdapterDeploymentSettingsComponent implements OnInit { + @Input() + deploymentConfiguration: DeploymentConfiguration; + + availableServiceTags: SpServiceTag[] = []; + availableServiceTagValues: string[] = []; + + deploymentMode = 'all'; + + separatorKeysCodes: number[] = [ENTER, COMMA]; + serviceTagCtrl = new FormControl(''); + filteredServiceTags: Observable<string[]>; + + @ViewChild('serviceTagInput') serviceTagInput: ElementRef<HTMLInputElement>; + + constructor(private serviceTagService: ServiceTagService) { + this.filteredServiceTags = this.serviceTagCtrl.valueChanges.pipe( + startWith(null), + map((serviceTagValue: string | null) => { + return serviceTagValue + ? this._filter(serviceTagValue) + : this.availableServiceTagValues.slice(); + }), + ); + } + + ngOnInit(): void { + if (this.deploymentConfiguration.desiredServiceTags.length > 0) { + this.deploymentMode = 'filter'; + } + this.serviceTagService.getCustomServiceTags().subscribe(res => { + this.availableServiceTags = res; + this.availableServiceTagValues = res.map(st => st.value); + }); + } + + add(event: MatChipInputEvent): void { + const value = (event.value || '').trim(); + + if (value) { + this.deploymentConfiguration.desiredServiceTags.push( + this.findTag(value), + ); + } + + event.chipInput!.clear(); + + this.serviceTagCtrl.setValue(null); + } + + findTag(value: string): SpServiceTag { + return this.availableServiceTags.find(st => st.value === value); + } + + remove(serviceTag: string): void { + const index = this.deploymentConfiguration.desiredServiceTags.findIndex( + st => st.value === serviceTag, + ); + + if (index >= 0) { + this.deploymentConfiguration.desiredServiceTags.splice(index, 1); + } + } + + selected(event: MatAutocompleteSelectedEvent): void { + this.deploymentConfiguration.desiredServiceTags.push( + this.findTag(event.option.viewValue), + ); + this.serviceTagInput.nativeElement.value = ''; + this.serviceTagCtrl.setValue(null); + } + + private _filter(value: string): string[] { + const filterValue = value.toLowerCase(); + + return this.availableServiceTagValues.filter(st => + st.toLowerCase().includes(filterValue), + ); + } + + handleSelectionChange(event: MatRadioChange): void { + if (event.value === 'all') { + this.deploymentConfiguration.desiredServiceTags = []; + } + } +} diff --git a/ui/src/app/connect/components/adapter-configuration/adapter-settings/adapter-settings.component.html b/ui/src/app/connect/components/adapter-configuration/adapter-settings/adapter-settings.component.html index 4b3be6f469..82bb3806b3 100644 --- a/ui/src/app/connect/components/adapter-configuration/adapter-settings/adapter-settings.component.html +++ b/ui/src/app/connect/components/adapter-configuration/adapter-settings/adapter-settings.component.html @@ -18,6 +18,17 @@ <div fxFlex="100" fxLayout="column"> <div fxFlex="100" fxLayout="column"> + <sp-basic-inner-panel + panelTitle="Service Selection" + outerMargin="20px 0px" + > + <sp-adapter-deployment-settings + [deploymentConfiguration]=" + adapterDescription.deploymentConfiguration + " + > + </sp-adapter-deployment-settings> + </sp-basic-inner-panel> <sp-basic-inner-panel panelTitle="Basic Settings" outerMargin="20px 0px" diff --git a/ui/src/app/connect/connect.module.ts b/ui/src/app/connect/connect.module.ts index 00f7197aee..143973821d 100644 --- a/ui/src/app/connect/connect.module.ts +++ b/ui/src/app/connect/connect.module.ts @@ -106,6 +106,7 @@ import { MatSortModule } from '@angular/material/sort'; import { MatTooltipModule } from '@angular/material/tooltip'; import { MatProgressBarModule } from '@angular/material/progress-bar'; import { MatButtonToggleModule } from '@angular/material/button-toggle'; +import { SpAdapterDeploymentSettingsComponent } from './components/adapter-configuration/adapter-settings/adapter-deployment-settings/adapter-deployment-settings.component'; @NgModule({ imports: [ @@ -210,6 +211,7 @@ import { MatButtonToggleModule } from '@angular/material/button-toggle'; SchemaEditorHeaderComponent, SpEpSettingsSectionComponent, StartAdapterConfigurationComponent, + SpAdapterDeploymentSettingsComponent, SpAdapterDetailsLogsComponent, SpAdapterDetailsMetricsComponent, SpAdapterOptionsPanelComponent,
