This is an automated email from the ASF dual-hosted git repository.

wiener pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git


The following commit(s) were added to refs/heads/dev by this push:
     new cf0423b  [hotfix] consolidated consul config and service registration
cf0423b is described below

commit cf0423b5577d14639f707bafd33035867c52a783
Author: Patrick Wiener <[email protected]>
AuthorDate: Wed Aug 5 20:55:10 2020 +0200

    [hotfix] consolidated consul config and service registration
---
 .../api/resources/{processor.py => dummy.py}       | 25 +++---
 .../streampipes/api/resources/processor.py         |  1 +
 .../streampipes/api/resources/welcome.py           |  2 +
 streampipes-wrapper-python/streampipes/api/rest.py |  6 +-
 .../streampipes/api/templates/index.html           |  2 +-
 streampipes-wrapper-python/streampipes/core.py     | 15 ----
 streampipes-wrapper-python/streampipes/declarer.py | 60 ++++++++++++++
 streampipes-wrapper-python/streampipes/manager.py  | 25 ++----
 .../streampipes/model/pipeline_element_config.py   | 48 ++++++++++-
 .../streampipes/submitter.py                       | 40 ++++++++++
 .../streampipes/utils/register.py                  | 93 ++++------------------
 11 files changed, 188 insertions(+), 129 deletions(-)

diff --git a/streampipes-wrapper-python/streampipes/api/resources/processor.py 
b/streampipes-wrapper-python/streampipes/api/resources/dummy.py
similarity index 59%
copy from streampipes-wrapper-python/streampipes/api/resources/processor.py
copy to streampipes-wrapper-python/streampipes/api/resources/dummy.py
index 14602f6..598d91d 100644
--- a/streampipes-wrapper-python/streampipes/api/resources/processor.py
+++ b/streampipes-wrapper-python/streampipes/api/resources/dummy.py
@@ -14,20 +14,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from streampipes.api.resources.base import InvocableElement
+from flask import request
+from flask_classful import FlaskView, route
+from streampipes.manager import ProcessorDispatcher
 
 
-class SepaElementResource(InvocableElement):
+# TODO: Delete when finished
+# will be deleted and is only necessary for interims working example still 
relying on Java
+class DummyInterimsResource(FlaskView):
 
-    def get_instance_id(self, uri: str, element_id: str):
-        pass
-
-    def get_element_declarers(self):
-        pass
-
-    def get_extractor(self, graph):
-        pass
-
-    def create_grounding_debug_information(self, graph):
-        pass
+    @route('/invoke', methods=['POST'])
+    def start(self):
+        return ProcessorDispatcher.start(**request.get_json())
 
+    @route('/detach', methods=['POST'])
+    def stop(self):
+        return ProcessorDispatcher.stop(**request.get_json())
\ No newline at end of file
diff --git a/streampipes-wrapper-python/streampipes/api/resources/processor.py 
b/streampipes-wrapper-python/streampipes/api/resources/processor.py
index 14602f6..e3f5820 100644
--- a/streampipes-wrapper-python/streampipes/api/resources/processor.py
+++ b/streampipes-wrapper-python/streampipes/api/resources/processor.py
@@ -17,6 +17,7 @@
 from streampipes.api.resources.base import InvocableElement
 
 
+# TODO: implement
 class SepaElementResource(InvocableElement):
 
     def get_instance_id(self, uri: str, element_id: str):
diff --git a/streampipes-wrapper-python/streampipes/api/resources/welcome.py 
b/streampipes-wrapper-python/streampipes/api/resources/welcome.py
index 4343f47..381a0cd 100644
--- a/streampipes-wrapper-python/streampipes/api/resources/welcome.py
+++ b/streampipes-wrapper-python/streampipes/api/resources/welcome.py
@@ -34,6 +34,8 @@ class WelcomeResource(FlaskView):
     @route('/', methods=['GET'])
     @produces('application/json','text/html')
     def welcome(self):
+        # TODO: get description of all declared semantic event processor 
agents (sepa)
+        # TODO: DeclarerSingleton().get_declarers()
         if request.accept_mimetypes['text/html']:
             return render_template('index.html', 
processors=self.dummy_processors)
 
diff --git a/streampipes-wrapper-python/streampipes/api/rest.py 
b/streampipes-wrapper-python/streampipes/api/rest.py
index 8d15b8d..61d828d 100644
--- a/streampipes-wrapper-python/streampipes/api/rest.py
+++ b/streampipes-wrapper-python/streampipes/api/rest.py
@@ -19,6 +19,8 @@ import threading
 
 import bjoern
 from flask import Flask
+
+from streampipes.api.resources.dummy import DummyInterimsResource
 from streampipes.api.resources.processor import SepaElementResource
 from streampipes.api.resources.welcome import WelcomeResource
 
@@ -38,8 +40,10 @@ class PipelineElementApi(object):
             # register resources
             SepaElementResource.register(self.app, route_base='/', 
route_prefix='sepa')
             WelcomeResource.register(self.app, route_base='/')
+            # TODO: delete when finished
+            DummyInterimsResource.register(self.app, route_base='/')
 
     def run(self, port: int):
-        print('serving API via bjoern WSGI server ... {}:{}'.format('0.0.0.0', 
port))
+        print('serving API via bjoern WSGI server ... 
http://{}:{}'.format('0.0.0.0', port))
         threading.Thread(target=bjoern.run, args=(self.app,), kwargs={'host': 
'0.0.0.0', 'port': int(port)}).start()
 
diff --git a/streampipes-wrapper-python/streampipes/api/templates/index.html 
b/streampipes-wrapper-python/streampipes/api/templates/index.html
index dac26a7..78ae21b 100644
--- a/streampipes-wrapper-python/streampipes/api/templates/index.html
+++ b/streampipes-wrapper-python/streampipes/api/templates/index.html
@@ -24,7 +24,7 @@
     </h4>
             {% for processor in processors %}
             <h3>{{ processor.name }}</h3>
-            <h4>URI: {{ processor.uri }}</h4>
+            <h4>URI: <a href="{{ processor.uri }}">{{ processor.uri }}</a></h4>
             <h4>Description: {{ processor.description }}</h4>
             {% endfor %}
 </div>
diff --git a/streampipes-wrapper-python/streampipes/core.py 
b/streampipes-wrapper-python/streampipes/core.py
index f8140a3..b2fa5c3 100644
--- a/streampipes-wrapper-python/streampipes/core.py
+++ b/streampipes-wrapper-python/streampipes/core.py
@@ -21,21 +21,6 @@ import threading
 from abc import ABC, abstractmethod
 from confluent_kafka.admin import AdminClient
 from confluent_kafka import Producer, Consumer
-from streampipes.api.rest import PipelineElementApi
-from streampipes.base.banner import banner
-from streampipes.model.pipeline_element_config import Config
-from streampipes.utils.register import ConsulUtils
-
-
-class StandaloneModelSubmitter(ABC):
-    @classmethod
-    def init(cls, config: Config):
-        # print banner
-        print(banner)
-        # start api
-        PipelineElementApi().run(port=config.port)
-        # register pipeline element service
-        ConsulUtils().register_service(app_id=config.app_id, host=config.host, 
port=int(config.port))
 
 
 class EventProcessor(ABC):
diff --git a/streampipes-wrapper-python/streampipes/declarer.py 
b/streampipes-wrapper-python/streampipes/declarer.py
new file mode 100644
index 0000000..35c0b17
--- /dev/null
+++ b/streampipes-wrapper-python/streampipes/declarer.py
@@ -0,0 +1,60 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+class Singleton(type):
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super(Singleton, cls).__call__(*args, 
**kwargs)
+        return cls._instances[cls]
+
+
+class DeclarerSingleton(metaclass=Singleton):
+
+    """ EventProcessorManager holds running processor instances """
+    _processors = {}
+
+    def __init__(self):
+        self.sepa_declarers = {}
+        self.supported_protocols = {}
+        self.supported_formats = {}
+        self.route = '/'
+        self.host = None
+        self.port = None
+
+    @classmethod
+    def add(cls, processors=None):
+        """ holds dict with <app_id, processor class>"""
+        cls._processors = processors
+
+    @classmethod
+    def get_processor(cls, key):
+        return cls._processors[key]
+
+    @classmethod
+    def get(cls):
+        return cls._processors
+
+    # def get_declarers(self):
+    #     return self.sepa_declarers
+    #
+    # def get_base_uri(self):
+    #     return 'http://' + self.host + ':' + self.port + self.route
+
+
diff --git a/streampipes-wrapper-python/streampipes/manager.py 
b/streampipes-wrapper-python/streampipes/manager.py
index e151e8d..3892544 100644
--- a/streampipes-wrapper-python/streampipes/manager.py
+++ b/streampipes-wrapper-python/streampipes/manager.py
@@ -16,28 +16,13 @@
 #
 """Manages processor life cycle"""
 import logging
-from abc import ABC
+import abc
+from streampipes.declarer import DeclarerSingleton
 
 
-class Declarer(ABC):
-    """ EventProcessorManager holds running processor instances """
-    _processors = {}
+class ProcessorDispatcher(object):
+    __metaclass__ = abc.ABC
 
-    @classmethod
-    def add(cls, processors=None):
-        """ holds dict with <app_id, processor class>"""
-        cls._processors = processors
-
-    @classmethod
-    def get_processor(cls, key):
-        return cls._processors[key]
-
-    @classmethod
-    def get(cls):
-        return cls._processors
-
-
-class ProcessorDispatcher(ABC):
     _running_instances = {}
 
     logger = logging.getLogger(__name__)
@@ -46,7 +31,7 @@ class ProcessorDispatcher(ABC):
     def start(cls, **kwargs):
         processor_id = kwargs.get('processor_id')
         try:
-            processor = Declarer.get_processor(processor_id)(**kwargs)
+            processor = 
DeclarerSingleton().get_processor(processor_id)(**kwargs)
             processor.init()
             cls._running_instances[processor.invocation_id] = processor
 
diff --git 
a/streampipes-wrapper-python/streampipes/model/pipeline_element_config.py 
b/streampipes-wrapper-python/streampipes/model/pipeline_element_config.py
index 0976967..c0f3839 100644
--- a/streampipes-wrapper-python/streampipes/model/pipeline_element_config.py
+++ b/streampipes-wrapper-python/streampipes/model/pipeline_element_config.py
@@ -15,8 +15,7 @@
 # limitations under the License.
 #
 import os
-
-from streampipes.utils.register import ConsulUtils
+from streampipes.model.config_item import ConfigItem
 
 
 class Config(object):
@@ -28,6 +27,7 @@ class Config(object):
         self.host: str = ""
         self.port: int = 0
         self.service: str = ""
+        self.config_items = {}
 
     def register(self, type: str, env_key: str, default, description: str, 
configuration_scope=None, is_password=None):
         if not type:
@@ -44,7 +44,38 @@ class Config(object):
         elif type is 'service':
             self.service = self._env_or_default(env_key, default)
 
-        ConsulUtils().register_config(self.app_id, env_key, default, 
description, configuration_scope, is_password)
+        config_item = self._create_config_item(env_key, default, description, 
configuration_scope, is_password)
+        self.config_items[env_key] = config_item
+
+    def _create_config_item(self, env_key: str, default, description: str, 
configuration_scope=None, is_password=None):
+        config_item = ConfigItem()
+
+        if env_key is not None:
+            if os.getenv(env_key):
+                env_value = os.getenv(env_key)
+                config_item.value = env_value
+                config_item.value_type = self._check_default_type(env_value)
+            else:
+                config_item.value = default
+                config_item.value_type = self._check_default_type(default)
+        else:
+            config_item.value = default
+            config_item.value_type = self._check_default_type(default)
+
+        config_item.description = description
+
+        if configuration_scope is not None:
+            config_item.configuration_scope = configuration_scope
+        else:
+            # TODO: configuration_scope needed? Currently manually set
+            config_item.configuration_scope = 'CONTAINER_STARTUP_CONFIG'
+
+        if is_password is not None:
+            config_item.is_password = is_password
+        else:
+            config_item.is_password = False
+
+        return config_item
 
     @staticmethod
     def _env_or_default(key, default):
@@ -52,3 +83,14 @@ class Config(object):
             if os.getenv(key):
                 return os.getenv(key)
         return default
+
+    @staticmethod
+    def _check_default_type(value) -> str:
+        if isinstance(value, int):
+            return 'xs:integer'
+        elif isinstance(value, str):
+            return 'xs:string'
+        elif isinstance(value, bool):
+            return 'xs:boolean'
+        elif isinstance(value, float):
+            return 'xs:float'
diff --git a/streampipes-wrapper-python/streampipes/submitter.py 
b/streampipes-wrapper-python/streampipes/submitter.py
new file mode 100644
index 0000000..0defc9e
--- /dev/null
+++ b/streampipes-wrapper-python/streampipes/submitter.py
@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import abc
+
+from streampipes.api.rest import PipelineElementApi
+from streampipes.base.banner import banner
+from streampipes.declarer import DeclarerSingleton
+from streampipes.model.pipeline_element_config import Config
+from streampipes.utils.register import ConsulUtils
+
+
+class StandaloneModelSubmitter(object):
+    __metaclass__ = abc.ABC
+
+    @classmethod
+    def init(cls, config: Config):
+        # print banner
+        print(banner)
+        # add host to declarer singleton
+        DeclarerSingleton().host = config.host
+        DeclarerSingleton().port = config.port
+        # start api
+        PipelineElementApi().run(port=config.port)
+        # register pipeline element config and service
+        ConsulUtils().register_configs(config=config)
+        ConsulUtils().register_service(app_id=config.app_id, host=config.host, 
port=int(config.port))
\ No newline at end of file
diff --git a/streampipes-wrapper-python/streampipes/utils/register.py 
b/streampipes-wrapper-python/streampipes/utils/register.py
index a4a663c..f38e26f 100644
--- a/streampipes-wrapper-python/streampipes/utils/register.py
+++ b/streampipes-wrapper-python/streampipes/utils/register.py
@@ -18,7 +18,7 @@
 import os
 import consul
 from consul import Check
-from streampipes.model.config_item import ConfigItem
+from streampipes.model.pipeline_element_config import Config
 
 
 class ConsulUtils(object):
@@ -40,84 +40,25 @@ class ConsulUtils(object):
         if not port:
             raise ValueError
 
-        self.consul.agent.service.register(name='pe',
-                                           service_id=app_id,
-                                           address=host,
-                                           port=port,
+        self.consul.agent.service.register(name='pe', service_id=app_id, 
address=host, port=port,
                                            tags=['pe', 'python', app_id],
-                                           check=Check.http(url='http://' + 
host + ':' + str(port),
-                                                            
interval=self._DEFAULT_CONSUL_CONFIG[
-                                                                
'HEALTHCHECK_INTERVAL'],
-                                                            header={"Accept": 
["application/json"]}))
-
-    def register_config(self, app_id: str, env_key: str, default, description: 
str, configuration_scope=None,
-                        is_password=None):
-        if not app_id:
-            raise ValueError
-        if not env_key:
-            raise ValueError
-        if not default:
-            raise ValueError
-        if not description:
-            raise ValueError
-
-        config_item = ConfigItem()
-
-        if env_key is not None:
-            if os.getenv(env_key):
-                env_value = os.getenv(env_key)
-                config_item.value = env_value
-                config_item.value_type = self._check_default_type(env_value)
-            else:
-                config_item.value = default
-                config_item.value_type = self._check_default_type(default)
-        else:
-            config_item.value = default
-            config_item.value_type = self._check_default_type(default)
-
-        config_item.description = description
-
-        if configuration_scope is not None:
-            config_item.configuration_scope = configuration_scope
-        else:
-            # TODO: configuration_scope needed? Currently manually set
-            config_item.configuration_scope = 'CONTAINER_STARTUP_CONFIG'
-
-        if is_password is not None:
-            config_item.is_password = is_password
-        else:
-            config_item.is_password = False
-
-        key = self._get_consul_key(app_id, env_key)
-        index, data = self.consul.kv.get(key)
-        # TODO: update existing keys?
-        if data is None:
-            self.consul.kv.put(key, config_item.to_json())
-
-    def _get_consul_key(self, app_id, key):
+                                           check=Check.http(
+                                               url='http://' + host + ':' + 
str(port),
+                                               
interval=self._DEFAULT_CONSUL_CONFIG['HEALTHCHECK_INTERVAL'],
+                                               header={"Accept": 
["application/json"]}
+                                           ))
+
+    def register_configs(self, config: Config):
+        for key, config_item in config.config_items.items():
+            key_route = self._get_consul_key_route(config.app_id, key)
+            index, data = self.consul.kv.get(key_route)
+            # TODO: update existing keys?
+            if data is None:
+                self.consul.kv.put(key_route, config_item.to_json())
+
+    def _get_consul_key_route(self, app_id, key):
         return self._DEFAULT_CONSUL_CONFIG['CONSUL_BASIC_ROUTE'] + app_id + 
'/' + key
 
-    @staticmethod
-    def _env_or_default(key, default):
-        if key is not None:
-            if os.getenv(key):
-                return os.getenv(key)
-            else:
-                return default
-        else:
-            return default
-
-    @staticmethod
-    def _check_default_type(value) -> str:
-        if isinstance(value, int):
-            return 'xs:integer'
-        elif isinstance(value, str):
-            return 'xs:string'
-        elif isinstance(value, bool):
-            return 'xs:boolean'
-        elif isinstance(value, float):
-            return 'xs:float'
-
     def _consul(self):
         if os.getenv('CONSUL_LOCATION'):
             return consul.Consul(host=os.getenv('CONSUL_LOCATION'),

Reply via email to