This is an automated email from the ASF dual-hosted git repository.
wiener pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
The following commit(s) were added to refs/heads/dev by this push:
new cf0423b [hotfix] consolidated consul config and service registration
cf0423b is described below
commit cf0423b5577d14639f707bafd33035867c52a783
Author: Patrick Wiener <[email protected]>
AuthorDate: Wed Aug 5 20:55:10 2020 +0200
[hotfix] consolidated consul config and service registration
---
.../api/resources/{processor.py => dummy.py} | 25 +++---
.../streampipes/api/resources/processor.py | 1 +
.../streampipes/api/resources/welcome.py | 2 +
streampipes-wrapper-python/streampipes/api/rest.py | 6 +-
.../streampipes/api/templates/index.html | 2 +-
streampipes-wrapper-python/streampipes/core.py | 15 ----
streampipes-wrapper-python/streampipes/declarer.py | 60 ++++++++++++++
streampipes-wrapper-python/streampipes/manager.py | 25 ++----
.../streampipes/model/pipeline_element_config.py | 48 ++++++++++-
.../streampipes/submitter.py | 40 ++++++++++
.../streampipes/utils/register.py | 93 ++++------------------
11 files changed, 188 insertions(+), 129 deletions(-)
diff --git a/streampipes-wrapper-python/streampipes/api/resources/processor.py
b/streampipes-wrapper-python/streampipes/api/resources/dummy.py
similarity index 59%
copy from streampipes-wrapper-python/streampipes/api/resources/processor.py
copy to streampipes-wrapper-python/streampipes/api/resources/dummy.py
index 14602f6..598d91d 100644
--- a/streampipes-wrapper-python/streampipes/api/resources/processor.py
+++ b/streampipes-wrapper-python/streampipes/api/resources/dummy.py
@@ -14,20 +14,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-from streampipes.api.resources.base import InvocableElement
+from flask import request
+from flask_classful import FlaskView, route
+from streampipes.manager import ProcessorDispatcher
-class SepaElementResource(InvocableElement):
+# TODO: Delete when finished
+# will be deleted and is only necessary for interims working example still
relying on Java
+class DummyInterimsResource(FlaskView):
- def get_instance_id(self, uri: str, element_id: str):
- pass
-
- def get_element_declarers(self):
- pass
-
- def get_extractor(self, graph):
- pass
-
- def create_grounding_debug_information(self, graph):
- pass
+ @route('/invoke', methods=['POST'])
+ def start(self):
+ return ProcessorDispatcher.start(**request.get_json())
+ @route('/detach', methods=['POST'])
+ def stop(self):
+ return ProcessorDispatcher.stop(**request.get_json())
\ No newline at end of file
diff --git a/streampipes-wrapper-python/streampipes/api/resources/processor.py
b/streampipes-wrapper-python/streampipes/api/resources/processor.py
index 14602f6..e3f5820 100644
--- a/streampipes-wrapper-python/streampipes/api/resources/processor.py
+++ b/streampipes-wrapper-python/streampipes/api/resources/processor.py
@@ -17,6 +17,7 @@
from streampipes.api.resources.base import InvocableElement
+# TODO: implement
class SepaElementResource(InvocableElement):
def get_instance_id(self, uri: str, element_id: str):
diff --git a/streampipes-wrapper-python/streampipes/api/resources/welcome.py
b/streampipes-wrapper-python/streampipes/api/resources/welcome.py
index 4343f47..381a0cd 100644
--- a/streampipes-wrapper-python/streampipes/api/resources/welcome.py
+++ b/streampipes-wrapper-python/streampipes/api/resources/welcome.py
@@ -34,6 +34,8 @@ class WelcomeResource(FlaskView):
@route('/', methods=['GET'])
@produces('application/json','text/html')
def welcome(self):
+ # TODO: get description of all declared semantic event processor
agents (sepa)
+ # TODO: DeclarerSingleton().get_declarers()
if request.accept_mimetypes['text/html']:
return render_template('index.html',
processors=self.dummy_processors)
diff --git a/streampipes-wrapper-python/streampipes/api/rest.py
b/streampipes-wrapper-python/streampipes/api/rest.py
index 8d15b8d..61d828d 100644
--- a/streampipes-wrapper-python/streampipes/api/rest.py
+++ b/streampipes-wrapper-python/streampipes/api/rest.py
@@ -19,6 +19,8 @@ import threading
import bjoern
from flask import Flask
+
+from streampipes.api.resources.dummy import DummyInterimsResource
from streampipes.api.resources.processor import SepaElementResource
from streampipes.api.resources.welcome import WelcomeResource
@@ -38,8 +40,10 @@ class PipelineElementApi(object):
# register resources
SepaElementResource.register(self.app, route_base='/',
route_prefix='sepa')
WelcomeResource.register(self.app, route_base='/')
+ # TODO: delete when finished
+ DummyInterimsResource.register(self.app, route_base='/')
def run(self, port: int):
- print('serving API via bjoern WSGI server ... {}:{}'.format('0.0.0.0',
port))
+ print('serving API via bjoern WSGI server ...
http://{}:{}'.format('0.0.0.0', port))
threading.Thread(target=bjoern.run, args=(self.app,), kwargs={'host':
'0.0.0.0', 'port': int(port)}).start()
diff --git a/streampipes-wrapper-python/streampipes/api/templates/index.html
b/streampipes-wrapper-python/streampipes/api/templates/index.html
index dac26a7..78ae21b 100644
--- a/streampipes-wrapper-python/streampipes/api/templates/index.html
+++ b/streampipes-wrapper-python/streampipes/api/templates/index.html
@@ -24,7 +24,7 @@
</h4>
{% for processor in processors %}
<h3>{{ processor.name }}</h3>
- <h4>URI: {{ processor.uri }}</h4>
+ <h4>URI: <a href="{{ processor.uri }}">{{ processor.uri }}</a></h4>
<h4>Description: {{ processor.description }}</h4>
{% endfor %}
</div>
diff --git a/streampipes-wrapper-python/streampipes/core.py
b/streampipes-wrapper-python/streampipes/core.py
index f8140a3..b2fa5c3 100644
--- a/streampipes-wrapper-python/streampipes/core.py
+++ b/streampipes-wrapper-python/streampipes/core.py
@@ -21,21 +21,6 @@ import threading
from abc import ABC, abstractmethod
from confluent_kafka.admin import AdminClient
from confluent_kafka import Producer, Consumer
-from streampipes.api.rest import PipelineElementApi
-from streampipes.base.banner import banner
-from streampipes.model.pipeline_element_config import Config
-from streampipes.utils.register import ConsulUtils
-
-
-class StandaloneModelSubmitter(ABC):
- @classmethod
- def init(cls, config: Config):
- # print banner
- print(banner)
- # start api
- PipelineElementApi().run(port=config.port)
- # register pipeline element service
- ConsulUtils().register_service(app_id=config.app_id, host=config.host,
port=int(config.port))
class EventProcessor(ABC):
diff --git a/streampipes-wrapper-python/streampipes/declarer.py
b/streampipes-wrapper-python/streampipes/declarer.py
new file mode 100644
index 0000000..35c0b17
--- /dev/null
+++ b/streampipes-wrapper-python/streampipes/declarer.py
@@ -0,0 +1,60 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+class Singleton(type):
+ _instances = {}
+
+ def __call__(cls, *args, **kwargs):
+ if cls not in cls._instances:
+ cls._instances[cls] = super(Singleton, cls).__call__(*args,
**kwargs)
+ return cls._instances[cls]
+
+
+class DeclarerSingleton(metaclass=Singleton):
+
+ """ EventProcessorManager holds running processor instances """
+ _processors = {}
+
+ def __init__(self):
+ self.sepa_declarers = {}
+ self.supported_protocols = {}
+ self.supported_formats = {}
+ self.route = '/'
+ self.host = None
+ self.port = None
+
+ @classmethod
+ def add(cls, processors=None):
+ """ holds dict with <app_id, processor class>"""
+ cls._processors = processors
+
+ @classmethod
+ def get_processor(cls, key):
+ return cls._processors[key]
+
+ @classmethod
+ def get(cls):
+ return cls._processors
+
+ # def get_declarers(self):
+ # return self.sepa_declarers
+ #
+ # def get_base_uri(self):
+ # return 'http://' + self.host + ':' + self.port + self.route
+
+
diff --git a/streampipes-wrapper-python/streampipes/manager.py
b/streampipes-wrapper-python/streampipes/manager.py
index e151e8d..3892544 100644
--- a/streampipes-wrapper-python/streampipes/manager.py
+++ b/streampipes-wrapper-python/streampipes/manager.py
@@ -16,28 +16,13 @@
#
"""Manages processor life cycle"""
import logging
-from abc import ABC
+import abc
+from streampipes.declarer import DeclarerSingleton
-class Declarer(ABC):
- """ EventProcessorManager holds running processor instances """
- _processors = {}
+class ProcessorDispatcher(object):
+ __metaclass__ = abc.ABC
- @classmethod
- def add(cls, processors=None):
- """ holds dict with <app_id, processor class>"""
- cls._processors = processors
-
- @classmethod
- def get_processor(cls, key):
- return cls._processors[key]
-
- @classmethod
- def get(cls):
- return cls._processors
-
-
-class ProcessorDispatcher(ABC):
_running_instances = {}
logger = logging.getLogger(__name__)
@@ -46,7 +31,7 @@ class ProcessorDispatcher(ABC):
def start(cls, **kwargs):
processor_id = kwargs.get('processor_id')
try:
- processor = Declarer.get_processor(processor_id)(**kwargs)
+ processor =
DeclarerSingleton().get_processor(processor_id)(**kwargs)
processor.init()
cls._running_instances[processor.invocation_id] = processor
diff --git
a/streampipes-wrapper-python/streampipes/model/pipeline_element_config.py
b/streampipes-wrapper-python/streampipes/model/pipeline_element_config.py
index 0976967..c0f3839 100644
--- a/streampipes-wrapper-python/streampipes/model/pipeline_element_config.py
+++ b/streampipes-wrapper-python/streampipes/model/pipeline_element_config.py
@@ -15,8 +15,7 @@
# limitations under the License.
#
import os
-
-from streampipes.utils.register import ConsulUtils
+from streampipes.model.config_item import ConfigItem
class Config(object):
@@ -28,6 +27,7 @@ class Config(object):
self.host: str = ""
self.port: int = 0
self.service: str = ""
+ self.config_items = {}
def register(self, type: str, env_key: str, default, description: str,
configuration_scope=None, is_password=None):
if not type:
@@ -44,7 +44,38 @@ class Config(object):
elif type is 'service':
self.service = self._env_or_default(env_key, default)
- ConsulUtils().register_config(self.app_id, env_key, default,
description, configuration_scope, is_password)
+ config_item = self._create_config_item(env_key, default, description,
configuration_scope, is_password)
+ self.config_items[env_key] = config_item
+
+ def _create_config_item(self, env_key: str, default, description: str,
configuration_scope=None, is_password=None):
+ config_item = ConfigItem()
+
+ if env_key is not None:
+ if os.getenv(env_key):
+ env_value = os.getenv(env_key)
+ config_item.value = env_value
+ config_item.value_type = self._check_default_type(env_value)
+ else:
+ config_item.value = default
+ config_item.value_type = self._check_default_type(default)
+ else:
+ config_item.value = default
+ config_item.value_type = self._check_default_type(default)
+
+ config_item.description = description
+
+ if configuration_scope is not None:
+ config_item.configuration_scope = configuration_scope
+ else:
+ # TODO: configuration_scope needed? Currently manually set
+ config_item.configuration_scope = 'CONTAINER_STARTUP_CONFIG'
+
+ if is_password is not None:
+ config_item.is_password = is_password
+ else:
+ config_item.is_password = False
+
+ return config_item
@staticmethod
def _env_or_default(key, default):
@@ -52,3 +83,14 @@ class Config(object):
if os.getenv(key):
return os.getenv(key)
return default
+
+ @staticmethod
+ def _check_default_type(value) -> str:
+ if isinstance(value, int):
+ return 'xs:integer'
+ elif isinstance(value, str):
+ return 'xs:string'
+ elif isinstance(value, bool):
+ return 'xs:boolean'
+ elif isinstance(value, float):
+ return 'xs:float'
diff --git a/streampipes-wrapper-python/streampipes/submitter.py
b/streampipes-wrapper-python/streampipes/submitter.py
new file mode 100644
index 0000000..0defc9e
--- /dev/null
+++ b/streampipes-wrapper-python/streampipes/submitter.py
@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import abc
+
+from streampipes.api.rest import PipelineElementApi
+from streampipes.base.banner import banner
+from streampipes.declarer import DeclarerSingleton
+from streampipes.model.pipeline_element_config import Config
+from streampipes.utils.register import ConsulUtils
+
+
+class StandaloneModelSubmitter(object):
+ __metaclass__ = abc.ABC
+
+ @classmethod
+ def init(cls, config: Config):
+ # print banner
+ print(banner)
+ # add host to declarer singleton
+ DeclarerSingleton().host = config.host
+ DeclarerSingleton().port = config.port
+ # start api
+ PipelineElementApi().run(port=config.port)
+ # register pipeline element config and service
+ ConsulUtils().register_configs(config=config)
+ ConsulUtils().register_service(app_id=config.app_id, host=config.host,
port=int(config.port))
\ No newline at end of file
diff --git a/streampipes-wrapper-python/streampipes/utils/register.py
b/streampipes-wrapper-python/streampipes/utils/register.py
index a4a663c..f38e26f 100644
--- a/streampipes-wrapper-python/streampipes/utils/register.py
+++ b/streampipes-wrapper-python/streampipes/utils/register.py
@@ -18,7 +18,7 @@
import os
import consul
from consul import Check
-from streampipes.model.config_item import ConfigItem
+from streampipes.model.pipeline_element_config import Config
class ConsulUtils(object):
@@ -40,84 +40,25 @@ class ConsulUtils(object):
if not port:
raise ValueError
- self.consul.agent.service.register(name='pe',
- service_id=app_id,
- address=host,
- port=port,
+ self.consul.agent.service.register(name='pe', service_id=app_id,
address=host, port=port,
tags=['pe', 'python', app_id],
- check=Check.http(url='http://' +
host + ':' + str(port),
-
interval=self._DEFAULT_CONSUL_CONFIG[
-
'HEALTHCHECK_INTERVAL'],
- header={"Accept":
["application/json"]}))
-
- def register_config(self, app_id: str, env_key: str, default, description:
str, configuration_scope=None,
- is_password=None):
- if not app_id:
- raise ValueError
- if not env_key:
- raise ValueError
- if not default:
- raise ValueError
- if not description:
- raise ValueError
-
- config_item = ConfigItem()
-
- if env_key is not None:
- if os.getenv(env_key):
- env_value = os.getenv(env_key)
- config_item.value = env_value
- config_item.value_type = self._check_default_type(env_value)
- else:
- config_item.value = default
- config_item.value_type = self._check_default_type(default)
- else:
- config_item.value = default
- config_item.value_type = self._check_default_type(default)
-
- config_item.description = description
-
- if configuration_scope is not None:
- config_item.configuration_scope = configuration_scope
- else:
- # TODO: configuration_scope needed? Currently manually set
- config_item.configuration_scope = 'CONTAINER_STARTUP_CONFIG'
-
- if is_password is not None:
- config_item.is_password = is_password
- else:
- config_item.is_password = False
-
- key = self._get_consul_key(app_id, env_key)
- index, data = self.consul.kv.get(key)
- # TODO: update existing keys?
- if data is None:
- self.consul.kv.put(key, config_item.to_json())
-
- def _get_consul_key(self, app_id, key):
+ check=Check.http(
+ url='http://' + host + ':' +
str(port),
+
interval=self._DEFAULT_CONSUL_CONFIG['HEALTHCHECK_INTERVAL'],
+ header={"Accept":
["application/json"]}
+ ))
+
+ def register_configs(self, config: Config):
+ for key, config_item in config.config_items.items():
+ key_route = self._get_consul_key_route(config.app_id, key)
+ index, data = self.consul.kv.get(key_route)
+ # TODO: update existing keys?
+ if data is None:
+ self.consul.kv.put(key_route, config_item.to_json())
+
+ def _get_consul_key_route(self, app_id, key):
return self._DEFAULT_CONSUL_CONFIG['CONSUL_BASIC_ROUTE'] + app_id +
'/' + key
- @staticmethod
- def _env_or_default(key, default):
- if key is not None:
- if os.getenv(key):
- return os.getenv(key)
- else:
- return default
- else:
- return default
-
- @staticmethod
- def _check_default_type(value) -> str:
- if isinstance(value, int):
- return 'xs:integer'
- elif isinstance(value, str):
- return 'xs:string'
- elif isinstance(value, bool):
- return 'xs:boolean'
- elif isinstance(value, float):
- return 'xs:float'
-
def _consul(self):
if os.getenv('CONSUL_LOCATION'):
return consul.Consul(host=os.getenv('CONSUL_LOCATION'),