This is an automated email from the ASF dual-hosted git repository.
wiener pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
The following commit(s) were added to refs/heads/dev by this push:
new 1fa62ca [STREAMPIPES-185] some minor changes to consul registration
and refactoring
1fa62ca is described below
commit 1fa62cac19d11640f6a1b470e8e8d9f034d4d233
Author: Patrick Wiener <[email protected]>
AuthorDate: Sun Aug 2 10:06:44 2020 +0200
[STREAMPIPES-185] some minor changes to consul registration and refactoring
---
streampipes-wrapper-python/streampipes/api.py | 81 ---------------
streampipes-wrapper-python/streampipes/api/rest.py | 111 +++++++++++++++++++++
.../streampipes/base/__init__.py | 0
.../streampipes/{ => base}/banner.py | 0
streampipes-wrapper-python/streampipes/core.py | 17 ++--
...cessor_config.py => pipeline_element_config.py} | 4 +-
.../streampipes/utils/register.py | 60 ++++++-----
7 files changed, 155 insertions(+), 118 deletions(-)
diff --git a/streampipes-wrapper-python/streampipes/api.py
b/streampipes-wrapper-python/streampipes/api.py
deleted file mode 100644
index 36f3735..0000000
--- a/streampipes-wrapper-python/streampipes/api.py
+++ /dev/null
@@ -1,81 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-""" API endpoints """
-import json
-
-from flask import request, jsonify, Flask, Response, make_response
-from waitress import serve
-
-from streampipes.manager import ProcessorDispatcher
-
-
-class EndpointAction(object):
- def __init__(self, action):
- self.action = action
-
- def __call__(self, *args, **kwargs):
- """ call corresponding handler method """
- if request.get_json() is not None:
- resp = self.action(**request.get_json())
- else:
- resp = self.action()
-
- return make_response(jsonify(resp), 200)
-
-
-class API(object):
- """ EventProcessorAPI contains relevant RESTful endpoints to start and
stop """
- app = None
-
- def __init__(self, port: int):
- self.app = Flask('python-processor')
- self.port = port
- self.add_endpoints()
-
- def run(self):
- serve(self.app, host='0.0.0.0', port=self.port)
-
- def add_endpoints(self):
- """ define and add event processor API endpoints """
- self.add_endpoint(endpoint='/', endpoint_name='/', methods=['GET'],
handler=self.welcome)
- self.add_endpoint(endpoint='/invoke', endpoint_name='/invoke',
methods=['POST'], handler=self.invoke)
- self.add_endpoint(endpoint='/detach', endpoint_name='/detach',
methods=['POST'], handler=self.detach)
-
- def add_endpoint(self, endpoint=None, endpoint_name=None, handler=None,
methods=None):
- self.app.add_url_rule(endpoint, endpoint_name,
EndpointAction(handler), methods=methods)
-
- @staticmethod
- def welcome():
- return {'welcome': 'hello-world!'}
-
- @staticmethod
- def invoke(**kwargs):
- """ Receives invocation graph from pipeline management in the backend
-
- :param json: contains invocation graph.
- :return: None.
- """
- return ProcessorDispatcher.start(**kwargs)
-
- @staticmethod
- def detach(**kwargs):
- """ Receives elementId to be terminated from pipeline management in
the backend
-
- :param json: contains elementId to be terminated
- :return: None.
- """
- return ProcessorDispatcher.stop(**kwargs)
\ No newline at end of file
diff --git a/streampipes-wrapper-python/streampipes/api/rest.py
b/streampipes-wrapper-python/streampipes/api/rest.py
new file mode 100644
index 0000000..6871f01
--- /dev/null
+++ b/streampipes-wrapper-python/streampipes/api/rest.py
@@ -0,0 +1,111 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+""" API endpoints """
+import json
+
+from flask import Flask
+from waitress import serve
+
+
+class FlaskProductionConfig(object):
+ DEBUG = False
+ DEVELOPMENT = False
+
+
+class API(object):
+ """ EventProcessorAPI contains relevant RESTful endpoints to start and
stop """
+ app = None
+
+ def __init__(self, port: int):
+ self.port = port
+ self.app = Flask(__name__, instance_relative_config=False)
+ self.app.config.from_object(FlaskProductionConfig)
+
+ with self.app.app_context():
+ # import endpoints
+ from streampipes.api import endpoints
+
+ # register blueprints
+ self.app.register_blueprint(endpoints.welcome_blueprint)
+ self.app.register_blueprint(endpoints.sepa_blueprint)
+
+ def run(self):
+ serve(self.app, host='0.0.0.0', port=self.port)
+
+
+# from flask import request, jsonify, Flask, Response, make_response
+# from waitress import serve
+#
+# from streampipes.manager import ProcessorDispatcher
+#
+#
+# class EndpointAction(object):
+# def __init__(self, action):
+# self.action = action
+#
+# def __call__(self, *args, **kwargs):
+# """ call corresponding handler method """
+# if request.get_json() is not None:
+# resp = self.action(**request.get_json())
+# else:
+# resp = self.action()
+#
+# return make_response(jsonify(resp), 200)
+#
+#
+# class API(object):
+# """ EventProcessorAPI contains relevant RESTful endpoints to start and
stop """
+# app = None
+#
+# def __init__(self, port: int):
+# self.app = Flask('python-processor')
+# self.port = port
+# self.add_endpoints()
+#
+# def run(self):
+# serve(self.app, host='0.0.0.0', port=self.port)
+#
+# def add_endpoints(self):
+# """ define and add event processor API endpoints """
+# self.add_endpoint(endpoint='/', endpoint_name='/', methods=['GET'],
handler=self.welcome)
+# self.add_endpoint(endpoint='/invoke', endpoint_name='/invoke',
methods=['POST'], handler=self.invoke)
+# self.add_endpoint(endpoint='/detach', endpoint_name='/detach',
methods=['POST'], handler=self.detach)
+#
+# def add_endpoint(self, endpoint=None, endpoint_name=None, handler=None,
methods=None):
+# self.app.add_url_rule(endpoint, endpoint_name,
EndpointAction(handler), methods=methods)
+#
+# @staticmethod
+# def welcome():
+# return {'welcome': 'hello-world!'}
+#
+# @staticmethod
+# def invoke(**kwargs):
+# """ Receives invocation graph from pipeline management in the backend
+#
+# :param json: contains invocation graph.
+# :return: None.
+# """
+# return ProcessorDispatcher.start(**kwargs)
+#
+# @staticmethod
+# def detach(**kwargs):
+# """ Receives elementId to be terminated from pipeline management in
the backend
+#
+# :param json: contains elementId to be terminated
+# :return: None.
+# """
+# return ProcessorDispatcher.stop(**kwargs)
\ No newline at end of file
diff --git a/streampipes-wrapper-python/streampipes/base/__init__.py
b/streampipes-wrapper-python/streampipes/base/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/streampipes-wrapper-python/streampipes/banner.py
b/streampipes-wrapper-python/streampipes/base/banner.py
similarity index 100%
rename from streampipes-wrapper-python/streampipes/banner.py
rename to streampipes-wrapper-python/streampipes/base/banner.py
diff --git a/streampipes-wrapper-python/streampipes/core.py
b/streampipes-wrapper-python/streampipes/core.py
index 08bb665..4fd2f4d 100644
--- a/streampipes-wrapper-python/streampipes/core.py
+++ b/streampipes-wrapper-python/streampipes/core.py
@@ -21,20 +21,22 @@ import threading
from abc import ABC, abstractmethod
from confluent_kafka.admin import AdminClient
from confluent_kafka import Producer, Consumer
-from streampipes.api import API
-from streampipes.banner import banner
-from streampipes.model.processor_config import Config
+
+from streampipes.api.rest import PipelineElementApi
+from streampipes.base.banner import banner
+from streampipes.model.pipeline_element_config import Config
from streampipes.utils.register import ConsulUtils
class StandaloneModelSubmitter(ABC):
@classmethod
def init(cls, config: Config):
+ # print banner
print(banner)
- api = API(port=config.port)
- threading.Thread(target=api.run()).start()
-
- ConsulUtils.register_service(app_id=config.app_id, host=config.host,
port=int(config.port))
+ # start api
+ PipelineElementApi().run(port=config.port)
+ # register pipeline element service
+ ConsulUtils().register_service(app_id=config.app_id, host=config.host,
port=int(config.port))
class EventProcessor(ABC):
@@ -115,7 +117,6 @@ class EventProcessor(ABC):
if result is not None:
self.__produce(result)
- #@threaded
def __consume(self):
""" retrieve events from kafka """
self._consumer.subscribe(topics=[self._input_topics])
diff --git a/streampipes-wrapper-python/streampipes/model/processor_config.py
b/streampipes-wrapper-python/streampipes/model/pipeline_element_config.py
similarity index 92%
rename from streampipes-wrapper-python/streampipes/model/processor_config.py
rename to
streampipes-wrapper-python/streampipes/model/pipeline_element_config.py
index 702e206..af0a40e 100644
--- a/streampipes-wrapper-python/streampipes/model/processor_config.py
+++ b/streampipes-wrapper-python/streampipes/model/pipeline_element_config.py
@@ -37,7 +37,7 @@ class Config(object):
elif type is 'service':
self.service = self.__env_or_default(env_key, default)
- ConsulUtils.register_config(self.app_id, env_key, default,
description, configuration_scope, is_password)
+ ConsulUtils().register_config(self.app_id, env_key, default,
description, configuration_scope, is_password)
@staticmethod
def __env_or_default(key, default):
@@ -47,4 +47,4 @@ class Config(object):
else:
return default
else:
- return default
\ No newline at end of file
+ return default
diff --git a/streampipes-wrapper-python/streampipes/utils/register.py
b/streampipes-wrapper-python/streampipes/utils/register.py
index 117e102..5ab3242 100644
--- a/streampipes-wrapper-python/streampipes/utils/register.py
+++ b/streampipes-wrapper-python/streampipes/utils/register.py
@@ -30,39 +30,35 @@ class ConsulUtils(object):
'CONSUL_BASIC_ROUTE': 'sp/v1/'
}
- if os.getenv('CONSUL_LOCATION'):
- consul = consul.Consul(host=os.getenv('CONSUL_LOCATION'),
port=_DEFAULT_CONSUL_CONFIG['CONSUL_PORT'])
- elif os.getenv('SP_DEBUG'):
- consul = consul.Consul(host='localhost',
port=_DEFAULT_CONSUL_CONFIG['CONSUL_PORT'])
- else:
- consul = consul.Consul(host=_DEFAULT_CONSUL_CONFIG['CONSUL_HOST'],
port=_DEFAULT_CONSUL_CONFIG['CONSUL_PORT'])
+ def __init__(self):
+ self.consul = self.__consul()
- @classmethod
- def register_service(cls, app_id: str, host: str, port: int):
- # TODO: add service tags
- cls.consul.agent.service.register(name='pe',
- service_id=app_id,
- address=host,
- port=port,
- tags=['pe', 'python', app_id],
- check=Check.http(url='http://' +
host + ':' + str(port),
-
interval=cls._DEFAULT_CONSUL_CONFIG['HEALTHCHECK_INTERVAL']))
+ def register_service(self, app_id: str, host: str, port: int):
+ self.consul.agent.service.register(name='pe',
+ service_id=app_id,
+ address=host,
+ port=port,
+ tags=['pe', 'python', app_id],
+ check=Check.http(url='http://' +
host + ':' + str(port),
+
interval=self._DEFAULT_CONSUL_CONFIG[
+
'HEALTHCHECK_INTERVAL'],
+
header={"Content-Type": ["application/json"]}))
- @classmethod
- def register_config(cls, app_id: str, env_key: str, default, description:
str, configuration_scope=None, is_password=None):
+ def register_config(self, app_id: str, env_key: str, default, description:
str, configuration_scope=None,
+ is_password=None):
config_item = ConfigItem()
if env_key is not None:
if os.getenv(env_key):
env_value = os.getenv(env_key)
config_item.value = env_value
- config_item.value_type = cls.__check_default_type(env_value)
+ config_item.value_type = self.__check_default_type(env_value)
else:
config_item.value = default
- config_item.value_type = cls.__check_default_type(default)
+ config_item.value_type = self.__check_default_type(default)
else:
config_item.value = default
- config_item.value_type = cls.__check_default_type(default)
+ config_item.value_type = self.__check_default_type(default)
config_item.description = description
@@ -77,15 +73,14 @@ class ConsulUtils(object):
else:
config_item.is_password = False
- key = cls.__get_consul_key(app_id, env_key)
- data = cls.consul.kv.get(key)
+ key = self.__get_consul_key(app_id, env_key)
+ index, data = self.consul.kv.get(key)
# TODO: update existing keys?
if data is None:
- cls.consul.kv.put(key, config_item.to_json())
+ self.consul.kv.put(key, config_item.to_json())
- @classmethod
- def __get_consul_key(cls, app_id, key):
- return cls._DEFAULT_CONSUL_CONFIG['CONSUL_BASIC_ROUTE'] + app_id + '/'
+ key
+ def __get_consul_key(self, app_id, key):
+ return self._DEFAULT_CONSUL_CONFIG['CONSUL_BASIC_ROUTE'] + app_id +
'/' + key
@staticmethod
def __env_or_default(key, default):
@@ -107,3 +102,14 @@ class ConsulUtils(object):
return 'xs:boolean'
elif isinstance(value, float):
return 'xs:float'
+
+ def __consul(self):
+ if os.getenv('CONSUL_LOCATION'):
+ return consul.Consul(host=os.getenv('CONSUL_LOCATION'),
+
port=self._DEFAULT_CONSUL_CONFIG['CONSUL_PORT'])
+ elif os.getenv('SP_DEBUG'):
+ return consul.Consul(host='localhost',
+
port=self._DEFAULT_CONSUL_CONFIG['CONSUL_PORT'])
+ else:
+ return
consul.Consul(host=self._DEFAULT_CONSUL_CONFIG['CONSUL_HOST'],
+
port=self._DEFAULT_CONSUL_CONFIG['CONSUL_PORT'])