This is an automated email from the ASF dual-hosted git repository.

wiener pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git


The following commit(s) were added to refs/heads/dev by this push:
     new 1fa62ca  [STREAMPIPES-185] some minor changes to consul registration 
and refactoring
1fa62ca is described below

commit 1fa62cac19d11640f6a1b470e8e8d9f034d4d233
Author: Patrick Wiener <[email protected]>
AuthorDate: Sun Aug 2 10:06:44 2020 +0200

    [STREAMPIPES-185] some minor changes to consul registration and refactoring
---
 streampipes-wrapper-python/streampipes/api.py      |  81 ---------------
 streampipes-wrapper-python/streampipes/api/rest.py | 111 +++++++++++++++++++++
 .../streampipes/base/__init__.py                   |   0
 .../streampipes/{ => base}/banner.py               |   0
 streampipes-wrapper-python/streampipes/core.py     |  17 ++--
 ...cessor_config.py => pipeline_element_config.py} |   4 +-
 .../streampipes/utils/register.py                  |  60 ++++++-----
 7 files changed, 155 insertions(+), 118 deletions(-)

diff --git a/streampipes-wrapper-python/streampipes/api.py 
b/streampipes-wrapper-python/streampipes/api.py
deleted file mode 100644
index 36f3735..0000000
--- a/streampipes-wrapper-python/streampipes/api.py
+++ /dev/null
@@ -1,81 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-""" API endpoints """
-import json
-
-from flask import request, jsonify, Flask, Response, make_response
-from waitress import serve
-
-from streampipes.manager import ProcessorDispatcher
-
-
-class EndpointAction(object):
-    def __init__(self, action):
-        self.action = action
-
-    def __call__(self, *args, **kwargs):
-        """ call corresponding handler method """
-        if request.get_json() is not None:
-            resp = self.action(**request.get_json())
-        else:
-            resp = self.action()
-
-        return make_response(jsonify(resp), 200)
-
-
-class API(object):
-    """ EventProcessorAPI contains relevant RESTful endpoints to start and 
stop """
-    app = None
-
-    def __init__(self, port: int):
-        self.app = Flask('python-processor')
-        self.port = port
-        self.add_endpoints()
-
-    def run(self):
-        serve(self.app, host='0.0.0.0', port=self.port)
-
-    def add_endpoints(self):
-        """ define and add event processor API endpoints """
-        self.add_endpoint(endpoint='/', endpoint_name='/', methods=['GET'], 
handler=self.welcome)
-        self.add_endpoint(endpoint='/invoke', endpoint_name='/invoke', 
methods=['POST'], handler=self.invoke)
-        self.add_endpoint(endpoint='/detach', endpoint_name='/detach', 
methods=['POST'], handler=self.detach)
-
-    def add_endpoint(self, endpoint=None, endpoint_name=None, handler=None, 
methods=None):
-        self.app.add_url_rule(endpoint, endpoint_name, 
EndpointAction(handler), methods=methods)
-
-    @staticmethod
-    def welcome():
-        return {'welcome': 'hello-world!'}
-
-    @staticmethod
-    def invoke(**kwargs):
-        """ Receives invocation graph from pipeline management in the backend
-
-        :param json: contains invocation graph.
-        :return: None.
-        """
-        return ProcessorDispatcher.start(**kwargs)
-
-    @staticmethod
-    def detach(**kwargs):
-        """ Receives elementId to be terminated from pipeline management in 
the backend
-
-        :param json: contains elementId to be terminated
-        :return: None.
-        """
-        return ProcessorDispatcher.stop(**kwargs)
\ No newline at end of file
diff --git a/streampipes-wrapper-python/streampipes/api/rest.py 
b/streampipes-wrapper-python/streampipes/api/rest.py
new file mode 100644
index 0000000..6871f01
--- /dev/null
+++ b/streampipes-wrapper-python/streampipes/api/rest.py
@@ -0,0 +1,111 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+""" API endpoints """
+import json
+
+from flask import Flask
+from waitress import serve
+
+
+class FlaskProductionConfig(object):
+    DEBUG = False
+    DEVELOPMENT = False
+
+
+class API(object):
+    """ EventProcessorAPI contains relevant RESTful endpoints to start and 
stop """
+    app = None
+
+    def __init__(self, port: int):
+        self.port = port
+        self.app = Flask(__name__, instance_relative_config=False)
+        self.app.config.from_object(FlaskProductionConfig)
+
+        with self.app.app_context():
+            # import endpoints
+            from streampipes.api import endpoints
+
+            # register blueprints
+            self.app.register_blueprint(endpoints.welcome_blueprint)
+            self.app.register_blueprint(endpoints.sepa_blueprint)
+
+    def run(self):
+        serve(self.app, host='0.0.0.0', port=self.port)
+
+
+# from flask import request, jsonify, Flask, Response, make_response
+# from waitress import serve
+#
+# from streampipes.manager import ProcessorDispatcher
+#
+#
+# class EndpointAction(object):
+#     def __init__(self, action):
+#         self.action = action
+#
+#     def __call__(self, *args, **kwargs):
+#         """ call corresponding handler method """
+#         if request.get_json() is not None:
+#             resp = self.action(**request.get_json())
+#         else:
+#             resp = self.action()
+#
+#         return make_response(jsonify(resp), 200)
+#
+#
+# class API(object):
+#     """ EventProcessorAPI contains relevant RESTful endpoints to start and 
stop """
+#     app = None
+#
+#     def __init__(self, port: int):
+#         self.app = Flask('python-processor')
+#         self.port = port
+#         self.add_endpoints()
+#
+#     def run(self):
+#         serve(self.app, host='0.0.0.0', port=self.port)
+#
+#     def add_endpoints(self):
+#         """ define and add event processor API endpoints """
+#         self.add_endpoint(endpoint='/', endpoint_name='/', methods=['GET'], 
handler=self.welcome)
+#         self.add_endpoint(endpoint='/invoke', endpoint_name='/invoke', 
methods=['POST'], handler=self.invoke)
+#         self.add_endpoint(endpoint='/detach', endpoint_name='/detach', 
methods=['POST'], handler=self.detach)
+#
+#     def add_endpoint(self, endpoint=None, endpoint_name=None, handler=None, 
methods=None):
+#         self.app.add_url_rule(endpoint, endpoint_name, 
EndpointAction(handler), methods=methods)
+#
+#     @staticmethod
+#     def welcome():
+#         return {'welcome': 'hello-world!'}
+#
+#     @staticmethod
+#     def invoke(**kwargs):
+#         """ Receives invocation graph from pipeline management in the backend
+#
+#         :param json: contains invocation graph.
+#         :return: None.
+#         """
+#         return ProcessorDispatcher.start(**kwargs)
+#
+#     @staticmethod
+#     def detach(**kwargs):
+#         """ Receives elementId to be terminated from pipeline management in 
the backend
+#
+#         :param json: contains elementId to be terminated
+#         :return: None.
+#         """
+#         return ProcessorDispatcher.stop(**kwargs)
\ No newline at end of file
diff --git a/streampipes-wrapper-python/streampipes/base/__init__.py 
b/streampipes-wrapper-python/streampipes/base/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/streampipes-wrapper-python/streampipes/banner.py 
b/streampipes-wrapper-python/streampipes/base/banner.py
similarity index 100%
rename from streampipes-wrapper-python/streampipes/banner.py
rename to streampipes-wrapper-python/streampipes/base/banner.py
diff --git a/streampipes-wrapper-python/streampipes/core.py 
b/streampipes-wrapper-python/streampipes/core.py
index 08bb665..4fd2f4d 100644
--- a/streampipes-wrapper-python/streampipes/core.py
+++ b/streampipes-wrapper-python/streampipes/core.py
@@ -21,20 +21,22 @@ import threading
 from abc import ABC, abstractmethod
 from confluent_kafka.admin import AdminClient
 from confluent_kafka import Producer, Consumer
-from streampipes.api import API
-from streampipes.banner import banner
-from streampipes.model.processor_config import Config
+
+from streampipes.api.rest import PipelineElementApi
+from streampipes.base.banner import banner
+from streampipes.model.pipeline_element_config import Config
 from streampipes.utils.register import ConsulUtils
 
 
 class StandaloneModelSubmitter(ABC):
     @classmethod
     def init(cls, config: Config):
+        # print banner
         print(banner)
-        api = API(port=config.port)
-        threading.Thread(target=api.run()).start()
-
-        ConsulUtils.register_service(app_id=config.app_id, host=config.host, 
port=int(config.port))
+        # start api
+        PipelineElementApi().run(port=config.port)
+        # register pipeline element service
+        ConsulUtils().register_service(app_id=config.app_id, host=config.host, 
port=int(config.port))
 
 
 class EventProcessor(ABC):
@@ -115,7 +117,6 @@ class EventProcessor(ABC):
         if result is not None:
             self.__produce(result)
 
-    #@threaded
     def __consume(self):
         """ retrieve events from kafka """
         self._consumer.subscribe(topics=[self._input_topics])
diff --git a/streampipes-wrapper-python/streampipes/model/processor_config.py 
b/streampipes-wrapper-python/streampipes/model/pipeline_element_config.py
similarity index 92%
rename from streampipes-wrapper-python/streampipes/model/processor_config.py
rename to 
streampipes-wrapper-python/streampipes/model/pipeline_element_config.py
index 702e206..af0a40e 100644
--- a/streampipes-wrapper-python/streampipes/model/processor_config.py
+++ b/streampipes-wrapper-python/streampipes/model/pipeline_element_config.py
@@ -37,7 +37,7 @@ class Config(object):
         elif type is 'service':
             self.service = self.__env_or_default(env_key, default)
 
-        ConsulUtils.register_config(self.app_id, env_key, default, 
description, configuration_scope, is_password)
+        ConsulUtils().register_config(self.app_id, env_key, default, 
description, configuration_scope, is_password)
 
     @staticmethod
     def __env_or_default(key, default):
@@ -47,4 +47,4 @@ class Config(object):
             else:
                 return default
         else:
-            return default
\ No newline at end of file
+            return default
diff --git a/streampipes-wrapper-python/streampipes/utils/register.py 
b/streampipes-wrapper-python/streampipes/utils/register.py
index 117e102..5ab3242 100644
--- a/streampipes-wrapper-python/streampipes/utils/register.py
+++ b/streampipes-wrapper-python/streampipes/utils/register.py
@@ -30,39 +30,35 @@ class ConsulUtils(object):
         'CONSUL_BASIC_ROUTE': 'sp/v1/'
     }
 
-    if os.getenv('CONSUL_LOCATION'):
-        consul = consul.Consul(host=os.getenv('CONSUL_LOCATION'), 
port=_DEFAULT_CONSUL_CONFIG['CONSUL_PORT'])
-    elif os.getenv('SP_DEBUG'):
-        consul = consul.Consul(host='localhost', 
port=_DEFAULT_CONSUL_CONFIG['CONSUL_PORT'])
-    else:
-        consul = consul.Consul(host=_DEFAULT_CONSUL_CONFIG['CONSUL_HOST'], 
port=_DEFAULT_CONSUL_CONFIG['CONSUL_PORT'])
+    def __init__(self):
+        self.consul = self.__consul()
 
-    @classmethod
-    def register_service(cls, app_id: str, host: str, port: int):
-        # TODO: add service tags
-        cls.consul.agent.service.register(name='pe',
-                                          service_id=app_id,
-                                          address=host,
-                                          port=port,
-                                          tags=['pe', 'python', app_id],
-                                          check=Check.http(url='http://' + 
host + ':' + str(port),
-                                                           
interval=cls._DEFAULT_CONSUL_CONFIG['HEALTHCHECK_INTERVAL']))
+    def register_service(self, app_id: str, host: str, port: int):
+        self.consul.agent.service.register(name='pe',
+                                           service_id=app_id,
+                                           address=host,
+                                           port=port,
+                                           tags=['pe', 'python', app_id],
+                                           check=Check.http(url='http://' + 
host + ':' + str(port),
+                                                            
interval=self._DEFAULT_CONSUL_CONFIG[
+                                                                
'HEALTHCHECK_INTERVAL'],
+                                                            
header={"Content-Type": ["application/json"]}))
 
-    @classmethod
-    def register_config(cls, app_id: str, env_key: str, default, description: 
str, configuration_scope=None, is_password=None):
+    def register_config(self, app_id: str, env_key: str, default, description: 
str, configuration_scope=None,
+                        is_password=None):
         config_item = ConfigItem()
 
         if env_key is not None:
             if os.getenv(env_key):
                 env_value = os.getenv(env_key)
                 config_item.value = env_value
-                config_item.value_type = cls.__check_default_type(env_value)
+                config_item.value_type = self.__check_default_type(env_value)
             else:
                 config_item.value = default
-                config_item.value_type = cls.__check_default_type(default)
+                config_item.value_type = self.__check_default_type(default)
         else:
             config_item.value = default
-            config_item.value_type = cls.__check_default_type(default)
+            config_item.value_type = self.__check_default_type(default)
 
         config_item.description = description
 
@@ -77,15 +73,14 @@ class ConsulUtils(object):
         else:
             config_item.is_password = False
 
-        key = cls.__get_consul_key(app_id, env_key)
-        data = cls.consul.kv.get(key)
+        key = self.__get_consul_key(app_id, env_key)
+        index, data = self.consul.kv.get(key)
         # TODO: update existing keys?
         if data is None:
-            cls.consul.kv.put(key, config_item.to_json())
+            self.consul.kv.put(key, config_item.to_json())
 
-    @classmethod
-    def __get_consul_key(cls, app_id, key):
-        return cls._DEFAULT_CONSUL_CONFIG['CONSUL_BASIC_ROUTE'] + app_id + '/' 
+ key
+    def __get_consul_key(self, app_id, key):
+        return self._DEFAULT_CONSUL_CONFIG['CONSUL_BASIC_ROUTE'] + app_id + 
'/' + key
 
     @staticmethod
     def __env_or_default(key, default):
@@ -107,3 +102,14 @@ class ConsulUtils(object):
             return 'xs:boolean'
         elif isinstance(value, float):
             return 'xs:float'
+
+    def __consul(self):
+        if os.getenv('CONSUL_LOCATION'):
+            return consul.Consul(host=os.getenv('CONSUL_LOCATION'),
+                                 
port=self._DEFAULT_CONSUL_CONFIG['CONSUL_PORT'])
+        elif os.getenv('SP_DEBUG'):
+            return consul.Consul(host='localhost',
+                                 
port=self._DEFAULT_CONSUL_CONFIG['CONSUL_PORT'])
+        else:
+            return 
consul.Consul(host=self._DEFAULT_CONSUL_CONFIG['CONSUL_HOST'],
+                                 
port=self._DEFAULT_CONSUL_CONFIG['CONSUL_PORT'])

Reply via email to