This is an automated email from the ASF dual-hosted git repository.

wiener pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git


The following commit(s) were added to refs/heads/dev by this push:
     new 3e60e56  [STREAMPIPES-174] Cannot start multiple instances of processor
3e60e56 is described below

commit 3e60e56652065a7b1e7081bd8fd0342bc1268c46
Author: Patrick Wiener <[email protected]>
AuthorDate: Mon Jul 20 07:58:09 2020 +0200

    [STREAMPIPES-174] Cannot start multiple instances of processor
---
 streampipes-wrapper-python/streampipes/api.py      |  4 +--
 .../streampipes/configuration.py                   |  2 +-
 streampipes-wrapper-python/streampipes/core.py     | 41 +++++++++-------------
 streampipes-wrapper-python/streampipes/manager.py  | 20 +++++++----
 4 files changed, 33 insertions(+), 34 deletions(-)

diff --git a/streampipes-wrapper-python/streampipes/api.py 
b/streampipes-wrapper-python/streampipes/api.py
index 39dbe6e..f6baf08 100644
--- a/streampipes-wrapper-python/streampipes/api.py
+++ b/streampipes-wrapper-python/streampipes/api.py
@@ -58,7 +58,7 @@ class API(object):
         :param json: contains invocation graph.
         :return: None.
         """
-        return ProcessorDispatcher.start(app_id=kwargs.get('processor_id'), 
**kwargs)
+        return ProcessorDispatcher.start(**kwargs)
 
     @staticmethod
     def detach(**kwargs):
@@ -67,4 +67,4 @@ class API(object):
         :param json: contains elementId to be terminated
         :return: None.
         """
-        return 
ProcessorDispatcher.stop(invocation_id=kwargs.get('invocation_id'))
\ No newline at end of file
+        return ProcessorDispatcher.stop(**kwargs)
\ No newline at end of file
diff --git a/streampipes-wrapper-python/streampipes/configuration.py 
b/streampipes-wrapper-python/streampipes/configuration.py
index 1a6ae11..719b6e1 100644
--- a/streampipes-wrapper-python/streampipes/configuration.py
+++ b/streampipes-wrapper-python/streampipes/configuration.py
@@ -19,7 +19,7 @@ import os
 
 flask_host = os.getenv('FLASK_HOST', '0.0.0.0')
 port = os.getenv('SP_PORT', 5000)
-kafka_thread = 'kafka_thread'
+kafka_consumer_thread = 'kafka_consumer_thread'
 
 banner="""\
  _______ __                              ______ __
diff --git a/streampipes-wrapper-python/streampipes/core.py 
b/streampipes-wrapper-python/streampipes/core.py
index 346e04f..c31cd9e 100644
--- a/streampipes-wrapper-python/streampipes/core.py
+++ b/streampipes-wrapper-python/streampipes/core.py
@@ -17,13 +17,13 @@
 """contains relevant base classes"""
 import json
 import logging
-import queue
-import uuid
 from abc import ABC, abstractmethod
+from confluent_kafka.admin import AdminClient
+
 from streampipes.api import API
-from streampipes.configuration import kafka_thread, banner
-from confluent_kafka import Producer, Consumer
+from streampipes.configuration import banner, kafka_consumer_thread
 from streampipes.helper import threaded
+from confluent_kafka import Producer, Consumer
 
 
 class StandaloneSubmitter(ABC):
@@ -46,7 +46,6 @@ class EventProcessor(ABC):
         'session.timeout.ms': 30000,
         'fetch.max.bytes': 5000012,
         'auto.offset.reset': 'latest',
-        'group.id': 'streampipes_python_' + str(uuid.uuid4()),
     }
 
     _DEFAULT_KAFKA_PRODUCER_CONFIG = {
@@ -59,14 +58,13 @@ class EventProcessor(ABC):
     def __init__(self, **kwargs):
         """initialize EventProcessor with Kafka Prodcuer and Consumer"""
         self.logger = logging.getLogger(__name__)
+
         self._input_topics = kwargs.get('input_topics')
         self._output_topics = kwargs.get('output_topics')
         self._invocation_id = kwargs.get('invocation_id')
         self._bootstrap_servers = kwargs.get('bootstrap_servers')
         self.static_properties = kwargs.get('static_properties')
 
-        # exchange events via queue - maybe register callback would be better
-        self.queue = queue.Queue(maxsize=0)
         self._running = False
         self._threads = {}
 
@@ -74,21 +72,23 @@ class EventProcessor(ABC):
             self._DEFAULT_KAFKA_CONSUMER_CONFIG['bootstrap.servers'] = 
self._bootstrap_servers
             self._DEFAULT_KAFKA_PRODUCER_CONFIG['bootstrap.servers'] = 
self._bootstrap_servers
 
-        self._producer = Producer(**self._DEFAULT_KAFKA_PRODUCER_CONFIG)
-        self._consumer = Consumer(**self._DEFAULT_KAFKA_CONSUMER_CONFIG)
+        self._DEFAULT_KAFKA_CONSUMER_CONFIG['group.id'] = 
'streampipes_python_' + self._invocation_id
+
+        self._producer = Producer(self._DEFAULT_KAFKA_PRODUCER_CONFIG)
+        self._consumer = Consumer(self._DEFAULT_KAFKA_CONSUMER_CONFIG)
         #self._create_topic(topic=self._output_topics, 
conf=self._DEFAULT_KAFKA_PRODUCER_CONFIG)
 
         self.on_invocation()
 
     def init(self):
-        self.logger.info('start processor {}'.format(self._invocation_id))
-        self._threads[kafka_thread] = self.kafka_thread_main()
+        self.logger.info('start processor {}'.format(self.invocation_id))
+        self._threads[kafka_consumer_thread] = 
self._consume(self._input_topics)
 
     def active_threads(self):
         return self._threads
 
     @property
-    def invoke_id(self):
+    def invocation_id(self):
         return self._invocation_id
 
     def __del__(self):
@@ -115,18 +115,7 @@ class EventProcessor(ABC):
             self._produce(result)
 
     @threaded
-    def kafka_thread_main(self):
-        """ start threaded kafka consumer """
-        try:
-            self._consumer_run(self._input_topics)
-        except KeyboardInterrupt:
-            self.logger.info("consumer: aborted by user")
-            self._running = False
-        except Exception as ex:
-            self.logger.fatal("consumer: fatal exception: {}".format(ex))
-            self._running = False
-
-    def _consumer_run(self, topics):
+    def _consume(self, topics):
         """ retrieve events from kafka """
         self._consumer.subscribe(topics=[topics])
         self._running = True
@@ -143,6 +132,7 @@ class EventProcessor(ABC):
                     continue
             else:
                 try:
+                    # json -> dict
                     event = json.loads(msg.value().decode('utf-8'))
                     if isinstance(event, int):
                         self.logger.info("Integer not allowed 
{}".format(event))
@@ -157,6 +147,7 @@ class EventProcessor(ABC):
         """ send events to kafka """
         event = json.dumps(result).encode('utf-8')
         try:
+            # dict -> json
             self._producer.produce(self._output_topics, value=event)
         except BufferError:
             self._producer.poll(1)
@@ -175,7 +166,7 @@ class EventProcessor(ABC):
     #             raise
 
     def stop(self):
-        self.logger.info('stop processor {}'.format(self._invocation_id))
+        self.logger.info('stop processor {}'.format(self.invocation_id))
         self._running = False
         self._consumer.close()
         self._producer.flush()
diff --git a/streampipes-wrapper-python/streampipes/manager.py 
b/streampipes-wrapper-python/streampipes/manager.py
index d7a2d50..64a2a4e 100644
--- a/streampipes-wrapper-python/streampipes/manager.py
+++ b/streampipes-wrapper-python/streampipes/manager.py
@@ -17,7 +17,8 @@
 """Manages processor life cycle"""
 import logging
 from abc import ABC
-from streampipes.configuration import kafka_thread
+
+from streampipes.configuration import kafka_consumer_thread
 
 
 class Declarer(ABC):
@@ -40,33 +41,40 @@ class Declarer(ABC):
 
 class ProcessorDispatcher(ABC):
     _running_instances = {}
+
     logger = logging.getLogger(__name__)
 
     @classmethod
-    def start(cls, processor_id=None, **kwargs):
+    def start(cls, **kwargs):
+        processor_id = kwargs.get('processor_id')
         try:
             processor = Declarer.get_processor(processor_id)(**kwargs)
             processor.init()
-            cls._running_instances[processor.invoke_id] = processor
+            cls._running_instances[processor.invocation_id] = processor
+
             return {'status': 'success'}
 
         except KeyError:
             err = "KeyError. processor_id not found"
-            cls.logger.info('{}: {}'.format(err,processor_id))
+            cls.logger.info('{}: {}'.format(err, processor_id))
             return {'status': err}
 
     @classmethod
-    def stop(cls, invocation_id=None):
+    def stop(cls, **kwargs):
+        invocation_id = kwargs.get('invocation_id')
         try:
             processor = cls._running_instances[invocation_id]
             active_threads = processor.active_threads()
             processor.stop()
-            active_threads[kafka_thread].join()
+            active_threads[kafka_consumer_thread].join()
+
             del processor
             cls._running_instances.pop(invocation_id)
+
             return {'status': 'success'}
 
         except KeyError:
             err = "KeyError. invocation_id not found"
             cls.logger.info('{}: {}'.format(err,invocation_id))
+
             return {'status': err}
\ No newline at end of file

Reply via email to