xintongsong commented on code in PR #32: URL: https://github.com/apache/flink-agents/pull/32#discussion_r2163244558
########## python/flink_agents/examples/workflow_example.py: ########## @@ -0,0 +1,68 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################# +from collections import deque +from typing import Any + +from flink_agents.api.decorators import action +from flink_agents.api.event import Event, InputEvent, OutputEvent +from flink_agents.api.execution_enviroment import AgentsExecutionEnvironment +from flink_agents.api.runner_context import RunnerContext +from flink_agents.api.workflow import Workflow + + +class MyEvent(Event): #noqa D101 + value: Any + +#TODO: Replace this workflow with more practical example. +class MyWorkflow(Workflow): + """An example of Workflow to show the basic usage. Currently, this workflow doesn't + really make sense, and it's mainly for developing validation. + """ + @action(InputEvent) + @staticmethod + def first_action(event: Event, ctx: RunnerContext): #noqa D102 + input = event.input + content = input + ' first_action' + ctx.send_event(MyEvent(value=content)) + ctx.send_event(OutputEvent(output=content)) + + @action(MyEvent) + @staticmethod + def second_action(event: Event, ctx: RunnerContext): #noqa D102 + input = event.value + content = input + ' second_action' + ctx.send_event(OutputEvent(output=content)) + + +if __name__ == "__main__": + env = AgentsExecutionEnvironment.get_execution_environment(module=__name__) Review Comment: User should not need to specify this. ########## python/flink_agents/examples/workflow_example.py: ########## @@ -0,0 +1,68 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################# +from collections import deque +from typing import Any + +from flink_agents.api.decorators import action +from flink_agents.api.event import Event, InputEvent, OutputEvent +from flink_agents.api.execution_enviroment import AgentsExecutionEnvironment +from flink_agents.api.runner_context import RunnerContext +from flink_agents.api.workflow import Workflow + + +class MyEvent(Event): #noqa D101 + value: Any + +#TODO: Replace this workflow with more practical example. +class MyWorkflow(Workflow): + """An example of Workflow to show the basic usage. Currently, this workflow doesn't + really make sense, and it's mainly for developing validation. + """ + @action(InputEvent) + @staticmethod + def first_action(event: Event, ctx: RunnerContext): #noqa D102 + input = event.input + content = input + ' first_action' + ctx.send_event(MyEvent(value=content)) + ctx.send_event(OutputEvent(output=content)) + + @action(MyEvent) + @staticmethod + def second_action(event: Event, ctx: RunnerContext): #noqa D102 + input = event.value + content = input + ' second_action' + ctx.send_event(OutputEvent(output=content)) + + +if __name__ == "__main__": + env = AgentsExecutionEnvironment.get_execution_environment(module=__name__) + + input_queue = deque() Review Comment: Might be better to use list rather than deque. List is more lightweight and do not confuse people with potential concurrent supports. ########## python/flink_agents/runtime/local_runner.py: ########## @@ -0,0 +1,204 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################# +import logging +import uuid +from collections import deque +from typing import Any, Dict + +from typing_extensions import override + +from flink_agents.api.event import Event, InputEvent, OutputEvent +from flink_agents.api.runner_context import RunnerContext +from flink_agents.api.workflow import Workflow +from flink_agents.plan.workflow_plan import WorkflowPlan +from flink_agents.runtime.workflow_runner import WorkflowRunner + +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger(__name__) + + +class LocalRunnerContext(RunnerContext): + """Implementation of WorkflowRunnerContext for local workflow execution. + + Attributes: + ---------- + __workflow_plan : WorkflowPlan + Internal workflow plan for this context. + __key : Any + Unique identifier for the context, correspond to the key in flink KeyedStream. + events : deque[Event] + Queue of events to be processed in this context. + outputs : deque[Any] + Queue of outputs generated by workflow execution. + """ + + __workflow_plan: WorkflowPlan + __key: Any + events: deque[Event] + outputs: deque[Any] + + def __init__(self, workflow_plan: WorkflowPlan, key: Any) -> None: + """Initialize a new session with the given workflow and key. + + Parameters + ---------- + workflow : Workflow + Workflow plan used for this context. + key : Any + Unique context identifier, which is corresponding to the key in flink + KeyedStream when agents receive DataStream as input. + """ + self.__workflow_plan = workflow_plan + self.__key = key + self.events = deque() + self.outputs = deque() + + @property + def key(self) -> Any: + """Get the unique identifier for this context. + + Returns: + ------- + Any + The unique identifier for this context. + """ + return self.__key + + @override + def send_event(self, event: Event) -> None: + """Send an event to the context's event queue and log it. + + Parameters + ---------- + event : Event + The event to be added to the queue. + """ + logger.info("key: %s, send_event: %s", self.__key, event) + self.events.append(event) + + def add_output(self, output: Any) -> None: Review Comment: ```suggestion def emit_output(self, output: Any) -> None: ``` ########## python/flink_agents/runtime/local_runner.py: ########## @@ -0,0 +1,204 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################# +import logging +import uuid +from collections import deque +from typing import Any, Dict + +from typing_extensions import override + +from flink_agents.api.event import Event, InputEvent, OutputEvent +from flink_agents.api.runner_context import RunnerContext +from flink_agents.api.workflow import Workflow +from flink_agents.plan.workflow_plan import WorkflowPlan +from flink_agents.runtime.workflow_runner import WorkflowRunner + +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger(__name__) + + +class LocalRunnerContext(RunnerContext): + """Implementation of WorkflowRunnerContext for local workflow execution. + + Attributes: + ---------- + __workflow_plan : WorkflowPlan + Internal workflow plan for this context. + __key : Any + Unique identifier for the context, correspond to the key in flink KeyedStream. + events : deque[Event] + Queue of events to be processed in this context. + outputs : deque[Any] + Queue of outputs generated by workflow execution. + """ + + __workflow_plan: WorkflowPlan + __key: Any + events: deque[Event] + outputs: deque[Any] + + def __init__(self, workflow_plan: WorkflowPlan, key: Any) -> None: + """Initialize a new session with the given workflow and key. + + Parameters + ---------- + workflow : Workflow + Workflow plan used for this context. + key : Any + Unique context identifier, which is corresponding to the key in flink + KeyedStream when agents receive DataStream as input. + """ + self.__workflow_plan = workflow_plan + self.__key = key + self.events = deque() + self.outputs = deque() + + @property + def key(self) -> Any: + """Get the unique identifier for this context. + + Returns: + ------- + Any + The unique identifier for this context. + """ + return self.__key + + @override + def send_event(self, event: Event) -> None: + """Send an event to the context's event queue and log it. + + Parameters + ---------- + event : Event + The event to be added to the queue. + """ + logger.info("key: %s, send_event: %s", self.__key, event) + self.events.append(event) + + def add_output(self, output: Any) -> None: Review Comment: And why do we need to emit outputs through the context? Why having separate output queues for each context? ########## python/flink_agents/api/execution_enviroment.py: ########## @@ -0,0 +1,106 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################# +from __future__ import annotations + +import importlib +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any, Dict + +if TYPE_CHECKING: + from collections import deque + + from pyflink.datastream import DataStream + + from flink_agents.api.workflow import Workflow + + +class AgentsExecutionEnvironment(ABC): + """Base class for workflow execution environment.""" + + @classmethod + def get_execution_environment(cls, **kwargs: Dict[str, Any]) -> AgentsExecutionEnvironment: + """Get agents execution environment. + + Returns: + ------- + AgentsExecutionEnvironment + local ExecutionEnvironment for local testing or debugging, or cluster + ExecutionEnvironment for running on flink cluster. + """ + # use local environment when run python file directly. + if 'module' in kwargs and kwargs['module'] == "__main__": + return importlib.import_module( + "flink_agents.runtime.local_execution_environment" + ).get_execution_environment(**kwargs) + else: + #TODO: implement flink execution environment + raise RuntimeError() + + @abstractmethod + def from_queue(self, input: deque) -> AgentsExecutionEnvironment: + """Set input for agents. + + Parameters + ---------- + input : deque + Receive a queue as input, should run on local. + """ + + @abstractmethod + def from_datastream(self, input: DataStream) -> AgentsExecutionEnvironment: Review Comment: Should not introduce interfaces that we don't need and have not think through at the moment. ########## python/flink_agents/api/tests/test_decorators.py: ########## @@ -0,0 +1,51 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################# +import pytest + +from flink_agents.api.decorators import action +from flink_agents.api.event import Event, InputEvent, OutputEvent +from flink_agents.api.runner_context import RunnerContext + + +def test_action_decorator() -> None: #noqa D103 + @action(InputEvent) + def forward_action(event: Event, ctx: RunnerContext) -> None: + input = event.input + ctx.send_event(OutputEvent(output=input)) + + assert hasattr(forward_action, '_listen_events') + listen_events = forward_action._listen_events + assert listen_events == (InputEvent,) + +def test_action_decorator_listen_muli_events() -> None: #noqa D103 Review Comment: ```suggestion def test_action_decorator_listen_multi_events() -> None: #noqa D103 ``` ########## python/flink_agents/runtime/local_runner.py: ########## @@ -0,0 +1,204 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################# +import logging +import uuid +from collections import deque +from typing import Any, Dict + +from typing_extensions import override + +from flink_agents.api.event import Event, InputEvent, OutputEvent +from flink_agents.api.runner_context import RunnerContext +from flink_agents.api.workflow import Workflow +from flink_agents.plan.workflow_plan import WorkflowPlan +from flink_agents.runtime.workflow_runner import WorkflowRunner + +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger(__name__) + + +class LocalRunnerContext(RunnerContext): + """Implementation of WorkflowRunnerContext for local workflow execution. + + Attributes: + ---------- + __workflow_plan : WorkflowPlan + Internal workflow plan for this context. + __key : Any + Unique identifier for the context, correspond to the key in flink KeyedStream. + events : deque[Event] + Queue of events to be processed in this context. + outputs : deque[Any] + Queue of outputs generated by workflow execution. + """ + + __workflow_plan: WorkflowPlan + __key: Any + events: deque[Event] + outputs: deque[Any] + + def __init__(self, workflow_plan: WorkflowPlan, key: Any) -> None: + """Initialize a new session with the given workflow and key. + + Parameters + ---------- + workflow : Workflow + Workflow plan used for this context. + key : Any + Unique context identifier, which is corresponding to the key in flink + KeyedStream when agents receive DataStream as input. + """ + self.__workflow_plan = workflow_plan + self.__key = key + self.events = deque() + self.outputs = deque() + + @property + def key(self) -> Any: + """Get the unique identifier for this context. + + Returns: + ------- + Any + The unique identifier for this context. + """ + return self.__key + + @override + def send_event(self, event: Event) -> None: + """Send an event to the context's event queue and log it. + + Parameters + ---------- + event : Event + The event to be added to the queue. + """ + logger.info("key: %s, send_event: %s", self.__key, event) + self.events.append(event) + + def add_output(self, output: Any) -> None: + """Add an output generate by workflow execution to the context. + + Parameters + ---------- + output : Any + The output to be added to the queue. + """ + self.outputs.append(output) + + def get_outputs(self) -> deque[Any]: + """Add an output generate by workflow execution to the context. + + Returns: + ------- + deque[Any] + The outputs generated by workflow execution on this context. + """ + return self.outputs + + def clear_output(self) -> None: + """Clear outputs stored in this context.""" + self.outputs.clear() + + +class LocalRunner(WorkflowRunner): + """Workflow runner implementation for local execution, which is + convenient for debugging. + + Attributes: + ---------- + __workflow_plan : WorkflowPlan + Internal workflow plan. + __sessions : dict[Any, LocalRunnerContext] Review Comment: Should not have the concept session. ########## python/flink_agents/runtime/local_runner.py: ########## @@ -0,0 +1,204 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################# +import logging +import uuid +from collections import deque +from typing import Any, Dict + +from typing_extensions import override + +from flink_agents.api.event import Event, InputEvent, OutputEvent +from flink_agents.api.runner_context import RunnerContext +from flink_agents.api.workflow import Workflow +from flink_agents.plan.workflow_plan import WorkflowPlan +from flink_agents.runtime.workflow_runner import WorkflowRunner + +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger(__name__) + + +class LocalRunnerContext(RunnerContext): + """Implementation of WorkflowRunnerContext for local workflow execution. + + Attributes: + ---------- + __workflow_plan : WorkflowPlan + Internal workflow plan for this context. + __key : Any + Unique identifier for the context, correspond to the key in flink KeyedStream. + events : deque[Event] + Queue of events to be processed in this context. + outputs : deque[Any] + Queue of outputs generated by workflow execution. + """ + + __workflow_plan: WorkflowPlan + __key: Any + events: deque[Event] + outputs: deque[Any] + + def __init__(self, workflow_plan: WorkflowPlan, key: Any) -> None: + """Initialize a new session with the given workflow and key. + + Parameters + ---------- + workflow : Workflow + Workflow plan used for this context. + key : Any + Unique context identifier, which is corresponding to the key in flink + KeyedStream when agents receive DataStream as input. + """ + self.__workflow_plan = workflow_plan + self.__key = key + self.events = deque() + self.outputs = deque() + + @property + def key(self) -> Any: + """Get the unique identifier for this context. + + Returns: + ------- + Any + The unique identifier for this context. + """ + return self.__key + + @override + def send_event(self, event: Event) -> None: + """Send an event to the context's event queue and log it. + + Parameters + ---------- + event : Event + The event to be added to the queue. + """ + logger.info("key: %s, send_event: %s", self.__key, event) + self.events.append(event) + + def add_output(self, output: Any) -> None: + """Add an output generate by workflow execution to the context. + + Parameters + ---------- + output : Any + The output to be added to the queue. + """ + self.outputs.append(output) + + def get_outputs(self) -> deque[Any]: + """Add an output generate by workflow execution to the context. + + Returns: + ------- + deque[Any] + The outputs generated by workflow execution on this context. + """ + return self.outputs + + def clear_output(self) -> None: + """Clear outputs stored in this context.""" + self.outputs.clear() + + +class LocalRunner(WorkflowRunner): + """Workflow runner implementation for local execution, which is + convenient for debugging. + + Attributes: + ---------- + __workflow_plan : WorkflowPlan + Internal workflow plan. + __sessions : dict[Any, LocalRunnerContext] + Dictionary of active sessions indexed by key. + """ + + __workflow_plan: WorkflowPlan + __sessions: dict[Any, LocalRunnerContext] + + def __init__(self, workflow: Workflow) -> None: + """Initialize the runner with the provided workflow. + + Parameters + ---------- + workflow : flink_agent_api.workflow.Workflow + The workflow class to convert and run. + """ + self.__workflow_plan = WorkflowPlan.from_workflow(workflow) + self.__sessions = {} + + @override + def run(self, **data: Dict[str, Any]) -> Any: + """Execute the workflow for a specific session. + + Parameters + ---------- + **data : dict[str, Any] + input record from upstream. + + Returns: + ------- + key + The key of the input that was processed. + """ + if 'key' in data: + key = data['key'] + elif 'k' in data: + key = data['k'] + else: + key = uuid.uuid4() + + if key not in self.__sessions: + self.__sessions[key] = LocalRunnerContext(self.__workflow_plan, key) + session = self.__sessions[key] + + if 'value' in data: + input_event = InputEvent(input=data['value']) + elif 'v' in data: + input_event = InputEvent(input=data['v']) + else: + msg = "Input data must be dict has 'v' or 'value' field" + raise RuntimeError(msg) + + for k, v in data.items(): + input_event.__setattr__(k, v) Review Comment: what is this for? ########## python/flink_agents/api/tests/test_decorators.py: ########## @@ -0,0 +1,51 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################# +import pytest + +from flink_agents.api.decorators import action +from flink_agents.api.event import Event, InputEvent, OutputEvent +from flink_agents.api.runner_context import RunnerContext + + +def test_action_decorator() -> None: #noqa D103 Review Comment: Still missing case for listening non-event types. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
