letaoj commented on code in PR #85:
URL: https://github.com/apache/flink-agents/pull/85#discussion_r2258338250
##########
examples/pom.xml:
##########
@@ -28,4 +28,27 @@ under the License.
<artifactId>flink-agents-examples</artifactId>
<name>Flink Agents : Examples</name>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-agents-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-agents-runtime</artifactId>
+ <version>${project.version}</version>
+ </dependency>
Review Comment:
I think I was testing out something in this folder, which requires to use
the `LocalExecutionEnvironment` (which is now removed). I will remove this
dependency for good.
##########
runtime/src/main/java/org/apache/flink/agents/runtime/env/LocalExecutionEnvironment.java:
##########
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.agents.runtime.env;
+
+import org.apache.flink.agents.api.Action;
+import org.apache.flink.agents.api.Agent;
+import org.apache.flink.agents.api.AgentBuilder;
+import org.apache.flink.agents.api.AgentsExecutionEnvironment;
+import org.apache.flink.agents.api.Event;
+import org.apache.flink.agents.api.InputEvent;
+import org.apache.flink.agents.api.OutputEvent;
+import org.apache.flink.agents.api.context.RunnerContext;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.table.api.Schema;
+import org.apache.flink.table.api.Table;
+import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
+
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+
+/**
+ * Implementation of AgentsExecutionEnvironment for local execution.
+ *
+ * <p>This environment is primarily used for testing and development, allowing
agents to be executed
+ * locally without a Flink cluster.
+ */
+public class LocalExecutionEnvironment extends AgentsExecutionEnvironment {
+
+ @Override
+ public AgentBuilder fromList(List<Object> input) {
+ return new LocalAgentBuilder(input);
+ }
+
+ @Override
+ public <T, K> AgentBuilder fromDataStream(DataStream<T> input,
KeySelector<T, K> keySelector) {
+ throw new UnsupportedOperationException(
+ "LocalExecutionEnvironment does not support fromDataStream.
Use RemoteExecutionEnvironment instead.");
+ }
+
+ @Override
+ public <K> AgentBuilder fromTable(
+ Table input, StreamTableEnvironment tableEnv, KeySelector<Object,
K> keySelector) {
+ throw new UnsupportedOperationException(
+ "LocalExecutionEnvironment does not support fromTable. Use
RemoteExecutionEnvironment instead.");
+ }
+
+ @Override
+ public void execute() throws Exception {
+ // For local execution, the execution happens immediately when
toList() is
+ // called
+ // This method is kept for API compatibility
+ }
+
+ /** Implementation of AgentBuilder for local execution environment. */
+ private static class LocalAgentBuilder implements AgentBuilder {
+
+ private final List<Object> input;
+ private Agent agent;
+ private List<Map<String, Object>> output;
+
+ public LocalAgentBuilder(List<Object> input) {
+ this.input = input;
+ }
+
+ @Override
+ public AgentBuilder apply(Agent agent) {
+ this.agent = agent;
+ return this;
+ }
+
+ @Override
+ public List<Map<String, Object>> toList() {
+ if (agent == null) {
+ throw new IllegalStateException("Must apply agent before
calling toList");
+ }
+
+ if (output == null) {
+ output = executeAgent();
+ }
+
+ return output;
+ }
+
+ @Override
+ public DataStream<Object> toDataStream() {
+ throw new UnsupportedOperationException(
+ "LocalAgentBuilder does not support toDataStream. Use
toList instead.");
+ }
+
+ @Override
+ public Table toTable(Schema schema) {
+ throw new UnsupportedOperationException(
+ "LocalAgentBuilder does not support toTable. Use toList
instead.");
+ }
+
+ private List<Map<String, Object>> executeAgent() {
Review Comment:
Thanks for the suggestion. I'm still new to Flink so I do not have much
experience with Local testing. I will remove this file for now and look into
the MiniCluster that you referred in a follow-up PR for the local testing
environment
##########
runtime/src/main/java/org/apache/flink/agents/runtime/env/RemoteExecutionEnvironment.java:
##########
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.agents.runtime.env;
+
+import org.apache.flink.agents.api.Agent;
+import org.apache.flink.agents.api.AgentBuilder;
+import org.apache.flink.agents.api.AgentsExecutionEnvironment;
+import org.apache.flink.agents.plan.AgentPlan;
+import org.apache.flink.agents.runtime.CompileUtils;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.table.api.Schema;
+import org.apache.flink.table.api.Table;
+import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Implementation of AgentsExecutionEnvironment for remote execution with
Flink.
+ *
+ * <p>This environment integrates agents with Flink's streaming runtime,
enabling agents to process
+ * DataStreams and Tables within a Flink cluster.
+ */
+public class RemoteExecutionEnvironment extends AgentsExecutionEnvironment {
+
+ private final StreamExecutionEnvironment env;
+
+ public RemoteExecutionEnvironment(StreamExecutionEnvironment env) {
+ this.env = env;
+ }
+
+ @Override
+ public AgentBuilder fromList(List<Object> input) {
+ throw new UnsupportedOperationException(
+ "RemoteExecutionEnvironment does not support fromList. Use
fromDataStream or fromTable instead.");
+ }
+
+ @Override
+ public <T, K> AgentBuilder fromDataStream(DataStream<T> input,
KeySelector<T, K> keySelector) {
+ return new RemoteAgentBuilder<>(input, keySelector, env);
+ }
+
+ @Override
+ public <K> AgentBuilder fromTable(
+ Table input, StreamTableEnvironment tableEnv, KeySelector<Object,
K> keySelector) {
+ return new RemoteAgentBuilder<>(input, tableEnv, keySelector, env);
+ }
+
+ @Override
+ public void execute() throws Exception {
+ env.execute();
+ }
+
+ /** Implementation of AgentBuilder for remote execution environment. */
+ private static class RemoteAgentBuilder<T, K> implements AgentBuilder {
+
+ private final DataStream<T> inputDataStream;
+ private final KeySelector<T, K> keySelector;
+ private final StreamExecutionEnvironment env;
+ private final StreamTableEnvironment tableEnv;
+
+ private AgentPlan agentPlan;
+ private DataStream<Object> outputDataStream;
+
+ // Constructor for DataStream input
+ public RemoteAgentBuilder(
+ DataStream<T> inputDataStream,
+ KeySelector<T, K> keySelector,
+ StreamExecutionEnvironment env) {
+ this.inputDataStream = inputDataStream;
+ this.keySelector = keySelector;
+ this.env = env;
+ this.tableEnv = null;
+ }
+
+ // Constructor for Table input
+ @SuppressWarnings("unchecked")
+ public RemoteAgentBuilder(
+ Table inputTable,
+ StreamTableEnvironment tableEnv,
+ KeySelector<Object, K> keySelector,
+ StreamExecutionEnvironment env) {
+ this.inputDataStream = (DataStream<T>)
tableEnv.toDataStream(inputTable);
+ this.keySelector = (KeySelector<T, K>) keySelector;
+ this.env = env;
+ this.tableEnv = tableEnv;
+ }
+
+ @Override
+ public AgentBuilder apply(Agent agent) {
+ try {
+ this.agentPlan = new AgentPlan(agent);
+ return this;
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to create agent plan from
agent", e);
+ }
+ }
+
+ @Override
+ public List<Map<String, Object>> toList() {
+ throw new UnsupportedOperationException(
+ "RemoteAgentBuilder does not support toList. Use
toDataStream or toTable instead.");
+ }
+
+ @Override
+ public DataStream<Object> toDataStream() {
+ if (agentPlan == null) {
+ throw new IllegalStateException("Must apply agent before
calling toDataStream");
+ }
+
+ if (outputDataStream == null) {
+ if (keySelector != null) {
+ outputDataStream =
+ CompileUtils.connectToAgent(inputDataStream,
keySelector, agentPlan);
+ } else {
+ // If no key selector provided, use a simple pass-through
key selector
+ outputDataStream =
+ CompileUtils.connectToAgent(inputDataStream, x ->
x, agentPlan);
+ }
+ }
+
+ return outputDataStream;
+ }
+
+ @Override
+ public Table toTable(Schema schema) {
+ if (tableEnv == null) {
+ throw new IllegalStateException(
+ "Table environment not available. Use fromTable()
method to enable table output.");
+ }
Review Comment:
Should the correct logic be, if the table env is null, then we will create
one using `StreamTableEnvironment.create(env)` and then use it?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
