xintongsong commented on code in PR #85: URL: https://github.com/apache/flink-agents/pull/85#discussion_r2255714983
########## runtime/src/main/java/org/apache/flink/agents/runtime/env/LocalExecutionEnvironment.java: ########## @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.runtime.env; + +import org.apache.flink.agents.api.Action; +import org.apache.flink.agents.api.Agent; +import org.apache.flink.agents.api.AgentBuilder; +import org.apache.flink.agents.api.AgentsExecutionEnvironment; +import org.apache.flink.agents.api.Event; +import org.apache.flink.agents.api.InputEvent; +import org.apache.flink.agents.api.OutputEvent; +import org.apache.flink.agents.api.context.RunnerContext; +import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; + +/** + * Implementation of AgentsExecutionEnvironment for local execution. + * + * <p>This environment is primarily used for testing and development, allowing agents to be executed + * locally without a Flink cluster. + */ +public class LocalExecutionEnvironment extends AgentsExecutionEnvironment { + + @Override + public AgentBuilder fromList(List<Object> input) { + return new LocalAgentBuilder(input); + } + + @Override + public <T, K> AgentBuilder fromDataStream(DataStream<T> input, KeySelector<T, K> keySelector) { + throw new UnsupportedOperationException( + "LocalExecutionEnvironment does not support fromDataStream. Use RemoteExecutionEnvironment instead."); + } + + @Override + public <K> AgentBuilder fromTable( + Table input, StreamTableEnvironment tableEnv, KeySelector<Object, K> keySelector) { + throw new UnsupportedOperationException( + "LocalExecutionEnvironment does not support fromTable. Use RemoteExecutionEnvironment instead."); + } + + @Override + public void execute() throws Exception { + // For local execution, the execution happens immediately when toList() is + // called + // This method is kept for API compatibility + } + + /** Implementation of AgentBuilder for local execution environment. */ + private static class LocalAgentBuilder implements AgentBuilder { + + private final List<Object> input; + private Agent agent; + private List<Map<String, Object>> output; + + public LocalAgentBuilder(List<Object> input) { + this.input = input; + } + + @Override + public AgentBuilder apply(Agent agent) { + this.agent = agent; + return this; + } + + @Override + public List<Map<String, Object>> toList() { + if (agent == null) { + throw new IllegalStateException("Must apply agent before calling toList"); + } + + if (output == null) { + output = executeAgent(); + } + + return output; + } + + @Override + public DataStream<Object> toDataStream() { + throw new UnsupportedOperationException( + "LocalAgentBuilder does not support toDataStream. Use toList instead."); + } + + @Override + public Table toTable(Schema schema) { + throw new UnsupportedOperationException( + "LocalAgentBuilder does not support toTable. Use toList instead."); + } + + private List<Map<String, Object>> executeAgent() { Review Comment: I'm not sure about implementing this in the LocalExecutionEnvironment. As the projects evolves, the agent execution logics can become more and more complex. In python, we introduced the local mode so that users can easily execute the agent in their python environment, without relying on a JDK/JVM or a Flink cluster. For java users, they already have the java environment, and Flink also supports local execution (MiniCluster). So I think it might make sense to rely on Flink's MiniCluster for executing the agents locally. In this way, we can reuse the same agent execution implementation as in the remote mode, rather than re-implement it here. ########## examples/pom.xml: ########## @@ -28,4 +28,27 @@ under the License. <artifactId>flink-agents-examples</artifactId> <name>Flink Agents : Examples</name> + <dependencies> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-agents-api</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-agents-runtime</artifactId> + <version>${project.version}</version> + </dependency> Review Comment: Which part of this module needs to depend on flink-agents-runtime? I'm asking because by design users should not need to depend on non-api modules when building their agents. If there're something they need, we probably should consider moving it to the api module. ########## runtime/src/main/java/org/apache/flink/agents/runtime/env/LocalExecutionEnvironment.java: ########## @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.runtime.env; + +import org.apache.flink.agents.api.Action; +import org.apache.flink.agents.api.Agent; +import org.apache.flink.agents.api.AgentBuilder; +import org.apache.flink.agents.api.AgentsExecutionEnvironment; +import org.apache.flink.agents.api.Event; +import org.apache.flink.agents.api.InputEvent; +import org.apache.flink.agents.api.OutputEvent; +import org.apache.flink.agents.api.context.RunnerContext; +import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; + +/** + * Implementation of AgentsExecutionEnvironment for local execution. + * + * <p>This environment is primarily used for testing and development, allowing agents to be executed + * locally without a Flink cluster. + */ +public class LocalExecutionEnvironment extends AgentsExecutionEnvironment { + + @Override + public AgentBuilder fromList(List<Object> input) { + return new LocalAgentBuilder(input); + } + + @Override + public <T, K> AgentBuilder fromDataStream(DataStream<T> input, KeySelector<T, K> keySelector) { + throw new UnsupportedOperationException( + "LocalExecutionEnvironment does not support fromDataStream. Use RemoteExecutionEnvironment instead."); + } + + @Override + public <K> AgentBuilder fromTable( + Table input, StreamTableEnvironment tableEnv, KeySelector<Object, K> keySelector) { + throw new UnsupportedOperationException( + "LocalExecutionEnvironment does not support fromTable. Use RemoteExecutionEnvironment instead."); + } + + @Override + public void execute() throws Exception { + // For local execution, the execution happens immediately when toList() is + // called + // This method is kept for API compatibility + } Review Comment: I think this is against intuition. In python, `to_list()` provides an empty output list, which will be filled in after `execute()`. We may want to keep the same behavior here. ########## runtime/src/main/java/org/apache/flink/agents/runtime/env/RemoteExecutionEnvironment.java: ########## @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.runtime.env; + +import org.apache.flink.agents.api.Agent; +import org.apache.flink.agents.api.AgentBuilder; +import org.apache.flink.agents.api.AgentsExecutionEnvironment; +import org.apache.flink.agents.plan.AgentPlan; +import org.apache.flink.agents.runtime.CompileUtils; +import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; + +import java.util.List; +import java.util.Map; + +/** + * Implementation of AgentsExecutionEnvironment for remote execution with Flink. + * + * <p>This environment integrates agents with Flink's streaming runtime, enabling agents to process + * DataStreams and Tables within a Flink cluster. + */ +public class RemoteExecutionEnvironment extends AgentsExecutionEnvironment { + + private final StreamExecutionEnvironment env; + + public RemoteExecutionEnvironment(StreamExecutionEnvironment env) { + this.env = env; + } + + @Override + public AgentBuilder fromList(List<Object> input) { + throw new UnsupportedOperationException( + "RemoteExecutionEnvironment does not support fromList. Use fromDataStream or fromTable instead."); + } + + @Override + public <T, K> AgentBuilder fromDataStream(DataStream<T> input, KeySelector<T, K> keySelector) { + return new RemoteAgentBuilder<>(input, keySelector, env); + } + + @Override + public <K> AgentBuilder fromTable( + Table input, StreamTableEnvironment tableEnv, KeySelector<Object, K> keySelector) { + return new RemoteAgentBuilder<>(input, tableEnv, keySelector, env); + } + + @Override + public void execute() throws Exception { + env.execute(); + } + + /** Implementation of AgentBuilder for remote execution environment. */ + private static class RemoteAgentBuilder<T, K> implements AgentBuilder { + + private final DataStream<T> inputDataStream; + private final KeySelector<T, K> keySelector; + private final StreamExecutionEnvironment env; + private final StreamTableEnvironment tableEnv; + + private AgentPlan agentPlan; + private DataStream<Object> outputDataStream; + + // Constructor for DataStream input + public RemoteAgentBuilder( + DataStream<T> inputDataStream, + KeySelector<T, K> keySelector, + StreamExecutionEnvironment env) { + this.inputDataStream = inputDataStream; + this.keySelector = keySelector; + this.env = env; + this.tableEnv = null; + } + + // Constructor for Table input + @SuppressWarnings("unchecked") + public RemoteAgentBuilder( + Table inputTable, + StreamTableEnvironment tableEnv, + KeySelector<Object, K> keySelector, + StreamExecutionEnvironment env) { + this.inputDataStream = (DataStream<T>) tableEnv.toDataStream(inputTable); + this.keySelector = (KeySelector<T, K>) keySelector; + this.env = env; + this.tableEnv = tableEnv; + } + + @Override + public AgentBuilder apply(Agent agent) { + try { + this.agentPlan = new AgentPlan(agent); + return this; + } catch (Exception e) { + throw new RuntimeException("Failed to create agent plan from agent", e); + } + } + + @Override + public List<Map<String, Object>> toList() { + throw new UnsupportedOperationException( + "RemoteAgentBuilder does not support toList. Use toDataStream or toTable instead."); + } + + @Override + public DataStream<Object> toDataStream() { + if (agentPlan == null) { + throw new IllegalStateException("Must apply agent before calling toDataStream"); + } + + if (outputDataStream == null) { + if (keySelector != null) { + outputDataStream = + CompileUtils.connectToAgent(inputDataStream, keySelector, agentPlan); + } else { + // If no key selector provided, use a simple pass-through key selector + outputDataStream = + CompileUtils.connectToAgent(inputDataStream, x -> x, agentPlan); + } + } + + return outputDataStream; + } + + @Override + public Table toTable(Schema schema) { + if (tableEnv == null) { + throw new IllegalStateException( + "Table environment not available. Use fromTable() method to enable table output."); + } Review Comment: Nice catch~! I checked the python implementation, and it is indeed a problem that if you create the agent builder with `from_datastream`, you won't be able output an table with `to_table`. But this is not the desired behavior @wenjin272, I think we should fix this. Let's first create an issue to track it. ########## runtime/src/main/java/org/apache/flink/agents/runtime/env/LocalExecutionEnvironment.java: ########## @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.agents.runtime.env; + +import org.apache.flink.agents.api.Action; +import org.apache.flink.agents.api.Agent; +import org.apache.flink.agents.api.AgentBuilder; +import org.apache.flink.agents.api.AgentsExecutionEnvironment; +import org.apache.flink.agents.api.Event; +import org.apache.flink.agents.api.InputEvent; +import org.apache.flink.agents.api.OutputEvent; +import org.apache.flink.agents.api.context.RunnerContext; +import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; + +/** + * Implementation of AgentsExecutionEnvironment for local execution. + * + * <p>This environment is primarily used for testing and development, allowing agents to be executed + * locally without a Flink cluster. + */ +public class LocalExecutionEnvironment extends AgentsExecutionEnvironment { + + @Override + public AgentBuilder fromList(List<Object> input) { + return new LocalAgentBuilder(input); + } + + @Override + public <T, K> AgentBuilder fromDataStream(DataStream<T> input, KeySelector<T, K> keySelector) { + throw new UnsupportedOperationException( + "LocalExecutionEnvironment does not support fromDataStream. Use RemoteExecutionEnvironment instead."); + } + + @Override + public <K> AgentBuilder fromTable( + Table input, StreamTableEnvironment tableEnv, KeySelector<Object, K> keySelector) { + throw new UnsupportedOperationException( + "LocalExecutionEnvironment does not support fromTable. Use RemoteExecutionEnvironment instead."); + } + + @Override + public void execute() throws Exception { + // For local execution, the execution happens immediately when toList() is + // called + // This method is kept for API compatibility + } + + /** Implementation of AgentBuilder for local execution environment. */ + private static class LocalAgentBuilder implements AgentBuilder { + + private final List<Object> input; + private Agent agent; + private List<Map<String, Object>> output; + + public LocalAgentBuilder(List<Object> input) { + this.input = input; + } + + @Override + public AgentBuilder apply(Agent agent) { + this.agent = agent; + return this; + } + + @Override + public List<Map<String, Object>> toList() { + if (agent == null) { + throw new IllegalStateException("Must apply agent before calling toList"); + } + + if (output == null) { + output = executeAgent(); + } + + return output; + } + + @Override + public DataStream<Object> toDataStream() { + throw new UnsupportedOperationException( + "LocalAgentBuilder does not support toDataStream. Use toList instead."); + } + + @Override + public Table toTable(Schema schema) { + throw new UnsupportedOperationException( + "LocalAgentBuilder does not support toTable. Use toList instead."); + } + + private List<Map<String, Object>> executeAgent() { Review Comment: I would also suggest to keep this PR concise and leave supporting local execution in java as a separate follow-up issue. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
