This is an automated email from the ASF dual-hosted git repository.
sxnan pushed a commit to branch release-0.2
in repository https://gitbox.apache.org/repos/asf/flink-agents.git
The following commit(s) were added to refs/heads/release-0.2 by this push:
new 0b91f9b1 [infra] Deduplicate common jar dependency to reduce wheel
size (#557)
0b91f9b1 is described below
commit 0b91f9b1af2d76ab8033b1557c4e2315b7c6904a
Author: Xuannan <[email protected]>
AuthorDate: Thu Mar 5 21:59:53 2026 +0800
[infra] Deduplicate common jar dependency to reduce wheel size (#557)
---
dist/{ => common}/pom.xml | 126 ++++++++++-------------
dist/flink-1.20/pom.xml | 44 ++++++--
dist/flink-2.0/pom.xml | 42 +++++++-
dist/flink-2.1/pom.xml | 43 +++++++-
dist/flink-2.2/pom.xml | 43 ++++++++
dist/pom.xml | 1 +
python/flink_agents/api/execution_environment.py | 15 ++-
tools/build.sh | 26 +++--
tools/releasing/create_binary_release.sh | 6 +-
9 files changed, 254 insertions(+), 92 deletions(-)
diff --git a/dist/pom.xml b/dist/common/pom.xml
similarity index 50%
copy from dist/pom.xml
copy to dist/common/pom.xml
index 0d5dc1cd..2be34afc 100644
--- a/dist/pom.xml
+++ b/dist/common/pom.xml
@@ -7,7 +7,9 @@ regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
+
http://www.apache.org/licenses/LICENSE-2.0
+
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -21,90 +23,63 @@ under the License.
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-agents</artifactId>
- <version>0.2-SNAPSHOT</version>
+ <artifactId>flink-agents-dist</artifactId>
+ <version>0.3-SNAPSHOT</version>
</parent>
- <artifactId>flink-agents-dist</artifactId>
- <name>Flink Agents : Dist</name>
- <packaging>pom</packaging>
-
- <properties>
- <flink.1.20.version>1.20.3</flink.1.20.version>
- <flink.2.0.version>2.0.1</flink.2.0.version>
- <flink.2.1.version>2.1.1</flink.2.1.version>
- <flink.2.2.version>2.2.0</flink.2.2.version>
- </properties>
-
- <modules>
- <module>flink-1.20</module>
- <module>flink-2.0</module>
- <module>flink-2.1</module>
- <module>flink-2.2</module>
- </modules>
+ <artifactId>flink-agents-dist-common</artifactId>
+ <name>Flink Agents : Dist : Common</name>
+ <description>Common dependencies shared across all Flink versions for
Python wheel</description>
<dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-agents-api</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-agents-plan</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-agents-runtime</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
-
<artifactId>flink-agents-integrations-chat-models-ollama</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
-
<artifactId>flink-agents-integrations-chat-models-openai</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
-
<artifactId>flink-agents-integrations-chat-models-anthropic</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
-
<artifactId>flink-agents-integrations-chat-models-azureai</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
-
<artifactId>flink-agents-integrations-embedding-models-ollama</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
-
<artifactId>flink-agents-integrations-vector-stores-elasticsearch</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-agents-integrations-mcp</artifactId>
- <version>${project.version}</version>
- </dependency>
+ <!-- All integrations dependencies - same as parent pom -->
+ <!-- These will be bundled into the common JAR -->
</dependencies>
<build>
<plugins>
- <!-- Build uber jar -->
+ <!-- Skip deployment - this module is for Python wheel
optimization only -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <!-- Copy the shared LICENSE/NOTICE files from parent -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-shared-resources</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+
<outputDirectory>${project.build.outputDirectory}</outputDirectory>
+ <resources>
+ <resource>
+
<directory>${project.parent.basedir}/src/main/resources</directory>
+ <includes>
+ <include>META-INF/**</include>
+ </includes>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- Build uber jar with all dependencies except flink-agents own
code -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
- <id>shade-flink-agents</id>
+ <id>shade-common</id>
<phase>package</phase>
<goals>
<goal>shade</goal>
@@ -113,6 +88,7 @@ under the License.
<shadeTestJar>false</shadeTestJar>
<shadedArtifactAttached>false</shadedArtifactAttached>
<filters>
+ <!-- Exclude signature files -->
<filter>
<artifact>*:*</artifact>
<excludes>
@@ -121,6 +97,13 @@ under the License.
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
+ <!-- Exclude flink-agents own code from common
JAR -->
+ <filter>
+
<artifact>org.apache.flink:flink-agents-*</artifact>
+ <excludes>
+
<exclude>org/apache/flink/agents/**</exclude>
+ </excludes>
+ </filter>
</filters>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
@@ -128,6 +111,7 @@ under the License.
<Multi-Release>true</Multi-Release>
</manifestEntries>
</transformer>
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
</transformers>
</configuration>
</execution>
diff --git a/dist/flink-1.20/pom.xml b/dist/flink-1.20/pom.xml
index fe4fc4f2..19102cb9 100644
--- a/dist/flink-1.20/pom.xml
+++ b/dist/flink-1.20/pom.xml
@@ -138,7 +138,7 @@ under the License.
</executions>
</plugin>
- <!-- Extend parent's shade plugin with flink-1.20 specific
exclusions -->
+ <!-- Extend parent's shade plugin with flink-1.20 specific
configurations -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
@@ -146,17 +146,43 @@ under the License.
<execution>
<id>shade-flink-agents</id>
<configuration>
- <!-- Append flink-1.20 specific filter to parent's
filters -->
- <filters combine.children="append">
- <!-- Exclude the conflict classes of the
runtime module, using version flink-1.20. -->
+
<createDependencyReducedPom>false</createDependencyReducedPom>
+ </configuration>
+ </execution>
+ <!-- Build thin jar for Python wheel (only flink-agents
code) -->
+ <execution>
+ <id>shade-flink-agents-thin</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>false</shadeTestJar>
+
<shadedArtifactAttached>false</shadedArtifactAttached>
+
<finalName>${project.artifactId}-${project.version}-thin</finalName>
+ <!-- Only include flink-agents artifacts -->
+ <artifactSet>
+ <includes>
+
<include>org.apache.flink:flink-agents-*</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <!-- Only include flink-agents classes -->
<filter>
-
<artifact>org.apache.flink:flink-agents-runtime</artifact>
- <excludes>
-
<exclude>org/apache/flink/agents/runtime/operator/OperatorUtils.class</exclude>
- </excludes>
+ <artifact>*:*</artifact>
+ <includes>
+
<include>org/apache/flink/agents/**</include>
+ </includes>
</filter>
</filters>
-
<createDependencyReducedPom>false</createDependencyReducedPom>
+ <transformers>
+ <transformer
+
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <manifestEntries>
+ <Multi-Release>true</Multi-Release>
+ </manifestEntries>
+ </transformer>
+ </transformers>
</configuration>
</execution>
</executions>
diff --git a/dist/flink-2.0/pom.xml b/dist/flink-2.0/pom.xml
index 2b306ffd..3e72c2dd 100644
--- a/dist/flink-2.0/pom.xml
+++ b/dist/flink-2.0/pom.xml
@@ -84,7 +84,47 @@
</executions>
</plugin>
- <!-- Shade plugin configuration inherits from the parent pom
(dist/pom.xml) -->
+ <!-- Build thin jar for Python wheel (only flink-agents code) -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink-agents-thin</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>false</shadeTestJar>
+
<shadedArtifactAttached>false</shadedArtifactAttached>
+
<finalName>${project.artifactId}-${project.version}-thin</finalName>
+ <!-- Only include flink-agents artifacts -->
+ <artifactSet>
+ <includes>
+
<include>org.apache.flink:flink-agents-*</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <!-- Only include flink-agents classes -->
+ <filter>
+ <artifact>*:*</artifact>
+ <includes>
+
<include>org/apache/flink/agents/**</include>
+ </includes>
+ </filter>
+ </filters>
+ <transformers>
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <manifestEntries>
+ <Multi-Release>true</Multi-Release>
+ </manifestEntries>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
diff --git a/dist/flink-2.1/pom.xml b/dist/flink-2.1/pom.xml
index da903424..b5c6b54d 100644
--- a/dist/flink-2.1/pom.xml
+++ b/dist/flink-2.1/pom.xml
@@ -84,7 +84,48 @@
</executions>
</plugin>
- <!-- Shade plugin configuration inherits from the parent pom
(dist/pom.xml) -->
+ <!-- Build thin jar for Python wheel (only flink-agents code) -->
+ <!-- Not attached as Maven artifact to avoid deployment -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink-agents-thin</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>false</shadeTestJar>
+
<shadedArtifactAttached>false</shadedArtifactAttached>
+
<finalName>${project.artifactId}-${project.version}-thin</finalName>
+ <!-- Only include flink-agents artifacts -->
+ <artifactSet>
+ <includes>
+
<include>org.apache.flink:flink-agents-*</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <!-- Only include flink-agents classes -->
+ <filter>
+ <artifact>*:*</artifact>
+ <includes>
+
<include>org/apache/flink/agents/**</include>
+ </includes>
+ </filter>
+ </filters>
+ <transformers>
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <manifestEntries>
+ <Multi-Release>true</Multi-Release>
+ </manifestEntries>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
diff --git a/dist/flink-2.2/pom.xml b/dist/flink-2.2/pom.xml
index a328c69f..8cff9757 100644
--- a/dist/flink-2.2/pom.xml
+++ b/dist/flink-2.2/pom.xml
@@ -54,6 +54,49 @@ under the License.
</execution>
</executions>
</plugin>
+
+ <!-- Build thin jar for Python wheel (only flink-agents code) -->
+ <!-- Not attached as Maven artifact to avoid deployment -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink-agents-thin</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>false</shadeTestJar>
+
<shadedArtifactAttached>false</shadedArtifactAttached>
+
<finalName>${project.artifactId}-${project.version}-thin</finalName>
+ <!-- Only include flink-agents artifacts -->
+ <artifactSet>
+ <includes>
+
<include>org.apache.flink:flink-agents-*</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <!-- Only include flink-agents classes -->
+ <filter>
+ <artifact>*:*</artifact>
+ <includes>
+
<include>org/apache/flink/agents/**</include>
+ </includes>
+ </filter>
+ </filters>
+ <transformers>
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <manifestEntries>
+ <Multi-Release>true</Multi-Release>
+ </manifestEntries>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/dist/pom.xml b/dist/pom.xml
index 0d5dc1cd..602e0600 100644
--- a/dist/pom.xml
+++ b/dist/pom.xml
@@ -37,6 +37,7 @@ under the License.
</properties>
<modules>
+ <module>common</module>
<module>flink-1.20</module>
<module>flink-2.0</module>
<module>flink-2.1</module>
diff --git a/python/flink_agents/api/execution_environment.py
b/python/flink_agents/api/execution_environment.py
index 4b4c80b6..11de9ab2 100644
--- a/python/flink_agents/api/execution_environment.py
+++ b/python/flink_agents/api/execution_environment.py
@@ -130,9 +130,20 @@ class AgentsExecutionEnvironment(ABC):
else:
major_version = flink_version_manager.major_version
if major_version:
- # Determine the version-specific lib directory
- version_dir = f"flink-{major_version}"
lib_base = files("flink_agents.lib")
+
+ # Load the common JAR (shared dependencies)
+ common_lib = lib_base / "common"
+ if common_lib.is_dir():
+ for jar_file in common_lib.iterdir():
+ if jar_file.is_file() and
str(jar_file).endswith(".jar"):
+ env.add_jars(f"file://{jar_file}")
+ else:
+ err_msg = "Flink Agents common JAR not found."
+ raise FileNotFoundError(err_msg)
+
+ # Load the version-specific thin JAR
+ version_dir = f"flink-{major_version}"
version_lib = lib_base / version_dir
# Check if version-specific directory exists
diff --git a/tools/build.sh b/tools/build.sh
index 718adf55..0e17a8f3 100755
--- a/tools/build.sh
+++ b/tools/build.sh
@@ -53,8 +53,20 @@ if $build_python; then
PROJECT_VERSION=$(sed -n 's/.*<version>\(.*\)<\/version>.*/\1/p' pom.xml |
head -n 2 | tail -n 1)
- # Automatically detect and copy all Flink version JARs from dist
subdirectories
DIST_DIR="${PROJECT_ROOT}/dist"
+
+ # Copy common JAR (shared dependencies, ~110MB)
+ echo "Processing common dependencies..."
+ mkdir -p "${PYTHON_LIB_DIR}/common"
+
common_jar="${DIST_DIR}/common/target/flink-agents-dist-common-${PROJECT_VERSION}.jar"
+ if [ -f "$common_jar" ]; then
+ cp "$common_jar" "${PYTHON_LIB_DIR}/common/"
+ echo " Copied: flink-agents-dist-common-${PROJECT_VERSION}.jar"
+ else
+ echo " Warning: Common JAR file not found at $common_jar"
+ fi
+
+ # Copy thin JARs for each Flink version (only flink-agents code, ~1MB each)
for version_dir in "${DIST_DIR}"/flink-*; do
if [ -d "$version_dir" ]; then
version_name=$(basename "$version_dir")
@@ -63,13 +75,13 @@ if $build_python; then
# Create corresponding lib subdirectory
mkdir -p "${PYTHON_LIB_DIR}/${version_name}"
- # Find and copy the JAR file
-
jar_file="${version_dir}/target/flink-agents-dist-${version_name}-${PROJECT_VERSION}.jar"
- if [ -f "$jar_file" ]; then
- cp "$jar_file" "${PYTHON_LIB_DIR}/${version_name}/"
- echo " Copied:
flink-agents-dist-${version_name}-${PROJECT_VERSION}.jar"
+ # Find and copy the thin JAR file
+
thin_jar="${version_dir}/target/flink-agents-dist-${version_name}-${PROJECT_VERSION}-thin.jar"
+ if [ -f "$thin_jar" ]; then
+ cp "$thin_jar" "${PYTHON_LIB_DIR}/${version_name}/"
+ echo " Copied:
flink-agents-dist-${version_name}-${PROJECT_VERSION}-thin.jar"
else
- echo " Warning: JAR file not found at $jar_file"
+ echo " Warning: Thin JAR file not found at $thin_jar"
fi
fi
done
diff --git a/tools/releasing/create_binary_release.sh
b/tools/releasing/create_binary_release.sh
index eee8f53f..8c053c75 100755
--- a/tools/releasing/create_binary_release.sh
+++ b/tools/releasing/create_binary_release.sh
@@ -61,10 +61,14 @@ make_binary_release() {
echo "Creating binary release"
# Dynamically discover dist sub-modules (directories containing pom.xml)
+ # Exclude 'common' module - it's for Python wheel optimization only, not for
Maven release
DIST_MODULES=()
for module_dir in dist/*/; do
if [ -f "${module_dir}pom.xml" ]; then
- DIST_MODULES+=("$(basename "${module_dir}")")
+ module_name=$(basename "${module_dir}")
+ if [ "$module_name" != "common" ]; then
+ DIST_MODULES+=("$module_name")
+ fi
fi
done
