This is an automated email from the ASF dual-hosted git repository.
ahmedabu98 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 849fcf89c81 [Beam SQL Shell] Add Iceberg deps when using icebergio
(#38994)
849fcf89c81 is described below
commit 849fcf89c81e7b8ac2ccdf8e3596d35993ff33ab
Author: Ahmed Abualsaud <[email protected]>
AuthorDate: Wed Jun 17 17:45:08 2026 -0400
[Beam SQL Shell] Add Iceberg deps when using icebergio (#38994)
* add iceberg sql dep when using icebergio
* add maven snapshot repo
---
scripts/beam-sql.sh | 73 ++++++++++++++++++++++++++++++++++++++++++++++++-----
1 file changed, 67 insertions(+), 6 deletions(-)
diff --git a/scripts/beam-sql.sh b/scripts/beam-sql.sh
index c9ec365b99b..9df48f6c18d 100755
--- a/scripts/beam-sql.sh
+++ b/scripts/beam-sql.sh
@@ -36,6 +36,8 @@
MAVEN_DISTRIBUTION_URL="https://repo.maven.apache.org/maven2/org/apache/maven/ap
# Maven Plugin Configuration
MAVEN_SHADE_PLUGIN_VERSION="3.5.1"
+ICEBERG_VERSION="${ICEBERG_VERSION:-1.10.0}"
+APACHE_SNAPSHOT_REPOSITORY_URL="https://repository.apache.org/content/repositories/snapshots/";
mkdir -p "${CACHE_DIR}"
# Create a temporary directory for our Maven project.
@@ -84,6 +86,38 @@ function setup_maven_wrapper() {
MAVEN_CMD="${mvnw_script}"
}
+function add_beam_dependency() {
+ local artifact_id="$1"
+ cat >> "${POM_FILE}" << EOL
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>${artifact_id}</artifactId>
+ <version>\${beam.version}</version>
+ </dependency>
+EOL
+}
+
+function add_dependency() {
+ local group_id="$1"
+ local artifact_id="$2"
+ local version="$3"
+ cat >> "${POM_FILE}" << EOL
+ <dependency>
+ <groupId>${group_id}</groupId>
+ <artifactId>${artifact_id}</artifactId>
+ <version>${version}</version>
+ </dependency>
+EOL
+}
+
+function normalize_beam_version() {
+ case "${BEAM_VERSION}" in
+ *-[sS][nN][aA][pP][sS][hH][oO][tT])
+ BEAM_VERSION="${BEAM_VERSION%-*}-SNAPSHOT"
+ ;;
+ esac
+}
+
function usage() {
echo "Usage: $0 [--version <beam_version>] [--runner <runner_name>] [--io
<io_connector>] [--list-versions] [--list-ios] [--list-runners] [--debug]
[-h|--help]"
echo ""
@@ -91,6 +125,7 @@ function usage() {
echo ""
echo "Options:"
echo " --version Specify the Apache Beam version (default:
${DEFAULT_BEAM_VERSION})."
+ echo " SNAPSHOT versions are resolved from Apache's Maven
snapshot repository."
echo " --runner Specify the Beam runner to use (default: direct)."
echo " Supported runners:"
echo " direct - DirectRunner (runs locally, good for
development)"
@@ -208,7 +243,7 @@ function list_runners() {
echo "✅ Available runners for Beam ${BEAM_VERSION}:"
echo ""
-
+
# Process each runner and provide descriptions
while IFS= read -r runner; do
case "$runner" in
@@ -217,7 +252,7 @@ function list_runners() {
echo " Runs locally on your machine. Good for
development and testing."
;;
"google-cloud-dataflow-java")
- echo " dataflow - DataflowRunner"
+ echo " dataflow - DataflowRunner"
echo " Runs on Google Cloud Dataflow for
production workloads."
;;
flink-*)
@@ -273,7 +308,7 @@ function list_runners() {
;;
esac
done <<< "$runners"
-
+
echo ""
echo "💡 Usage: ./beam-sql.sh --runner <runner_name>"
echo " Default: direct"
@@ -291,7 +326,7 @@ DEBUG_MODE=false
while [[ "$#" -gt 0 ]]; do
case $1 in
- --version) BEAM_VERSION="$2"; shift ;;
+ --version) BEAM_VERSION="$2"; normalize_beam_version; shift ;;
--runner) BEAM_RUNNER=$(echo "$2" | tr '[:upper:]' '[:lower:]'); shift ;;
--io) IO_CONNECTORS+=("$2"); shift ;;
--list-versions) list_versions; exit 0 ;;
@@ -363,7 +398,15 @@ else
EOL
# Add IO and Runner dependencies
for io in "${IO_CONNECTORS[@]}"; do
- echo "
<dependency><groupId>org.apache.beam</groupId><artifactId>beam-sdks-java-io-${io}</artifactId><version>\${beam.version}</version></dependency>"
>> "${POM_FILE}"
+ add_beam_dependency "beam-sdks-java-io-${io}"
+ case "${io}" in
+ iceberg)
+ add_beam_dependency "beam-sdks-java-extensions-sql-iceberg"
+ add_dependency "org.apache.iceberg" "iceberg-aws" "${ICEBERG_VERSION}"
+ add_dependency "org.apache.iceberg" "iceberg-aws-bundle"
"${ICEBERG_VERSION}"
+ add_dependency "org.apache.iceberg" "iceberg-gcp" "${ICEBERG_VERSION}"
+ ;;
+ esac
done
RUNNER_ARTIFACT=""
case "${BEAM_RUNNER}" in
@@ -372,12 +415,25 @@ EOL
*) echo "❌ Error: Unsupported runner '${BEAM_RUNNER}'." >&2; exit 1 ;;
esac
if [ -n "${RUNNER_ARTIFACT}" ]; then
- echo "
<dependency><groupId>org.apache.beam</groupId><artifactId>${RUNNER_ARTIFACT}</artifactId><version>\${beam.version}</version></dependency>"
>> "${POM_FILE}"
+ add_beam_dependency "${RUNNER_ARTIFACT}"
fi
# Complete the POM with the build section for the maven-shade-plugin
cat >> "${POM_FILE}" << EOL
</dependencies>
+ <repositories>
+ <repository>
+ <id>apache.snapshots</id>
+ <url>${APACHE_SNAPSHOT_REPOSITORY_URL}</url>
+ <releases>
+ <enabled>false</enabled>
+ </releases>
+ <snapshots>
+ <enabled>true</enabled>
+ <updatePolicy>always</updatePolicy>
+ </snapshots>
+ </repository>
+ </repositories>
<properties>
<beam.version>${BEAM_VERSION}</beam.version>
</properties>
@@ -395,6 +451,11 @@ cat >> "${POM_FILE}" << EOL
</goals>
<configuration>
<transformers>
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <manifestEntries>
+ <Multi-Release>true</Multi-Release>
+ </manifestEntries>
+ </transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
</transformers>
<filters>
