nryanov commented on issue #12521:
URL: https://github.com/apache/iceberg/issues/12521#issuecomment-2722409077
```
dependencies {
compileOnly libs.spark.sql
compileOnly libs.spark.core
compileOnly libs.spark.avro
compileOnly libs.iceberg.spark.runtime
compileOnly libs.iceberg.aws.bundle
compileOnly libs.iceberg.hive.metastore
compileOnly libs.hadoop.aws
compileOnly libs.hadoop.client
compileOnly libs.aws.sdk.bundle
compileOnly libs.logj4.api
compileOnly libs.logj4.client
implementation libs.jcommander
implementation libs.caffeine
}
jar {
manifest.attributes["Main-Class"] = "Application"
}
tasks.named('shadowJar',
com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
archiveBaseName = 'jobs'
archiveClassifier = ''
archiveVersion = ''
manifest {
inheritFrom(project.tasks.jar.manifest)
}
}
```
```
[versions]
# lib versions should be in sync according to
https://github.com/apache/iceberg/blob/apache-iceberg-1.7.1/gradle/libs.versions.toml
spark = "3.5.2"
iceberg = "1.7.1"
hadoop = "3.4.1"
aws = "2.29.1"
log4j = "2.24.3"
jcommander = "2.0"
caffeine = "2.9.3"
[libraries]
spark-hive = { group = "org.apache.spark", name = "spark-hive_2.12",
version.ref = "spark" }
spark-sql = { group = "org.apache.spark", name = "spark-sql_2.12",
version.ref = "spark" }
spark-core = { group = "org.apache.spark", name = "spark-core_2.12",
version.ref = "spark" }
spark-avro = { group = "org.apache.spark", name = "spark-avro_2.12",
version.ref = "spark" }
iceberg-spark-runtime = { group = "org.apache.iceberg", name =
"iceberg-spark-runtime-3.5_2.12", version.ref = "iceberg" }
iceberg-hive-metastore = { group = "org.apache.iceberg", name =
"iceberg-hive-metastore", version.ref = "iceberg" }
iceberg-hive-runtime = { group = "org.apache.iceberg", name =
"iceberg-hive-runtime", version.ref = "iceberg" }
iceberg-aws-bundle = { group = "org.apache.iceberg", name =
"iceberg-aws-bundle", version.ref = "iceberg" }
hadoop-aws = { group = "org.apache.hadoop", name = "hadoop-aws", version.ref
= "hadoop" }
hadoop-client = { group = "org.apache.hadoop", name = "hadoop-client",
version.ref = "hadoop" }
aws-sdk-bundle = { group = "software.amazon.awssdk", name = "bundle",
version.ref = "aws" }
logj4-api = { group = "org.apache.logging.log4j", name = "log4j-api",
version.ref = "log4j" }
logj4-client = { group = "org.apache.logging.log4j", name = "log4j-core",
version.ref = "log4j" }
jcommander = { group = "org.jcommander", name = "jcommander", version.ref =
"jcommander" }
caffeine = { group = "com.github.ben-manes.caffeine", name = "caffeine",
version.ref = "caffeine" }
```
These are dependencies in my app. And other runtime libs are installed like
this:
```
FROM apache/spark:3.5.2-scala2.12-java17-ubuntu
ENV SPARK_VERSION=3.5.2
ENV SPARK_MAJOR_VERSION=3.5
ENV SCALA_VERSION=2.12
ENV ICEBERG_VERSION=1.7.1
ENV AWS_SDK_VERSION=2.29.1
ENV HADOOP_AWS_VERSION=3.4.1
RUN wget -O /opt/spark/jars/bundle-${AWS_SDK_VERSION}.jar
https://repo1.maven.org/maven2/software/amazon/awssdk/bundle/${AWS_SDK_VERSION}/bundle-${AWS_SDK_VERSION}.jar
RUN wget -O /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar
https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar
&& \
wget -O /opt/spark/jars/iceberg-hive-metastore-${ICEBERG_VERSION}.jar
https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-hive-metastore/${ICEBERG_VERSION}/iceberg-hive-metastore-${ICEBERG_VERSION}.jar
&& \
wget -O /opt/spark/jars/hadoop-aws-${HADOOP_AWS_VERSION}.jar
https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_AWS_VERSION}/hadoop-aws-${HADOOP_AWS_VERSION}.jar
&& \
wget -O /opt/spark/jars/hadoop-client-${HADOOP_AWS_VERSION}.jar
https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-client/${HADOOP_AWS_VERSION}/hadoop-client-${HADOOP_AWS_VERSION}.jar
&& \
wget -O /opt/spark/jars/hadoop-client-api-${HADOOP_AWS_VERSION}.jar
https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-client-api/${HADOOP_AWS_VERSION}/hadoop-client-api-${HADOOP_AWS_VERSION}.jar
&& \
wget -O /opt/spark/jars/hadoop-client-runtime-${HADOOP_AWS_VERSION}.jar
https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-client-runtime/${HADOOP_AWS_VERSION}/hadoop-client-runtime-${HADOOP_AWS_VERSION}.jar
&& \
wget -O /opt/spark/jars/spark-avro_${SCALA_VERSION}-${SPARK_VERSION}.jar
https://repo1.maven.org/maven2/org/apache/spark/spark-avro_${SCALA_VERSION}/${SPARK_VERSION}/spark-avro_${SCALA_VERSION}-${SPARK_VERSION}.jar
&& \
wget -O /opt/spark/jars/caffeine-2.9.3.jar
https://repo1.maven.org/maven2/com/github/ben-manes/caffeine/caffeine/2.9.3/caffeine-2.9.3.jar
&& \
wget -O
/opt/spark/jars/iceberg-spark-runtime-${SPARK_MAJOR_VERSION}_${SCALA_VERSION}-${ICEBERG_VERSION}.jar
https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${SPARK_MAJOR_VERSION}_${SCALA_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${SPARK_MAJOR_VERSION}_${SCALA_VERSION}-${ICEBERG_VERSION}.jar
RUN rm /opt/spark/jars/hadoop-client-api-3.3.4.jar && \
rm /opt/spark/jars/hadoop-client-runtime-3.3.4.jar && \
rm /opt/spark/jars/hadoop-yarn-server-web-proxy-3.3.4.jar
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
