gh-yzou commented on code in PR #1303:
URL: https://github.com/apache/polaris/pull/1303#discussion_r2029673198
##########
plugins/spark/v3.5/build.gradle.kts:
##########
@@ -41,18 +41,37 @@ val scalaVersion = getAndUseScalaVersionForProject()
val icebergVersion = pluginlibs.versions.iceberg.get()
val spark35Version = pluginlibs.versions.spark35.get()
+val scalaLibraryVersion =
+ if (scalaVersion.equals("2.12")) {
+ pluginlibs.versions.scala212.get()
+ } else {
+ pluginlibs.versions.scala213.get()
+ }
+
dependencies {
implementation(project(":polaris-api-iceberg-service")) {
- // exclude the iceberg and jackson dependencies, use the
- // dependencies packed in the iceberg-spark dependency
+ // exclude the iceberg dependencies, use the ones pulled
+ // by iceberg-core
exclude("org.apache.iceberg", "*")
- exclude("com.fasterxml.jackson.core", "*")
}
+ implementation(project(":polaris-api-catalog-service"))
+ implementation(project(":polaris-core")) { exclude("org.apache.iceberg",
"*") }
+
+ implementation("org.apache.iceberg:iceberg-core:${icebergVersion}")
+ implementation("com.fasterxml.jackson.core:jackson-annotations")
+ implementation("com.fasterxml.jackson.core:jackson-core")
+ implementation("com.fasterxml.jackson.core:jackson-databind")
implementation(
"org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}:${icebergVersion}"
- )
+ ) {
+ // exclude the iceberg rest dependencies, use the ones pulled
+ // with iceberg-core dependency
+ exclude("org.apache.iceberg.rest", "*")
Review Comment:
You are right. The reason I exclude it here is because I want to make sure I
am using the classes in the rest modular from polaris-core, instead of the one
packed in the iceberg-spark-runtime.
The reason is that the request and response class generated by Polaris are
compiled with "com.fasterxml.jackson.annotation.*". However, all rest class in
packed in the iceberg-spark-runtime are compiled with
org.apache.iceberg.shaded.com.fasterxml.jackson.annotation.*.
The problem raised due to this is the annotation detection doesn't work
anymore during deserialization because of the class mismatch, where the
deserialization class is looking for
org.apache.iceberg.shaded.com.fasterxml.jackson.annotation.JsonProperty, but
the response have com.fasterxml.jackson.annotation.JsonProperty.
Therefore, i simply switched to use all non-shaded and non-relocated classes
for now.
In near future, one way i am thinking we can potentially do if we want to
use all shaded class provided in the iceberg package is regenerate the response
and request class for client plugin only using shaded jackson library. Then
they can all be compiled with the same library, and the only contract we will
have with server is the spec.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
