yihua commented on code in PR #19674: URL: https://github.com/apache/hudi/pull/19674#discussion_r3833880189
########## packaging/hudi-native-spark-bundle/pom.xml: ########## @@ -0,0 +1,493 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>hudi</artifactId> + <groupId>org.apache.hudi</groupId> + <version>1.3.0-SNAPSHOT</version> + <relativePath>../../pom.xml</relativePath> + </parent> + <modelVersion>4.0.0</modelVersion> + <artifactId>hudi-native-spark${sparkbundle.version}-bundle_${scala.binary.version}</artifactId> + <packaging>jar</packaging> + + <properties> + <checkstyle.skip>true</checkstyle.skip> + <main.basedir>${project.parent.basedir}</main.basedir> + <skipTests>true</skipTests> + <javax.servlet.version>3.1.0</javax.servlet.version> + <!-- Apache DataFusion Comet releases one artifact per Spark minor version, and does not + cover every Spark version Hudi builds against. Default to producing nothing, so that a + build for an uncovered Spark version yields no bundle at all rather than one named + "native" that carries no native code. The comet-spark<version> profiles below opt in. --> + <hudi.native.bundle.skip>true</hudi.native.bundle.skip> + <hudi.native.bundle.jar.phase>none</hudi.native.bundle.jar.phase> + <maven.install.skip>${hudi.native.bundle.skip}</maven.install.skip> + <maven.deploy.skip>${hudi.native.bundle.skip}</maven.deploy.skip> + </properties> + + <build> + <plugins> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <executions> + <execution> + <id>default-jar</id> + <!-- none when this module produces no bundle. Without it maven-jar-plugin still emits + a jar named as if it were a bundle, holding nothing but the generated META-INF. + skipIfEmpty does not help: remote-resources populates target/classes regardless. --> + <phase>${hudi.native.bundle.jar.phase}</phase> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <version>${maven-shade-plugin.version}</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <skip>${hudi.native.bundle.skip}</skip> + <createSourcesJar>${shadeSources}</createSourcesJar> + <dependencyReducedPomLocation>${project.build.directory}/dependency-reduced-pom.xml + </dependencyReducedPomLocation> + <transformers> + <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer" /> + <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"> + <addHeader>true</addHeader> + </transformer> + <transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer"> + <resource>META-INF/LICENSE</resource> + <file>target/classes/META-INF/LICENSE</file> + </transformer> + <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer"> + <resource>META-INF/services/org.apache.spark.sql.sources.DataSourceRegister</resource> + </transformer> + <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> + </transformers> + <artifactSet> + <includes combine.children="append"> + <include>org.apache.hudi:hudi-hadoop-common</include> + <include>org.apache.hudi:hudi-common</include> + <include>org.apache.hudi:hudi-client-common</include> + <include>org.apache.hudi:hudi-spark-client</include> + <include>org.apache.hudi:hudi-spark-common_${scala.binary.version}</include> + <include>org.apache.hudi:hudi-spark_${scala.binary.version}</include> + <include>org.apache.hudi:${hudi.spark.module}_${scala.binary.version}</include> + <include>org.apache.hudi:${hudi.spark.common.module}</include> + <include>org.apache.hudi:hudi-hive-sync</include> + <include>org.apache.hudi:hudi-sync-common</include> + <include>org.apache.hudi:hudi-hadoop-mr</include> + <include>org.apache.hudi:hudi-timeline-service</include> + + <!-- NOTE: Comet must never be relocated. Its native library is bound through JNI + symbols named Java_org_apache_comet_Native_*, and the shared objects are + resolved as classpath resources under org/apache/comet/<os>/<arch>/, so a + relocation of org.apache.comet builds cleanly and then fails on the first + native call. The same applies to org.apache.arrow.c, which Comet carries + unshaded for the Arrow C data interface JNI bindings. --> + <include>org.apache.datafusion:*</include> + + <include>javax.servlet:javax.servlet-api</include> + <include>com.beust:jcommander</include> + <include>io.javalin:javalin</include> + <!-- Spark only has mortbay jetty --> + <include>org.eclipse.jetty:*</include> + <include>org.eclipse.jetty.websocket:*</include> + <include>org.jetbrains.kotlin:*</include> + <include>org.rocksdb:rocksdbjni</include> + <!-- Bundle Jackson JSR310 library since it is not present in spark 2.x. For spark 3.x this will + bundle the same JSR310 version that is included in spark runtime --> + <include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310</include> + <include>com.lmax:disruptor</include> + <include>com.github.davidmoten:guava-mini</include> + <include>com.github.davidmoten:hilbert-curve</include> + <include>com.github.ben-manes.caffeine:caffeine</include> + <include>org.apache.parquet:parquet-avro</include> + <include>com.twitter:chill-protobuf</include> + + <include>io.dropwizard.metrics:metrics-core</include> + <include>io.dropwizard.metrics:metrics-graphite</include> + <include>io.dropwizard.metrics:metrics-jmx</include> + <include>io.prometheus:simpleclient</include> + <include>io.prometheus:simpleclient_httpserver</include> + <include>io.prometheus:simpleclient_dropwizard</include> + <include>io.prometheus:simpleclient_pushgateway</include> + <include>io.prometheus:simpleclient_common</include> + <include>com.uber.m3:tally-m3</include> + <include>com.uber.m3:tally-core</include> + + <include>org.apache.hive:hive-common</include> + <include>org.apache.hive:hive-service</include> + <include>org.apache.hive:hive-service-rpc</include> + <include>org.apache.hive:hive-metastore</include> + <include>org.apache.hive:hive-jdbc</include> + + <include>org.apache.curator:curator-framework</include> + <include>org.apache.curator:curator-client</include> + <include>org.apache.curator:curator-recipes</include> + <include>commons-codec:commons-codec</include> + <include>commons-io:commons-io</include> + <include>org.openjdk.jol:jol-core</include> + </includes> + </artifactSet> + <relocations combine.children="append"> + <!-- NOTE: We have to relocate all classes w/in org.apache.spark.sql.avro to avoid + potential classpath collisions in case users would like to also use "spark-avro" w/in + their runtime, since Hudi carries some of the same classes as "spark-avro" --> + <relocation> + <pattern>org.apache.spark.sql.avro.</pattern> + <shadedPattern>org.apache.hudi.org.apache.spark.sql.avro.</shadedPattern> + </relocation> + <relocation> + <pattern>javax.servlet.</pattern> + <shadedPattern>org.apache.hudi.javax.servlet.</shadedPattern> + </relocation> + <relocation> + <pattern>com.beust.jcommander.</pattern> + <shadedPattern>org.apache.hudi.com.beust.jcommander.</shadedPattern> + </relocation> + <relocation> + <pattern>org.apache.commons.io.</pattern> + <shadedPattern>org.apache.hudi.org.apache.commons.io.</shadedPattern> + </relocation> + <!-- TODO: clean up hive dep - Revisit GH ISSUE #533 & PR#633--> + <relocation> + <pattern>org.apache.hive.jdbc.</pattern> + <shadedPattern>${spark.bundle.hive.shade.prefix}org.apache.hive.jdbc.</shadedPattern> + </relocation> + <relocation> + <pattern>org.apache.hadoop.hive.metastore.</pattern> + <shadedPattern>${spark.bundle.hive.shade.prefix}org.apache.hadoop.hive.metastore.</shadedPattern> + </relocation> + <relocation> + <pattern>org.apache.hive.common.</pattern> + <shadedPattern>${spark.bundle.hive.shade.prefix}org.apache.hive.common.</shadedPattern> + </relocation> + <relocation> + <pattern>org.apache.hadoop.hive.common.</pattern> + <shadedPattern>${spark.bundle.hive.shade.prefix}org.apache.hadoop.hive.common.</shadedPattern> + </relocation> + <relocation> + <pattern>org.apache.hadoop.hive.conf.</pattern> + <shadedPattern>${spark.bundle.hive.shade.prefix}org.apache.hadoop.hive.conf.</shadedPattern> + </relocation> + <relocation> + <pattern>org.apache.hive.service.</pattern> + <shadedPattern>${spark.bundle.hive.shade.prefix}org.apache.hive.service.</shadedPattern> + </relocation> + <relocation> + <pattern>org.apache.hadoop.hive.service.</pattern> + <shadedPattern>${spark.bundle.hive.shade.prefix}org.apache.hadoop.hive.service.</shadedPattern> + </relocation> + <relocation> + <pattern>com.codahale.metrics.</pattern> + <shadedPattern>org.apache.hudi.com.codahale.metrics.</shadedPattern> + </relocation> + <relocation> + <pattern>org.apache.commons.codec.</pattern> + <shadedPattern>org.apache.hudi.org.apache.commons.codec.</shadedPattern> + </relocation> + <relocation> + <pattern>org.eclipse.jetty.</pattern> + <shadedPattern>org.apache.hudi.org.apache.jetty.</shadedPattern> + </relocation> + <relocation> + <pattern>org.openjdk.jol.</pattern> + <shadedPattern>org.apache.hudi.org.openjdk.jol.</shadedPattern> + </relocation> + <relocation> + <pattern>com.uber.m3.</pattern> + <shadedPattern>org.apache.hudi.com.uber.m3.</shadedPattern> + </relocation> + </relocations> + <filters> Review Comment: I think this is a Comet problem that does not affect Hudi in a materialized way now. I'd leave it as is. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
