This is an automated email from the ASF dual-hosted git repository. yhu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new a38d9b94a73 Vendor grpc 1.54.0 (#26271) a38d9b94a73 is described below commit a38d9b94a738e4c488e7339ae3710fd5e1dc119e Author: Yi Hu <ya...@google.com> AuthorDate: Fri Apr 14 15:06:52 2023 -0400 Vendor grpc 1.54.0 (#26271) --- .../org/apache/beam/gradle/BeamModulePlugin.groovy | 1 + .../apache/beam/gradle/GrpcVendoring_1_54_0.groovy | 204 +++++++++++++++++++++ settings.gradle.kts | 1 + vendor/README.md | 8 +- vendor/grpc-1_54_0/build.gradle | 37 ++++ 5 files changed, 247 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 9f8537aed3f..60f01846709 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -815,6 +815,7 @@ class BeamModulePlugin implements Plugin<Project> { testcontainers_gcloud : "org.testcontainers:gcloud:$testcontainers_version", testcontainers_rabbitmq : "org.testcontainers:rabbitmq:$testcontainers_version", vendored_grpc_1_48_1 : "org.apache.beam:beam-vendor-grpc-1_48_1:0.1", + vendored_grpc_1_54_0 : "org.apache.beam:beam-vendor-grpc-1_54_0:0.1", vendored_guava_26_0_jre : "org.apache.beam:beam-vendor-guava-26_0-jre:0.1", vendored_calcite_1_28_0 : "org.apache.beam:beam-vendor-calcite-1_28_0:0.2", woodstox_core_asl : "org.codehaus.woodstox:woodstox-core-asl:4.4.1", diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_54_0.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_54_0.groovy new file mode 100644 index 00000000000..ddb317b8929 --- /dev/null +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_54_0.groovy @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.gradle + +/** + * Utilities for working with our vendored version of gRPC. + * + * To update: + * 1. Determine the set of io.grpc libraries we want to include, most likely a superset of + * of the previous vendored gRPC version. + * 2. Use mvn dependency:tree and https://search.maven.org/search?q=g:io.grpc + * to determine dependency tree. You may need to search for optional dependencies + * and determine if they are needed (e.g. conscrypt). + * 3. Validate built artifacts by running linkage tool + * (https://github.com/apache/beam/tree/master/vendor#how-to-validate-the-vendored-dependencies) + * and unit and integration tests in a PR (e.g. https://github.com/apache/beam/pull/16460, + * https://github.com/apache/beam/pull/16459) + */ +class GrpcVendoring_1_54_0 { + static def grpc_version = "1.54.0" + + // See https://github.com/grpc/grpc-java/blob/v1.54.0/gradle/libs.versions.toml + // or https://search.maven.org/search?q=g:io.grpc%201.54.0 + static def guava_version = "31.1-jre" + static def protobuf_version = "3.21.7" + static def gson_version = "2.9.0" + static def google_auth_version = "1.4.0" + static def opencensus_version = "0.31.0" + static def conscrypt_version = "2.5.2" + static def proto_google_common_protos_version = "2.9.0" + static def netty_version = "4.1.87.Final" + static def netty_tcnative_version = "2.0.56.Final" + + /** Returns the list of implementation time dependencies. */ + static List<String> dependencies() { + return [ + "com.google.guava:guava:$guava_version", + "com.google.protobuf:protobuf-java:$protobuf_version", + "com.google.protobuf:protobuf-java-util:$protobuf_version", + "com.google.code.gson:gson:$gson_version", + "io.grpc:grpc-auth:$grpc_version", + "io.grpc:grpc-core:$grpc_version", + "io.grpc:grpc-context:$grpc_version", + "io.grpc:grpc-netty:$grpc_version", + "io.grpc:grpc-protobuf:$grpc_version", + "io.grpc:grpc-stub:$grpc_version", + "io.grpc:grpc-testing:$grpc_version", + // Use a classifier to ensure we get the jar containing native libraries. In the future + // hopefully netty releases a single jar containing native libraries for all architectures. + "io.netty:netty-transport-native-epoll:$netty_version:linux-x86_64", + "io.netty:netty-tcnative-boringssl-static:$netty_tcnative_version", + "com.google.auth:google-auth-library-credentials:$google_auth_version", + "com.google.api.grpc:proto-google-common-protos:$proto_google_common_protos_version", + "io.opencensus:opencensus-api:$opencensus_version", + "io.opencensus:opencensus-contrib-grpc-metrics:$opencensus_version", + ] + } + + /** + * Returns the list of runtime time dependencies that should be exported as runtime + * dependencies within the vendored jar. + */ + static List<String> runtimeDependencies() { + return [ + 'com.google.errorprone:error_prone_annotations:2.14.0', + // TODO(BEAM-9288): Enable relocation for conscrypt + "org.conscrypt:conscrypt-openjdk-uber:$conscrypt_version" + ] + } + + /** + * Returns the list of test dependencies. + */ + static List<String> testDependencies() { + return [ + 'junit:junit:4.12', + ] + } + + static Map<String, String> relocations() { + // The relocation paths below specifically use gRPC and the full version string as + // the code relocation prefix. See https://lists.apache.org/thread.html/4c12db35b40a6d56e170cd6fc8bb0ac4c43a99aa3cb7dbae54176815@%3Cdev.beam.apache.org%3E + // for further details. + + // To produce the list of necessary relocations, one needs to start with a set of target + // packages that one wants to vendor, find all necessary transitive dependencies of that + // set and provide relocations for each such that all necessary packages and their + // dependencies are relocated. Any optional dependency that doesn't need relocation + // must be excluded via an 'exclude' rule. There is additional complexity of libraries that use + // JNI or reflection and have to be handled on case by case basis by learning whether + // they support relocation and how would one go about doing it by reading any documentation + // those libraries may provide. The 'validateShadedJarDoesntLeakNonOrgApacheBeamClasses' + // ensures that there are no classes outside of the 'org.apache.beam' namespace. + + String version = "v" + grpc_version.replace(".", "p") + String prefix = "org.apache.beam.vendor.grpc.${version}"; + List<String> packagesToRelocate = [ + // guava uses the com.google.common and com.google.thirdparty package namespaces + "com.google.common", + "com.google.thirdparty", + "com.google.protobuf", + "com.google.gson", + "com.google.auth", + "com.google.api", + "com.google.cloud", + "com.google.logging", + "com.google.longrunning", + "com.google.rpc", + "com.google.type", + "com.google.geo.type", + "io.grpc", + "io.netty", + "io.opencensus", + "io.perfmark", + ] + + return packagesToRelocate.collectEntries { + [ (it): "${prefix}.${it}" ] + } + [ + // Adapted from https://github.com/grpc/grpc-java/blob/e283f70ad91f99c7fee8b31b605ef12a4f9b1690/netty/shaded/build.gradle#L41 + // We "io.netty": "${prefix}.io.netty",have to be careful with these replacements as they must not match any + // string in NativeLibraryLoader, else they cause corruption. Note that + // this includes concatenation of string literals and constants. + 'META-INF/native/libnetty': "META-INF/native/liborg_apache_beam_vendor_grpc_${version}_netty", + 'META-INF/native/netty': "META-INF/native/org_apache_beam_vendor_grpc_${version}_netty", + 'META-INF/native/lib-netty': "META-INF/native/lib-org-apache-beam-vendor-grpc-${version}-netty", + ] + } + + /** Returns the list of shading exclusions. */ + static List<String> exclusions() { + return [ + // Don't include in the vendored jar: + // android annotations, errorprone, checkerframework, JDK8 annotations, objenesis, junit, + // commons-logging, log4j, slf4j and mockito + "android/annotation/**/", + "com/google/errorprone/**", + "com/google/instrumentation/**", + "com/google/j2objc/annotations/**", + "io/netty/handler/codec/marshalling/**", + "io/netty/handler/codec/spdy/**", + "io/netty/handler/codec/compression/JZlib*", + "io/netty/handler/codec/compression/Lz4*", + "io/netty/handler/codec/compression/Lzf*", + "io/netty/handler/codec/compression/Lzma*", + "io/netty/handler/codec/protobuf/Protobuf*Nano.class", + "io/netty/util/internal/logging/CommonsLogger*", + "io/netty/util/internal/logging/LocationAwareSlf4JLogger*", + "io/netty/util/internal/logging/Log4JLogger*", + "io/netty/util/internal/logging/Log4J2Logger*", + "javax/annotation/**", + "junit/**", + "module-info.class", + "org/checkerframework/**", + "org/codehaus/mojo/animal_sniffer/**", + "org/conscrypt/**", + "META-INF/native/libconscrypt**", + "META-INF/native/conscrypt**", + "org/hamcrest/**", + "org/junit/**", + "org/mockito/**", + "org/objenesis/**", + ] + } + + /** + * Returns a closure contaning the dependencies map used for shading gRPC within the main + * Apache Beam project. + */ + static Object dependenciesClosure() { + return { + dependencies().each { implementation it } + runtimeDependencies().each { shadow it } + } + } + + /** + * Returns a closure with the code relocation configuration for shading gRPC within the main + * Apache Beam project. + */ + static Object shadowClosure() { + return { + relocations().each { srcNamespace, destNamespace -> + relocate srcNamespace, destNamespace + } + exclusions().each { exclude it } + } + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index ecf1387e403..75f56e6faae 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -269,6 +269,7 @@ include(":sdks:typescript") include(":sdks:typescript:container") include(":vendor:bytebuddy-1_12_8") include(":vendor:grpc-1_48_1") +include(":vendor:grpc-1_54_0") include(":vendor:calcite-1_28_0") include(":vendor:guava-26_0-jre") include(":website") diff --git a/vendor/README.md b/vendor/README.md index f7e75591406..360d009a193 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -36,8 +36,8 @@ For example, when we upgrade the version of gRPC to 1.48.1 and the version of th we could run the linkage tool as following: ``` -$ ./gradlew -p vendor/grpc-1_48_1 publishMavenJavaPublicationToMavenLocal -Ppublishing -PvendoredDependenciesOnly -$ ./gradlew -PvendoredDependenciesOnly -Ppublishing -PjavaLinkageArtifactIds=beam-vendor-grpc-1_48_1:0.1-SNAPSHOT :checkJavaLinkage +$ ./gradlew -p vendor/grpc-1_54_0 publishMavenJavaPublicationToMavenLocal -Ppublishing -PvendoredDependenciesOnly +$ ./gradlew -PvendoredDependenciesOnly -Ppublishing -PjavaLinkageArtifactIds=beam-vendor-grpc-1_54_0:0.1-SNAPSHOT :checkJavaLinkage ``` ### Known Linkage Errors in the Vendored gRPC Dependencies @@ -86,12 +86,12 @@ Steps: copy to the `tempLib` folder in Beam: ``` -./gradlew -p vendor/grpc-1_48_1 publishMavenJavaPublicationToMavenLocal -Ppublishing -PvendoredDependenciesOnly +./gradlew -p vendor/grpc-1_54_0 publishMavenJavaPublicationToMavenLocal -Ppublishing -PvendoredDependenciesOnly mkdir -p tempLib/org/apache/beam # Copy files (jar/poms/metadata) to your beam repository -cp -R ~/.m2/repository/org/apache/beam/beam-vendor-grpc-1_48_1/ \ +cp -R ~/.m2/repository/org/apache/beam/beam-vendor-grpc-1_54_0/ \ tempLib/org/apache/beam ``` diff --git a/vendor/grpc-1_54_0/build.gradle b/vendor/grpc-1_54_0/build.gradle new file mode 100644 index 00000000000..69e638fdff7 --- /dev/null +++ b/vendor/grpc-1_54_0/build.gradle @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.beam.gradle.GrpcVendoring_1_54_0 + +plugins { id 'org.apache.beam.vendor-java' } + +description = "Apache Beam :: Vendored Dependencies :: gRPC :: 1.54.0" + +group = "org.apache.beam" +version = "0.1" + +vendorJava( + dependencies: GrpcVendoring_1_54_0.dependencies(), + runtimeDependencies: GrpcVendoring_1_54_0.runtimeDependencies(), + testDependencies: GrpcVendoring_1_54_0.testDependencies(), + relocations: GrpcVendoring_1_54_0.relocations(), + exclusions: GrpcVendoring_1_54_0.exclusions(), + artifactId: "beam-vendor-grpc-${GrpcVendoring_1_54_0.grpc_version.replace(".", "_")}", + groupId: group, + version: version, +)