This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push: new 9e88b4d Update pulsar netty/grpc dependencies (#1844) 9e88b4d is described below commit 9e88b4de751f0fd3c6e1f7d5aaa176c1cf84c52d Author: Sijie Guo <guosi...@gmail.com> AuthorDate: Fri May 25 16:14:49 2018 -0700 Update pulsar netty/grpc dependencies (#1844) *Motivation* apache/pulsar#1816 breaks the pulsar functions running in process mode. This because it removes the shading and introduces the conflicts between the netty version that grpc/bk depends and the netty version pulsar depends. the grpc doesn't work which fails the health check on functions *Solution* Update the netty/grpc/protobuf to align the versions to avoid conflicts between versions. --- pom.xml | 34 ++++++++++++-- .../pulsar/broker/namespace/OwnershipCache.java | 6 +-- pulsar-client-tools/pom.xml | 16 +++++++ .../pulsar/common/policies/data/FailureDomain.java | 6 +-- pulsar-functions/instance/pom.xml | 11 +++++ pulsar-functions/runtime-all/pom.xml | 54 ---------------------- pulsar-functions/runtime/pom.xml | 5 ++ .../pulsar/functions/runtime/ProcessRuntime.java | 1 + .../pulsar/functions/worker/FunctionActioner.java | 6 ++- 9 files changed, 70 insertions(+), 69 deletions(-) diff --git a/pom.xml b/pom.xml index 90fa1fa..a59c94f 100644 --- a/pom.xml +++ b/pom.xml @@ -123,7 +123,7 @@ flexible messaging model and an intuitive client API.</description> <bookkeeper.version>4.7.0</bookkeeper.version> <zookeeper.version>3.5.4-beta</zookeeper.version> - <netty.version>4.1.21.Final</netty.version> + <netty.version>4.1.22.Final</netty.version> <storm.version>1.0.5</storm.version> <jetty.version>9.3.11.v20160721</jetty.version> <jersey.version>2.25</jersey.version> @@ -140,8 +140,8 @@ flexible messaging model and an intuitive client API.</description> <typetools.version>0.5.0</typetools.version> <jboss-reflect.version>2.2.1.SP1</jboss-reflect.version> <protobuf2.version>2.4.1</protobuf2.version> - <protobuf3.version>3.4.0</protobuf3.version> - <grpc.version>1.5.0</grpc.version> + <protobuf3.version>3.5.1</protobuf3.version> + <grpc.version>1.12.0</grpc.version> <protoc-gen-grpc-java.version>1.0.0</protoc-gen-grpc-java.version> <gson.version>2.8.2</gson.version> <sketches.version>0.8.3</sketches.version> @@ -266,6 +266,11 @@ flexible messaging model and an intuitive client API.</description> <groupId>org.jboss.netty</groupId> <artifactId>netty</artifactId> </exclusion> + <!-- exclude all netty dependencies, use whatever pulsar is using --> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-*</artifactId> + </exclusion> </exclusions> </dependency> @@ -273,11 +278,23 @@ flexible messaging model and an intuitive client API.</description> <groupId>org.apache.bookkeeper</groupId> <artifactId>stream-storage-java-client</artifactId> <version>${bookkeeper.version}</version> + <exclusions> + <exclusion> + <groupId>*</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.bookkeeper</groupId> - <artifactId>bookkeeper-bookkeeper-stats-api</artifactId> + <artifactId>bookkeeper-common</artifactId> + <version>${bookkeeper.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.bookkeeper.stats</groupId> + <artifactId>bookkeeper-stats-api</artifactId> <version>${bookkeeper.version}</version> </dependency> @@ -356,7 +373,7 @@ flexible messaging model and an intuitive client API.</description> <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> - <version>20.0</version> + <version>21.0</version> </dependency> <dependency> @@ -725,6 +742,13 @@ flexible messaging model and an intuitive client API.</description> <groupId>org.apache.distributedlog</groupId> <artifactId>distributedlog-core</artifactId> <version>${bookkeeper.version}</version> + <exclusions> + <!-- exclude bookkeeper, reply on the bookkeeper version that pulsar uses --> + <exclusion> + <groupId>org.apache.bookkeeper</groupId> + <artifactId>bookkeeper-server</artifactId> + </exclusion> + </exclusions> </dependency> <!-- test dependencies --> diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java index e681fe7..f7dce01 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java @@ -18,8 +18,6 @@ */ package org.apache.pulsar.broker.namespace; -import static com.google.common.base.Preconditions.checkState; - import java.util.List; import java.util.Map; import java.util.Optional; @@ -47,8 +45,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.github.benmanes.caffeine.cache.AsyncCacheLoader; import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.github.benmanes.caffeine.cache.Caffeine; -import com.github.benmanes.caffeine.cache.RemovalCause; -import com.github.benmanes.caffeine.cache.RemovalListener; import com.google.common.collect.Lists; import com.google.common.util.concurrent.MoreExecutors; @@ -163,7 +159,7 @@ public class OwnershipCache { this.localZkCache = pulsar.getLocalZkCache(); this.ownershipReadOnlyCache = pulsar.getLocalZkCacheService().ownerInfoCache(); // ownedBundlesCache contains all namespaces that are owned by the local broker - this.ownedBundlesCache = Caffeine.newBuilder().executor(MoreExecutors.sameThreadExecutor()) + this.ownedBundlesCache = Caffeine.newBuilder().executor(MoreExecutors.directExecutor()) .buildAsync(new OwnedServiceUnitCacheLoader()); } diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index 5f044f5..8f94210 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -80,6 +80,22 @@ <version>${project.version}</version> </dependency> + <dependency> + <groupId>net.jodah</groupId> + <artifactId>typetools</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.bookkeeper</groupId> + <artifactId>stream-storage-java-client</artifactId> + <exclusions> + <exclusion> + <groupId>*</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> + <!-- functions related dependencies (end) --> </dependencies> diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/FailureDomain.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/FailureDomain.java index dd1d09b..408b3aa 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/FailureDomain.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/FailureDomain.java @@ -18,11 +18,11 @@ */ package org.apache.pulsar.common.policies.data; +import com.google.common.base.MoreObjects; +import com.google.common.base.Objects; import java.util.HashSet; import java.util.Set; -import com.google.common.base.Objects; - public class FailureDomain { public Set<String> brokers = new HashSet<String>(); @@ -47,6 +47,6 @@ public class FailureDomain { @Override public String toString() { - return Objects.toStringHelper(this).add("brokers", brokers).toString(); + return MoreObjects.toStringHelper(this).add("brokers", brokers).toString(); } } diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index e5cc988..11cefa5 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -74,6 +74,17 @@ <dependency> <groupId>org.apache.bookkeeper</groupId> <artifactId>stream-storage-java-client</artifactId> + <exclusions> + <exclusion> + <groupId>*</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.apache.bookkeeper</groupId> + <artifactId>bookkeeper-common</artifactId> </dependency> <dependency> diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml index 813f4c6..9219c4b 100644 --- a/pulsar-functions/runtime-all/pom.xml +++ b/pulsar-functions/runtime-all/pom.xml @@ -38,60 +38,6 @@ <groupId>${project.groupId}</groupId> <artifactId>pulsar-functions-runtime</artifactId> <version>${project.parent.version}</version> - <exclusions> - <exclusion> - <groupId>io.grpc</groupId> - <artifactId>grpc-all</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-lite</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.protobuf.nano</groupId> - <artifactId>protobuf-javanano</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java-util</artifactId> - </exclusion> - <exclusion> - <groupId>io.grpc</groupId> - <artifactId>grpc-protobuf</artifactId> - </exclusion> - <exclusion> - <groupId>io.grpc</groupId> - <artifactId>grpc-protobuf-lite</artifactId> - </exclusion> - <exclusion> - <groupId>io.grpc</groupId> - <artifactId>grpc-protobuf-nano</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.pulsar</groupId> - <artifactId>pulsar-functions-api</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.pulsar</groupId> - <artifactId>pulsar-functions-utils</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.pulsar</groupId> - <artifactId>pulsar-functions-metrics</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.pulsar</groupId> - <artifactId>pulsar-functions-runtime</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.pulsar</groupId> - <artifactId>pulsar-functions-instance</artifactId> - </exclusion> - </exclusions> </dependency> <!-- logging --> diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index 419d77b..61af0ee 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -50,6 +50,11 @@ <artifactId>grpc-all</artifactId> </dependency> + <dependency> + <groupId>com.beust</groupId> + <artifactId>jcommander</artifactId> + </dependency> + </dependencies> <build> diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java index 977b9b1..c5d0109 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java @@ -215,6 +215,7 @@ class ProcessRuntime implements Runtime { public void run() { CompletableFuture<InstanceCommunication.HealthCheckResult> result = healthCheck(); try { + result.get(); } catch (Exception e) { log.error("Health check failed for {}-{}", instanceConfig.getFunctionDetails().getName(), diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java index 5586114..4edee60 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.functions.worker; +import com.google.common.io.MoreFiles; +import com.google.common.io.RecursiveDeleteOption; import java.io.IOException; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; @@ -25,7 +27,6 @@ import java.nio.file.Paths; import lombok.*; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.distributedlog.api.namespace.Namespace; import org.apache.pulsar.functions.proto.Function; @@ -189,7 +190,8 @@ public class FunctionActioner implements AutoCloseable { if (pkgDir.exists()) { try { - FileUtils.deleteDirectory(pkgDir); + MoreFiles.deleteRecursively( + Paths.get(pkgDir.toURI()), RecursiveDeleteOption.ALLOW_INSECURE); } catch (IOException e) { log.warn("Failed to delete package for function: {}", FunctionDetailsUtils.getFullyQualifiedName(functionMetaData.getFunctionDetails()), e); -- To stop receiving notification emails like this one, please contact si...@apache.org.